From 61b02e7a2894e75974908843f42f9af938dee95e Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Mon, 15 Sep 2025 17:07:50 -0400 Subject: [PATCH 01/17] feat(switch): configure host network and switch network --- Cargo.lock | 10 + Cargo.toml | 14 +- harmony/Cargo.toml | 1 + harmony/src/domain/topology/ha_cluster.rs | 19 ++ harmony/src/domain/topology/network.rs | 45 +++- .../modules/okd/bootstrap_03_control_plane.rs | 28 ++- harmony/src/modules/okd/host_network.rs | 205 ++++++++++++++++++ harmony/src/modules/okd/mod.rs | 1 + 8 files changed, 313 insertions(+), 10 deletions(-) create mode 100644 harmony/src/modules/okd/host_network.rs diff --git a/Cargo.lock b/Cargo.lock index 2af94a0..1ec5d60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,6 +429,15 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "assertor" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff24d87260733dc86d38a11c60d9400ce4a74a05d0dafa2a6f5ab249cd857cb" +dependencies = [ + "num-traits", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -2305,6 +2314,7 @@ name = "harmony" version = "0.1.0" dependencies = [ "askama", + "assertor", "async-trait", "base64 0.22.1", "bollard", diff --git a/Cargo.toml b/Cargo.toml index d92c0e7..32231d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,8 @@ members = [ "harmony_composer", "harmony_inventory_agent", "harmony_secret_derive", - "harmony_secret", "adr/agent_discovery/mdns", + "harmony_secret", + "adr/agent_discovery/mdns", ] [workspace.package] @@ -66,5 +67,12 @@ thiserror = "2.0.14" serde = { version = "1.0.209", features = ["derive", "rc"] } serde_json = "1.0.127" askama = "0.14" -sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite" ] } -reqwest = { version = "0.12", features = ["blocking", "stream", "rustls-tls", "http2", "json"], default-features = false } +sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } +reqwest = { version = "0.12", features = [ + "blocking", + "stream", + "rustls-tls", + "http2", + "json", +], default-features = false } +assertor = "0.0.4" diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index ad57db1..f9e671a 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -80,3 +80,4 @@ inquire.workspace = true [dev-dependencies] pretty_assertions.workspace = true +assertor.workspace = true diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index c9f565e..ac53d01 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -7,6 +7,7 @@ use log::info; use crate::data::FileContent; use crate::executors::ExecutorError; +use crate::hardware::PhysicalHost; use crate::topology::PxeOptions; use super::DHCPStaticEntry; @@ -15,6 +16,7 @@ use super::DnsRecord; use super::DnsRecordType; use super::DnsServer; use super::Firewall; +use super::HostNetworkConfig; use super::HttpServer; use super::IpAddress; use super::K8sclient; @@ -24,6 +26,8 @@ use super::LogicalHost; use super::PreparationError; use super::PreparationOutcome; use super::Router; +use super::Switch; +use super::SwitchError; use super::TftpServer; use super::Topology; @@ -263,6 +267,21 @@ impl HttpServer for HAClusterTopology { } } +#[async_trait] +impl Switch for HAClusterTopology { + async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { + todo!() + } + + async fn configure_host_network( + &self, + _host: &PhysicalHost, + _config: HostNetworkConfig, + ) -> Result<(), SwitchError> { + todo!() + } +} + #[derive(Debug)] pub struct DummyInfra; diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index c7ab5cc..a6127c8 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -1,10 +1,11 @@ -use std::{net::Ipv4Addr, str::FromStr, sync::Arc}; +use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc}; use async_trait::async_trait; +use derive_new::new; use harmony_types::net::{IpAddress, MacAddress}; use serde::Serialize; -use crate::executors::ExecutorError; +use crate::{executors::ExecutorError, hardware::PhysicalHost}; use super::{LogicalHost, k8s::K8sClient}; @@ -172,6 +173,46 @@ impl FromStr for DnsRecordType { } } +#[async_trait] +pub trait Switch: Send + Sync { + async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option; + + async fn configure_host_network( + &self, + _host: &PhysicalHost, + _config: HostNetworkConfig, + ) -> Result<(), SwitchError>; +} + +#[derive(Clone, Debug, PartialEq)] +pub struct HostNetworkConfig { + pub bond: Bond, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Bond { + pub interfaces: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SlaveInterface { + pub mac_address: MacAddress, + // FIXME: Should we add speed as well? And other params +} + +#[derive(Debug, Clone, new)] +pub struct SwitchError { + msg: String, +} + +impl std::fmt::Display for SwitchError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.msg) + } +} + +impl Error for SwitchError {} + #[cfg(test)] mod test { use std::sync::Arc; diff --git a/harmony/src/modules/okd/bootstrap_03_control_plane.rs b/harmony/src/modules/okd/bootstrap_03_control_plane.rs index ba9e12d..dd0fdee 100644 --- a/harmony/src/modules/okd/bootstrap_03_control_plane.rs +++ b/harmony/src/modules/okd/bootstrap_03_control_plane.rs @@ -5,11 +5,13 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::{HostRole, Inventory}, modules::{ - dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore, - inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl, + dhcp::DhcpHostBindingScore, + http::IPxeMacBootFileScore, + inventory::DiscoverHostForRoleScore, + okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl}, }, score::Score, - topology::{HAClusterTopology, HostBinding}, + topology::{self, HAClusterTopology, HostBinding}, }; use async_trait::async_trait; use derive_new::new; @@ -209,8 +211,23 @@ impl OKDSetup03ControlPlaneInterpret { Ok(()) } + // TODO: Apply host network configuration. + // Delegate to a score: HostNetworkConfigurationScore { host: physical_host } qui manipule Switch dans Topology + // Use-case Affilium: remplacement carte reseau, pas juste installation clean + // /// Placeholder for automating network bonding configuration. - async fn persist_network_bond(&self) -> Result<(), InterpretError> { + async fn persist_network_bond( + &self, + inventory: &Inventory, + topology: &HAClusterTopology, + hosts: &Vec, + ) -> Result<(), InterpretError> { + let score = HostNetworkConfigurationScore { + hosts: hosts.clone(), // FIXME: Avoid clone if possible + }; + + score.interpret(inventory, topology); + // Generate MC or NNCP from inventory NIC data; apply via ignition or post-join. info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP"); inquire::Confirm::new( @@ -260,7 +277,8 @@ impl Interpret for OKDSetup03ControlPlaneInterpret { self.reboot_targets(&nodes).await?; // 5. Placeholder for post-boot network configuration (e.g., bonding). - self.persist_network_bond().await?; + self.persist_network_bond(inventory, topology, &nodes) + .await?; // TODO: Implement a step to wait for the control plane nodes to join the cluster // and for the cluster operators to become available. This would be similar to diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs new file mode 100644 index 0000000..60a89e7 --- /dev/null +++ b/harmony/src/modules/okd/host_network.rs @@ -0,0 +1,205 @@ +use async_trait::async_trait; +use harmony_types::{id::Id, net::MacAddress}; +use serde::Serialize; + +use crate::{ + data::Version, + hardware::PhysicalHost, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + score::Score, + topology::{self, Bond, HostNetworkConfig, SlaveInterface, Switch, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct HostNetworkConfigurationScore { + pub hosts: Vec, +} + +impl Score for HostNetworkConfigurationScore { + fn name(&self) -> String { + "HostNetworkConfigurationScore".into() + } + + fn create_interpret(&self) -> Box> { + Box::new(HostNetworkConfigurationInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug)] +pub struct HostNetworkConfigurationInterpret { + score: HostNetworkConfigurationScore, +} + +#[async_trait] +impl Interpret for HostNetworkConfigurationInterpret { + fn get_name(&self) -> InterpretName { + InterpretName::Custom("HostNetworkConfigurationInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + vec![] + } + + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + let host = self.score.hosts.first().unwrap(); + let mac_addresses = host.get_mac_address(); + let mac_address = mac_addresses.first().unwrap(); + let host_network_config = HostNetworkConfig { + bond: Bond { + interfaces: vec![SlaveInterface { + mac_address: *mac_address, + }], + }, + }; + + let _ = topology + .configure_host_network(host, host_network_config) + .await; + + // foreach hosts + // foreach mac addresses + // let port = topology.get_port_for_mac_address(); // si pas de port -> mac address pas connectee + // create port channel for all ports found + // create bond for all valid addresses (port found) + // apply network to host first, then switch (to avoid losing hosts that are already connected) + // topology.configure_host_network(host, config) <--- will create bonds + // topology.configure_switch_network(port, config) <--- will create port channels + + Ok(Outcome::success("".into())) + } +} + +struct PortMapping { + port: String, + mac_address: MacAddress, +} + +#[cfg(test)] +mod tests { + use assertor::*; + use harmony_inventory_agent::hwinfo::NetworkInterface; + use lazy_static::lazy_static; + + use crate::{ + hardware::HostCategory, + topology::{ + Bond, HostNetworkConfig, PreparationError, PreparationOutcome, SlaveInterface, + SwitchError, + }, + }; + use std::{ + str::FromStr, + sync::{Arc, Mutex}, + }; + + use super::*; + + lazy_static! { + pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap(); + pub static ref INTERFACE: MacAddress = + MacAddress::try_from("00:11:22:33:44:55".to_string()).unwrap(); + } + + #[tokio::test] + async fn one_host_one_mac_address_should_create_bond_with_one_interface() { + let host = given_host(&HOST_ID, *INTERFACE); + let score = given_score(vec![host]); + let topology = SwitchWithPortTopology::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![( + HOST_ID.clone(), + HostNetworkConfig { + bond: Bond { + interfaces: vec![SlaveInterface { + mac_address: *INTERFACE, + }], + }, + }, + )]); + } + + fn given_host(id: &Id, mac_address: MacAddress) -> PhysicalHost { + PhysicalHost { + id: id.clone(), + category: HostCategory::Server, + network: vec![NetworkInterface { + name: "interface-1".into(), + mac_address, + speed_mbps: None, + is_up: true, + mtu: 1, + ipv4_addresses: vec![], + ipv6_addresses: vec![], + driver: "driver".into(), + firmware_version: None, + }], + storage: vec![], + labels: vec![], + memory_modules: vec![], + cpus: vec![], + } + } + + fn given_score(hosts: Vec) -> HostNetworkConfigurationScore { + HostNetworkConfigurationScore { hosts } + } + + struct SwitchWithPortTopology { + configured_host_networks: Arc>>, + } + + impl SwitchWithPortTopology { + fn new() -> Self { + Self { + configured_host_networks: Arc::new(Mutex::new(vec![])), + } + } + } + + #[async_trait] + impl Topology for SwitchWithPortTopology { + fn name(&self) -> &str { + "SwitchWithPortTopology" + } + + async fn ensure_ready(&self) -> Result { + Ok(PreparationOutcome::Success { details: "".into() }) + } + } + + #[async_trait] + impl Switch for SwitchWithPortTopology { + async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { + Some("1/0/42".into()) + } + + async fn configure_host_network( + &self, + host: &PhysicalHost, + config: HostNetworkConfig, + ) -> Result<(), SwitchError> { + let mut configured_host_networks = self.configured_host_networks.lock().unwrap(); + configured_host_networks.push((host.id.clone(), config.clone())); + + Ok(()) + } + } +} diff --git a/harmony/src/modules/okd/mod.rs b/harmony/src/modules/okd/mod.rs index 1bd4514..1d052d5 100644 --- a/harmony/src/modules/okd/mod.rs +++ b/harmony/src/modules/okd/mod.rs @@ -19,3 +19,4 @@ pub use bootstrap_03_control_plane::*; pub use bootstrap_04_workers::*; pub use bootstrap_05_sanity_check::*; pub use bootstrap_06_installation_report::*; +pub mod host_network; -- 2.39.5 From fe0501b78480b13383175357d4f06387096dd974 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Mon, 15 Sep 2025 22:05:09 -0400 Subject: [PATCH 02/17] split host & switch config --- harmony/src/domain/topology/ha_cluster.rs | 9 + harmony/src/domain/topology/network.rs | 20 +- harmony/src/modules/okd/host_network.rs | 290 ++++++++++++++++++---- 3 files changed, 269 insertions(+), 50 deletions(-) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index ac53d01..b9a3b4e 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -28,6 +28,7 @@ use super::PreparationOutcome; use super::Router; use super::Switch; use super::SwitchError; +use super::SwitchNetworkConfig; use super::TftpServer; use super::Topology; @@ -280,6 +281,14 @@ impl Switch for HAClusterTopology { ) -> Result<(), SwitchError> { todo!() } + + async fn configure_switch_network( + &self, + _host: &PhysicalHost, + _config: SwitchNetworkConfig, + ) -> Result<(), SwitchError> { + todo!() + } } #[derive(Debug)] diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index a6127c8..4492667 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -179,8 +179,14 @@ pub trait Switch: Send + Sync { async fn configure_host_network( &self, - _host: &PhysicalHost, - _config: HostNetworkConfig, + host: &PhysicalHost, + config: HostNetworkConfig, + ) -> Result<(), SwitchError>; + + async fn configure_switch_network( + &self, + host: &PhysicalHost, + config: SwitchNetworkConfig, ) -> Result<(), SwitchError>; } @@ -200,6 +206,16 @@ pub struct SlaveInterface { // FIXME: Should we add speed as well? And other params } +#[derive(Clone, Debug, PartialEq)] +pub struct SwitchNetworkConfig { + pub port_channel: PortChannel, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PortChannel { + pub ports: Vec, +} + #[derive(Debug, Clone, new)] pub struct SwitchError { msg: String, diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs index 60a89e7..514a9fd 100644 --- a/harmony/src/modules/okd/host_network.rs +++ b/harmony/src/modules/okd/host_network.rs @@ -8,7 +8,10 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{self, Bond, HostNetworkConfig, SlaveInterface, Switch, Topology}, + topology::{ + self, Bond, HostNetworkConfig, PortChannel, SlaveInterface, Switch, SwitchNetworkConfig, + Topology, + }, }; #[derive(Debug, Clone, Serialize)] @@ -56,20 +59,34 @@ impl Interpret for HostNetworkConfigurationInterpret { _inventory: &Inventory, topology: &T, ) -> Result { - let host = self.score.hosts.first().unwrap(); - let mac_addresses = host.get_mac_address(); - let mac_address = mac_addresses.first().unwrap(); - let host_network_config = HostNetworkConfig { - bond: Bond { - interfaces: vec![SlaveInterface { - mac_address: *mac_address, - }], - }, - }; + for host in &self.score.hosts { + let mut interfaces = vec![]; + let mut ports = vec![]; - let _ = topology - .configure_host_network(host, host_network_config) - .await; + for mac_address in host.get_mac_address() { + if let Some(port) = topology.get_port_for_mac_address(&mac_address).await { + interfaces.push(SlaveInterface { mac_address }); + ports.push(port); + } + } + + let _ = topology + .configure_host_network( + host, + HostNetworkConfig { + bond: Bond { interfaces }, + }, + ) + .await; + let _ = topology + .configure_switch_network( + host, + SwitchNetworkConfig { + port_channel: PortChannel { ports }, + }, + ) + .await; + } // foreach hosts // foreach mac addresses @@ -78,17 +95,12 @@ impl Interpret for HostNetworkConfigurationInterpret { // create bond for all valid addresses (port found) // apply network to host first, then switch (to avoid losing hosts that are already connected) // topology.configure_host_network(host, config) <--- will create bonds - // topology.configure_switch_network(port, config) <--- will create port channels + // topology.configure_switch_network(host, config) <--- will create port channels Ok(Outcome::success("".into())) } } -struct PortMapping { - port: String, - mac_address: MacAddress, -} - #[cfg(test)] mod tests { use assertor::*; @@ -98,8 +110,8 @@ mod tests { use crate::{ hardware::HostCategory, topology::{ - Bond, HostNetworkConfig, PreparationError, PreparationOutcome, SlaveInterface, - SwitchError, + Bond, HostNetworkConfig, PortChannel, PreparationError, PreparationOutcome, + SlaveInterface, SwitchError, SwitchNetworkConfig, }, }; use std::{ @@ -111,15 +123,22 @@ mod tests { lazy_static! { pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap(); - pub static ref INTERFACE: MacAddress = - MacAddress::try_from("00:11:22:33:44:55".to_string()).unwrap(); + pub static ref ANOTHER_HOST_ID: Id = Id::from_str("host-2").unwrap(); + pub static ref EXISTING_INTERFACE: MacAddress = + MacAddress::try_from("00:00:00:00:00:00".to_string()).unwrap(); + pub static ref ANOTHER_EXISTING_INTERFACE: MacAddress = + MacAddress::try_from("42:42:42:42:42:42".to_string()).unwrap(); + pub static ref UNKNOWN_INTERFACE: MacAddress = + MacAddress::try_from("99:99:99:99:99:99".to_string()).unwrap(); + pub static ref PORT: String = "1/0/42".into(); + pub static ref ANOTHER_PORT: String = "2/0/42".into(); } #[tokio::test] - async fn one_host_one_mac_address_should_create_bond_with_one_interface() { - let host = given_host(&HOST_ID, *INTERFACE); + async fn host_with_one_mac_address_should_create_bond_with_one_interface() { + let host = given_host(&HOST_ID, vec![*EXISTING_INTERFACE]); let score = given_score(vec![host]); - let topology = SwitchWithPortTopology::new(); + let topology = TopologyWithSwitch::new(); let _ = score.interpret(&Inventory::empty(), &topology).await; @@ -129,28 +148,161 @@ mod tests { HostNetworkConfig { bond: Bond { interfaces: vec![SlaveInterface { - mac_address: *INTERFACE, + mac_address: *EXISTING_INTERFACE, }], }, }, )]); } - fn given_host(id: &Id, mac_address: MacAddress) -> PhysicalHost { + #[tokio::test] + async fn host_with_one_mac_address_should_create_port_channel_with_one_port() { + let host = given_host(&HOST_ID, vec![*EXISTING_INTERFACE]); + let score = given_score(vec![host]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); + assert_that!(*configured_switch_networks).contains_exactly(vec![( + HOST_ID.clone(), + SwitchNetworkConfig { + port_channel: PortChannel { + ports: vec![PORT.clone()], + }, + }, + )]); + } + + #[tokio::test] + async fn host_with_multiple_mac_addresses_should_create_one_bond_with_all_interfaces() { + let score = given_score(vec![given_host( + &HOST_ID, + vec![*EXISTING_INTERFACE, *ANOTHER_EXISTING_INTERFACE], + )]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![( + HOST_ID.clone(), + HostNetworkConfig { + bond: Bond { + interfaces: vec![ + SlaveInterface { + mac_address: *EXISTING_INTERFACE, + }, + SlaveInterface { + mac_address: *ANOTHER_EXISTING_INTERFACE, + }, + ], + }, + }, + )]); + } + + #[tokio::test] + async fn multiple_hosts_should_create_one_bond_per_host() { + let score = given_score(vec![ + given_host(&HOST_ID, vec![*EXISTING_INTERFACE]), + given_host(&ANOTHER_HOST_ID, vec![*ANOTHER_EXISTING_INTERFACE]), + ]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![ + ( + HOST_ID.clone(), + HostNetworkConfig { + bond: Bond { + interfaces: vec![SlaveInterface { + mac_address: *EXISTING_INTERFACE, + }], + }, + }, + ), + ( + ANOTHER_HOST_ID.clone(), + HostNetworkConfig { + bond: Bond { + interfaces: vec![SlaveInterface { + mac_address: *ANOTHER_EXISTING_INTERFACE, + }], + }, + }, + ), + ]); + } + + #[tokio::test] + async fn multiple_hosts_should_create_one_port_channel_per_host() { + let score = given_score(vec![ + given_host(&HOST_ID, vec![*EXISTING_INTERFACE]), + given_host(&ANOTHER_HOST_ID, vec![*ANOTHER_EXISTING_INTERFACE]), + ]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); + assert_that!(*configured_switch_networks).contains_exactly(vec![ + ( + HOST_ID.clone(), + SwitchNetworkConfig { + port_channel: PortChannel { + ports: vec![PORT.clone()], + }, + }, + ), + ( + ANOTHER_HOST_ID.clone(), + SwitchNetworkConfig { + port_channel: PortChannel { + ports: vec![ANOTHER_PORT.clone()], + }, + }, + ), + ]); + } + + #[tokio::test] + async fn port_not_found_for_mac_address_should_not_configure_interface() { + // FIXME: Should it still configure an empty bond/port channel? + let score = given_score(vec![given_host(&HOST_ID, vec![*UNKNOWN_INTERFACE])]); + let topology = TopologyWithSwitch::new_port_not_found(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![( + HOST_ID.clone(), + HostNetworkConfig { + bond: Bond { interfaces: vec![] }, + }, + )]); + let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); + assert_that!(*configured_switch_networks).contains_exactly(vec![( + HOST_ID.clone(), + SwitchNetworkConfig { + port_channel: PortChannel { ports: vec![] }, + }, + )]); + } + + fn given_score(hosts: Vec) -> HostNetworkConfigurationScore { + HostNetworkConfigurationScore { hosts } + } + + fn given_host(id: &Id, mac_addresses: Vec) -> PhysicalHost { + let network = mac_addresses.iter().map(|m| given_interface(*m)).collect(); + PhysicalHost { id: id.clone(), category: HostCategory::Server, - network: vec![NetworkInterface { - name: "interface-1".into(), - mac_address, - speed_mbps: None, - is_up: true, - mtu: 1, - ipv4_addresses: vec![], - ipv6_addresses: vec![], - driver: "driver".into(), - firmware_version: None, - }], + network, storage: vec![], labels: vec![], memory_modules: vec![], @@ -158,24 +310,46 @@ mod tests { } } - fn given_score(hosts: Vec) -> HostNetworkConfigurationScore { - HostNetworkConfigurationScore { hosts } + fn given_interface(mac_address: MacAddress) -> NetworkInterface { + NetworkInterface { + name: format!("{mac_address}"), + mac_address, + speed_mbps: None, + is_up: true, + mtu: 1, + ipv4_addresses: vec![], + ipv6_addresses: vec![], + driver: "driver".into(), + firmware_version: None, + } } - struct SwitchWithPortTopology { + struct TopologyWithSwitch { + available_ports: Vec, configured_host_networks: Arc>>, + configured_switch_networks: Arc>>, } - impl SwitchWithPortTopology { + impl TopologyWithSwitch { fn new() -> Self { Self { + available_ports: vec![PORT.clone(), ANOTHER_PORT.clone()], configured_host_networks: Arc::new(Mutex::new(vec![])), + configured_switch_networks: Arc::new(Mutex::new(vec![])), + } + } + + fn new_port_not_found() -> Self { + Self { + available_ports: vec![], + configured_host_networks: Arc::new(Mutex::new(vec![])), + configured_switch_networks: Arc::new(Mutex::new(vec![])), } } } #[async_trait] - impl Topology for SwitchWithPortTopology { + impl Topology for TopologyWithSwitch { fn name(&self) -> &str { "SwitchWithPortTopology" } @@ -186,9 +360,18 @@ mod tests { } #[async_trait] - impl Switch for SwitchWithPortTopology { - async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { - Some("1/0/42".into()) + impl Switch for TopologyWithSwitch { + async fn get_port_for_mac_address(&self, _mac_address: &MacAddress) -> Option { + if self.available_ports.is_empty() { + return None; + } + + Some( + self.available_ports + .get(self.configured_host_networks.lock().unwrap().len() % 2) + .unwrap() + .clone(), + ) } async fn configure_host_network( @@ -201,5 +384,16 @@ mod tests { Ok(()) } + + async fn configure_switch_network( + &self, + host: &PhysicalHost, + config: SwitchNetworkConfig, + ) -> Result<(), SwitchError> { + let mut configured_switch_networks = self.configured_switch_networks.lock().unwrap(); + configured_switch_networks.push((host.id.clone(), config.clone())); + + Ok(()) + } } } -- 2.39.5 From 427009bbfe410d660de8842337400fe2abd1e545 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Mon, 15 Sep 2025 22:47:56 -0400 Subject: [PATCH 03/17] merge 2 configure steps in 1 and let the topology handle it --- harmony/src/domain/topology/ha_cluster.rs | 11 +- harmony/src/domain/topology/network.rs | 26 +--- harmony/src/modules/okd/host_network.rs | 182 +++++----------------- 3 files changed, 47 insertions(+), 172 deletions(-) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index b9a3b4e..26ac28d 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -28,7 +28,6 @@ use super::PreparationOutcome; use super::Router; use super::Switch; use super::SwitchError; -use super::SwitchNetworkConfig; use super::TftpServer; use super::Topology; @@ -270,7 +269,7 @@ impl HttpServer for HAClusterTopology { #[async_trait] impl Switch for HAClusterTopology { - async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { + async fn get_port_for_mac_address(&self, _mac_address: &MacAddress) -> Option { todo!() } @@ -281,14 +280,6 @@ impl Switch for HAClusterTopology { ) -> Result<(), SwitchError> { todo!() } - - async fn configure_switch_network( - &self, - _host: &PhysicalHost, - _config: SwitchNetworkConfig, - ) -> Result<(), SwitchError> { - todo!() - } } #[derive(Debug)] diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index 4492667..9b42a22 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -182,40 +182,20 @@ pub trait Switch: Send + Sync { host: &PhysicalHost, config: HostNetworkConfig, ) -> Result<(), SwitchError>; - - async fn configure_switch_network( - &self, - host: &PhysicalHost, - config: SwitchNetworkConfig, - ) -> Result<(), SwitchError>; } #[derive(Clone, Debug, PartialEq)] pub struct HostNetworkConfig { - pub bond: Bond, + pub switch_ports: Vec, } #[derive(Clone, Debug, PartialEq)] -pub struct Bond { - pub interfaces: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct SlaveInterface { +pub struct SwitchPort { pub mac_address: MacAddress, + pub port_name: String, // FIXME: Should we add speed as well? And other params } -#[derive(Clone, Debug, PartialEq)] -pub struct SwitchNetworkConfig { - pub port_channel: PortChannel, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct PortChannel { - pub ports: Vec, -} - #[derive(Debug, Clone, new)] pub struct SwitchError { msg: String, diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs index 514a9fd..7a68115 100644 --- a/harmony/src/modules/okd/host_network.rs +++ b/harmony/src/modules/okd/host_network.rs @@ -8,10 +8,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{ - self, Bond, HostNetworkConfig, PortChannel, SlaveInterface, Switch, SwitchNetworkConfig, - Topology, - }, + topology::{HostNetworkConfig, Switch, SwitchPort, Topology}, }; #[derive(Debug, Clone, Serialize)] @@ -60,31 +57,19 @@ impl Interpret for HostNetworkConfigurationInterpret { topology: &T, ) -> Result { for host in &self.score.hosts { - let mut interfaces = vec![]; - let mut ports = vec![]; + let mut switch_ports = vec![]; for mac_address in host.get_mac_address() { - if let Some(port) = topology.get_port_for_mac_address(&mac_address).await { - interfaces.push(SlaveInterface { mac_address }); - ports.push(port); + if let Some(port_name) = topology.get_port_for_mac_address(&mac_address).await { + switch_ports.push(SwitchPort { + mac_address, + port_name, + }); } } let _ = topology - .configure_host_network( - host, - HostNetworkConfig { - bond: Bond { interfaces }, - }, - ) - .await; - let _ = topology - .configure_switch_network( - host, - SwitchNetworkConfig { - port_channel: PortChannel { ports }, - }, - ) + .configure_host_network(host, HostNetworkConfig { switch_ports }) .await; } @@ -93,9 +78,8 @@ impl Interpret for HostNetworkConfigurationInterpret { // let port = topology.get_port_for_mac_address(); // si pas de port -> mac address pas connectee // create port channel for all ports found // create bond for all valid addresses (port found) - // apply network to host first, then switch (to avoid losing hosts that are already connected) - // topology.configure_host_network(host, config) <--- will create bonds - // topology.configure_switch_network(host, config) <--- will create port channels + // apply network config + // topology.configure_host_network(host, config) <--- will create both bonds & port channels Ok(Outcome::success("".into())) } @@ -110,8 +94,7 @@ mod tests { use crate::{ hardware::HostCategory, topology::{ - Bond, HostNetworkConfig, PortChannel, PreparationError, PreparationOutcome, - SlaveInterface, SwitchError, SwitchNetworkConfig, + HostNetworkConfig, PreparationError, PreparationOutcome, SwitchError, SwitchPort, }, }; use std::{ @@ -125,11 +108,11 @@ mod tests { pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap(); pub static ref ANOTHER_HOST_ID: Id = Id::from_str("host-2").unwrap(); pub static ref EXISTING_INTERFACE: MacAddress = - MacAddress::try_from("00:00:00:00:00:00".to_string()).unwrap(); + MacAddress::try_from("AA:BB:CC:DD:EE:F1".to_string()).unwrap(); pub static ref ANOTHER_EXISTING_INTERFACE: MacAddress = - MacAddress::try_from("42:42:42:42:42:42".to_string()).unwrap(); + MacAddress::try_from("AA:BB:CC:DD:EE:F2".to_string()).unwrap(); pub static ref UNKNOWN_INTERFACE: MacAddress = - MacAddress::try_from("99:99:99:99:99:99".to_string()).unwrap(); + MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(); pub static ref PORT: String = "1/0/42".into(); pub static ref ANOTHER_PORT: String = "2/0/42".into(); } @@ -146,30 +129,10 @@ mod tests { assert_that!(*configured_host_networks).contains_exactly(vec![( HOST_ID.clone(), HostNetworkConfig { - bond: Bond { - interfaces: vec![SlaveInterface { - mac_address: *EXISTING_INTERFACE, - }], - }, - }, - )]); - } - - #[tokio::test] - async fn host_with_one_mac_address_should_create_port_channel_with_one_port() { - let host = given_host(&HOST_ID, vec![*EXISTING_INTERFACE]); - let score = given_score(vec![host]); - let topology = TopologyWithSwitch::new(); - - let _ = score.interpret(&Inventory::empty(), &topology).await; - - let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); - assert_that!(*configured_switch_networks).contains_exactly(vec![( - HOST_ID.clone(), - SwitchNetworkConfig { - port_channel: PortChannel { - ports: vec![PORT.clone()], - }, + switch_ports: vec![SwitchPort { + mac_address: *EXISTING_INTERFACE, + port_name: PORT.clone(), + }], }, )]); } @@ -188,16 +151,16 @@ mod tests { assert_that!(*configured_host_networks).contains_exactly(vec![( HOST_ID.clone(), HostNetworkConfig { - bond: Bond { - interfaces: vec![ - SlaveInterface { - mac_address: *EXISTING_INTERFACE, - }, - SlaveInterface { - mac_address: *ANOTHER_EXISTING_INTERFACE, - }, - ], - }, + switch_ports: vec![ + SwitchPort { + mac_address: *EXISTING_INTERFACE, + port_name: PORT.clone(), + }, + SwitchPort { + mac_address: *ANOTHER_EXISTING_INTERFACE, + port_name: ANOTHER_PORT.clone(), + }, + ], }, )]); } @@ -217,52 +180,19 @@ mod tests { ( HOST_ID.clone(), HostNetworkConfig { - bond: Bond { - interfaces: vec![SlaveInterface { - mac_address: *EXISTING_INTERFACE, - }], - }, + switch_ports: vec![SwitchPort { + mac_address: *EXISTING_INTERFACE, + port_name: PORT.clone(), + }], }, ), ( ANOTHER_HOST_ID.clone(), HostNetworkConfig { - bond: Bond { - interfaces: vec![SlaveInterface { - mac_address: *ANOTHER_EXISTING_INTERFACE, - }], - }, - }, - ), - ]); - } - - #[tokio::test] - async fn multiple_hosts_should_create_one_port_channel_per_host() { - let score = given_score(vec![ - given_host(&HOST_ID, vec![*EXISTING_INTERFACE]), - given_host(&ANOTHER_HOST_ID, vec![*ANOTHER_EXISTING_INTERFACE]), - ]); - let topology = TopologyWithSwitch::new(); - - let _ = score.interpret(&Inventory::empty(), &topology).await; - - let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); - assert_that!(*configured_switch_networks).contains_exactly(vec![ - ( - HOST_ID.clone(), - SwitchNetworkConfig { - port_channel: PortChannel { - ports: vec![PORT.clone()], - }, - }, - ), - ( - ANOTHER_HOST_ID.clone(), - SwitchNetworkConfig { - port_channel: PortChannel { - ports: vec![ANOTHER_PORT.clone()], - }, + switch_ports: vec![SwitchPort { + mac_address: *ANOTHER_EXISTING_INTERFACE, + port_name: ANOTHER_PORT.clone(), + }], }, ), ]); @@ -280,14 +210,7 @@ mod tests { assert_that!(*configured_host_networks).contains_exactly(vec![( HOST_ID.clone(), HostNetworkConfig { - bond: Bond { interfaces: vec![] }, - }, - )]); - let configured_switch_networks = topology.configured_switch_networks.lock().unwrap(); - assert_that!(*configured_switch_networks).contains_exactly(vec![( - HOST_ID.clone(), - SwitchNetworkConfig { - port_channel: PortChannel { ports: vec![] }, + switch_ports: vec![], }, )]); } @@ -325,25 +248,22 @@ mod tests { } struct TopologyWithSwitch { - available_ports: Vec, + available_ports: Arc>>, configured_host_networks: Arc>>, - configured_switch_networks: Arc>>, } impl TopologyWithSwitch { fn new() -> Self { Self { - available_ports: vec![PORT.clone(), ANOTHER_PORT.clone()], + available_ports: Arc::new(Mutex::new(vec![PORT.clone(), ANOTHER_PORT.clone()])), configured_host_networks: Arc::new(Mutex::new(vec![])), - configured_switch_networks: Arc::new(Mutex::new(vec![])), } } fn new_port_not_found() -> Self { Self { - available_ports: vec![], + available_ports: Arc::new(Mutex::new(vec![])), configured_host_networks: Arc::new(Mutex::new(vec![])), - configured_switch_networks: Arc::new(Mutex::new(vec![])), } } } @@ -362,16 +282,11 @@ mod tests { #[async_trait] impl Switch for TopologyWithSwitch { async fn get_port_for_mac_address(&self, _mac_address: &MacAddress) -> Option { - if self.available_ports.is_empty() { + let mut ports = self.available_ports.lock().unwrap(); + if ports.is_empty() { return None; } - - Some( - self.available_ports - .get(self.configured_host_networks.lock().unwrap().len() % 2) - .unwrap() - .clone(), - ) + Some(ports.remove(0)) } async fn configure_host_network( @@ -384,16 +299,5 @@ mod tests { Ok(()) } - - async fn configure_switch_network( - &self, - host: &PhysicalHost, - config: SwitchNetworkConfig, - ) -> Result<(), SwitchError> { - let mut configured_switch_networks = self.configured_switch_networks.lock().unwrap(); - configured_switch_networks.push((host.id.clone(), config.clone())); - - Ok(()) - } } } -- 2.39.5 From 0de52aedbf9c407870613e8104e2e1314d848181 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Tue, 16 Sep 2025 17:19:32 -0400 Subject: [PATCH 04/17] find ports in Brocade switch & configure port-channels (blind implementation) --- Cargo.lock | 12 ++ Cargo.toml | 2 +- brocade/Cargo.toml | 13 ++ brocade/src/lib.rs | 251 ++++++++++++++++++++++ harmony/Cargo.toml | 1 + harmony/src/domain/topology/ha_cluster.rs | 106 ++++++++- harmony_types/src/net.rs | 2 +- 7 files changed, 382 insertions(+), 5 deletions(-) create mode 100644 brocade/Cargo.toml create mode 100644 brocade/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1ec5d60..ffbd2c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -674,6 +674,17 @@ dependencies = [ "serde_with", ] +[[package]] +name = "brocade" +version = "0.1.0" +dependencies = [ + "async-trait", + "harmony_types", + "russh", + "russh-keys", + "tokio", +] + [[package]] name = "brotli" version = "8.0.2" @@ -2318,6 +2329,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bollard", + "brocade", "chrono", "cidr", "convert_case", diff --git a/Cargo.toml b/Cargo.toml index 32231d7..a10bf81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ "harmony_inventory_agent", "harmony_secret_derive", "harmony_secret", - "adr/agent_discovery/mdns", + "adr/agent_discovery/mdns", "brocade", ] [workspace.package] diff --git a/brocade/Cargo.toml b/brocade/Cargo.toml new file mode 100644 index 0000000..f80d1ac --- /dev/null +++ b/brocade/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "brocade" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +async-trait.workspace = true +harmony_types = { version = "0.1.0", path = "../harmony_types" } +russh.workspace = true +russh-keys.workspace = true +tokio.workspace = true diff --git a/brocade/src/lib.rs b/brocade/src/lib.rs new file mode 100644 index 0000000..86b75b8 --- /dev/null +++ b/brocade/src/lib.rs @@ -0,0 +1,251 @@ +use std::{ + fmt::{self, Display}, + sync::Arc, +}; + +use async_trait::async_trait; +use harmony_types::net::{IpAddress, MacAddress}; +use russh::client::{Handle, Handler}; +use russh_keys::key; +use std::str::FromStr; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub struct MacAddressEntry { + pub vlan: u16, + pub mac_address: MacAddress, + pub port_name: String, +} + +pub struct BrocadeClient { + client: Handle, +} + +impl BrocadeClient { + pub async fn init(ip: IpAddress, username: &str, password: &str) -> Result { + let config = russh::client::Config::default(); + let mut client = russh::client::connect(Arc::new(config), (ip, 22), Client {}).await?; + + match client.authenticate_password(username, password).await? { + true => Ok(Self { client }), + false => Err(Error::AuthenticationError( + "ssh authentication failed".to_string(), + )), + } + } + + pub async fn show_mac_address_table(&self) -> Result, Error> { + let output = self.run_command("show mac-address-table").await?; + let mut entries = Vec::new(); + + // The Brocade output usually has a header and then one entry per line. + // We will skip the header and parse each line. + // Sample line: "1234 AA:BB:CC:DD:EE:F1 GigabitEthernet1/1/1" + for line in output.lines().skip(1) { + // Skip the header row + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 3 { + // Assuming the format is: + if let Ok(vlan) = u16::from_str(parts[0]) { + let mac = MacAddress::try_from(parts[1].to_string()); + let port = parts[2].to_string(); + + if let Ok(mac_address) = mac { + entries.push(MacAddressEntry { + vlan, + mac_address, + port_name: port, + }); + } + } + } + } + + Ok(entries) + } + + pub async fn configure_port_channel(&self, ports: &[String]) -> Result { + let channel_id = self.find_available_channel_id().await?; + let mut commands = Vec::new(); + + // Start configuration mode. + commands.push("configure terminal".to_string()); + + // Create the port channel interface. + commands.push(format!("interface Port-channel {channel_id}")); + commands.push("no ip address".to_string()); + commands.push("exit".to_string()); + + // Configure each physical port to join the channel. + for port in ports { + commands.push(format!("interface {port}")); + // 'channel-group' command to add the interface to the port channel. + // Using 'mode active' enables LACP. + commands.push(format!("channel-group {channel_id} mode active")); + commands.push("exit".to_string()); + } + + // Save the configuration. + commands.push("write memory".to_string()); + + self.run_commands(commands).await?; + + Ok(channel_id) + } + + pub async fn find_available_channel_id(&self) -> Result { + // FIXME: The command might vary slightly by Brocade OS version. + let output = self.run_command("show port-channel summary").await?; + let mut used_ids = Vec::new(); + + // Sample output line: "3 Po3(SU) LACP Eth Yes 128/128 active " + // We're looking for the ID, which is the first number. + for line in output.lines() { + if line.trim().starts_with(|c: char| c.is_ascii_digit()) { + let parts: Vec<&str> = line.split_whitespace().collect(); + if let Ok(id) = u8::from_str(parts[0]) { + used_ids.push(id); + } + } + } + + // Sort the used IDs to find the next available number. + used_ids.sort(); + + let mut next_id = 1; + for &id in &used_ids { + if id == next_id { + next_id += 1; + } else { + // Found a gap, so this is our ID. + return Ok(next_id); + } + } + + Ok(next_id) + } + + async fn run_command(&self, command: &str) -> Result { + let mut channel = self.client.channel_open_session().await?; + let mut output = Vec::new(); + + channel.exec(true, command).await?; + + loop { + let Some(msg) = channel.wait().await else { + break; + }; + + match msg { + russh::ChannelMsg::ExtendedData { ref data, .. } + | russh::ChannelMsg::Data { ref data } => { + output.append(&mut data.to_vec()); + } + russh::ChannelMsg::ExitStatus { exit_status } => { + if exit_status != 0 { + return Err(Error::CommandError(format!( + "Command failed with exit status {exit_status}, output {}", + String::from_utf8(output).unwrap_or_default() + ))); + } + } + russh::ChannelMsg::Success + | russh::ChannelMsg::WindowAdjusted { .. } + | russh::ChannelMsg::Eof => {} + _ => { + return Err(Error::UnexpectedError(format!( + "Russh got unexpected msg {msg:?}" + ))); + } + } + } + + channel.close().await?; + + let output = String::from_utf8(output).expect("Output should be UTF-8 compatible"); + Ok(output) + } + + async fn run_commands(&self, commands: Vec) -> Result<(), Error> { + let mut channel = self.client.channel_open_session().await?; + + // Execute commands sequentially and check for errors immediately. + for command in commands { + let mut output = Vec::new(); + channel.exec(true, command.as_str()).await?; + + loop { + let Some(msg) = channel.wait().await else { + break; + }; + + match msg { + russh::ChannelMsg::ExtendedData { ref data, .. } + | russh::ChannelMsg::Data { ref data } => { + output.append(&mut data.to_vec()); + } + russh::ChannelMsg::ExitStatus { exit_status } => { + if exit_status != 0 { + let output_str = String::from_utf8(output).unwrap_or_default(); + return Err(Error::CommandError(format!( + "Command '{command}' failed with exit status {exit_status}: {output_str}", + ))); + } + } + _ => {} // Ignore other messages like success or EOF for now. + } + } + } + + channel.close().await?; + Ok(()) + } +} + +struct Client {} + +#[async_trait] +impl Handler for Client { + type Error = Error; + + async fn check_server_key( + &mut self, + _server_public_key: &key::PublicKey, + ) -> Result { + Ok(true) + } +} + +#[derive(Debug)] +pub enum Error { + NetworkError(String), + AuthenticationError(String), + ConfigurationError(String), + UnexpectedError(String), + CommandError(String), +} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::NetworkError(msg) => write!(f, "Network error: {msg}"), + Error::AuthenticationError(msg) => write!(f, "Authentication error: {msg}"), + Error::ConfigurationError(msg) => write!(f, "Configuration error: {msg}"), + Error::UnexpectedError(msg) => write!(f, "Unexpected error: {msg}"), + Error::CommandError(msg) => write!(f, "Command failed: {msg}"), + } + } +} + +impl From for String { + fn from(val: Error) -> Self { + format!("{val}") + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(_value: russh::Error) -> Self { + Error::NetworkError("Russh client error".to_string()) + } +} diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index f9e671a..f49210a 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -77,6 +77,7 @@ harmony_secret = { path = "../harmony_secret" } askama.workspace = true sqlx.workspace = true inquire.workspace = true +brocade = { version = "0.1.0", path = "../brocade" } [dev-dependencies] pretty_assertions.workspace = true diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 26ac28d..42cd7c5 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -1,9 +1,16 @@ use async_trait::async_trait; +use brocade::BrocadeClient; use harmony_macros::ip; +use harmony_secret::Secret; +use harmony_secret::SecretManager; use harmony_types::net::MacAddress; use harmony_types::net::Url; use log::debug; use log::info; +use russh::client; +use russh::client::Handler; +use serde::Deserialize; +use serde::Serialize; use crate::data::FileContent; use crate::executors::ExecutorError; @@ -28,10 +35,12 @@ use super::PreparationOutcome; use super::Router; use super::Switch; use super::SwitchError; +use super::SwitchPort; use super::TftpServer; use super::Topology; use super::k8s::K8sClient; +use std::error::Error; use std::sync::Arc; #[derive(Debug, Clone)] @@ -93,6 +102,39 @@ impl HAClusterTopology { .to_string() } + fn find_master_switch(&self) -> Option { + self.switch.first().cloned() // FIXME: Should we be smarter to find the master switch? + } + + async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { + todo!() + } + + async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result { + let auth = SecretManager::get_or_prompt::() + .await + .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?; + + let switch = self + .find_master_switch() + .ok_or(SwitchError::new("No switch found in topology".to_string()))?; + let client = BrocadeSwitchClient::init(switch.ip, &auth.username, &auth.password) + .await + .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?; + + let switch_ports: Vec = config + .switch_ports + .iter() + .map(|s| s.port_name.clone()) + .collect(); + let channel_id = client + .configure_port_channel(switch_ports) + .await + .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?; + + Ok(channel_id) + } + pub fn autoload() -> Self { let dummy_infra = Arc::new(DummyInfra {}); let dummy_host = LogicalHost { @@ -269,19 +311,77 @@ impl HttpServer for HAClusterTopology { #[async_trait] impl Switch for HAClusterTopology { - async fn get_port_for_mac_address(&self, _mac_address: &MacAddress) -> Option { - todo!() + async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { + let auth = SecretManager::get_or_prompt::() + .await + .unwrap(); + + let switch = self.find_master_switch()?; + let client = BrocadeSwitchClient::init(switch.ip, &auth.username, &auth.password).await; + + let Ok(client) = client else { + return None; + }; + + client.find_port(mac_address).await } async fn configure_host_network( &self, _host: &PhysicalHost, - _config: HostNetworkConfig, + config: HostNetworkConfig, ) -> Result<(), SwitchError> { + let _ = self.configure_bond(&config).await; + let channel_id = self.configure_port_channel(&config).await; + todo!() } } +#[async_trait] +trait SwitchClient { + async fn find_port(&self, mac_address: &MacAddress) -> Option; + async fn configure_port_channel(&self, switch_ports: Vec) -> Result; +} + +struct BrocadeSwitchClient { + brocade: BrocadeClient, +} + +impl BrocadeSwitchClient { + async fn init(ip: IpAddress, username: &str, password: &str) -> Result { + let brocade = BrocadeClient::init(ip, username, password).await?; + Ok(Self { brocade }) + } +} + +#[async_trait] +impl SwitchClient for BrocadeSwitchClient { + async fn find_port(&self, mac_address: &MacAddress) -> Option { + let Ok(table) = self.brocade.show_mac_address_table().await else { + return None; + }; + + table + .iter() + .find(|entry| entry.mac_address == *mac_address) + .map(|entry| entry.port_name.clone()) + } + + async fn configure_port_channel(&self, switch_ports: Vec) -> Result { + self.brocade + .configure_port_channel(&switch_ports) + .await + .map_err(|e| SwitchError::new(format!("Failed to configure port channel: {e}"))) + } +} + +#[derive(Secret, Serialize, Deserialize, Debug)] +struct BrocadeSwitchAuth { + username: String, + password: String, +} + #[derive(Debug)] pub struct DummyInfra; diff --git a/harmony_types/src/net.rs b/harmony_types/src/net.rs index 594a3e2..5b7449a 100644 --- a/harmony_types/src/net.rs +++ b/harmony_types/src/net.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct MacAddress(pub [u8; 6]); impl MacAddress { -- 2.39.5 From ffe3c09907ce00101068d0351eafa961f30c25f3 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 25 Sep 2025 16:31:40 -0400 Subject: [PATCH 05/17] configure bond with NMState --- Cargo.lock | 1 + brocade/Cargo.toml | 3 +- brocade/src/lib.rs | 24 +- harmony/Cargo.toml | 2 +- harmony/src/domain/topology/ha_cluster.rs | 253 +++++++++++++++--- harmony/src/domain/topology/network.rs | 11 +- .../modules/okd/bootstrap_03_control_plane.rs | 26 +- harmony/src/modules/okd/crd/machineconfig.rs | 43 +++ harmony/src/modules/okd/crd/mod.rs | 42 +++ harmony/src/modules/okd/crd/nmstate.rs | 251 +++++++++++++++++ harmony/src/modules/okd/host_network.rs | 90 ++++--- harmony/src/modules/okd/mod.rs | 1 + harmony_types/src/net.rs | 6 +- 13 files changed, 658 insertions(+), 95 deletions(-) create mode 100644 harmony/src/modules/okd/crd/machineconfig.rs create mode 100644 harmony/src/modules/okd/crd/mod.rs create mode 100644 harmony/src/modules/okd/crd/nmstate.rs diff --git a/Cargo.lock b/Cargo.lock index ffbd2c9..7e74492 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -680,6 +680,7 @@ version = "0.1.0" dependencies = [ "async-trait", "harmony_types", + "log", "russh", "russh-keys", "tokio", diff --git a/brocade/Cargo.toml b/brocade/Cargo.toml index f80d1ac..3112918 100644 --- a/brocade/Cargo.toml +++ b/brocade/Cargo.toml @@ -7,7 +7,8 @@ license.workspace = true [dependencies] async-trait.workspace = true -harmony_types = { version = "0.1.0", path = "../harmony_types" } +harmony_types = { path = "../harmony_types" } russh.workspace = true russh-keys.workspace = true tokio.workspace = true +log.workspace = true diff --git a/brocade/src/lib.rs b/brocade/src/lib.rs index 86b75b8..49cfaae 100644 --- a/brocade/src/lib.rs +++ b/brocade/src/lib.rs @@ -5,6 +5,7 @@ use std::{ use async_trait::async_trait; use harmony_types::net::{IpAddress, MacAddress}; +use log::{debug, info}; use russh::client::{Handle, Handler}; use russh_keys::key; use std::str::FromStr; @@ -21,7 +22,12 @@ pub struct BrocadeClient { } impl BrocadeClient { - pub async fn init(ip: IpAddress, username: &str, password: &str) -> Result { + pub async fn init( + ip_addresses: &[IpAddress], + username: &str, + password: &str, + ) -> Result { + let ip = ip_addresses[0]; let config = russh::client::Config::default(); let mut client = russh::client::connect(Arc::new(config), (ip, 22), Client {}).await?; @@ -64,6 +70,8 @@ impl BrocadeClient { } pub async fn configure_port_channel(&self, ports: &[String]) -> Result { + info!("[Brocade] Configuring port-channel with ports: {ports:?}"); + let channel_id = self.find_available_channel_id().await?; let mut commands = Vec::new(); @@ -93,7 +101,8 @@ impl BrocadeClient { } pub async fn find_available_channel_id(&self) -> Result { - // FIXME: The command might vary slightly by Brocade OS version. + debug!("[Brocade] Finding next available channel id..."); + let output = self.run_command("show port-channel summary").await?; let mut used_ids = Vec::new(); @@ -121,10 +130,13 @@ impl BrocadeClient { } } + debug!("[Brocade] Found channel id '{next_id}'"); Ok(next_id) } async fn run_command(&self, command: &str) -> Result { + debug!("[Brocade] Running command: '{command}'..."); + let mut channel = self.client.channel_open_session().await?; let mut output = Vec::new(); @@ -142,9 +154,9 @@ impl BrocadeClient { } russh::ChannelMsg::ExitStatus { exit_status } => { if exit_status != 0 { + let output_str = String::from_utf8(output).unwrap_or_default(); return Err(Error::CommandError(format!( - "Command failed with exit status {exit_status}, output {}", - String::from_utf8(output).unwrap_or_default() + "Command failed with exit status {exit_status}, output {output_str}", ))); } } @@ -170,6 +182,8 @@ impl BrocadeClient { // Execute commands sequentially and check for errors immediately. for command in commands { + debug!("[Brocade] Running command: '{command}'..."); + let mut output = Vec::new(); channel.exec(true, command.as_str()).await?; @@ -187,7 +201,7 @@ impl BrocadeClient { if exit_status != 0 { let output_str = String::from_utf8(output).unwrap_or_default(); return Err(Error::CommandError(format!( - "Command '{command}' failed with exit status {exit_status}: {output_str}", + "Command failed with exit status {exit_status}: {output_str}", ))); } } diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index f49210a..3633983 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -77,7 +77,7 @@ harmony_secret = { path = "../harmony_secret" } askama.workspace = true sqlx.workspace = true inquire.workspace = true -brocade = { version = "0.1.0", path = "../brocade" } +brocade = { path = "../brocade" } [dev-dependencies] pretty_assertions.workspace = true diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 42cd7c5..cc07b8b 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -5,16 +5,25 @@ use harmony_secret::Secret; use harmony_secret::SecretManager; use harmony_types::net::MacAddress; use harmony_types::net::Url; +use k8s_openapi::api::core::v1::Namespace; +use kube::api::ObjectMeta; use log::debug; use log::info; -use russh::client; -use russh::client::Handler; use serde::Deserialize; use serde::Serialize; use crate::data::FileContent; use crate::executors::ExecutorError; use crate::hardware::PhysicalHost; +use crate::modules::okd::crd::InstallPlanApproval; +use crate::modules::okd::crd::OperatorGroup; +use crate::modules::okd::crd::OperatorGroupSpec; +use crate::modules::okd::crd::Subscription; +use crate::modules::okd::crd::SubscriptionSpec; +use crate::modules::okd::crd::nmstate; +use crate::modules::okd::crd::nmstate::NMState; +use crate::modules::okd::crd::nmstate::NodeNetworkConfigurationPolicy; +use crate::modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec; use crate::topology::PxeOptions; use super::DHCPStaticEntry; @@ -35,12 +44,12 @@ use super::PreparationOutcome; use super::Router; use super::Switch; use super::SwitchError; -use super::SwitchPort; use super::TftpServer; use super::Topology; use super::k8s::K8sClient; -use std::error::Error; +use std::collections::BTreeMap; +use std::net::IpAddr; use std::sync::Arc; #[derive(Debug, Clone)] @@ -102,37 +111,224 @@ impl HAClusterTopology { .to_string() } - fn find_master_switch(&self) -> Option { - self.switch.first().cloned() // FIXME: Should we be smarter to find the master switch? + async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> { + // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate) + debug!("Installing NMState operator..."); + let k8s_client = self.k8s_client().await?; + + let nmstate_namespace = Namespace { + metadata: ObjectMeta { + name: Some("openshift-nmstate".to_string()), + finalizers: Some(vec!["kubernetes".to_string()]), + ..Default::default() + }, + ..Default::default() + }; + debug!("Creating NMState namespace: {nmstate_namespace:#?}"); + k8s_client + .apply(&nmstate_namespace, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate_operator_group = OperatorGroup { + metadata: ObjectMeta { + name: Some("openshift-nmstate".to_string()), + namespace: Some("openshift-nmstate".to_string()), + ..Default::default() + }, + spec: OperatorGroupSpec { + target_namespaces: vec!["openshift-nmstate".to_string()], + }, + }; + debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); + k8s_client + .apply(&nmstate_operator_group, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate_subscription = Subscription { + metadata: ObjectMeta { + name: Some("kubernetes-nmstate-operator".to_string()), + namespace: Some("openshift-nmstate".to_string()), + ..Default::default() + }, + spec: SubscriptionSpec { + channel: Some("stable".to_string()), + install_plan_approval: Some(InstallPlanApproval::Automatic), + name: "kubernetes-nmstate-operator".to_string(), + source: "redhat-operators".to_string(), + source_namespace: "openshift-marketplace".to_string(), + }, + }; + debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}"); + k8s_client + .apply(&nmstate_subscription, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate = NMState { + metadata: ObjectMeta { + name: Some("nmstate".to_string()), + ..Default::default() + }, + ..Default::default() + }; + debug!("Creating NMState: {nmstate:#?}"); + k8s_client + .apply(&nmstate, None) + .await + .map_err(|e| e.to_string())?; + + Ok(()) } - async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { + fn get_next_bond_id(&self) -> u8 { + 42 // FIXME: Find a better way to declare the bond id + } + + async fn configure_bond( + &self, + host: &PhysicalHost, + config: &HostNetworkConfig, + ) -> Result<(), SwitchError> { + self.ensure_nmstate_operator_installed() + .await + .map_err(|e| { + SwitchError::new(format!( + "Can't configure bond, NMState operator not available: {e}" + )) + })?; + + let bond_config = self.create_bond_configuration(host, config); + debug!("Configuring bond for host {host:?}: {bond_config:#?}"); + self.k8s_client() + .await + .unwrap() + .apply(&bond_config, None) + .await + .unwrap(); + todo!() } - async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result { + fn create_bond_configuration( + &self, + host: &PhysicalHost, + config: &HostNetworkConfig, + ) -> NodeNetworkConfigurationPolicy { + let host_name = host.id.clone(); + + let bond_id = self.get_next_bond_id(); + let bond_name = format!("bond{bond_id}"); + let mut bond_mtu: Option = None; + let mut bond_mac_address: Option = None; + let mut bond_ports = Vec::new(); + let mut interfaces: Vec = Vec::new(); + + for switch_port in &config.switch_ports { + let interface_name = switch_port.interface.name.clone(); + + interfaces.push(nmstate::InterfaceSpec { + name: interface_name.clone(), + description: Some(format!("Member of bond {bond_name}")), + r#type: "ethernet".to_string(), + state: "up".to_string(), + mtu: Some(switch_port.interface.mtu), + mac_address: Some(switch_port.interface.mac_address.to_string()), + ipv4: Some(nmstate::IpStackSpec { + enabled: Some(false), + ..Default::default() + }), + ipv6: Some(nmstate::IpStackSpec { + enabled: Some(false), + ..Default::default() + }), + link_aggregation: None, + ..Default::default() + }); + + bond_ports.push(interface_name); + + // Use the first port's details for the bond mtu and mac address + if bond_mtu.is_none() { + bond_mtu = Some(switch_port.interface.mtu); + } + if bond_mac_address.is_none() { + bond_mac_address = Some(switch_port.interface.mac_address.to_string()); + } + } + + interfaces.push(nmstate::InterfaceSpec { + name: bond_name.clone(), + description: Some(format!("Network bond for host {host_name}")), + r#type: "bond".to_string(), + state: "up".to_string(), + mtu: bond_mtu, + mac_address: bond_mac_address, + ipv4: Some(nmstate::IpStackSpec { + dhcp: Some(true), + enabled: Some(true), + ..Default::default() + }), + ipv6: Some(nmstate::IpStackSpec { + dhcp: Some(true), + autoconf: Some(true), + enabled: Some(true), + ..Default::default() + }), + link_aggregation: Some(nmstate::BondSpec { + mode: "802.3ad".to_string(), + ports: bond_ports, + ..Default::default() + }), + ..Default::default() + }); + + NodeNetworkConfigurationPolicy { + metadata: ObjectMeta { + name: Some(format!("{host_name}-bond-config")), + ..Default::default() + }, + spec: NodeNetworkConfigurationPolicySpec { + node_selector: Some(BTreeMap::from([( + "kubernetes.io/hostname".to_string(), + host_name.to_string(), + )])), + desired_state: nmstate::DesiredStateSpec { interfaces }, + }, + } + } + + async fn get_switch_client(&self) -> Result, SwitchError> { let auth = SecretManager::get_or_prompt::() .await .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?; - let switch = self - .find_master_switch() - .ok_or(SwitchError::new("No switch found in topology".to_string()))?; - let client = BrocadeSwitchClient::init(switch.ip, &auth.username, &auth.password) + // FIXME: We assume Brocade switches + let switches: Vec = self.switch.iter().map(|s| s.ip).collect(); + let client = BrocadeSwitchClient::init(&switches, &auth.username, &auth.password) .await .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?; + Ok(Box::new(client)) + } + + async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { + debug!("Configuring port channel: {config:#?}"); + let client = self.get_switch_client().await?; + let switch_ports: Vec = config .switch_ports .iter() .map(|s| s.port_name.clone()) .collect(); - let channel_id = client + + client .configure_port_channel(switch_ports) .await .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?; - Ok(channel_id) + Ok(()) } pub fn autoload() -> Self { @@ -312,12 +508,7 @@ impl HttpServer for HAClusterTopology { #[async_trait] impl Switch for HAClusterTopology { async fn get_port_for_mac_address(&self, mac_address: &MacAddress) -> Option { - let auth = SecretManager::get_or_prompt::() - .await - .unwrap(); - - let switch = self.find_master_switch()?; - let client = BrocadeSwitchClient::init(switch.ip, &auth.username, &auth.password).await; + let client = self.get_switch_client().await; let Ok(client) = client else { return None; @@ -328,18 +519,16 @@ impl Switch for HAClusterTopology { async fn configure_host_network( &self, - _host: &PhysicalHost, + host: &PhysicalHost, config: HostNetworkConfig, ) -> Result<(), SwitchError> { - let _ = self.configure_bond(&config).await; - let channel_id = self.configure_port_channel(&config).await; - - todo!() + self.configure_bond(host, &config).await?; + self.configure_port_channel(&config).await } } #[async_trait] -trait SwitchClient { +trait SwitchClient: Send + Sync { async fn find_port(&self, mac_address: &MacAddress) -> Option; async fn configure_port_channel(&self, switch_ports: Vec) -> Result; } @@ -349,8 +538,12 @@ struct BrocadeSwitchClient { } impl BrocadeSwitchClient { - async fn init(ip: IpAddress, username: &str, password: &str) -> Result { - let brocade = BrocadeClient::init(ip, username, password).await?; + async fn init( + ip_addresses: &[IpAddress], + username: &str, + password: &str, + ) -> Result { + let brocade = BrocadeClient::init(ip_addresses, username, password).await?; Ok(Self { brocade }) } } @@ -451,8 +644,8 @@ impl DhcpServer for DummyInfra { } async fn set_dhcp_range( &self, - start: &IpAddress, - end: &IpAddress, + _start: &IpAddress, + _end: &IpAddress, ) -> Result<(), ExecutorError> { unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA) } diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index 9b42a22..a1da8dd 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -191,9 +191,16 @@ pub struct HostNetworkConfig { #[derive(Clone, Debug, PartialEq)] pub struct SwitchPort { - pub mac_address: MacAddress, + pub interface: NetworkInterface, pub port_name: String, - // FIXME: Should we add speed as well? And other params +} + +#[derive(Clone, Debug, PartialEq)] +pub struct NetworkInterface { + pub name: String, + pub mac_address: MacAddress, + pub speed_mbps: Option, + pub mtu: u32, } #[derive(Debug, Clone, new)] diff --git a/harmony/src/modules/okd/bootstrap_03_control_plane.rs b/harmony/src/modules/okd/bootstrap_03_control_plane.rs index dd0fdee..a3fe4c2 100644 --- a/harmony/src/modules/okd/bootstrap_03_control_plane.rs +++ b/harmony/src/modules/okd/bootstrap_03_control_plane.rs @@ -11,7 +11,7 @@ use crate::{ okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl}, }, score::Score, - topology::{self, HAClusterTopology, HostBinding}, + topology::{HAClusterTopology, HostBinding}, }; use async_trait::async_trait; use derive_new::new; @@ -30,7 +30,7 @@ pub struct OKDSetup03ControlPlaneScore {} impl Score for OKDSetup03ControlPlaneScore { fn create_interpret(&self) -> Box> { - Box::new(OKDSetup03ControlPlaneInterpret::new(self.clone())) + Box::new(OKDSetup03ControlPlaneInterpret::new()) } fn name(&self) -> String { @@ -40,17 +40,15 @@ impl Score for OKDSetup03ControlPlaneScore { #[derive(Debug, Clone)] pub struct OKDSetup03ControlPlaneInterpret { - score: OKDSetup03ControlPlaneScore, version: Version, status: InterpretStatus, } impl OKDSetup03ControlPlaneInterpret { - pub fn new(score: OKDSetup03ControlPlaneScore) -> Self { + pub fn new() -> Self { let version = Version::from("1.0.0").unwrap(); Self { version, - score, status: InterpretStatus::QUEUED, } } @@ -161,7 +159,7 @@ impl OKDSetup03ControlPlaneInterpret { } .to_string(); - debug!("[ControlPlane] iPXE content template:\n{}", content); + debug!("[ControlPlane] iPXE content template:\n{content}"); // Create and apply an iPXE boot file for each node. for node in nodes { @@ -191,16 +189,13 @@ impl OKDSetup03ControlPlaneInterpret { /// Prompts the user to reboot the target control plane nodes. async fn reboot_targets(&self, nodes: &Vec) -> Result<(), InterpretError> { let node_ids: Vec = nodes.iter().map(|n| n.id.to_string()).collect(); - info!( - "[ControlPlane] Requesting reboot for control plane nodes: {:?}", - node_ids - ); + info!("[ControlPlane] Requesting reboot for control plane nodes: {node_ids:?}",); let confirmation = inquire::Confirm::new( &format!("Please reboot the {} control plane nodes ({}) to apply their PXE configuration. Press enter when ready.", nodes.len(), node_ids.join(", ")), ) .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?; + .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; if !confirmation { return Err(InterpretError::new( @@ -222,19 +217,18 @@ impl OKDSetup03ControlPlaneInterpret { topology: &HAClusterTopology, hosts: &Vec, ) -> Result<(), InterpretError> { + // Generate MC or NNCP from inventory NIC data; apply via ignition or post-join. + info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP"); let score = HostNetworkConfigurationScore { hosts: hosts.clone(), // FIXME: Avoid clone if possible }; + score.interpret(inventory, topology).await?; - score.interpret(inventory, topology); - - // Generate MC or NNCP from inventory NIC data; apply via ignition or post-join. - info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP"); inquire::Confirm::new( "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.", ) .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?; + .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; Ok(()) } diff --git a/harmony/src/modules/okd/crd/machineconfig.rs b/harmony/src/modules/okd/crd/machineconfig.rs new file mode 100644 index 0000000..df710e2 --- /dev/null +++ b/harmony/src/modules/okd/crd/machineconfig.rs @@ -0,0 +1,43 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "machineconfiguration.openshift.io", + version = "v1", + kind = "MachineConfig", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct MachineConfigSpec { + pub config: Config, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Config { + pub ignition: Ignition, + pub storage: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub struct Ignition { + pub version: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub struct Storage { + pub files: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub struct File { + pub path: String, + pub contents: FileContents, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub struct FileContents { + pub source: String, +} diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs new file mode 100644 index 0000000..aa35190 --- /dev/null +++ b/harmony/src/modules/okd/crd/mod.rs @@ -0,0 +1,42 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub mod machineconfig; +pub mod nmstate; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1", + kind = "OperatorGroup", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct OperatorGroupSpec { + pub target_namespaces: Vec, +} + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1alpha1", + kind = "Subscription", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionSpec { + pub name: String, + pub source: String, + pub source_namespace: String, + pub channel: Option, + pub install_plan_approval: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub enum InstallPlanApproval { + #[serde(rename = "Automatic")] + Automatic, + #[serde(rename = "Manual")] + Manual, +} diff --git a/harmony/src/modules/okd/crd/nmstate.rs b/harmony/src/modules/okd/crd/nmstate.rs new file mode 100644 index 0000000..b6117fd --- /dev/null +++ b/harmony/src/modules/okd/crd/nmstate.rs @@ -0,0 +1,251 @@ +use std::collections::BTreeMap; + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)] +#[serde(rename_all = "camelCase")] +pub struct NMStateSpec { + pub probe_configuration: Option, +} + +impl Default for NMState { + fn default() -> Self { + Self { + metadata: Default::default(), + spec: NMStateSpec { + probe_configuration: None, + }, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ProbeConfig { + pub dns: ProbeDns, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ProbeDns { + pub host: String, +} + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "nmstate.io", + version = "v1", + kind = "NodeNetworkConfigurationPolicy", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct NodeNetworkConfigurationPolicySpec { + pub node_selector: Option>, + pub desired_state: DesiredStateSpec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct DesiredStateSpec { + pub interfaces: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct InterfaceSpec { + pub name: String, + pub description: Option, + pub r#type: String, + pub state: String, + pub mac_address: Option, + pub mtu: Option, + pub controller: Option, + pub ipv4: Option, + pub ipv6: Option, + pub ethernet: Option, + pub link_aggregation: Option, + pub vlan: Option, + pub vxlan: Option, + pub mac_vtap: Option, + pub mac_vlan: Option, + pub infiniband: Option, + pub linux_bridge: Option, + pub ovs_bridge: Option, + pub ethtool: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct IpStackSpec { + pub enabled: Option, + pub dhcp: Option, + pub autoconf: Option, + pub address: Option>, + pub auto_dns: Option, + pub auto_gateway: Option, + pub auto_routes: Option, + pub dhcp_client_id: Option, + pub dhcp_duid: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct IpAddressSpec { + pub ip: String, + pub prefix_length: u8, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthernetSpec { + pub speed: Option, + pub duplex: Option, + pub auto_negotiation: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct BondSpec { + pub mode: String, + pub ports: Vec, + pub options: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanSpec { + pub base_iface: String, + pub id: u16, + pub protocol: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VxlanSpec { + pub base_iface: String, + pub id: u32, + pub remote: String, + pub local: Option, + pub learning: Option, + pub destination_port: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct MacVtapSpec { + pub base_iface: String, + pub mode: String, + pub promiscuous: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct MacVlanSpec { + pub base_iface: String, + pub mode: String, + pub promiscuous: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct InfinibandSpec { + pub base_iface: String, + pub pkey: String, + pub mode: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgeSpec { + pub options: Option, + pub ports: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgeOptions { + pub mac_ageing_time: Option, + pub multicast_snooping: Option, + pub stp: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct StpOptions { + pub enabled: Option, + pub forward_delay: Option, + pub hello_time: Option, + pub max_age: Option, + pub priority: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgePort { + pub name: String, + pub vlan: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgePortVlan { + pub mode: Option, + pub trunk_tags: Option>, + pub tag: Option, + pub enable_native: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanTag { + pub id: u16, + pub id_range: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanIdRange { + pub min: u16, + pub max: u16, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsBridgeSpec { + pub options: Option, + pub ports: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsBridgeOptions { + pub stp: Option, + pub rstp: Option, + pub mcast_snooping_enable: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsPortSpec { + pub name: String, + pub link_aggregation: Option, + pub vlan: Option, + pub r#type: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthtoolSpec { + // FIXME: Properly describe this spec +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthtoolFecSpec { + pub auto: Option, + pub mode: Option, +} diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs index 7a68115..f6318c0 100644 --- a/harmony/src/modules/okd/host_network.rs +++ b/harmony/src/modules/okd/host_network.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use harmony_types::{id::Id, net::MacAddress}; +use harmony_types::id::Id; use serde::Serialize; use crate::{ @@ -8,7 +8,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{HostNetworkConfig, Switch, SwitchPort, Topology}, + topology::{HostNetworkConfig, NetworkInterface, Switch, SwitchPort, Topology}, }; #[derive(Debug, Clone, Serialize)] @@ -59,10 +59,17 @@ impl Interpret for HostNetworkConfigurationInterpret { for host in &self.score.hosts { let mut switch_ports = vec![]; - for mac_address in host.get_mac_address() { + for network_interface in &host.network { + let mac_address = network_interface.mac_address; + if let Some(port_name) = topology.get_port_for_mac_address(&mac_address).await { switch_ports.push(SwitchPort { - mac_address, + interface: NetworkInterface { + name: network_interface.name.clone(), + mac_address, + speed_mbps: network_interface.speed_mbps, + mtu: network_interface.mtu, + }, port_name, }); } @@ -73,14 +80,6 @@ impl Interpret for HostNetworkConfigurationInterpret { .await; } - // foreach hosts - // foreach mac addresses - // let port = topology.get_port_for_mac_address(); // si pas de port -> mac address pas connectee - // create port channel for all ports found - // create bond for all valid addresses (port found) - // apply network config - // topology.configure_host_network(host, config) <--- will create both bonds & port channels - Ok(Outcome::success("".into())) } } @@ -88,7 +87,7 @@ impl Interpret for HostNetworkConfigurationInterpret { #[cfg(test)] mod tests { use assertor::*; - use harmony_inventory_agent::hwinfo::NetworkInterface; + use harmony_types::net::MacAddress; use lazy_static::lazy_static; use crate::{ @@ -107,19 +106,31 @@ mod tests { lazy_static! { pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap(); pub static ref ANOTHER_HOST_ID: Id = Id::from_str("host-2").unwrap(); - pub static ref EXISTING_INTERFACE: MacAddress = - MacAddress::try_from("AA:BB:CC:DD:EE:F1".to_string()).unwrap(); - pub static ref ANOTHER_EXISTING_INTERFACE: MacAddress = - MacAddress::try_from("AA:BB:CC:DD:EE:F2".to_string()).unwrap(); - pub static ref UNKNOWN_INTERFACE: MacAddress = - MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(); + pub static ref EXISTING_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F1".to_string()).unwrap(), + name: "interface-1".into(), + speed_mbps: None, + mtu: 1, + }; + pub static ref ANOTHER_EXISTING_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F2".to_string()).unwrap(), + name: "interface-2".into(), + speed_mbps: None, + mtu: 1, + }; + pub static ref UNKNOWN_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(), + name: "unknown-interface".into(), + speed_mbps: None, + mtu: 1, + }; pub static ref PORT: String = "1/0/42".into(); pub static ref ANOTHER_PORT: String = "2/0/42".into(); } #[tokio::test] async fn host_with_one_mac_address_should_create_bond_with_one_interface() { - let host = given_host(&HOST_ID, vec![*EXISTING_INTERFACE]); + let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]); let score = given_score(vec![host]); let topology = TopologyWithSwitch::new(); @@ -130,7 +141,7 @@ mod tests { HOST_ID.clone(), HostNetworkConfig { switch_ports: vec![SwitchPort { - mac_address: *EXISTING_INTERFACE, + interface: EXISTING_INTERFACE.clone(), port_name: PORT.clone(), }], }, @@ -141,7 +152,10 @@ mod tests { async fn host_with_multiple_mac_addresses_should_create_one_bond_with_all_interfaces() { let score = given_score(vec![given_host( &HOST_ID, - vec![*EXISTING_INTERFACE, *ANOTHER_EXISTING_INTERFACE], + vec![ + EXISTING_INTERFACE.clone(), + ANOTHER_EXISTING_INTERFACE.clone(), + ], )]); let topology = TopologyWithSwitch::new(); @@ -153,11 +167,11 @@ mod tests { HostNetworkConfig { switch_ports: vec![ SwitchPort { - mac_address: *EXISTING_INTERFACE, + interface: EXISTING_INTERFACE.clone(), port_name: PORT.clone(), }, SwitchPort { - mac_address: *ANOTHER_EXISTING_INTERFACE, + interface: ANOTHER_EXISTING_INTERFACE.clone(), port_name: ANOTHER_PORT.clone(), }, ], @@ -168,8 +182,8 @@ mod tests { #[tokio::test] async fn multiple_hosts_should_create_one_bond_per_host() { let score = given_score(vec![ - given_host(&HOST_ID, vec![*EXISTING_INTERFACE]), - given_host(&ANOTHER_HOST_ID, vec![*ANOTHER_EXISTING_INTERFACE]), + given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]), + given_host(&ANOTHER_HOST_ID, vec![ANOTHER_EXISTING_INTERFACE.clone()]), ]); let topology = TopologyWithSwitch::new(); @@ -181,7 +195,7 @@ mod tests { HOST_ID.clone(), HostNetworkConfig { switch_ports: vec![SwitchPort { - mac_address: *EXISTING_INTERFACE, + interface: EXISTING_INTERFACE.clone(), port_name: PORT.clone(), }], }, @@ -190,7 +204,7 @@ mod tests { ANOTHER_HOST_ID.clone(), HostNetworkConfig { switch_ports: vec![SwitchPort { - mac_address: *ANOTHER_EXISTING_INTERFACE, + interface: ANOTHER_EXISTING_INTERFACE.clone(), port_name: ANOTHER_PORT.clone(), }], }, @@ -201,7 +215,7 @@ mod tests { #[tokio::test] async fn port_not_found_for_mac_address_should_not_configure_interface() { // FIXME: Should it still configure an empty bond/port channel? - let score = given_score(vec![given_host(&HOST_ID, vec![*UNKNOWN_INTERFACE])]); + let score = given_score(vec![given_host(&HOST_ID, vec![UNKNOWN_INTERFACE.clone()])]); let topology = TopologyWithSwitch::new_port_not_found(); let _ = score.interpret(&Inventory::empty(), &topology).await; @@ -219,8 +233,8 @@ mod tests { HostNetworkConfigurationScore { hosts } } - fn given_host(id: &Id, mac_addresses: Vec) -> PhysicalHost { - let network = mac_addresses.iter().map(|m| given_interface(*m)).collect(); + fn given_host(id: &Id, network_interfaces: Vec) -> PhysicalHost { + let network = network_interfaces.iter().map(given_interface).collect(); PhysicalHost { id: id.clone(), @@ -233,13 +247,15 @@ mod tests { } } - fn given_interface(mac_address: MacAddress) -> NetworkInterface { - NetworkInterface { - name: format!("{mac_address}"), - mac_address, - speed_mbps: None, + fn given_interface( + interface: &NetworkInterface, + ) -> harmony_inventory_agent::hwinfo::NetworkInterface { + harmony_inventory_agent::hwinfo::NetworkInterface { + name: interface.name.clone(), + mac_address: interface.mac_address, + speed_mbps: interface.speed_mbps, is_up: true, - mtu: 1, + mtu: interface.mtu, ipv4_addresses: vec![], ipv6_addresses: vec![], driver: "driver".into(), diff --git a/harmony/src/modules/okd/mod.rs b/harmony/src/modules/okd/mod.rs index 1d052d5..a12f132 100644 --- a/harmony/src/modules/okd/mod.rs +++ b/harmony/src/modules/okd/mod.rs @@ -19,4 +19,5 @@ pub use bootstrap_03_control_plane::*; pub use bootstrap_04_workers::*; pub use bootstrap_05_sanity_check::*; pub use bootstrap_06_installation_report::*; +pub mod crd; pub mod host_network; diff --git a/harmony_types/src/net.rs b/harmony_types/src/net.rs index 5b7449a..51de86e 100644 --- a/harmony_types/src/net.rs +++ b/harmony_types/src/net.rs @@ -41,7 +41,7 @@ impl TryFrom for MacAddress { bytes[i] = u8::from_str_radix(part, 16).map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidInput, - format!("Invalid hex value in part {}: '{}'", i, part), + format!("Invalid hex value in part {i}: '{part}'"), ) })?; } @@ -106,8 +106,8 @@ impl Serialize for Url { impl std::fmt::Display for Url { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Url::LocalFolder(path) => write!(f, "{}", path), - Url::Url(url) => write!(f, "{}", url), + Url::LocalFolder(path) => write!(f, "{path}"), + Url::Url(url) => write!(f, "{url}"), } } } -- 2.39.5 From 2388f585f5992365afb48b3236d855d4c688ebdd Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 25 Sep 2025 17:13:42 -0400 Subject: [PATCH 06/17] reorganize modules --- brocade/src/lib.rs | 3 +- examples/okd_installation/src/topology.rs | 2 +- harmony/src/domain/topology/ha_cluster.rs | 55 ++--------------------- harmony/src/domain/topology/network.rs | 6 +++ harmony/src/infra/brocade.rs | 49 ++++++++++++++++++++ harmony/src/infra/mod.rs | 1 + 6 files changed, 62 insertions(+), 54 deletions(-) create mode 100644 harmony/src/infra/brocade.rs diff --git a/brocade/src/lib.rs b/brocade/src/lib.rs index 49cfaae..0df2ec4 100644 --- a/brocade/src/lib.rs +++ b/brocade/src/lib.rs @@ -27,7 +27,8 @@ impl BrocadeClient { username: &str, password: &str, ) -> Result { - let ip = ip_addresses[0]; + let ip = ip_addresses[0]; // FIXME: Find a better way to get master switch IP address + let config = russh::client::Config::default(); let mut client = russh::client::connect(Arc::new(config), (ip, 22), Client {}).await?; diff --git a/examples/okd_installation/src/topology.rs b/examples/okd_installation/src/topology.rs index 02553a5..31062f5 100644 --- a/examples/okd_installation/src/topology.rs +++ b/examples/okd_installation/src/topology.rs @@ -1,6 +1,6 @@ use cidr::Ipv4Cidr; use harmony::{ - hardware::{FirewallGroup, HostCategory, Location, PhysicalHost, SwitchGroup}, + hardware::{Location, SwitchGroup}, infra::opnsense::OPNSenseManagementInterface, inventory::Inventory, topology::{HAClusterTopology, LogicalHost, UnmanagedRouter}, diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index cc07b8b..27c640c 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -1,7 +1,5 @@ use async_trait::async_trait; -use brocade::BrocadeClient; use harmony_macros::ip; -use harmony_secret::Secret; use harmony_secret::SecretManager; use harmony_types::net::MacAddress; use harmony_types::net::Url; @@ -9,12 +7,12 @@ use k8s_openapi::api::core::v1::Namespace; use kube::api::ObjectMeta; use log::debug; use log::info; -use serde::Deserialize; -use serde::Serialize; use crate::data::FileContent; use crate::executors::ExecutorError; use crate::hardware::PhysicalHost; +use crate::infra::brocade::BrocadeSwitchAuth; +use crate::infra::brocade::BrocadeSwitchClient; use crate::modules::okd::crd::InstallPlanApproval; use crate::modules::okd::crd::OperatorGroup; use crate::modules::okd::crd::OperatorGroupSpec; @@ -43,6 +41,7 @@ use super::PreparationError; use super::PreparationOutcome; use super::Router; use super::Switch; +use super::SwitchClient; use super::SwitchError; use super::TftpServer; @@ -527,54 +526,6 @@ impl Switch for HAClusterTopology { } } -#[async_trait] -trait SwitchClient: Send + Sync { - async fn find_port(&self, mac_address: &MacAddress) -> Option; - async fn configure_port_channel(&self, switch_ports: Vec) -> Result; -} - -struct BrocadeSwitchClient { - brocade: BrocadeClient, -} - -impl BrocadeSwitchClient { - async fn init( - ip_addresses: &[IpAddress], - username: &str, - password: &str, - ) -> Result { - let brocade = BrocadeClient::init(ip_addresses, username, password).await?; - Ok(Self { brocade }) - } -} - -#[async_trait] -impl SwitchClient for BrocadeSwitchClient { - async fn find_port(&self, mac_address: &MacAddress) -> Option { - let Ok(table) = self.brocade.show_mac_address_table().await else { - return None; - }; - - table - .iter() - .find(|entry| entry.mac_address == *mac_address) - .map(|entry| entry.port_name.clone()) - } - - async fn configure_port_channel(&self, switch_ports: Vec) -> Result { - self.brocade - .configure_port_channel(&switch_ports) - .await - .map_err(|e| SwitchError::new(format!("Failed to configure port channel: {e}"))) - } -} - -#[derive(Secret, Serialize, Deserialize, Debug)] -struct BrocadeSwitchAuth { - username: String, - password: String, -} - #[derive(Debug)] pub struct DummyInfra; diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index a1da8dd..fe02a5e 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -216,6 +216,12 @@ impl std::fmt::Display for SwitchError { impl Error for SwitchError {} +#[async_trait] +pub trait SwitchClient: Send + Sync { + async fn find_port(&self, mac_address: &MacAddress) -> Option; + async fn configure_port_channel(&self, switch_ports: Vec) -> Result; +} + #[cfg(test)] mod test { use std::sync::Arc; diff --git a/harmony/src/infra/brocade.rs b/harmony/src/infra/brocade.rs new file mode 100644 index 0000000..7e03565 --- /dev/null +++ b/harmony/src/infra/brocade.rs @@ -0,0 +1,49 @@ +use async_trait::async_trait; +use brocade::BrocadeClient; +use harmony_secret::Secret; +use harmony_types::net::{IpAddress, MacAddress}; +use serde::{Deserialize, Serialize}; + +use crate::topology::{SwitchClient, SwitchError}; + +pub struct BrocadeSwitchClient { + brocade: BrocadeClient, +} + +impl BrocadeSwitchClient { + pub async fn init( + ip_addresses: &[IpAddress], + username: &str, + password: &str, + ) -> Result { + let brocade = BrocadeClient::init(ip_addresses, username, password).await?; + Ok(Self { brocade }) + } +} + +#[async_trait] +impl SwitchClient for BrocadeSwitchClient { + async fn find_port(&self, mac_address: &MacAddress) -> Option { + let Ok(table) = self.brocade.show_mac_address_table().await else { + return None; + }; + + table + .iter() + .find(|entry| entry.mac_address == *mac_address) + .map(|entry| entry.port_name.clone()) + } + + async fn configure_port_channel(&self, switch_ports: Vec) -> Result { + self.brocade + .configure_port_channel(&switch_ports) + .await + .map_err(|e| SwitchError::new(format!("Failed to configure port channel: {e}"))) + } +} + +#[derive(Secret, Serialize, Deserialize, Debug)] +pub struct BrocadeSwitchAuth { + pub username: String, + pub password: String, +} diff --git a/harmony/src/infra/mod.rs b/harmony/src/infra/mod.rs index c05c7b6..203cf90 100644 --- a/harmony/src/infra/mod.rs +++ b/harmony/src/infra/mod.rs @@ -1,3 +1,4 @@ +pub mod brocade; pub mod executors; pub mod hp_ilo; pub mod intel_amt; -- 2.39.5 From 58c1fd4a963db39cfaea556b94da99f197fb4bc4 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 25 Sep 2025 17:17:23 -0400 Subject: [PATCH 07/17] remove unused code --- .../modules/okd/bootstrap_03_control_plane.rs | 7 +-- harmony/src/modules/okd/crd/machineconfig.rs | 43 ------------------- harmony/src/modules/okd/crd/mod.rs | 1 - 3 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 harmony/src/modules/okd/crd/machineconfig.rs diff --git a/harmony/src/modules/okd/bootstrap_03_control_plane.rs b/harmony/src/modules/okd/bootstrap_03_control_plane.rs index a3fe4c2..5efb983 100644 --- a/harmony/src/modules/okd/bootstrap_03_control_plane.rs +++ b/harmony/src/modules/okd/bootstrap_03_control_plane.rs @@ -206,10 +206,6 @@ impl OKDSetup03ControlPlaneInterpret { Ok(()) } - // TODO: Apply host network configuration. - // Delegate to a score: HostNetworkConfigurationScore { host: physical_host } qui manipule Switch dans Topology - // Use-case Affilium: remplacement carte reseau, pas juste installation clean - // /// Placeholder for automating network bonding configuration. async fn persist_network_bond( &self, @@ -217,8 +213,7 @@ impl OKDSetup03ControlPlaneInterpret { topology: &HAClusterTopology, hosts: &Vec, ) -> Result<(), InterpretError> { - // Generate MC or NNCP from inventory NIC data; apply via ignition or post-join. - info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP"); + info!("[ControlPlane] Ensuring persistent bonding"); let score = HostNetworkConfigurationScore { hosts: hosts.clone(), // FIXME: Avoid clone if possible }; diff --git a/harmony/src/modules/okd/crd/machineconfig.rs b/harmony/src/modules/okd/crd/machineconfig.rs deleted file mode 100644 index df710e2..0000000 --- a/harmony/src/modules/okd/crd/machineconfig.rs +++ /dev/null @@ -1,43 +0,0 @@ -use kube::CustomResource; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube( - group = "machineconfiguration.openshift.io", - version = "v1", - kind = "MachineConfig", - namespaced -)] -#[serde(rename_all = "camelCase")] -pub struct MachineConfigSpec { - pub config: Config, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct Config { - pub ignition: Ignition, - pub storage: Option, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -pub struct Ignition { - pub version: String, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -pub struct Storage { - pub files: Vec, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -pub struct File { - pub path: String, - pub contents: FileContents, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -pub struct FileContents { - pub source: String, -} diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index aa35190..c1a68ce 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -2,7 +2,6 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -pub mod machineconfig; pub mod nmstate; #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -- 2.39.5 From 7b6ac6641ac2192b74f06ebabc38a7445948821e Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 25 Sep 2025 17:18:42 -0400 Subject: [PATCH 08/17] add link to finish describing EthtoolSpec --- harmony/src/modules/okd/crd/nmstate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harmony/src/modules/okd/crd/nmstate.rs b/harmony/src/modules/okd/crd/nmstate.rs index b6117fd..9506882 100644 --- a/harmony/src/modules/okd/crd/nmstate.rs +++ b/harmony/src/modules/okd/crd/nmstate.rs @@ -240,7 +240,7 @@ pub struct OvsPortSpec { #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct EthtoolSpec { - // FIXME: Properly describe this spec + // FIXME: Properly describe this spec (https://nmstate.io/devel/yaml_api.html#ethtool) } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] -- 2.39.5 From f2f55d98d48d38747a172fa4bc8d31e1b3ffbfd4 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Sun, 28 Sep 2025 12:55:37 -0400 Subject: [PATCH 09/17] add option to run brocade commands in dry-run --- brocade/src/lib.rs | 105 ++++++++++++++++++---- harmony/src/domain/hardware/mod.rs | 8 +- harmony/src/domain/topology/ha_cluster.rs | 27 +++--- harmony/src/domain/topology/network.rs | 7 +- harmony/src/infra/brocade.rs | 19 ++-- harmony/src/modules/okd/host_network.rs | 66 +++++++++----- harmony_composer/src/main.rs | 4 + 7 files changed, 171 insertions(+), 65 deletions(-) diff --git a/brocade/src/lib.rs b/brocade/src/lib.rs index 0df2ec4..411d9de 100644 --- a/brocade/src/lib.rs +++ b/brocade/src/lib.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, fmt::{self, Display}, sync::Arc, }; @@ -6,8 +7,11 @@ use std::{ use async_trait::async_trait; use harmony_types::net::{IpAddress, MacAddress}; use log::{debug, info}; -use russh::client::{Handle, Handler}; -use russh_keys::key; +use russh::{ + client::{Handle, Handler}, + kex::DH_G1_SHA1, +}; +use russh_keys::key::{self, SSH_RSA}; use std::str::FromStr; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] @@ -19,6 +23,30 @@ pub struct MacAddressEntry { pub struct BrocadeClient { client: Handle, + options: BrocadeOptions, +} + +#[derive(Default, Clone, Debug)] +pub struct BrocadeOptions { + pub dry_run: bool, + pub ssh: SshOptions, +} + +#[derive(Clone, Debug)] +pub struct SshOptions { + pub preferred_algorithms: russh::Preferred, +} + +impl Default for SshOptions { + fn default() -> Self { + Self { + preferred_algorithms: russh::Preferred { + kex: Cow::Borrowed(&[DH_G1_SHA1]), + key: Cow::Borrowed(&[SSH_RSA]), + ..Default::default() + }, + } + } } impl BrocadeClient { @@ -26,14 +54,25 @@ impl BrocadeClient { ip_addresses: &[IpAddress], username: &str, password: &str, + options: Option, ) -> Result { - let ip = ip_addresses[0]; // FIXME: Find a better way to get master switch IP address + if ip_addresses.is_empty() { + return Err(Error::ConfigurationError( + "No IP addresses provided".to_string(), + )); + } - let config = russh::client::Config::default(); + let ip = ip_addresses[0]; // FIXME: Find a better way to get master switch IP address + let options = options.unwrap_or_default(); + + let config = russh::client::Config { + preferred: options.ssh.preferred_algorithms.clone(), + ..Default::default() + }; let mut client = russh::client::connect(Arc::new(config), (ip, 22), Client {}).await?; match client.authenticate_password(username, password).await? { - true => Ok(Self { client }), + true => Ok(Self { client, options }), false => Err(Error::AuthenticationError( "ssh authentication failed".to_string(), )), @@ -41,7 +80,7 @@ impl BrocadeClient { } pub async fn show_mac_address_table(&self) -> Result, Error> { - let output = self.run_command("show mac-address-table").await?; + let output = self.run_command("show mac-address").await?; let mut entries = Vec::new(); // The Brocade output usually has a header and then one entry per line. @@ -104,7 +143,7 @@ impl BrocadeClient { pub async fn find_available_channel_id(&self) -> Result { debug!("[Brocade] Finding next available channel id..."); - let output = self.run_command("show port-channel summary").await?; + let output = self.run_command("show lag").await?; let mut used_ids = Vec::new(); // Sample output line: "3 Po3(SU) LACP Eth Yes 128/128 active " @@ -121,7 +160,7 @@ impl BrocadeClient { // Sort the used IDs to find the next available number. used_ids.sort(); - let mut next_id = 1; + let mut next_id = 0; for &id in &used_ids { if id == next_id { next_id += 1; @@ -136,6 +175,11 @@ impl BrocadeClient { } async fn run_command(&self, command: &str) -> Result { + if !command.starts_with("show") && self.options.dry_run { + info!("[Brocade] Dry-run mode enabled, skipping command: {command}"); + return Ok("".into()); + } + debug!("[Brocade] Running command: '{command}'..."); let mut channel = self.client.channel_open_session().await?; @@ -161,9 +205,13 @@ impl BrocadeClient { ))); } } - russh::ChannelMsg::Success - | russh::ChannelMsg::WindowAdjusted { .. } - | russh::ChannelMsg::Eof => {} + russh::ChannelMsg::Eof => { + channel.close().await?; + } + russh::ChannelMsg::Close => { + break; + } + russh::ChannelMsg::Success | russh::ChannelMsg::WindowAdjusted { .. } => {} _ => { return Err(Error::UnexpectedError(format!( "Russh got unexpected msg {msg:?}" @@ -172,20 +220,25 @@ impl BrocadeClient { } } - channel.close().await?; - let output = String::from_utf8(output).expect("Output should be UTF-8 compatible"); + debug!("[Brocade] Command output:\n{output}"); Ok(output) } async fn run_commands(&self, commands: Vec) -> Result<(), Error> { - let mut channel = self.client.channel_open_session().await?; - // Execute commands sequentially and check for errors immediately. for command in commands { + if !command.starts_with("show") && self.options.dry_run { + info!("[Brocade] Dry-run mode enabled, skipping command: {command}"); + continue; + } + debug!("[Brocade] Running command: '{command}'..."); + let mut channel = self.client.channel_open_session().await?; let mut output = Vec::new(); + let mut close_received = false; + channel.exec(true, command.as_str()).await?; loop { @@ -206,12 +259,30 @@ impl BrocadeClient { ))); } } - _ => {} // Ignore other messages like success or EOF for now. + russh::ChannelMsg::Eof => { + channel.close().await?; + } + russh::ChannelMsg::Close => { + close_received = true; + break; + } + russh::ChannelMsg::Success | russh::ChannelMsg::WindowAdjusted { .. } => {} + _ => { + return Err(Error::UnexpectedError(format!( + "Russh got unexpected msg {msg:?}" + ))); + } } } + + if !close_received { + return Err(Error::UnexpectedError(format!( + "Channel closed without receiving a final CLOSE message for command: {}", + command + ))); + } } - channel.close().await?; Ok(()) } } diff --git a/harmony/src/domain/hardware/mod.rs b/harmony/src/domain/hardware/mod.rs index 1b1a72c..a070b83 100644 --- a/harmony/src/domain/hardware/mod.rs +++ b/harmony/src/domain/hardware/mod.rs @@ -12,11 +12,11 @@ pub type FirewallGroup = Vec; pub struct PhysicalHost { pub id: Id, pub category: HostCategory, - pub network: Vec, - pub storage: Vec, + pub network: Vec, // FIXME: Don't use harmony_inventory_agent::NetworkInterface + pub storage: Vec, // FIXME: Don't use harmony_inventory_agent::StorageDrive pub labels: Vec