From ea39d93aa79037ac3bc847638a1ea837207fe0fc Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Mon, 15 Sep 2025 17:07:50 -0400 Subject: [PATCH] feat(host_network): configure bonds on the host and switch port channels --- Cargo.lock | 36 +- brocade/src/network_operating_system.rs | 2 +- examples/okd_installation/src/topology.rs | 2 +- harmony/Cargo.toml | 3 + harmony/src/domain/topology/ha_cluster.rs | 299 ++++++++++++- harmony/src/domain/topology/network.rs | 84 +++- harmony/src/infra/brocade.rs | 385 +++++++++++++++++ harmony/src/infra/mod.rs | 1 + .../modules/okd/bootstrap_03_control_plane.rs | 41 +- harmony/src/modules/okd/crd/mod.rs | 41 ++ harmony/src/modules/okd/crd/nmstate.rs | 251 +++++++++++ harmony/src/modules/okd/host_network.rs | 394 ++++++++++++++++++ harmony/src/modules/okd/mod.rs | 2 + harmony_composer/src/main.rs | 4 + 14 files changed, 1477 insertions(+), 68 deletions(-) create mode 100644 harmony/src/infra/brocade.rs create mode 100644 harmony/src/modules/okd/crd/mod.rs create mode 100644 harmony/src/modules/okd/crd/nmstate.rs create mode 100644 harmony/src/modules/okd/host_network.rs diff --git a/Cargo.lock b/Cargo.lock index f88b4a6..0b5972f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,6 +429,15 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "assertor" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff24d87260733dc86d38a11c60d9400ce4a74a05d0dafa2a6f5ab249cd857cb" +dependencies = [ + "num-traits", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -2333,9 +2342,11 @@ name = "harmony" version = "0.1.0" dependencies = [ "askama", + "assertor", "async-trait", "base64 0.22.1", "bollard", + "brocade", "chrono", "cidr", "convert_case", @@ -2366,6 +2377,7 @@ dependencies = [ "once_cell", "opnsense-config", "opnsense-config-xml", + "option-ext", "pretty_assertions", "reqwest 0.11.27", "russh", @@ -2432,17 +2444,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "harmony_derive" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d138bbb32bb346299c5f95fbb53532313f39927cb47c411c99c634ef8665ef7" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "harmony_inventory_agent" version = "0.1.0" @@ -3889,19 +3890,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "okd_host_network" -version = "0.1.0" -dependencies = [ - "harmony", - "harmony_cli", - "harmony_derive", - "harmony_inventory_agent", - "harmony_macros", - "harmony_types", - "tokio", -] - [[package]] name = "once_cell" version = "1.21.3" diff --git a/brocade/src/network_operating_system.rs b/brocade/src/network_operating_system.rs index 4b9b271..b14bc08 100644 --- a/brocade/src/network_operating_system.rs +++ b/brocade/src/network_operating_system.rs @@ -270,7 +270,7 @@ impl BrocadeClient for NetworkOperatingSystemClient { commands.push("no ip address".into()); commands.push("no fabric isl enable".into()); commands.push("no fabric trunk enable".into()); - commands.push(format!("channel-group {} mode active", channel_id)); + commands.push(format!("channel-group {channel_id} mode active")); commands.push("no shutdown".into()); commands.push("exit".into()); } diff --git a/examples/okd_installation/src/topology.rs b/examples/okd_installation/src/topology.rs index 02553a5..31062f5 100644 --- a/examples/okd_installation/src/topology.rs +++ b/examples/okd_installation/src/topology.rs @@ -1,6 +1,6 @@ use cidr::Ipv4Cidr; use harmony::{ - hardware::{FirewallGroup, HostCategory, Location, PhysicalHost, SwitchGroup}, + hardware::{Location, SwitchGroup}, infra::opnsense::OPNSenseManagementInterface, inventory::Inventory, topology::{HAClusterTopology, LogicalHost, UnmanagedRouter}, diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index ad57db1..634cbe9 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -77,6 +77,9 @@ harmony_secret = { path = "../harmony_secret" } askama.workspace = true sqlx.workspace = true inquire.workspace = true +brocade = { path = "../brocade" } +option-ext = "0.2.0" [dev-dependencies] pretty_assertions.workspace = true +assertor.workspace = true diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index c9f565e..7be2725 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -1,33 +1,36 @@ use async_trait::async_trait; +use brocade::BrocadeOptions; use harmony_macros::ip; -use harmony_types::net::MacAddress; -use harmony_types::net::Url; +use harmony_secret::SecretManager; +use harmony_types::{ + net::{MacAddress, Url}, + switch::PortLocation, +}; +use k8s_openapi::api::core::v1::Namespace; +use kube::api::ObjectMeta; use log::debug; use log::info; use crate::data::FileContent; use crate::executors::ExecutorError; +use crate::hardware::PhysicalHost; +use crate::infra::brocade::BrocadeSwitchAuth; +use crate::infra::brocade::BrocadeSwitchClient; +use crate::modules::okd::crd::{ + InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec, + nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec}, +}; use crate::topology::PxeOptions; -use super::DHCPStaticEntry; -use super::DhcpServer; -use super::DnsRecord; -use super::DnsRecordType; -use super::DnsServer; -use super::Firewall; -use super::HttpServer; -use super::IpAddress; -use super::K8sclient; -use super::LoadBalancer; -use super::LoadBalancerService; -use super::LogicalHost; -use super::PreparationError; -use super::PreparationOutcome; -use super::Router; -use super::TftpServer; +use super::{ + DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig, + HttpServer, IpAddress, K8sclient, LoadBalancer, LoadBalancerService, LogicalHost, + PreparationError, PreparationOutcome, Router, Switch, SwitchClient, SwitchError, TftpServer, + Topology, k8s::K8sClient, +}; -use super::Topology; -use super::k8s::K8sClient; +use std::collections::BTreeMap; +use std::net::IpAddr; use std::sync::Arc; #[derive(Debug, Clone)] @@ -89,6 +92,231 @@ impl HAClusterTopology { .to_string() } + async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> { + // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate) + debug!("Installing NMState operator..."); + let k8s_client = self.k8s_client().await?; + + let nmstate_namespace = Namespace { + metadata: ObjectMeta { + name: Some("openshift-nmstate".to_string()), + finalizers: Some(vec!["kubernetes".to_string()]), + ..Default::default() + }, + ..Default::default() + }; + debug!("Creating NMState namespace: {nmstate_namespace:#?}"); + k8s_client + .apply(&nmstate_namespace, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate_operator_group = OperatorGroup { + metadata: ObjectMeta { + name: Some("openshift-nmstate".to_string()), + namespace: Some("openshift-nmstate".to_string()), + ..Default::default() + }, + spec: OperatorGroupSpec { + target_namespaces: vec!["openshift-nmstate".to_string()], + }, + }; + debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); + k8s_client + .apply(&nmstate_operator_group, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate_subscription = Subscription { + metadata: ObjectMeta { + name: Some("kubernetes-nmstate-operator".to_string()), + namespace: Some("openshift-nmstate".to_string()), + ..Default::default() + }, + spec: SubscriptionSpec { + channel: Some("stable".to_string()), + install_plan_approval: Some(InstallPlanApproval::Automatic), + name: "kubernetes-nmstate-operator".to_string(), + source: "redhat-operators".to_string(), + source_namespace: "openshift-marketplace".to_string(), + }, + }; + debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}"); + k8s_client + .apply(&nmstate_subscription, None) + .await + .map_err(|e| e.to_string())?; + + let nmstate = NMState { + metadata: ObjectMeta { + name: Some("nmstate".to_string()), + ..Default::default() + }, + ..Default::default() + }; + debug!("Creating NMState: {nmstate:#?}"); + k8s_client + .apply(&nmstate, None) + .await + .map_err(|e| e.to_string())?; + + Ok(()) + } + + fn get_next_bond_id(&self) -> u8 { + 42 // FIXME: Find a better way to declare the bond id + } + + async fn configure_bond( + &self, + host: &PhysicalHost, + config: &HostNetworkConfig, + ) -> Result<(), SwitchError> { + self.ensure_nmstate_operator_installed() + .await + .map_err(|e| { + SwitchError::new(format!( + "Can't configure bond, NMState operator not available: {e}" + )) + })?; + + let bond_config = self.create_bond_configuration(host, config); + debug!("Configuring bond for host {host:?}: {bond_config:#?}"); + self.k8s_client() + .await + .unwrap() + .apply(&bond_config, None) + .await + .unwrap(); + + todo!() + } + + fn create_bond_configuration( + &self, + host: &PhysicalHost, + config: &HostNetworkConfig, + ) -> NodeNetworkConfigurationPolicy { + let host_name = host.id.clone(); + + let bond_id = self.get_next_bond_id(); + let bond_name = format!("bond{bond_id}"); + let mut bond_mtu: Option = None; + let mut bond_mac_address: Option = None; + let mut bond_ports = Vec::new(); + let mut interfaces: Vec = Vec::new(); + + for switch_port in &config.switch_ports { + let interface_name = switch_port.interface.name.clone(); + + interfaces.push(nmstate::InterfaceSpec { + name: interface_name.clone(), + description: Some(format!("Member of bond {bond_name}")), + r#type: "ethernet".to_string(), + state: "up".to_string(), + mtu: Some(switch_port.interface.mtu), + mac_address: Some(switch_port.interface.mac_address.to_string()), + ipv4: Some(nmstate::IpStackSpec { + enabled: Some(false), + ..Default::default() + }), + ipv6: Some(nmstate::IpStackSpec { + enabled: Some(false), + ..Default::default() + }), + link_aggregation: None, + ..Default::default() + }); + + bond_ports.push(interface_name); + + // Use the first port's details for the bond mtu and mac address + if bond_mtu.is_none() { + bond_mtu = Some(switch_port.interface.mtu); + } + if bond_mac_address.is_none() { + bond_mac_address = Some(switch_port.interface.mac_address.to_string()); + } + } + + interfaces.push(nmstate::InterfaceSpec { + name: bond_name.clone(), + description: Some(format!("Network bond for host {host_name}")), + r#type: "bond".to_string(), + state: "up".to_string(), + mtu: bond_mtu, + mac_address: bond_mac_address, + ipv4: Some(nmstate::IpStackSpec { + dhcp: Some(true), + enabled: Some(true), + ..Default::default() + }), + ipv6: Some(nmstate::IpStackSpec { + dhcp: Some(true), + autoconf: Some(true), + enabled: Some(true), + ..Default::default() + }), + link_aggregation: Some(nmstate::BondSpec { + mode: "802.3ad".to_string(), + ports: bond_ports, + ..Default::default() + }), + ..Default::default() + }); + + NodeNetworkConfigurationPolicy { + metadata: ObjectMeta { + name: Some(format!("{host_name}-bond-config")), + ..Default::default() + }, + spec: NodeNetworkConfigurationPolicySpec { + node_selector: Some(BTreeMap::from([( + "kubernetes.io/hostname".to_string(), + host_name.to_string(), + )])), + desired_state: nmstate::DesiredStateSpec { interfaces }, + }, + } + } + + async fn get_switch_client(&self) -> Result, SwitchError> { + let auth = SecretManager::get_or_prompt::() + .await + .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?; + + // FIXME: We assume Brocade switches + let switches: Vec = self.switch.iter().map(|s| s.ip).collect(); + let brocade_options = Some(BrocadeOptions { + dry_run: *crate::config::DRY_RUN, + ..Default::default() + }); + let client = + BrocadeSwitchClient::init(&switches, &auth.username, &auth.password, brocade_options) + .await + .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?; + + Ok(Box::new(client)) + } + + async fn configure_port_channel( + &self, + host: &PhysicalHost, + config: &HostNetworkConfig, + ) -> Result<(), SwitchError> { + debug!("Configuring port channel: {config:#?}"); + let client = self.get_switch_client().await?; + + let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect(); + + client + .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports) + .await + .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?; + + Ok(()) + } + pub fn autoload() -> Self { let dummy_infra = Arc::new(DummyInfra {}); let dummy_host = LogicalHost { @@ -263,6 +491,33 @@ impl HttpServer for HAClusterTopology { } } +#[async_trait] +impl Switch for HAClusterTopology { + async fn setup_switch(&self) -> Result<(), SwitchError> { + let client = self.get_switch_client().await?; + client.setup().await?; + Ok(()) + } + + async fn get_port_for_mac_address( + &self, + mac_address: &MacAddress, + ) -> Result, SwitchError> { + let client = self.get_switch_client().await?; + let port = client.find_port(mac_address).await?; + Ok(port) + } + + async fn configure_host_network( + &self, + host: &PhysicalHost, + config: HostNetworkConfig, + ) -> Result<(), SwitchError> { + self.configure_bond(host, &config).await?; + self.configure_port_channel(host, &config).await + } +} + #[derive(Debug)] pub struct DummyInfra; @@ -332,8 +587,8 @@ impl DhcpServer for DummyInfra { } async fn set_dhcp_range( &self, - start: &IpAddress, - end: &IpAddress, + _start: &IpAddress, + _end: &IpAddress, ) -> Result<(), ExecutorError> { unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA) } diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index c7ab5cc..99db03a 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -1,10 +1,14 @@ -use std::{net::Ipv4Addr, str::FromStr, sync::Arc}; +use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc}; use async_trait::async_trait; -use harmony_types::net::{IpAddress, MacAddress}; +use derive_new::new; +use harmony_types::{ + net::{IpAddress, MacAddress}, + switch::PortLocation, +}; use serde::Serialize; -use crate::executors::ExecutorError; +use crate::{executors::ExecutorError, hardware::PhysicalHost}; use super::{LogicalHost, k8s::K8sClient}; @@ -172,6 +176,80 @@ impl FromStr for DnsRecordType { } } +#[async_trait] +pub trait Switch: Send + Sync { + async fn setup_switch(&self) -> Result<(), SwitchError>; + + async fn get_port_for_mac_address( + &self, + mac_address: &MacAddress, + ) -> Result, SwitchError>; + + async fn configure_host_network( + &self, + host: &PhysicalHost, + config: HostNetworkConfig, + ) -> Result<(), SwitchError>; +} + +#[derive(Clone, Debug, PartialEq)] +pub struct HostNetworkConfig { + pub switch_ports: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SwitchPort { + pub interface: NetworkInterface, + pub port: PortLocation, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct NetworkInterface { + pub name: String, + pub mac_address: MacAddress, + pub speed_mbps: Option, + pub mtu: u32, +} + +#[derive(Debug, Clone, new)] +pub struct SwitchError { + msg: String, +} + +impl std::fmt::Display for SwitchError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.msg) + } +} + +impl Error for SwitchError {} + +#[async_trait] +pub trait SwitchClient: Send + Sync { + /// Executes essential, idempotent, one-time initial configuration steps. + /// + /// This is an opiniated procedure that setups a switch to provide high availability + /// capabilities as decided by the NationTech team. + /// + /// This includes tasks like enabling switchport for all interfaces + /// except the ones intended for Fabric Networking, etc. + /// + /// The implementation must ensure the operation is **idempotent** (safe to run multiple times) + /// and that it doesn't break existing configurations. + async fn setup(&self) -> Result<(), SwitchError>; + + async fn find_port( + &self, + mac_address: &MacAddress, + ) -> Result, SwitchError>; + + async fn configure_port_channel( + &self, + channel_name: &str, + switch_ports: Vec, + ) -> Result; +} + #[cfg(test)] mod test { use std::sync::Arc; diff --git a/harmony/src/infra/brocade.rs b/harmony/src/infra/brocade.rs new file mode 100644 index 0000000..f721328 --- /dev/null +++ b/harmony/src/infra/brocade.rs @@ -0,0 +1,385 @@ +use async_trait::async_trait; +use brocade::{BrocadeClient, BrocadeOptions, InterSwitchLink, InterfaceStatus, PortOperatingMode}; +use harmony_secret::Secret; +use harmony_types::{ + net::{IpAddress, MacAddress}, + switch::{PortDeclaration, PortLocation}, +}; +use option_ext::OptionExt; +use serde::{Deserialize, Serialize}; + +use crate::topology::{SwitchClient, SwitchError}; + +pub struct BrocadeSwitchClient { + brocade: Box, +} + +impl BrocadeSwitchClient { + pub async fn init( + ip_addresses: &[IpAddress], + username: &str, + password: &str, + options: Option, + ) -> Result { + let brocade = brocade::init(ip_addresses, 22, username, password, options).await?; + Ok(Self { brocade }) + } +} + +#[async_trait] +impl SwitchClient for BrocadeSwitchClient { + async fn setup(&self) -> Result<(), SwitchError> { + let stack_topology = self + .brocade + .get_stack_topology() + .await + .map_err(|e| SwitchError::new(e.to_string()))?; + + let interfaces = self + .brocade + .get_interfaces() + .await + .map_err(|e| SwitchError::new(e.to_string()))?; + + let interfaces: Vec<(String, PortOperatingMode)> = interfaces + .into_iter() + .filter(|interface| { + interface.operating_mode.is_none() && interface.status == InterfaceStatus::Connected + }) + .filter(|interface| { + !stack_topology.iter().any(|link: &InterSwitchLink| { + link.local_port == interface.port_location + || link.remote_port.contains(&interface.port_location) + }) + }) + .map(|interface| (interface.name.clone(), PortOperatingMode::Access)) + .collect(); + + if interfaces.is_empty() { + return Ok(()); + } + + self.brocade + .configure_interfaces(interfaces) + .await + .map_err(|e| SwitchError::new(e.to_string()))?; + + Ok(()) + } + + async fn find_port( + &self, + mac_address: &MacAddress, + ) -> Result, SwitchError> { + let table = self + .brocade + .get_mac_address_table() + .await + .map_err(|e| SwitchError::new(format!("{e}")))?; + + let port = table + .iter() + .find(|entry| entry.mac_address == *mac_address) + .map(|entry| match &entry.port { + PortDeclaration::Single(port_location) => Ok(port_location.clone()), + _ => Err(SwitchError::new( + "Multiple ports found for MAC address".into(), + )), + }); + + match port { + Some(Ok(p)) => Ok(Some(p)), + Some(Err(e)) => Err(e), + None => Ok(None), + } + } + + async fn configure_port_channel( + &self, + channel_name: &str, + switch_ports: Vec, + ) -> Result { + let channel_id = self + .brocade + .find_available_channel_id() + .await + .map_err(|e| SwitchError::new(format!("{e}")))?; + + self.brocade + .create_port_channel(channel_id, channel_name, &switch_ports) + .await + .map_err(|e| SwitchError::new(format!("{e}")))?; + + Ok(channel_id) + } +} + +#[derive(Secret, Serialize, Deserialize, Debug)] +pub struct BrocadeSwitchAuth { + pub username: String, + pub password: String, +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use assertor::*; + use async_trait::async_trait; + use brocade::{ + BrocadeClient, BrocadeInfo, Error, InterSwitchLink, InterfaceInfo, InterfaceStatus, + InterfaceType, MacAddressEntry, PortChannelId, PortOperatingMode, + }; + use harmony_types::switch::PortLocation; + + use crate::{infra::brocade::BrocadeSwitchClient, topology::SwitchClient}; + + #[tokio::test] + async fn setup_should_configure_ethernet_interfaces_as_access_ports() { + let first_interface = given_interface() + .with_port_location(PortLocation(1, 0, 1)) + .build(); + let second_interface = given_interface() + .with_port_location(PortLocation(1, 0, 4)) + .build(); + let brocade = Box::new(FakeBrocadeClient::new( + vec![], + vec![first_interface.clone(), second_interface.clone()], + )); + let client = BrocadeSwitchClient { + brocade: brocade.clone(), + }; + + client.setup().await.unwrap(); + + let configured_interfaces = brocade.configured_interfaces.lock().unwrap(); + assert_that!(*configured_interfaces).contains_exactly(vec![ + (first_interface.name.clone(), PortOperatingMode::Access), + (second_interface.name.clone(), PortOperatingMode::Access), + ]); + } + + #[tokio::test] + async fn setup_with_an_already_configured_interface_should_skip_configuration() { + let brocade = Box::new(FakeBrocadeClient::new( + vec![], + vec![ + given_interface() + .with_operating_mode(Some(PortOperatingMode::Access)) + .build(), + ], + )); + let client = BrocadeSwitchClient { + brocade: brocade.clone(), + }; + + client.setup().await.unwrap(); + + let configured_interfaces = brocade.configured_interfaces.lock().unwrap(); + assert_that!(*configured_interfaces).is_empty(); + } + + #[tokio::test] + async fn setup_with_a_disconnected_interface_should_skip_configuration() { + let brocade = Box::new(FakeBrocadeClient::new( + vec![], + vec![ + given_interface() + .with_status(InterfaceStatus::SfpAbsent) + .build(), + given_interface() + .with_status(InterfaceStatus::NotConnected) + .build(), + ], + )); + let client = BrocadeSwitchClient { + brocade: brocade.clone(), + }; + + client.setup().await.unwrap(); + + let configured_interfaces = brocade.configured_interfaces.lock().unwrap(); + assert_that!(*configured_interfaces).is_empty(); + } + + #[tokio::test] + async fn setup_with_inter_switch_links_should_not_configure_interfaces_used_to_form_stack() { + let brocade = Box::new(FakeBrocadeClient::new( + vec![ + given_inter_switch_link() + .between(PortLocation(1, 0, 1), PortLocation(2, 0, 1)) + .build(), + given_inter_switch_link() + .between(PortLocation(2, 0, 2), PortLocation(3, 0, 1)) + .build(), + ], + vec![ + given_interface() + .with_port_location(PortLocation(1, 0, 1)) + .build(), + given_interface() + .with_port_location(PortLocation(2, 0, 1)) + .build(), + given_interface() + .with_port_location(PortLocation(3, 0, 1)) + .build(), + ], + )); + let client = BrocadeSwitchClient { + brocade: brocade.clone(), + }; + + client.setup().await.unwrap(); + + let configured_interfaces = brocade.configured_interfaces.lock().unwrap(); + assert_that!(*configured_interfaces).is_empty(); + } + + #[derive(Clone)] + struct FakeBrocadeClient { + stack_topology: Vec, + interfaces: Vec, + configured_interfaces: Arc>>, + } + + #[async_trait] + impl BrocadeClient for FakeBrocadeClient { + async fn version(&self) -> Result { + todo!() + } + + async fn get_mac_address_table(&self) -> Result, Error> { + todo!() + } + + async fn get_stack_topology(&self) -> Result, Error> { + Ok(self.stack_topology.clone()) + } + + async fn get_interfaces(&self) -> Result, Error> { + Ok(self.interfaces.clone()) + } + + async fn configure_interfaces( + &self, + interfaces: Vec<(String, PortOperatingMode)>, + ) -> Result<(), Error> { + let mut configured_interfaces = self.configured_interfaces.lock().unwrap(); + *configured_interfaces = interfaces; + + Ok(()) + } + + async fn find_available_channel_id(&self) -> Result { + todo!() + } + + async fn create_port_channel( + &self, + _channel_id: PortChannelId, + _channel_name: &str, + _ports: &[PortLocation], + ) -> Result<(), Error> { + todo!() + } + + async fn clear_port_channel(&self, _channel_name: &str) -> Result<(), Error> { + todo!() + } + } + + impl FakeBrocadeClient { + fn new(stack_topology: Vec, interfaces: Vec) -> Self { + Self { + stack_topology, + interfaces, + configured_interfaces: Arc::new(Mutex::new(vec![])), + } + } + } + + struct InterfaceInfoBuilder { + port_location: Option, + interface_type: Option, + operating_mode: Option, + status: Option, + } + + impl InterfaceInfoBuilder { + fn build(&self) -> InterfaceInfo { + let interface_type = self + .interface_type + .clone() + .unwrap_or(InterfaceType::Ethernet("TenGigabitEthernet".into())); + let port_location = self.port_location.clone().unwrap_or(PortLocation(1, 0, 1)); + let name = format!("{interface_type} {port_location}"); + let status = self.status.clone().unwrap_or(InterfaceStatus::Connected); + + InterfaceInfo { + name, + port_location, + interface_type, + operating_mode: self.operating_mode.clone(), + status, + } + } + + fn with_port_location(self, port_location: PortLocation) -> Self { + Self { + port_location: Some(port_location), + ..self + } + } + + fn with_operating_mode(self, operating_mode: Option) -> Self { + Self { + operating_mode, + ..self + } + } + + fn with_status(self, status: InterfaceStatus) -> Self { + Self { + status: Some(status), + ..self + } + } + } + + struct InterSwitchLinkBuilder { + link: Option<(PortLocation, PortLocation)>, + } + + impl InterSwitchLinkBuilder { + fn build(&self) -> InterSwitchLink { + let link = self + .link + .clone() + .unwrap_or((PortLocation(1, 0, 1), PortLocation(2, 0, 1))); + + InterSwitchLink { + local_port: link.0, + remote_port: Some(link.1), + } + } + + fn between(self, local_port: PortLocation, remote_port: PortLocation) -> Self { + Self { + link: Some((local_port, remote_port)), + } + } + } + + fn given_interface() -> InterfaceInfoBuilder { + InterfaceInfoBuilder { + port_location: None, + interface_type: None, + operating_mode: None, + status: None, + } + } + + fn given_inter_switch_link() -> InterSwitchLinkBuilder { + InterSwitchLinkBuilder { link: None } + } +} diff --git a/harmony/src/infra/mod.rs b/harmony/src/infra/mod.rs index c05c7b6..203cf90 100644 --- a/harmony/src/infra/mod.rs +++ b/harmony/src/infra/mod.rs @@ -1,3 +1,4 @@ +pub mod brocade; pub mod executors; pub mod hp_ilo; pub mod intel_amt; diff --git a/harmony/src/modules/okd/bootstrap_03_control_plane.rs b/harmony/src/modules/okd/bootstrap_03_control_plane.rs index ba9e12d..af8e71f 100644 --- a/harmony/src/modules/okd/bootstrap_03_control_plane.rs +++ b/harmony/src/modules/okd/bootstrap_03_control_plane.rs @@ -5,8 +5,10 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::{HostRole, Inventory}, modules::{ - dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore, - inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl, + dhcp::DhcpHostBindingScore, + http::IPxeMacBootFileScore, + inventory::DiscoverHostForRoleScore, + okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl}, }, score::Score, topology::{HAClusterTopology, HostBinding}, @@ -28,7 +30,7 @@ pub struct OKDSetup03ControlPlaneScore {} impl Score for OKDSetup03ControlPlaneScore { fn create_interpret(&self) -> Box> { - Box::new(OKDSetup03ControlPlaneInterpret::new(self.clone())) + Box::new(OKDSetup03ControlPlaneInterpret::new()) } fn name(&self) -> String { @@ -38,17 +40,15 @@ impl Score for OKDSetup03ControlPlaneScore { #[derive(Debug, Clone)] pub struct OKDSetup03ControlPlaneInterpret { - score: OKDSetup03ControlPlaneScore, version: Version, status: InterpretStatus, } impl OKDSetup03ControlPlaneInterpret { - pub fn new(score: OKDSetup03ControlPlaneScore) -> Self { + pub fn new() -> Self { let version = Version::from("1.0.0").unwrap(); Self { version, - score, status: InterpretStatus::QUEUED, } } @@ -159,7 +159,7 @@ impl OKDSetup03ControlPlaneInterpret { } .to_string(); - debug!("[ControlPlane] iPXE content template:\n{}", content); + debug!("[ControlPlane] iPXE content template:\n{content}"); // Create and apply an iPXE boot file for each node. for node in nodes { @@ -189,16 +189,13 @@ impl OKDSetup03ControlPlaneInterpret { /// Prompts the user to reboot the target control plane nodes. async fn reboot_targets(&self, nodes: &Vec) -> Result<(), InterpretError> { let node_ids: Vec = nodes.iter().map(|n| n.id.to_string()).collect(); - info!( - "[ControlPlane] Requesting reboot for control plane nodes: {:?}", - node_ids - ); + info!("[ControlPlane] Requesting reboot for control plane nodes: {node_ids:?}",); let confirmation = inquire::Confirm::new( &format!("Please reboot the {} control plane nodes ({}) to apply their PXE configuration. Press enter when ready.", nodes.len(), node_ids.join(", ")), ) .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?; + .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; if !confirmation { return Err(InterpretError::new( @@ -210,14 +207,23 @@ impl OKDSetup03ControlPlaneInterpret { } /// Placeholder for automating network bonding configuration. - async fn persist_network_bond(&self) -> Result<(), InterpretError> { - // Generate MC or NNCP from inventory NIC data; apply via ignition or post-join. - info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP"); + async fn persist_network_bond( + &self, + inventory: &Inventory, + topology: &HAClusterTopology, + hosts: &Vec, + ) -> Result<(), InterpretError> { + info!("[ControlPlane] Ensuring persistent bonding"); + let score = HostNetworkConfigurationScore { + hosts: hosts.clone(), + }; + score.interpret(inventory, topology).await?; + inquire::Confirm::new( "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.", ) .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?; + .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; Ok(()) } @@ -260,7 +266,8 @@ impl Interpret for OKDSetup03ControlPlaneInterpret { self.reboot_targets(&nodes).await?; // 5. Placeholder for post-boot network configuration (e.g., bonding). - self.persist_network_bond().await?; + self.persist_network_bond(inventory, topology, &nodes) + .await?; // TODO: Implement a step to wait for the control plane nodes to join the cluster // and for the cluster operators to become available. This would be similar to diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs new file mode 100644 index 0000000..c1a68ce --- /dev/null +++ b/harmony/src/modules/okd/crd/mod.rs @@ -0,0 +1,41 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub mod nmstate; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1", + kind = "OperatorGroup", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct OperatorGroupSpec { + pub target_namespaces: Vec, +} + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1alpha1", + kind = "Subscription", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionSpec { + pub name: String, + pub source: String, + pub source_namespace: String, + pub channel: Option, + pub install_plan_approval: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +pub enum InstallPlanApproval { + #[serde(rename = "Automatic")] + Automatic, + #[serde(rename = "Manual")] + Manual, +} diff --git a/harmony/src/modules/okd/crd/nmstate.rs b/harmony/src/modules/okd/crd/nmstate.rs new file mode 100644 index 0000000..5f71e4e --- /dev/null +++ b/harmony/src/modules/okd/crd/nmstate.rs @@ -0,0 +1,251 @@ +use std::collections::BTreeMap; + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)] +#[serde(rename_all = "camelCase")] +pub struct NMStateSpec { + pub probe_configuration: Option, +} + +impl Default for NMState { + fn default() -> Self { + Self { + metadata: Default::default(), + spec: NMStateSpec { + probe_configuration: None, + }, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ProbeConfig { + pub dns: ProbeDns, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ProbeDns { + pub host: String, +} + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "nmstate.io", + version = "v1", + kind = "NodeNetworkConfigurationPolicy", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct NodeNetworkConfigurationPolicySpec { + pub node_selector: Option>, + pub desired_state: DesiredStateSpec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct DesiredStateSpec { + pub interfaces: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct InterfaceSpec { + pub name: String, + pub description: Option, + pub r#type: String, + pub state: String, + pub mac_address: Option, + pub mtu: Option, + pub controller: Option, + pub ipv4: Option, + pub ipv6: Option, + pub ethernet: Option, + pub link_aggregation: Option, + pub vlan: Option, + pub vxlan: Option, + pub mac_vtap: Option, + pub mac_vlan: Option, + pub infiniband: Option, + pub linux_bridge: Option, + pub ovs_bridge: Option, + pub ethtool: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct IpStackSpec { + pub enabled: Option, + pub dhcp: Option, + pub autoconf: Option, + pub address: Option>, + pub auto_dns: Option, + pub auto_gateway: Option, + pub auto_routes: Option, + pub dhcp_client_id: Option, + pub dhcp_duid: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct IpAddressSpec { + pub ip: String, + pub prefix_length: u8, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthernetSpec { + pub speed: Option, + pub duplex: Option, + pub auto_negotiation: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct BondSpec { + pub mode: String, + pub ports: Vec, + pub options: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanSpec { + pub base_iface: String, + pub id: u16, + pub protocol: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VxlanSpec { + pub base_iface: String, + pub id: u32, + pub remote: String, + pub local: Option, + pub learning: Option, + pub destination_port: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct MacVtapSpec { + pub base_iface: String, + pub mode: String, + pub promiscuous: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct MacVlanSpec { + pub base_iface: String, + pub mode: String, + pub promiscuous: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct InfinibandSpec { + pub base_iface: String, + pub pkey: String, + pub mode: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgeSpec { + pub options: Option, + pub ports: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgeOptions { + pub mac_ageing_time: Option, + pub multicast_snooping: Option, + pub stp: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct StpOptions { + pub enabled: Option, + pub forward_delay: Option, + pub hello_time: Option, + pub max_age: Option, + pub priority: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgePort { + pub name: String, + pub vlan: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct LinuxBridgePortVlan { + pub mode: Option, + pub trunk_tags: Option>, + pub tag: Option, + pub enable_native: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanTag { + pub id: u16, + pub id_range: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct VlanIdRange { + pub min: u16, + pub max: u16, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsBridgeSpec { + pub options: Option, + pub ports: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsBridgeOptions { + pub stp: Option, + pub rstp: Option, + pub mcast_snooping_enable: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct OvsPortSpec { + pub name: String, + pub link_aggregation: Option, + pub vlan: Option, + pub r#type: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthtoolSpec { + // TODO: Properly describe this spec (https://nmstate.io/devel/yaml_api.html#ethtool) +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct EthtoolFecSpec { + pub auto: Option, + pub mode: Option, +} diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs new file mode 100644 index 0000000..3bc8c3c --- /dev/null +++ b/harmony/src/modules/okd/host_network.rs @@ -0,0 +1,394 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use log::{debug, info}; +use serde::Serialize; + +use crate::{ + data::Version, + hardware::PhysicalHost, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + score::Score, + topology::{HostNetworkConfig, NetworkInterface, Switch, SwitchPort, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct HostNetworkConfigurationScore { + pub hosts: Vec, +} + +impl Score for HostNetworkConfigurationScore { + fn name(&self) -> String { + "HostNetworkConfigurationScore".into() + } + + fn create_interpret(&self) -> Box> { + Box::new(HostNetworkConfigurationInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug)] +pub struct HostNetworkConfigurationInterpret { + score: HostNetworkConfigurationScore, +} + +impl HostNetworkConfigurationInterpret { + async fn configure_network_for_host( + &self, + topology: &T, + host: &PhysicalHost, + ) -> Result<(), InterpretError> { + let switch_ports = self.collect_switch_ports_for_host(topology, host).await?; + if !switch_ports.is_empty() { + topology + .configure_host_network(host, HostNetworkConfig { switch_ports }) + .await + .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?; + } + + Ok(()) + } + + async fn collect_switch_ports_for_host( + &self, + topology: &T, + host: &PhysicalHost, + ) -> Result, InterpretError> { + let mut switch_ports = vec![]; + + for network_interface in &host.network { + let mac_address = network_interface.mac_address; + + match topology.get_port_for_mac_address(&mac_address).await { + Ok(Some(port)) => { + switch_ports.push(SwitchPort { + interface: NetworkInterface { + name: network_interface.name.clone(), + mac_address, + speed_mbps: network_interface.speed_mbps, + mtu: network_interface.mtu, + }, + port, + }); + } + Ok(None) => debug!("No port found for host '{}', skipping", host.id), + Err(e) => { + return Err(InterpretError::new(format!( + "Failed to get port for host '{}': {}", + host.id, e + ))); + } + } + } + + Ok(switch_ports) + } +} + +#[async_trait] +impl Interpret for HostNetworkConfigurationInterpret { + fn get_name(&self) -> InterpretName { + InterpretName::Custom("HostNetworkConfigurationInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + vec![] + } + + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + if self.score.hosts.is_empty() { + return Ok(Outcome::noop("No hosts to configure".into())); + } + + info!( + "Started network configuration for {} host(s)...", + self.score.hosts.len() + ); + + topology + .setup_switch() + .await + .map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?; + + let mut configured_host_count = 0; + for host in &self.score.hosts { + self.configure_network_for_host(topology, host).await?; + configured_host_count += 1; + } + + if configured_host_count > 0 { + Ok(Outcome::success(format!( + "Configured {configured_host_count}/{} host(s)", + self.score.hosts.len() + ))) + } else { + Ok(Outcome::noop("No hosts configured".into())) + } + } +} + +#[cfg(test)] +mod tests { + use assertor::*; + use harmony_types::{net::MacAddress, switch::PortLocation}; + use lazy_static::lazy_static; + + use crate::{ + hardware::HostCategory, + topology::{ + HostNetworkConfig, PreparationError, PreparationOutcome, SwitchError, SwitchPort, + }, + }; + use std::{ + str::FromStr, + sync::{Arc, Mutex}, + }; + + use super::*; + + lazy_static! { + pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap(); + pub static ref ANOTHER_HOST_ID: Id = Id::from_str("host-2").unwrap(); + pub static ref EXISTING_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F1".to_string()).unwrap(), + name: "interface-1".into(), + speed_mbps: None, + mtu: 1, + }; + pub static ref ANOTHER_EXISTING_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F2".to_string()).unwrap(), + name: "interface-2".into(), + speed_mbps: None, + mtu: 1, + }; + pub static ref UNKNOWN_INTERFACE: NetworkInterface = NetworkInterface { + mac_address: MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(), + name: "unknown-interface".into(), + speed_mbps: None, + mtu: 1, + }; + pub static ref PORT: PortLocation = PortLocation(1, 0, 42); + pub static ref ANOTHER_PORT: PortLocation = PortLocation(2, 0, 42); + } + + #[tokio::test] + async fn should_setup_switch() { + let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]); + let score = given_score(vec![host]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let switch_setup = topology.switch_setup.lock().unwrap(); + assert_that!(*switch_setup).is_true(); + } + + #[tokio::test] + async fn host_with_one_mac_address_should_create_bond_with_one_interface() { + let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]); + let score = given_score(vec![host]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![( + HOST_ID.clone(), + HostNetworkConfig { + switch_ports: vec![SwitchPort { + interface: EXISTING_INTERFACE.clone(), + port: PORT.clone(), + }], + }, + )]); + } + + #[tokio::test] + async fn host_with_multiple_mac_addresses_should_create_one_bond_with_all_interfaces() { + let score = given_score(vec![given_host( + &HOST_ID, + vec![ + EXISTING_INTERFACE.clone(), + ANOTHER_EXISTING_INTERFACE.clone(), + ], + )]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![( + HOST_ID.clone(), + HostNetworkConfig { + switch_ports: vec![ + SwitchPort { + interface: EXISTING_INTERFACE.clone(), + port: PORT.clone(), + }, + SwitchPort { + interface: ANOTHER_EXISTING_INTERFACE.clone(), + port: ANOTHER_PORT.clone(), + }, + ], + }, + )]); + } + + #[tokio::test] + async fn multiple_hosts_should_create_one_bond_per_host() { + let score = given_score(vec![ + given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]), + given_host(&ANOTHER_HOST_ID, vec![ANOTHER_EXISTING_INTERFACE.clone()]), + ]); + let topology = TopologyWithSwitch::new(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).contains_exactly(vec![ + ( + HOST_ID.clone(), + HostNetworkConfig { + switch_ports: vec![SwitchPort { + interface: EXISTING_INTERFACE.clone(), + port: PORT.clone(), + }], + }, + ), + ( + ANOTHER_HOST_ID.clone(), + HostNetworkConfig { + switch_ports: vec![SwitchPort { + interface: ANOTHER_EXISTING_INTERFACE.clone(), + port: ANOTHER_PORT.clone(), + }], + }, + ), + ]); + } + + #[tokio::test] + async fn port_not_found_for_mac_address_should_not_configure_interface() { + let score = given_score(vec![given_host(&HOST_ID, vec![UNKNOWN_INTERFACE.clone()])]); + let topology = TopologyWithSwitch::new_port_not_found(); + + let _ = score.interpret(&Inventory::empty(), &topology).await; + + let configured_host_networks = topology.configured_host_networks.lock().unwrap(); + assert_that!(*configured_host_networks).is_empty(); + } + + fn given_score(hosts: Vec) -> HostNetworkConfigurationScore { + HostNetworkConfigurationScore { hosts } + } + + fn given_host(id: &Id, network_interfaces: Vec) -> PhysicalHost { + let network = network_interfaces.iter().map(given_interface).collect(); + + PhysicalHost { + id: id.clone(), + category: HostCategory::Server, + network, + storage: vec![], + labels: vec![], + memory_modules: vec![], + cpus: vec![], + } + } + + fn given_interface( + interface: &NetworkInterface, + ) -> harmony_inventory_agent::hwinfo::NetworkInterface { + harmony_inventory_agent::hwinfo::NetworkInterface { + name: interface.name.clone(), + mac_address: interface.mac_address, + speed_mbps: interface.speed_mbps, + is_up: true, + mtu: interface.mtu, + ipv4_addresses: vec![], + ipv6_addresses: vec![], + driver: "driver".into(), + firmware_version: None, + } + } + + struct TopologyWithSwitch { + available_ports: Arc>>, + configured_host_networks: Arc>>, + switch_setup: Arc>, + } + + impl TopologyWithSwitch { + fn new() -> Self { + Self { + available_ports: Arc::new(Mutex::new(vec![PORT.clone(), ANOTHER_PORT.clone()])), + configured_host_networks: Arc::new(Mutex::new(vec![])), + switch_setup: Arc::new(Mutex::new(false)), + } + } + + fn new_port_not_found() -> Self { + Self { + available_ports: Arc::new(Mutex::new(vec![])), + configured_host_networks: Arc::new(Mutex::new(vec![])), + switch_setup: Arc::new(Mutex::new(false)), + } + } + } + + #[async_trait] + impl Topology for TopologyWithSwitch { + fn name(&self) -> &str { + "SwitchWithPortTopology" + } + + async fn ensure_ready(&self) -> Result { + Ok(PreparationOutcome::Success { details: "".into() }) + } + } + + #[async_trait] + impl Switch for TopologyWithSwitch { + async fn setup_switch(&self) -> Result<(), SwitchError> { + let mut switch_configured = self.switch_setup.lock().unwrap(); + *switch_configured = true; + Ok(()) + } + + async fn get_port_for_mac_address( + &self, + _mac_address: &MacAddress, + ) -> Result, SwitchError> { + let mut ports = self.available_ports.lock().unwrap(); + if ports.is_empty() { + return Ok(None); + } + Ok(Some(ports.remove(0))) + } + + async fn configure_host_network( + &self, + host: &PhysicalHost, + config: HostNetworkConfig, + ) -> Result<(), SwitchError> { + let mut configured_host_networks = self.configured_host_networks.lock().unwrap(); + configured_host_networks.push((host.id.clone(), config.clone())); + + Ok(()) + } + } +} diff --git a/harmony/src/modules/okd/mod.rs b/harmony/src/modules/okd/mod.rs index 1bd4514..a12f132 100644 --- a/harmony/src/modules/okd/mod.rs +++ b/harmony/src/modules/okd/mod.rs @@ -19,3 +19,5 @@ pub use bootstrap_03_control_plane::*; pub use bootstrap_04_workers::*; pub use bootstrap_05_sanity_check::*; pub use bootstrap_06_installation_report::*; +pub mod crd; +pub mod host_network; diff --git a/harmony_composer/src/main.rs b/harmony_composer/src/main.rs index 4119460..817bb8b 100644 --- a/harmony_composer/src/main.rs +++ b/harmony_composer/src/main.rs @@ -54,6 +54,9 @@ struct DeployArgs { #[arg(long = "profile", short = 'p', default_value = "dev")] harmony_profile: HarmonyProfile, + + #[arg(long = "dry-run", short = 'd', default_value = "false")] + dry_run: bool, } #[derive(Args, Clone, Debug)] @@ -178,6 +181,7 @@ async fn main() { command .env("HARMONY_USE_LOCAL_K3D", format!("{use_local_k3d}")) .env("HARMONY_PROFILE", format!("{}", args.harmony_profile)) + .env("HARMONY_DRY_RUN", format!("{}", args.dry_run)) .arg("-y") .arg("-a");