Compare commits
3 Commits
6989d208cf
...
2d891e4463
Author | SHA1 | Date | |
---|---|---|---|
2d891e4463 | |||
f66e58b9ca | |||
ea39d93aa7 |
36
Cargo.lock
generated
36
Cargo.lock
generated
@ -429,6 +429,15 @@ dependencies = [
|
||||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assertor"
|
||||
version = "0.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ff24d87260733dc86d38a11c60d9400ce4a74a05d0dafa2a6f5ab249cd857cb"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-broadcast"
|
||||
version = "0.7.2"
|
||||
@ -2333,9 +2342,11 @@ name = "harmony"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"askama",
|
||||
"assertor",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bollard",
|
||||
"brocade",
|
||||
"chrono",
|
||||
"cidr",
|
||||
"convert_case",
|
||||
@ -2366,6 +2377,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"opnsense-config",
|
||||
"opnsense-config-xml",
|
||||
"option-ext",
|
||||
"pretty_assertions",
|
||||
"reqwest 0.11.27",
|
||||
"russh",
|
||||
@ -2432,17 +2444,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "harmony_derive"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d138bbb32bb346299c5f95fbb53532313f39927cb47c411c99c634ef8665ef7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "harmony_inventory_agent"
|
||||
version = "0.1.0"
|
||||
@ -3889,19 +3890,6 @@ dependencies = [
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "okd_host_network"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"harmony",
|
||||
"harmony_cli",
|
||||
"harmony_derive",
|
||||
"harmony_inventory_agent",
|
||||
"harmony_macros",
|
||||
"harmony_types",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
|
@ -270,7 +270,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
|
||||
commands.push("no ip address".into());
|
||||
commands.push("no fabric isl enable".into());
|
||||
commands.push("no fabric trunk enable".into());
|
||||
commands.push(format!("channel-group {} mode active", channel_id));
|
||||
commands.push(format!("channel-group {channel_id} mode active"));
|
||||
commands.push("no shutdown".into());
|
||||
commands.push("exit".into());
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
use cidr::Ipv4Cidr;
|
||||
use harmony::{
|
||||
hardware::{FirewallGroup, HostCategory, Location, PhysicalHost, SwitchGroup},
|
||||
hardware::{Location, SwitchGroup},
|
||||
infra::opnsense::OPNSenseManagementInterface,
|
||||
inventory::Inventory,
|
||||
topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
|
||||
|
@ -77,6 +77,9 @@ harmony_secret = { path = "../harmony_secret" }
|
||||
askama.workspace = true
|
||||
sqlx.workspace = true
|
||||
inquire.workspace = true
|
||||
brocade = { path = "../brocade" }
|
||||
option-ext = "0.2.0"
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions.workspace = true
|
||||
assertor.workspace = true
|
||||
|
@ -1,33 +1,36 @@
|
||||
use async_trait::async_trait;
|
||||
use brocade::BrocadeOptions;
|
||||
use harmony_macros::ip;
|
||||
use harmony_types::net::MacAddress;
|
||||
use harmony_types::net::Url;
|
||||
use harmony_secret::SecretManager;
|
||||
use harmony_types::{
|
||||
net::{MacAddress, Url},
|
||||
switch::PortLocation,
|
||||
};
|
||||
use k8s_openapi::api::core::v1::Namespace;
|
||||
use kube::api::ObjectMeta;
|
||||
use log::debug;
|
||||
use log::info;
|
||||
|
||||
use crate::data::FileContent;
|
||||
use crate::executors::ExecutorError;
|
||||
use crate::hardware::PhysicalHost;
|
||||
use crate::infra::brocade::BrocadeSwitchAuth;
|
||||
use crate::infra::brocade::BrocadeSwitchClient;
|
||||
use crate::modules::okd::crd::{
|
||||
InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
|
||||
nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
|
||||
};
|
||||
use crate::topology::PxeOptions;
|
||||
|
||||
use super::DHCPStaticEntry;
|
||||
use super::DhcpServer;
|
||||
use super::DnsRecord;
|
||||
use super::DnsRecordType;
|
||||
use super::DnsServer;
|
||||
use super::Firewall;
|
||||
use super::HttpServer;
|
||||
use super::IpAddress;
|
||||
use super::K8sclient;
|
||||
use super::LoadBalancer;
|
||||
use super::LoadBalancerService;
|
||||
use super::LogicalHost;
|
||||
use super::PreparationError;
|
||||
use super::PreparationOutcome;
|
||||
use super::Router;
|
||||
use super::TftpServer;
|
||||
use super::{
|
||||
DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
|
||||
HttpServer, IpAddress, K8sclient, LoadBalancer, LoadBalancerService, LogicalHost,
|
||||
PreparationError, PreparationOutcome, Router, Switch, SwitchClient, SwitchError, TftpServer,
|
||||
Topology, k8s::K8sClient,
|
||||
};
|
||||
|
||||
use super::Topology;
|
||||
use super::k8s::K8sClient;
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@ -89,6 +92,231 @@ impl HAClusterTopology {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
|
||||
// FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate)
|
||||
debug!("Installing NMState operator...");
|
||||
let k8s_client = self.k8s_client().await?;
|
||||
|
||||
let nmstate_namespace = Namespace {
|
||||
metadata: ObjectMeta {
|
||||
name: Some("openshift-nmstate".to_string()),
|
||||
finalizers: Some(vec!["kubernetes".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
debug!("Creating NMState namespace: {nmstate_namespace:#?}");
|
||||
k8s_client
|
||||
.apply(&nmstate_namespace, None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let nmstate_operator_group = OperatorGroup {
|
||||
metadata: ObjectMeta {
|
||||
name: Some("openshift-nmstate".to_string()),
|
||||
namespace: Some("openshift-nmstate".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
spec: OperatorGroupSpec {
|
||||
target_namespaces: vec!["openshift-nmstate".to_string()],
|
||||
},
|
||||
};
|
||||
debug!("Creating NMState operator group: {nmstate_operator_group:#?}");
|
||||
k8s_client
|
||||
.apply(&nmstate_operator_group, None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let nmstate_subscription = Subscription {
|
||||
metadata: ObjectMeta {
|
||||
name: Some("kubernetes-nmstate-operator".to_string()),
|
||||
namespace: Some("openshift-nmstate".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
spec: SubscriptionSpec {
|
||||
channel: Some("stable".to_string()),
|
||||
install_plan_approval: Some(InstallPlanApproval::Automatic),
|
||||
name: "kubernetes-nmstate-operator".to_string(),
|
||||
source: "redhat-operators".to_string(),
|
||||
source_namespace: "openshift-marketplace".to_string(),
|
||||
},
|
||||
};
|
||||
debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}");
|
||||
k8s_client
|
||||
.apply(&nmstate_subscription, None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let nmstate = NMState {
|
||||
metadata: ObjectMeta {
|
||||
name: Some("nmstate".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
debug!("Creating NMState: {nmstate:#?}");
|
||||
k8s_client
|
||||
.apply(&nmstate, None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_next_bond_id(&self) -> u8 {
|
||||
42 // FIXME: Find a better way to declare the bond id
|
||||
}
|
||||
|
||||
async fn configure_bond(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: &HostNetworkConfig,
|
||||
) -> Result<(), SwitchError> {
|
||||
self.ensure_nmstate_operator_installed()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
SwitchError::new(format!(
|
||||
"Can't configure bond, NMState operator not available: {e}"
|
||||
))
|
||||
})?;
|
||||
|
||||
let bond_config = self.create_bond_configuration(host, config);
|
||||
debug!("Configuring bond for host {host:?}: {bond_config:#?}");
|
||||
self.k8s_client()
|
||||
.await
|
||||
.unwrap()
|
||||
.apply(&bond_config, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn create_bond_configuration(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: &HostNetworkConfig,
|
||||
) -> NodeNetworkConfigurationPolicy {
|
||||
let host_name = host.id.clone();
|
||||
|
||||
let bond_id = self.get_next_bond_id();
|
||||
let bond_name = format!("bond{bond_id}");
|
||||
let mut bond_mtu: Option<u32> = None;
|
||||
let mut bond_mac_address: Option<String> = None;
|
||||
let mut bond_ports = Vec::new();
|
||||
let mut interfaces: Vec<nmstate::InterfaceSpec> = Vec::new();
|
||||
|
||||
for switch_port in &config.switch_ports {
|
||||
let interface_name = switch_port.interface.name.clone();
|
||||
|
||||
interfaces.push(nmstate::InterfaceSpec {
|
||||
name: interface_name.clone(),
|
||||
description: Some(format!("Member of bond {bond_name}")),
|
||||
r#type: "ethernet".to_string(),
|
||||
state: "up".to_string(),
|
||||
mtu: Some(switch_port.interface.mtu),
|
||||
mac_address: Some(switch_port.interface.mac_address.to_string()),
|
||||
ipv4: Some(nmstate::IpStackSpec {
|
||||
enabled: Some(false),
|
||||
..Default::default()
|
||||
}),
|
||||
ipv6: Some(nmstate::IpStackSpec {
|
||||
enabled: Some(false),
|
||||
..Default::default()
|
||||
}),
|
||||
link_aggregation: None,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
bond_ports.push(interface_name);
|
||||
|
||||
// Use the first port's details for the bond mtu and mac address
|
||||
if bond_mtu.is_none() {
|
||||
bond_mtu = Some(switch_port.interface.mtu);
|
||||
}
|
||||
if bond_mac_address.is_none() {
|
||||
bond_mac_address = Some(switch_port.interface.mac_address.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
interfaces.push(nmstate::InterfaceSpec {
|
||||
name: bond_name.clone(),
|
||||
description: Some(format!("Network bond for host {host_name}")),
|
||||
r#type: "bond".to_string(),
|
||||
state: "up".to_string(),
|
||||
mtu: bond_mtu,
|
||||
mac_address: bond_mac_address,
|
||||
ipv4: Some(nmstate::IpStackSpec {
|
||||
dhcp: Some(true),
|
||||
enabled: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
ipv6: Some(nmstate::IpStackSpec {
|
||||
dhcp: Some(true),
|
||||
autoconf: Some(true),
|
||||
enabled: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
link_aggregation: Some(nmstate::BondSpec {
|
||||
mode: "802.3ad".to_string(),
|
||||
ports: bond_ports,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
NodeNetworkConfigurationPolicy {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(format!("{host_name}-bond-config")),
|
||||
..Default::default()
|
||||
},
|
||||
spec: NodeNetworkConfigurationPolicySpec {
|
||||
node_selector: Some(BTreeMap::from([(
|
||||
"kubernetes.io/hostname".to_string(),
|
||||
host_name.to_string(),
|
||||
)])),
|
||||
desired_state: nmstate::DesiredStateSpec { interfaces },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_switch_client(&self) -> Result<Box<dyn SwitchClient>, SwitchError> {
|
||||
let auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?;
|
||||
|
||||
// FIXME: We assume Brocade switches
|
||||
let switches: Vec<IpAddr> = self.switch.iter().map(|s| s.ip).collect();
|
||||
let brocade_options = Some(BrocadeOptions {
|
||||
dry_run: *crate::config::DRY_RUN,
|
||||
..Default::default()
|
||||
});
|
||||
let client =
|
||||
BrocadeSwitchClient::init(&switches, &auth.username, &auth.password, brocade_options)
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?;
|
||||
|
||||
Ok(Box::new(client))
|
||||
}
|
||||
|
||||
async fn configure_port_channel(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: &HostNetworkConfig,
|
||||
) -> Result<(), SwitchError> {
|
||||
debug!("Configuring port channel: {config:#?}");
|
||||
let client = self.get_switch_client().await?;
|
||||
|
||||
let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
|
||||
|
||||
client
|
||||
.configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn autoload() -> Self {
|
||||
let dummy_infra = Arc::new(DummyInfra {});
|
||||
let dummy_host = LogicalHost {
|
||||
@ -263,6 +491,33 @@ impl HttpServer for HAClusterTopology {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Switch for HAClusterTopology {
|
||||
async fn setup_switch(&self) -> Result<(), SwitchError> {
|
||||
let client = self.get_switch_client().await?;
|
||||
client.setup().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_port_for_mac_address(
|
||||
&self,
|
||||
mac_address: &MacAddress,
|
||||
) -> Result<Option<PortLocation>, SwitchError> {
|
||||
let client = self.get_switch_client().await?;
|
||||
let port = client.find_port(mac_address).await?;
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
async fn configure_host_network(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: HostNetworkConfig,
|
||||
) -> Result<(), SwitchError> {
|
||||
self.configure_bond(host, &config).await?;
|
||||
self.configure_port_channel(host, &config).await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DummyInfra;
|
||||
|
||||
@ -332,8 +587,8 @@ impl DhcpServer for DummyInfra {
|
||||
}
|
||||
async fn set_dhcp_range(
|
||||
&self,
|
||||
start: &IpAddress,
|
||||
end: &IpAddress,
|
||||
_start: &IpAddress,
|
||||
_end: &IpAddress,
|
||||
) -> Result<(), ExecutorError> {
|
||||
unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
|
||||
}
|
||||
|
@ -1,10 +1,14 @@
|
||||
use std::{net::Ipv4Addr, str::FromStr, sync::Arc};
|
||||
use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use harmony_types::net::{IpAddress, MacAddress};
|
||||
use derive_new::new;
|
||||
use harmony_types::{
|
||||
net::{IpAddress, MacAddress},
|
||||
switch::PortLocation,
|
||||
};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::executors::ExecutorError;
|
||||
use crate::{executors::ExecutorError, hardware::PhysicalHost};
|
||||
|
||||
use super::{LogicalHost, k8s::K8sClient};
|
||||
|
||||
@ -172,6 +176,80 @@ impl FromStr for DnsRecordType {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Switch: Send + Sync {
|
||||
async fn setup_switch(&self) -> Result<(), SwitchError>;
|
||||
|
||||
async fn get_port_for_mac_address(
|
||||
&self,
|
||||
mac_address: &MacAddress,
|
||||
) -> Result<Option<PortLocation>, SwitchError>;
|
||||
|
||||
async fn configure_host_network(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: HostNetworkConfig,
|
||||
) -> Result<(), SwitchError>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct HostNetworkConfig {
|
||||
pub switch_ports: Vec<SwitchPort>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SwitchPort {
|
||||
pub interface: NetworkInterface,
|
||||
pub port: PortLocation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct NetworkInterface {
|
||||
pub name: String,
|
||||
pub mac_address: MacAddress,
|
||||
pub speed_mbps: Option<u32>,
|
||||
pub mtu: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, new)]
|
||||
pub struct SwitchError {
|
||||
msg: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SwitchError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.msg)
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for SwitchError {}
|
||||
|
||||
#[async_trait]
|
||||
pub trait SwitchClient: Send + Sync {
|
||||
/// Executes essential, idempotent, one-time initial configuration steps.
|
||||
///
|
||||
/// This is an opiniated procedure that setups a switch to provide high availability
|
||||
/// capabilities as decided by the NationTech team.
|
||||
///
|
||||
/// This includes tasks like enabling switchport for all interfaces
|
||||
/// except the ones intended for Fabric Networking, etc.
|
||||
///
|
||||
/// The implementation must ensure the operation is **idempotent** (safe to run multiple times)
|
||||
/// and that it doesn't break existing configurations.
|
||||
async fn setup(&self) -> Result<(), SwitchError>;
|
||||
|
||||
async fn find_port(
|
||||
&self,
|
||||
mac_address: &MacAddress,
|
||||
) -> Result<Option<PortLocation>, SwitchError>;
|
||||
|
||||
async fn configure_port_channel(
|
||||
&self,
|
||||
channel_name: &str,
|
||||
switch_ports: Vec<PortLocation>,
|
||||
) -> Result<u8, SwitchError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::sync::Arc;
|
||||
|
385
harmony/src/infra/brocade.rs
Normal file
385
harmony/src/infra/brocade.rs
Normal file
@ -0,0 +1,385 @@
|
||||
use async_trait::async_trait;
|
||||
use brocade::{BrocadeClient, BrocadeOptions, InterSwitchLink, InterfaceStatus, PortOperatingMode};
|
||||
use harmony_secret::Secret;
|
||||
use harmony_types::{
|
||||
net::{IpAddress, MacAddress},
|
||||
switch::{PortDeclaration, PortLocation},
|
||||
};
|
||||
use option_ext::OptionExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::topology::{SwitchClient, SwitchError};
|
||||
|
||||
pub struct BrocadeSwitchClient {
|
||||
brocade: Box<dyn BrocadeClient + Send + Sync>,
|
||||
}
|
||||
|
||||
impl BrocadeSwitchClient {
|
||||
pub async fn init(
|
||||
ip_addresses: &[IpAddress],
|
||||
username: &str,
|
||||
password: &str,
|
||||
options: Option<BrocadeOptions>,
|
||||
) -> Result<Self, brocade::Error> {
|
||||
let brocade = brocade::init(ip_addresses, 22, username, password, options).await?;
|
||||
Ok(Self { brocade })
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SwitchClient for BrocadeSwitchClient {
|
||||
async fn setup(&self) -> Result<(), SwitchError> {
|
||||
let stack_topology = self
|
||||
.brocade
|
||||
.get_stack_topology()
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(e.to_string()))?;
|
||||
|
||||
let interfaces = self
|
||||
.brocade
|
||||
.get_interfaces()
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(e.to_string()))?;
|
||||
|
||||
let interfaces: Vec<(String, PortOperatingMode)> = interfaces
|
||||
.into_iter()
|
||||
.filter(|interface| {
|
||||
interface.operating_mode.is_none() && interface.status == InterfaceStatus::Connected
|
||||
})
|
||||
.filter(|interface| {
|
||||
!stack_topology.iter().any(|link: &InterSwitchLink| {
|
||||
link.local_port == interface.port_location
|
||||
|| link.remote_port.contains(&interface.port_location)
|
||||
})
|
||||
})
|
||||
.map(|interface| (interface.name.clone(), PortOperatingMode::Access))
|
||||
.collect();
|
||||
|
||||
if interfaces.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.brocade
|
||||
.configure_interfaces(interfaces)
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_port(
|
||||
&self,
|
||||
mac_address: &MacAddress,
|
||||
) -> Result<Option<PortLocation>, SwitchError> {
|
||||
let table = self
|
||||
.brocade
|
||||
.get_mac_address_table()
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("{e}")))?;
|
||||
|
||||
let port = table
|
||||
.iter()
|
||||
.find(|entry| entry.mac_address == *mac_address)
|
||||
.map(|entry| match &entry.port {
|
||||
PortDeclaration::Single(port_location) => Ok(port_location.clone()),
|
||||
_ => Err(SwitchError::new(
|
||||
"Multiple ports found for MAC address".into(),
|
||||
)),
|
||||
});
|
||||
|
||||
match port {
|
||||
Some(Ok(p)) => Ok(Some(p)),
|
||||
Some(Err(e)) => Err(e),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn configure_port_channel(
|
||||
&self,
|
||||
channel_name: &str,
|
||||
switch_ports: Vec<PortLocation>,
|
||||
) -> Result<u8, SwitchError> {
|
||||
let channel_id = self
|
||||
.brocade
|
||||
.find_available_channel_id()
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("{e}")))?;
|
||||
|
||||
self.brocade
|
||||
.create_port_channel(channel_id, channel_name, &switch_ports)
|
||||
.await
|
||||
.map_err(|e| SwitchError::new(format!("{e}")))?;
|
||||
|
||||
Ok(channel_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Secret, Serialize, Deserialize, Debug)]
|
||||
pub struct BrocadeSwitchAuth {
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use assertor::*;
|
||||
use async_trait::async_trait;
|
||||
use brocade::{
|
||||
BrocadeClient, BrocadeInfo, Error, InterSwitchLink, InterfaceInfo, InterfaceStatus,
|
||||
InterfaceType, MacAddressEntry, PortChannelId, PortOperatingMode,
|
||||
};
|
||||
use harmony_types::switch::PortLocation;
|
||||
|
||||
use crate::{infra::brocade::BrocadeSwitchClient, topology::SwitchClient};
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_should_configure_ethernet_interfaces_as_access_ports() {
|
||||
let first_interface = given_interface()
|
||||
.with_port_location(PortLocation(1, 0, 1))
|
||||
.build();
|
||||
let second_interface = given_interface()
|
||||
.with_port_location(PortLocation(1, 0, 4))
|
||||
.build();
|
||||
let brocade = Box::new(FakeBrocadeClient::new(
|
||||
vec![],
|
||||
vec![first_interface.clone(), second_interface.clone()],
|
||||
));
|
||||
let client = BrocadeSwitchClient {
|
||||
brocade: brocade.clone(),
|
||||
};
|
||||
|
||||
client.setup().await.unwrap();
|
||||
|
||||
let configured_interfaces = brocade.configured_interfaces.lock().unwrap();
|
||||
assert_that!(*configured_interfaces).contains_exactly(vec![
|
||||
(first_interface.name.clone(), PortOperatingMode::Access),
|
||||
(second_interface.name.clone(), PortOperatingMode::Access),
|
||||
]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_with_an_already_configured_interface_should_skip_configuration() {
|
||||
let brocade = Box::new(FakeBrocadeClient::new(
|
||||
vec![],
|
||||
vec![
|
||||
given_interface()
|
||||
.with_operating_mode(Some(PortOperatingMode::Access))
|
||||
.build(),
|
||||
],
|
||||
));
|
||||
let client = BrocadeSwitchClient {
|
||||
brocade: brocade.clone(),
|
||||
};
|
||||
|
||||
client.setup().await.unwrap();
|
||||
|
||||
let configured_interfaces = brocade.configured_interfaces.lock().unwrap();
|
||||
assert_that!(*configured_interfaces).is_empty();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_with_a_disconnected_interface_should_skip_configuration() {
|
||||
let brocade = Box::new(FakeBrocadeClient::new(
|
||||
vec![],
|
||||
vec![
|
||||
given_interface()
|
||||
.with_status(InterfaceStatus::SfpAbsent)
|
||||
.build(),
|
||||
given_interface()
|
||||
.with_status(InterfaceStatus::NotConnected)
|
||||
.build(),
|
||||
],
|
||||
));
|
||||
let client = BrocadeSwitchClient {
|
||||
brocade: brocade.clone(),
|
||||
};
|
||||
|
||||
client.setup().await.unwrap();
|
||||
|
||||
let configured_interfaces = brocade.configured_interfaces.lock().unwrap();
|
||||
assert_that!(*configured_interfaces).is_empty();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_with_inter_switch_links_should_not_configure_interfaces_used_to_form_stack() {
|
||||
let brocade = Box::new(FakeBrocadeClient::new(
|
||||
vec![
|
||||
given_inter_switch_link()
|
||||
.between(PortLocation(1, 0, 1), PortLocation(2, 0, 1))
|
||||
.build(),
|
||||
given_inter_switch_link()
|
||||
.between(PortLocation(2, 0, 2), PortLocation(3, 0, 1))
|
||||
.build(),
|
||||
],
|
||||
vec![
|
||||
given_interface()
|
||||
.with_port_location(PortLocation(1, 0, 1))
|
||||
.build(),
|
||||
given_interface()
|
||||
.with_port_location(PortLocation(2, 0, 1))
|
||||
.build(),
|
||||
given_interface()
|
||||
.with_port_location(PortLocation(3, 0, 1))
|
||||
.build(),
|
||||
],
|
||||
));
|
||||
let client = BrocadeSwitchClient {
|
||||
brocade: brocade.clone(),
|
||||
};
|
||||
|
||||
client.setup().await.unwrap();
|
||||
|
||||
let configured_interfaces = brocade.configured_interfaces.lock().unwrap();
|
||||
assert_that!(*configured_interfaces).is_empty();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FakeBrocadeClient {
|
||||
stack_topology: Vec<InterSwitchLink>,
|
||||
interfaces: Vec<InterfaceInfo>,
|
||||
configured_interfaces: Arc<Mutex<Vec<(String, PortOperatingMode)>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BrocadeClient for FakeBrocadeClient {
|
||||
async fn version(&self) -> Result<BrocadeInfo, Error> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error> {
|
||||
Ok(self.stack_topology.clone())
|
||||
}
|
||||
|
||||
async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error> {
|
||||
Ok(self.interfaces.clone())
|
||||
}
|
||||
|
||||
async fn configure_interfaces(
|
||||
&self,
|
||||
interfaces: Vec<(String, PortOperatingMode)>,
|
||||
) -> Result<(), Error> {
|
||||
let mut configured_interfaces = self.configured_interfaces.lock().unwrap();
|
||||
*configured_interfaces = interfaces;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_available_channel_id(&self) -> Result<PortChannelId, Error> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn create_port_channel(
|
||||
&self,
|
||||
_channel_id: PortChannelId,
|
||||
_channel_name: &str,
|
||||
_ports: &[PortLocation],
|
||||
) -> Result<(), Error> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn clear_port_channel(&self, _channel_name: &str) -> Result<(), Error> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
impl FakeBrocadeClient {
|
||||
fn new(stack_topology: Vec<InterSwitchLink>, interfaces: Vec<InterfaceInfo>) -> Self {
|
||||
Self {
|
||||
stack_topology,
|
||||
interfaces,
|
||||
configured_interfaces: Arc::new(Mutex::new(vec![])),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct InterfaceInfoBuilder {
|
||||
port_location: Option<PortLocation>,
|
||||
interface_type: Option<InterfaceType>,
|
||||
operating_mode: Option<PortOperatingMode>,
|
||||
status: Option<InterfaceStatus>,
|
||||
}
|
||||
|
||||
impl InterfaceInfoBuilder {
|
||||
fn build(&self) -> InterfaceInfo {
|
||||
let interface_type = self
|
||||
.interface_type
|
||||
.clone()
|
||||
.unwrap_or(InterfaceType::Ethernet("TenGigabitEthernet".into()));
|
||||
let port_location = self.port_location.clone().unwrap_or(PortLocation(1, 0, 1));
|
||||
let name = format!("{interface_type} {port_location}");
|
||||
let status = self.status.clone().unwrap_or(InterfaceStatus::Connected);
|
||||
|
||||
InterfaceInfo {
|
||||
name,
|
||||
port_location,
|
||||
interface_type,
|
||||
operating_mode: self.operating_mode.clone(),
|
||||
status,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_port_location(self, port_location: PortLocation) -> Self {
|
||||
Self {
|
||||
port_location: Some(port_location),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
fn with_operating_mode(self, operating_mode: Option<PortOperatingMode>) -> Self {
|
||||
Self {
|
||||
operating_mode,
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
fn with_status(self, status: InterfaceStatus) -> Self {
|
||||
Self {
|
||||
status: Some(status),
|
||||
..self
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct InterSwitchLinkBuilder {
|
||||
link: Option<(PortLocation, PortLocation)>,
|
||||
}
|
||||
|
||||
impl InterSwitchLinkBuilder {
|
||||
fn build(&self) -> InterSwitchLink {
|
||||
let link = self
|
||||
.link
|
||||
.clone()
|
||||
.unwrap_or((PortLocation(1, 0, 1), PortLocation(2, 0, 1)));
|
||||
|
||||
InterSwitchLink {
|
||||
local_port: link.0,
|
||||
remote_port: Some(link.1),
|
||||
}
|
||||
}
|
||||
|
||||
fn between(self, local_port: PortLocation, remote_port: PortLocation) -> Self {
|
||||
Self {
|
||||
link: Some((local_port, remote_port)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn given_interface() -> InterfaceInfoBuilder {
|
||||
InterfaceInfoBuilder {
|
||||
port_location: None,
|
||||
interface_type: None,
|
||||
operating_mode: None,
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn given_inter_switch_link() -> InterSwitchLinkBuilder {
|
||||
InterSwitchLinkBuilder { link: None }
|
||||
}
|
||||
}
|
@ -1,3 +1,4 @@
|
||||
pub mod brocade;
|
||||
pub mod executors;
|
||||
pub mod hp_ilo;
|
||||
pub mod intel_amt;
|
||||
|
@ -5,8 +5,10 @@ use crate::{
|
||||
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
||||
inventory::{HostRole, Inventory},
|
||||
modules::{
|
||||
dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
|
||||
inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
|
||||
dhcp::DhcpHostBindingScore,
|
||||
http::IPxeMacBootFileScore,
|
||||
inventory::DiscoverHostForRoleScore,
|
||||
okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl},
|
||||
},
|
||||
score::Score,
|
||||
topology::{HAClusterTopology, HostBinding},
|
||||
@ -28,7 +30,7 @@ pub struct OKDSetup03ControlPlaneScore {}
|
||||
|
||||
impl Score<HAClusterTopology> for OKDSetup03ControlPlaneScore {
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
|
||||
Box::new(OKDSetup03ControlPlaneInterpret::new(self.clone()))
|
||||
Box::new(OKDSetup03ControlPlaneInterpret::new())
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
@ -38,17 +40,15 @@ impl Score<HAClusterTopology> for OKDSetup03ControlPlaneScore {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OKDSetup03ControlPlaneInterpret {
|
||||
score: OKDSetup03ControlPlaneScore,
|
||||
version: Version,
|
||||
status: InterpretStatus,
|
||||
}
|
||||
|
||||
impl OKDSetup03ControlPlaneInterpret {
|
||||
pub fn new(score: OKDSetup03ControlPlaneScore) -> Self {
|
||||
pub fn new() -> Self {
|
||||
let version = Version::from("1.0.0").unwrap();
|
||||
Self {
|
||||
version,
|
||||
score,
|
||||
status: InterpretStatus::QUEUED,
|
||||
}
|
||||
}
|
||||
@ -159,7 +159,7 @@ impl OKDSetup03ControlPlaneInterpret {
|
||||
}
|
||||
.to_string();
|
||||
|
||||
debug!("[ControlPlane] iPXE content template:\n{}", content);
|
||||
debug!("[ControlPlane] iPXE content template:\n{content}");
|
||||
|
||||
// Create and apply an iPXE boot file for each node.
|
||||
for node in nodes {
|
||||
@ -189,16 +189,13 @@ impl OKDSetup03ControlPlaneInterpret {
|
||||
/// Prompts the user to reboot the target control plane nodes.
|
||||
async fn reboot_targets(&self, nodes: &Vec<PhysicalHost>) -> Result<(), InterpretError> {
|
||||
let node_ids: Vec<String> = nodes.iter().map(|n| n.id.to_string()).collect();
|
||||
info!(
|
||||
"[ControlPlane] Requesting reboot for control plane nodes: {:?}",
|
||||
node_ids
|
||||
);
|
||||
info!("[ControlPlane] Requesting reboot for control plane nodes: {node_ids:?}",);
|
||||
|
||||
let confirmation = inquire::Confirm::new(
|
||||
&format!("Please reboot the {} control plane nodes ({}) to apply their PXE configuration. Press enter when ready.", nodes.len(), node_ids.join(", ")),
|
||||
)
|
||||
.prompt()
|
||||
.map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?;
|
||||
.map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
|
||||
|
||||
if !confirmation {
|
||||
return Err(InterpretError::new(
|
||||
@ -210,14 +207,23 @@ impl OKDSetup03ControlPlaneInterpret {
|
||||
}
|
||||
|
||||
/// Placeholder for automating network bonding configuration.
|
||||
async fn persist_network_bond(&self) -> Result<(), InterpretError> {
|
||||
// Generate MC or NNCP from inventory NIC data; apply via ignition or post-join.
|
||||
info!("[ControlPlane] Ensuring persistent bonding via MachineConfig/NNCP");
|
||||
async fn persist_network_bond(
|
||||
&self,
|
||||
inventory: &Inventory,
|
||||
topology: &HAClusterTopology,
|
||||
hosts: &Vec<PhysicalHost>,
|
||||
) -> Result<(), InterpretError> {
|
||||
info!("[ControlPlane] Ensuring persistent bonding");
|
||||
let score = HostNetworkConfigurationScore {
|
||||
hosts: hosts.clone(),
|
||||
};
|
||||
score.interpret(inventory, topology).await?;
|
||||
|
||||
inquire::Confirm::new(
|
||||
"Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
|
||||
)
|
||||
.prompt()
|
||||
.map_err(|e| InterpretError::new(format!("User prompt failed: {}", e)))?;
|
||||
.map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@ -260,7 +266,8 @@ impl Interpret<HAClusterTopology> for OKDSetup03ControlPlaneInterpret {
|
||||
self.reboot_targets(&nodes).await?;
|
||||
|
||||
// 5. Placeholder for post-boot network configuration (e.g., bonding).
|
||||
self.persist_network_bond().await?;
|
||||
self.persist_network_bond(inventory, topology, &nodes)
|
||||
.await?;
|
||||
|
||||
// TODO: Implement a step to wait for the control plane nodes to join the cluster
|
||||
// and for the cluster operators to become available. This would be similar to
|
||||
|
41
harmony/src/modules/okd/crd/mod.rs
Normal file
41
harmony/src/modules/okd/crd/mod.rs
Normal file
@ -0,0 +1,41 @@
|
||||
use kube::CustomResource;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod nmstate;
|
||||
|
||||
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[kube(
|
||||
group = "operators.coreos.com",
|
||||
version = "v1",
|
||||
kind = "OperatorGroup",
|
||||
namespaced
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OperatorGroupSpec {
|
||||
pub target_namespaces: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[kube(
|
||||
group = "operators.coreos.com",
|
||||
version = "v1alpha1",
|
||||
kind = "Subscription",
|
||||
namespaced
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SubscriptionSpec {
|
||||
pub name: String,
|
||||
pub source: String,
|
||||
pub source_namespace: String,
|
||||
pub channel: Option<String>,
|
||||
pub install_plan_approval: Option<InstallPlanApproval>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
pub enum InstallPlanApproval {
|
||||
#[serde(rename = "Automatic")]
|
||||
Automatic,
|
||||
#[serde(rename = "Manual")]
|
||||
Manual,
|
||||
}
|
251
harmony/src/modules/okd/crd/nmstate.rs
Normal file
251
harmony/src/modules/okd/crd/nmstate.rs
Normal file
@ -0,0 +1,251 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use kube::CustomResource;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NMStateSpec {
|
||||
pub probe_configuration: Option<ProbeConfig>,
|
||||
}
|
||||
|
||||
impl Default for NMState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
metadata: Default::default(),
|
||||
spec: NMStateSpec {
|
||||
probe_configuration: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ProbeConfig {
|
||||
pub dns: ProbeDns,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ProbeDns {
|
||||
pub host: String,
|
||||
}
|
||||
|
||||
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[kube(
|
||||
group = "nmstate.io",
|
||||
version = "v1",
|
||||
kind = "NodeNetworkConfigurationPolicy",
|
||||
namespaced
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NodeNetworkConfigurationPolicySpec {
|
||||
pub node_selector: Option<BTreeMap<String, String>>,
|
||||
pub desired_state: DesiredStateSpec,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct DesiredStateSpec {
|
||||
pub interfaces: Vec<InterfaceSpec>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct InterfaceSpec {
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub r#type: String,
|
||||
pub state: String,
|
||||
pub mac_address: Option<String>,
|
||||
pub mtu: Option<u32>,
|
||||
pub controller: Option<String>,
|
||||
pub ipv4: Option<IpStackSpec>,
|
||||
pub ipv6: Option<IpStackSpec>,
|
||||
pub ethernet: Option<EthernetSpec>,
|
||||
pub link_aggregation: Option<BondSpec>,
|
||||
pub vlan: Option<VlanSpec>,
|
||||
pub vxlan: Option<VxlanSpec>,
|
||||
pub mac_vtap: Option<MacVtapSpec>,
|
||||
pub mac_vlan: Option<MacVlanSpec>,
|
||||
pub infiniband: Option<InfinibandSpec>,
|
||||
pub linux_bridge: Option<LinuxBridgeSpec>,
|
||||
pub ovs_bridge: Option<OvsBridgeSpec>,
|
||||
pub ethtool: Option<EthtoolSpec>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct IpStackSpec {
|
||||
pub enabled: Option<bool>,
|
||||
pub dhcp: Option<bool>,
|
||||
pub autoconf: Option<bool>,
|
||||
pub address: Option<Vec<IpAddressSpec>>,
|
||||
pub auto_dns: Option<bool>,
|
||||
pub auto_gateway: Option<bool>,
|
||||
pub auto_routes: Option<bool>,
|
||||
pub dhcp_client_id: Option<String>,
|
||||
pub dhcp_duid: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct IpAddressSpec {
|
||||
pub ip: String,
|
||||
pub prefix_length: u8,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct EthernetSpec {
|
||||
pub speed: Option<u32>,
|
||||
pub duplex: Option<String>,
|
||||
pub auto_negotiation: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct BondSpec {
|
||||
pub mode: String,
|
||||
pub ports: Vec<String>,
|
||||
pub options: Option<BTreeMap<String, Value>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct VlanSpec {
|
||||
pub base_iface: String,
|
||||
pub id: u16,
|
||||
pub protocol: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct VxlanSpec {
|
||||
pub base_iface: String,
|
||||
pub id: u32,
|
||||
pub remote: String,
|
||||
pub local: Option<String>,
|
||||
pub learning: Option<bool>,
|
||||
pub destination_port: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct MacVtapSpec {
|
||||
pub base_iface: String,
|
||||
pub mode: String,
|
||||
pub promiscuous: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct MacVlanSpec {
|
||||
pub base_iface: String,
|
||||
pub mode: String,
|
||||
pub promiscuous: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct InfinibandSpec {
|
||||
pub base_iface: String,
|
||||
pub pkey: String,
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct LinuxBridgeSpec {
|
||||
pub options: Option<LinuxBridgeOptions>,
|
||||
pub ports: Option<Vec<LinuxBridgePort>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct LinuxBridgeOptions {
|
||||
pub mac_ageing_time: Option<u32>,
|
||||
pub multicast_snooping: Option<bool>,
|
||||
pub stp: Option<StpOptions>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct StpOptions {
|
||||
pub enabled: Option<bool>,
|
||||
pub forward_delay: Option<u16>,
|
||||
pub hello_time: Option<u16>,
|
||||
pub max_age: Option<u16>,
|
||||
pub priority: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct LinuxBridgePort {
|
||||
pub name: String,
|
||||
pub vlan: Option<LinuxBridgePortVlan>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct LinuxBridgePortVlan {
|
||||
pub mode: Option<String>,
|
||||
pub trunk_tags: Option<Vec<VlanTag>>,
|
||||
pub tag: Option<u16>,
|
||||
pub enable_native: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct VlanTag {
|
||||
pub id: u16,
|
||||
pub id_range: Option<VlanIdRange>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct VlanIdRange {
|
||||
pub min: u16,
|
||||
pub max: u16,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct OvsBridgeSpec {
|
||||
pub options: Option<OvsBridgeOptions>,
|
||||
pub ports: Option<Vec<OvsPortSpec>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct OvsBridgeOptions {
|
||||
pub stp: Option<bool>,
|
||||
pub rstp: Option<bool>,
|
||||
pub mcast_snooping_enable: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct OvsPortSpec {
|
||||
pub name: String,
|
||||
pub link_aggregation: Option<BondSpec>,
|
||||
pub vlan: Option<LinuxBridgePortVlan>,
|
||||
pub r#type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct EthtoolSpec {
|
||||
// TODO: Properly describe this spec (https://nmstate.io/devel/yaml_api.html#ethtool)
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub struct EthtoolFecSpec {
|
||||
pub auto: Option<bool>,
|
||||
pub mode: Option<String>,
|
||||
}
|
394
harmony/src/modules/okd/host_network.rs
Normal file
394
harmony/src/modules/okd/host_network.rs
Normal file
@ -0,0 +1,394 @@
|
||||
use async_trait::async_trait;
|
||||
use harmony_types::id::Id;
|
||||
use log::{debug, info};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{
|
||||
data::Version,
|
||||
hardware::PhysicalHost,
|
||||
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
||||
inventory::Inventory,
|
||||
score::Score,
|
||||
topology::{HostNetworkConfig, NetworkInterface, Switch, SwitchPort, Topology},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HostNetworkConfigurationScore {
|
||||
pub hosts: Vec<PhysicalHost>,
|
||||
}
|
||||
|
||||
impl<T: Topology + Switch> Score<T> for HostNetworkConfigurationScore {
|
||||
fn name(&self) -> String {
|
||||
"HostNetworkConfigurationScore".into()
|
||||
}
|
||||
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||
Box::new(HostNetworkConfigurationInterpret {
|
||||
score: self.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct HostNetworkConfigurationInterpret {
|
||||
score: HostNetworkConfigurationScore,
|
||||
}
|
||||
|
||||
impl HostNetworkConfigurationInterpret {
|
||||
async fn configure_network_for_host<T: Topology + Switch>(
|
||||
&self,
|
||||
topology: &T,
|
||||
host: &PhysicalHost,
|
||||
) -> Result<(), InterpretError> {
|
||||
let switch_ports = self.collect_switch_ports_for_host(topology, host).await?;
|
||||
if !switch_ports.is_empty() {
|
||||
topology
|
||||
.configure_host_network(host, HostNetworkConfig { switch_ports })
|
||||
.await
|
||||
.map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn collect_switch_ports_for_host<T: Topology + Switch>(
|
||||
&self,
|
||||
topology: &T,
|
||||
host: &PhysicalHost,
|
||||
) -> Result<Vec<SwitchPort>, InterpretError> {
|
||||
let mut switch_ports = vec![];
|
||||
|
||||
for network_interface in &host.network {
|
||||
let mac_address = network_interface.mac_address;
|
||||
|
||||
match topology.get_port_for_mac_address(&mac_address).await {
|
||||
Ok(Some(port)) => {
|
||||
switch_ports.push(SwitchPort {
|
||||
interface: NetworkInterface {
|
||||
name: network_interface.name.clone(),
|
||||
mac_address,
|
||||
speed_mbps: network_interface.speed_mbps,
|
||||
mtu: network_interface.mtu,
|
||||
},
|
||||
port,
|
||||
});
|
||||
}
|
||||
Ok(None) => debug!("No port found for host '{}', skipping", host.id),
|
||||
Err(e) => {
|
||||
return Err(InterpretError::new(format!(
|
||||
"Failed to get port for host '{}': {}",
|
||||
host.id, e
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(switch_ports)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
|
||||
fn get_name(&self) -> InterpretName {
|
||||
InterpretName::Custom("HostNetworkConfigurationInterpret")
|
||||
}
|
||||
|
||||
fn get_version(&self) -> Version {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_status(&self) -> InterpretStatus {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_children(&self) -> Vec<Id> {
|
||||
vec![]
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
_inventory: &Inventory,
|
||||
topology: &T,
|
||||
) -> Result<Outcome, InterpretError> {
|
||||
if self.score.hosts.is_empty() {
|
||||
return Ok(Outcome::noop("No hosts to configure".into()));
|
||||
}
|
||||
|
||||
info!(
|
||||
"Started network configuration for {} host(s)...",
|
||||
self.score.hosts.len()
|
||||
);
|
||||
|
||||
topology
|
||||
.setup_switch()
|
||||
.await
|
||||
.map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?;
|
||||
|
||||
let mut configured_host_count = 0;
|
||||
for host in &self.score.hosts {
|
||||
self.configure_network_for_host(topology, host).await?;
|
||||
configured_host_count += 1;
|
||||
}
|
||||
|
||||
if configured_host_count > 0 {
|
||||
Ok(Outcome::success(format!(
|
||||
"Configured {configured_host_count}/{} host(s)",
|
||||
self.score.hosts.len()
|
||||
)))
|
||||
} else {
|
||||
Ok(Outcome::noop("No hosts configured".into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use assertor::*;
|
||||
use harmony_types::{net::MacAddress, switch::PortLocation};
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
use crate::{
|
||||
hardware::HostCategory,
|
||||
topology::{
|
||||
HostNetworkConfig, PreparationError, PreparationOutcome, SwitchError, SwitchPort,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
str::FromStr,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref HOST_ID: Id = Id::from_str("host-1").unwrap();
|
||||
pub static ref ANOTHER_HOST_ID: Id = Id::from_str("host-2").unwrap();
|
||||
pub static ref EXISTING_INTERFACE: NetworkInterface = NetworkInterface {
|
||||
mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F1".to_string()).unwrap(),
|
||||
name: "interface-1".into(),
|
||||
speed_mbps: None,
|
||||
mtu: 1,
|
||||
};
|
||||
pub static ref ANOTHER_EXISTING_INTERFACE: NetworkInterface = NetworkInterface {
|
||||
mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F2".to_string()).unwrap(),
|
||||
name: "interface-2".into(),
|
||||
speed_mbps: None,
|
||||
mtu: 1,
|
||||
};
|
||||
pub static ref UNKNOWN_INTERFACE: NetworkInterface = NetworkInterface {
|
||||
mac_address: MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(),
|
||||
name: "unknown-interface".into(),
|
||||
speed_mbps: None,
|
||||
mtu: 1,
|
||||
};
|
||||
pub static ref PORT: PortLocation = PortLocation(1, 0, 42);
|
||||
pub static ref ANOTHER_PORT: PortLocation = PortLocation(2, 0, 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn should_setup_switch() {
|
||||
let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]);
|
||||
let score = given_score(vec![host]);
|
||||
let topology = TopologyWithSwitch::new();
|
||||
|
||||
let _ = score.interpret(&Inventory::empty(), &topology).await;
|
||||
|
||||
let switch_setup = topology.switch_setup.lock().unwrap();
|
||||
assert_that!(*switch_setup).is_true();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_with_one_mac_address_should_create_bond_with_one_interface() {
|
||||
let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]);
|
||||
let score = given_score(vec![host]);
|
||||
let topology = TopologyWithSwitch::new();
|
||||
|
||||
let _ = score.interpret(&Inventory::empty(), &topology).await;
|
||||
|
||||
let configured_host_networks = topology.configured_host_networks.lock().unwrap();
|
||||
assert_that!(*configured_host_networks).contains_exactly(vec![(
|
||||
HOST_ID.clone(),
|
||||
HostNetworkConfig {
|
||||
switch_ports: vec![SwitchPort {
|
||||
interface: EXISTING_INTERFACE.clone(),
|
||||
port: PORT.clone(),
|
||||
}],
|
||||
},
|
||||
)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_with_multiple_mac_addresses_should_create_one_bond_with_all_interfaces() {
|
||||
let score = given_score(vec![given_host(
|
||||
&HOST_ID,
|
||||
vec![
|
||||
EXISTING_INTERFACE.clone(),
|
||||
ANOTHER_EXISTING_INTERFACE.clone(),
|
||||
],
|
||||
)]);
|
||||
let topology = TopologyWithSwitch::new();
|
||||
|
||||
let _ = score.interpret(&Inventory::empty(), &topology).await;
|
||||
|
||||
let configured_host_networks = topology.configured_host_networks.lock().unwrap();
|
||||
assert_that!(*configured_host_networks).contains_exactly(vec![(
|
||||
HOST_ID.clone(),
|
||||
HostNetworkConfig {
|
||||
switch_ports: vec![
|
||||
SwitchPort {
|
||||
interface: EXISTING_INTERFACE.clone(),
|
||||
port: PORT.clone(),
|
||||
},
|
||||
SwitchPort {
|
||||
interface: ANOTHER_EXISTING_INTERFACE.clone(),
|
||||
port: ANOTHER_PORT.clone(),
|
||||
},
|
||||
],
|
||||
},
|
||||
)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multiple_hosts_should_create_one_bond_per_host() {
|
||||
let score = given_score(vec![
|
||||
given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]),
|
||||
given_host(&ANOTHER_HOST_ID, vec![ANOTHER_EXISTING_INTERFACE.clone()]),
|
||||
]);
|
||||
let topology = TopologyWithSwitch::new();
|
||||
|
||||
let _ = score.interpret(&Inventory::empty(), &topology).await;
|
||||
|
||||
let configured_host_networks = topology.configured_host_networks.lock().unwrap();
|
||||
assert_that!(*configured_host_networks).contains_exactly(vec![
|
||||
(
|
||||
HOST_ID.clone(),
|
||||
HostNetworkConfig {
|
||||
switch_ports: vec![SwitchPort {
|
||||
interface: EXISTING_INTERFACE.clone(),
|
||||
port: PORT.clone(),
|
||||
}],
|
||||
},
|
||||
),
|
||||
(
|
||||
ANOTHER_HOST_ID.clone(),
|
||||
HostNetworkConfig {
|
||||
switch_ports: vec![SwitchPort {
|
||||
interface: ANOTHER_EXISTING_INTERFACE.clone(),
|
||||
port: ANOTHER_PORT.clone(),
|
||||
}],
|
||||
},
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn port_not_found_for_mac_address_should_not_configure_interface() {
|
||||
let score = given_score(vec![given_host(&HOST_ID, vec![UNKNOWN_INTERFACE.clone()])]);
|
||||
let topology = TopologyWithSwitch::new_port_not_found();
|
||||
|
||||
let _ = score.interpret(&Inventory::empty(), &topology).await;
|
||||
|
||||
let configured_host_networks = topology.configured_host_networks.lock().unwrap();
|
||||
assert_that!(*configured_host_networks).is_empty();
|
||||
}
|
||||
|
||||
fn given_score(hosts: Vec<PhysicalHost>) -> HostNetworkConfigurationScore {
|
||||
HostNetworkConfigurationScore { hosts }
|
||||
}
|
||||
|
||||
fn given_host(id: &Id, network_interfaces: Vec<NetworkInterface>) -> PhysicalHost {
|
||||
let network = network_interfaces.iter().map(given_interface).collect();
|
||||
|
||||
PhysicalHost {
|
||||
id: id.clone(),
|
||||
category: HostCategory::Server,
|
||||
network,
|
||||
storage: vec![],
|
||||
labels: vec![],
|
||||
memory_modules: vec![],
|
||||
cpus: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn given_interface(
|
||||
interface: &NetworkInterface,
|
||||
) -> harmony_inventory_agent::hwinfo::NetworkInterface {
|
||||
harmony_inventory_agent::hwinfo::NetworkInterface {
|
||||
name: interface.name.clone(),
|
||||
mac_address: interface.mac_address,
|
||||
speed_mbps: interface.speed_mbps,
|
||||
is_up: true,
|
||||
mtu: interface.mtu,
|
||||
ipv4_addresses: vec![],
|
||||
ipv6_addresses: vec![],
|
||||
driver: "driver".into(),
|
||||
firmware_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct TopologyWithSwitch {
|
||||
available_ports: Arc<Mutex<Vec<PortLocation>>>,
|
||||
configured_host_networks: Arc<Mutex<Vec<(Id, HostNetworkConfig)>>>,
|
||||
switch_setup: Arc<Mutex<bool>>,
|
||||
}
|
||||
|
||||
impl TopologyWithSwitch {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
available_ports: Arc::new(Mutex::new(vec![PORT.clone(), ANOTHER_PORT.clone()])),
|
||||
configured_host_networks: Arc::new(Mutex::new(vec![])),
|
||||
switch_setup: Arc::new(Mutex::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn new_port_not_found() -> Self {
|
||||
Self {
|
||||
available_ports: Arc::new(Mutex::new(vec![])),
|
||||
configured_host_networks: Arc::new(Mutex::new(vec![])),
|
||||
switch_setup: Arc::new(Mutex::new(false)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Topology for TopologyWithSwitch {
|
||||
fn name(&self) -> &str {
|
||||
"SwitchWithPortTopology"
|
||||
}
|
||||
|
||||
async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
|
||||
Ok(PreparationOutcome::Success { details: "".into() })
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Switch for TopologyWithSwitch {
|
||||
async fn setup_switch(&self) -> Result<(), SwitchError> {
|
||||
let mut switch_configured = self.switch_setup.lock().unwrap();
|
||||
*switch_configured = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_port_for_mac_address(
|
||||
&self,
|
||||
_mac_address: &MacAddress,
|
||||
) -> Result<Option<PortLocation>, SwitchError> {
|
||||
let mut ports = self.available_ports.lock().unwrap();
|
||||
if ports.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(ports.remove(0)))
|
||||
}
|
||||
|
||||
async fn configure_host_network(
|
||||
&self,
|
||||
host: &PhysicalHost,
|
||||
config: HostNetworkConfig,
|
||||
) -> Result<(), SwitchError> {
|
||||
let mut configured_host_networks = self.configured_host_networks.lock().unwrap();
|
||||
configured_host_networks.push((host.id.clone(), config.clone()));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
@ -19,3 +19,5 @@ pub use bootstrap_03_control_plane::*;
|
||||
pub use bootstrap_04_workers::*;
|
||||
pub use bootstrap_05_sanity_check::*;
|
||||
pub use bootstrap_06_installation_report::*;
|
||||
pub mod crd;
|
||||
pub mod host_network;
|
||||
|
@ -54,6 +54,9 @@ struct DeployArgs {
|
||||
|
||||
#[arg(long = "profile", short = 'p', default_value = "dev")]
|
||||
harmony_profile: HarmonyProfile,
|
||||
|
||||
#[arg(long = "dry-run", short = 'd', default_value = "false")]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Args, Clone, Debug)]
|
||||
@ -178,6 +181,7 @@ async fn main() {
|
||||
command
|
||||
.env("HARMONY_USE_LOCAL_K3D", format!("{use_local_k3d}"))
|
||||
.env("HARMONY_PROFILE", format!("{}", args.harmony_profile))
|
||||
.env("HARMONY_DRY_RUN", format!("{}", args.dry_run))
|
||||
.arg("-y")
|
||||
.arg("-a");
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user