feat: Refactor load balancer to remove side effect and improve types #258
@@ -600,7 +600,6 @@ fn build_all_scores() -> Result<Vec<Box<dyn Score<OPNSenseFirewall>>>, Box<dyn s
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
private_services: vec![],
|
private_services: vec![],
|
||||||
wan_firewall_ports: vec![],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let dhcp_score = DhcpScore::new(
|
let dhcp_score = DhcpScore::new(
|
||||||
|
|||||||
@@ -69,6 +69,5 @@ fn build_large_score() -> LoadBalancerScore {
|
|||||||
lb_service.clone(),
|
lb_service.clone(),
|
||||||
lb_service.clone(),
|
lb_service.clone(),
|
||||||
],
|
],
|
||||||
wan_firewall_ports: vec![],
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use async_trait::async_trait;
|
|||||||
use harmony_types::firewall::VipMode;
|
use harmony_types::firewall::VipMode;
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
use harmony_types::net::{IpAddress, MacAddress};
|
use harmony_types::net::{IpAddress, MacAddress};
|
||||||
use log::info;
|
use log::{info, warn};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::config::secret::{OPNSenseApiCredentials, OPNSenseFirewallCredentials};
|
use crate::config::secret::{OPNSenseApiCredentials, OPNSenseFirewallCredentials};
|
||||||
@@ -176,8 +176,24 @@ impl DhcpServer for FirewallPairTopology {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn list_static_mappings(&self) -> Vec<(MacAddress, IpAddress)> {
|
async fn list_static_mappings(&self) -> Vec<(MacAddress, IpAddress)> {
|
||||||
// Return primary's view — both should be identical
|
let primary_mappings = self.primary.list_static_mappings().await;
|
||||||
self.primary.list_static_mappings().await
|
let backup_mappings = self.backup.list_static_mappings().await;
|
||||||
|
|
||||||
|
let primary_set: std::collections::HashSet<_> = primary_mappings.iter().collect();
|
||||||
|
let backup_set: std::collections::HashSet<_> = backup_mappings.iter().collect();
|
||||||
|
|
||||||
|
let only_primary: Vec<_> = primary_set.difference(&backup_set).collect();
|
||||||
|
let only_backup: Vec<_> = backup_set.difference(&primary_set).collect();
|
||||||
|
|
||||||
|
if !only_primary.is_empty() || !only_backup.is_empty() {
|
||||||
|
warn!(
|
||||||
|
"DHCP static mapping mismatch between primary and backup firewalls! \
|
||||||
|
Only on primary: {:?}, Only on backup: {:?}",
|
||||||
|
only_primary, only_backup
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
primary_mappings
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the primary firewall's IP. In a CARP setup, callers
|
/// Returns the primary firewall's IP. In a CARP setup, callers
|
||||||
|
|||||||
@@ -489,6 +489,9 @@ impl LoadBalancer for DummyInfra {
|
|||||||
async fn reload_restart(&self) -> Result<(), ExecutorError> {
|
async fn reload_restart(&self) -> Result<(), ExecutorError> {
|
||||||
unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
|
unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
|
||||||
}
|
}
|
||||||
|
async fn ensure_wan_access(&self, _port: u16) -> Result<(), ExecutorError> {
|
||||||
|
unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
|
|||||||
@@ -37,11 +37,9 @@ pub trait LoadBalancer: Send + Sync {
|
|||||||
/// from the WAN interface. Used by load balancers that need to receive
|
/// from the WAN interface. Used by load balancers that need to receive
|
||||||
/// external traffic (e.g., OKD ingress on ports 80/443).
|
/// external traffic (e.g., OKD ingress on ports 80/443).
|
||||||
///
|
///
|
||||||
/// Default implementation is a no-op for topologies that don't manage
|
/// Topologies that don't manage firewall rules (e.g., cloud environments
|
||||||
/// firewall rules (e.g., cloud environments with security groups).
|
/// with security groups) should return `Ok(())`.
|
||||||
async fn ensure_wan_access(&self, _port: u16) -> Result<(), ExecutorError> {
|
async fn ensure_wan_access(&self, port: u16) -> Result<(), ExecutorError>;
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Clone, Serialize)]
|
#[derive(Debug, PartialEq, Clone, Serialize)]
|
||||||
|
|||||||
@@ -111,12 +111,16 @@ impl LoadBalancer for OPNSenseFirewall {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn haproxy_service_to_harmony(svc: &HaproxyService) -> Option<LoadBalancerService> {
|
fn haproxy_service_to_harmony(svc: &HaproxyService) -> Option<LoadBalancerService> {
|
||||||
let listening_port = svc.bind.parse().unwrap_or_else(|_| {
|
let listening_port = match svc.bind.parse() {
|
||||||
panic!(
|
Ok(addr) => addr,
|
||||||
"HAProxy frontend address should be a valid SocketAddr, got {}",
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
"Skipping HAProxy service: bind address '{}' is not a valid SocketAddr: {e}",
|
||||||
svc.bind
|
svc.bind
|
||||||
)
|
);
|
||||||
});
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let backend_servers: Vec<BackendServer> = svc
|
let backend_servers: Vec<BackendServer> = svc
|
||||||
.servers
|
.servers
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use virt::sys;
|
|||||||
use super::error::KvmError;
|
use super::error::KvmError;
|
||||||
use super::types::{CdromConfig, NetworkConfig, VmConfig, VmInterface, VmStatus};
|
use super::types::{CdromConfig, NetworkConfig, VmConfig, VmInterface, VmStatus};
|
||||||
use super::xml;
|
use super::xml;
|
||||||
|
use harmony_types::net::MacAddress;
|
||||||
|
|
||||||
/// A handle to a libvirt hypervisor.
|
/// A handle to a libvirt hypervisor.
|
||||||
///
|
///
|
||||||
@@ -374,14 +375,15 @@ impl KvmExecutor {
|
|||||||
pub async fn set_interface_link(
|
pub async fn set_interface_link(
|
||||||
&self,
|
&self,
|
||||||
vm_name: &str,
|
vm_name: &str,
|
||||||
mac: &str,
|
mac: &MacAddress,
|
||||||
up: bool,
|
up: bool,
|
||||||
) -> Result<(), KvmError> {
|
) -> Result<(), KvmError> {
|
||||||
let state = if up { "up" } else { "down" };
|
let state = if up { "up" } else { "down" };
|
||||||
info!("Setting {vm_name} interface {mac} link {state}");
|
let mac_str = mac.to_string();
|
||||||
|
info!("Setting {vm_name} interface {mac_str} link {state}");
|
||||||
|
|
||||||
let output = tokio::process::Command::new("virsh")
|
let output = tokio::process::Command::new("virsh")
|
||||||
.args(["-c", &self.uri, "domif-setlink", vm_name, mac, state])
|
.args(["-c", &self.uri, "domif-setlink", vm_name, &mac_str, state])
|
||||||
.output()
|
.output()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -420,11 +422,18 @@ impl KvmExecutor {
|
|||||||
// virsh domiflist columns: Interface, Type, Source, Model, MAC
|
// virsh domiflist columns: Interface, Type, Source, Model, MAC
|
||||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||||
if parts.len() >= 5 {
|
if parts.len() >= 5 {
|
||||||
|
let mac = match MacAddress::try_from(parts[4].to_string()) {
|
||||||
|
Ok(m) => m,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Skipping interface with invalid MAC '{}': {e}", parts[4]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
interfaces.push(VmInterface {
|
interfaces.push(VmInterface {
|
||||||
interface_type: parts[1].to_string(),
|
interface_type: parts[1].to_string(),
|
||||||
source: parts[2].to_string(),
|
source: parts[2].to_string(),
|
||||||
model: parts[3].to_string(),
|
model: parts[3].to_string(),
|
||||||
mac: parts[4].to_string(),
|
mac,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use harmony_types::net::MacAddress;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
/// Information about a VM's network interface, as reported by `virsh domiflist`.
|
/// Information about a VM's network interface, as reported by `virsh domiflist`.
|
||||||
@@ -10,7 +11,7 @@ pub struct VmInterface {
|
|||||||
/// Device model (e.g. "virtio")
|
/// Device model (e.g. "virtio")
|
||||||
pub model: String,
|
pub model: String,
|
||||||
/// MAC address
|
/// MAC address
|
||||||
pub mac: String,
|
pub mac: MacAddress,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Specifies how a KVM host is accessed.
|
/// Specifies how a KVM host is accessed.
|
||||||
@@ -95,7 +96,7 @@ pub struct NetworkRef {
|
|||||||
pub name: String,
|
pub name: String,
|
||||||
/// Optional fixed MAC address for this interface. When `None`, libvirt
|
/// Optional fixed MAC address for this interface. When `None`, libvirt
|
||||||
/// assigns one automatically.
|
/// assigns one automatically.
|
||||||
pub mac: Option<String>,
|
pub mac: Option<MacAddress>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NetworkRef {
|
impl NetworkRef {
|
||||||
@@ -106,8 +107,8 @@ impl NetworkRef {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_mac(mut self, mac: impl Into<String>) -> Self {
|
pub fn with_mac(mut self, mac: MacAddress) -> Self {
|
||||||
self.mac = Some(mac.into());
|
self.mac = Some(mac);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -260,7 +261,7 @@ impl VmConfigBuilder {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct DhcpHost {
|
pub struct DhcpHost {
|
||||||
/// MAC address (e.g. `"52:54:00:00:50:01"`).
|
/// MAC address (e.g. `"52:54:00:00:50:01"`).
|
||||||
pub mac: String,
|
pub mac: MacAddress,
|
||||||
/// IP to assign (e.g. `"10.50.0.2"`).
|
/// IP to assign (e.g. `"10.50.0.2"`).
|
||||||
pub ip: String,
|
pub ip: String,
|
||||||
/// Optional hostname.
|
/// Optional hostname.
|
||||||
@@ -356,12 +357,12 @@ impl NetworkConfigBuilder {
|
|||||||
/// Add a static DHCP host entry (MAC → fixed IP).
|
/// Add a static DHCP host entry (MAC → fixed IP).
|
||||||
pub fn dhcp_host(
|
pub fn dhcp_host(
|
||||||
mut self,
|
mut self,
|
||||||
mac: impl Into<String>,
|
mac: MacAddress,
|
||||||
ip: impl Into<String>,
|
ip: impl Into<String>,
|
||||||
name: Option<String>,
|
name: Option<String>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
self.dhcp_hosts.push(DhcpHost {
|
self.dhcp_hosts.push(DhcpHost {
|
||||||
mac: mac.into(),
|
mac,
|
||||||
ip: ip.into(),
|
ip: ip.into(),
|
||||||
name,
|
name,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ fn nic_devices(vm: &VmConfig) -> String {
|
|||||||
.map(|net| {
|
.map(|net| {
|
||||||
let mac_line = net
|
let mac_line = net
|
||||||
.mac
|
.mac
|
||||||
.as_deref()
|
.as_ref()
|
||||||
.map(|m| format!("\n <mac address='{m}'/>"))
|
.map(|m| format!("\n <mac address='{m}'/>"))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
format!(
|
format!(
|
||||||
@@ -221,6 +221,7 @@ mod tests {
|
|||||||
use crate::modules::kvm::types::{
|
use crate::modules::kvm::types::{
|
||||||
BootDevice, ForwardMode, NetworkConfig, NetworkRef, VmConfig,
|
BootDevice, ForwardMode, NetworkConfig, NetworkRef, VmConfig,
|
||||||
};
|
};
|
||||||
|
use harmony_types::net::MacAddress;
|
||||||
|
|
||||||
// ── Domain XML ──────────────────────────────────────────────────────
|
// ── Domain XML ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -284,12 +285,13 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn domain_xml_nic_with_mac_address() {
|
fn domain_xml_nic_with_mac_address() {
|
||||||
|
let mac: MacAddress = "52:54:00:aa:bb:cc".to_string().try_into().unwrap();
|
||||||
let vm = VmConfig::builder("mac-test")
|
let vm = VmConfig::builder("mac-test")
|
||||||
.network(NetworkRef::named("mynet").with_mac("52:54:00:AA:BB:CC"))
|
.network(NetworkRef::named("mynet").with_mac(mac))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
let xml = domain_xml(&vm, "/tmp");
|
let xml = domain_xml(&vm, "/tmp");
|
||||||
assert!(xml.contains("mac address='52:54:00:AA:BB:CC'"));
|
assert!(xml.contains("mac address='52:54:00:aa:bb:cc'"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -454,14 +456,11 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn network_xml_with_dhcp_host() {
|
fn network_xml_with_dhcp_host() {
|
||||||
|
let mac: MacAddress = "52:54:00:00:50:01".to_string().try_into().unwrap();
|
||||||
let cfg = NetworkConfig::builder("hostnet")
|
let cfg = NetworkConfig::builder("hostnet")
|
||||||
.subnet("10.50.0.1", 24)
|
.subnet("10.50.0.1", 24)
|
||||||
.dhcp_range("10.50.0.100", "10.50.0.200")
|
.dhcp_range("10.50.0.100", "10.50.0.200")
|
||||||
.dhcp_host(
|
.dhcp_host(mac, "10.50.0.2", Some("opnsense".to_string()))
|
||||||
"52:54:00:00:50:01",
|
|
||||||
"10.50.0.2",
|
|
||||||
Some("opnsense".to_string()),
|
|
||||||
)
|
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
let xml = network_xml(&cfg);
|
let xml = network_xml(&cfg);
|
||||||
|
|||||||
@@ -19,12 +19,6 @@ pub struct LoadBalancerScore {
|
|||||||
// (listen_interface, LoadBalancerService) tuples or something like that
|
// (listen_interface, LoadBalancerService) tuples or something like that
|
||||||
// I am not sure what to use as listen_interface, should it be interface name, ip address,
|
// I am not sure what to use as listen_interface, should it be interface name, ip address,
|
||||||
// uuid?
|
// uuid?
|
||||||
/// TCP ports that must be open for inbound WAN traffic.
|
|
||||||
///
|
|
||||||
/// The load balancer interpret will call `ensure_wan_access` for each port
|
|
||||||
/// before configuring services, so that the load balancer is reachable
|
|
||||||
/// from outside the LAN.
|
|
||||||
pub wan_firewall_ports: Vec<u16>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: Topology + LoadBalancer> Score<T> for LoadBalancerScore {
|
impl<T: Topology + LoadBalancer> Score<T> for LoadBalancerScore {
|
||||||
@@ -66,11 +60,6 @@ impl<T: Topology + LoadBalancer> Interpret<T> for LoadBalancerInterpret {
|
|||||||
load_balancer.ensure_initialized().await?
|
load_balancer.ensure_initialized().await?
|
||||||
);
|
);
|
||||||
|
|
||||||
for port in &self.score.wan_firewall_ports {
|
|
||||||
info!("Ensuring WAN access for port {port}");
|
|
||||||
load_balancer.ensure_wan_access(*port).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
for service in self.score.public_services.iter() {
|
for service in self.score.public_services.iter() {
|
||||||
info!("Ensuring service exists {service:?}");
|
info!("Ensuring service exists {service:?}");
|
||||||
|
|
||||||
|
|||||||
@@ -350,13 +350,20 @@ impl OKDSetup02BootstrapInterpret {
|
|||||||
&self,
|
&self,
|
||||||
inventory: &Inventory,
|
inventory: &Inventory,
|
||||||
) -> Result<(), InterpretError> {
|
) -> Result<(), InterpretError> {
|
||||||
info!("[Stage 02/Bootstrap] Waiting for bootstrap to complete...");
|
let timeout_minutes: u64 = std::env::var("HARMONY_OKD_BOOTSTRAP_TIMEOUT_MINUTES")
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(90);
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"[Stage 02/Bootstrap] Waiting for bootstrap to complete (timeout: {timeout_minutes}m)..."
|
||||||
|
);
|
||||||
info!("[Stage 02/Bootstrap] Running: openshift-install wait-for bootstrap-complete");
|
info!("[Stage 02/Bootstrap] Running: openshift-install wait-for bootstrap-complete");
|
||||||
|
|
||||||
let okd_installation_path =
|
let okd_installation_path =
|
||||||
format!("./data/okd/installation_files_{}", inventory.location.name);
|
format!("./data/okd/installation_files_{}", inventory.location.name);
|
||||||
|
|
||||||
let output = Command::new("./data/okd/bin/openshift-install")
|
let child = Command::new("./data/okd/bin/openshift-install")
|
||||||
.args([
|
.args([
|
||||||
"wait-for",
|
"wait-for",
|
||||||
"bootstrap-complete",
|
"bootstrap-complete",
|
||||||
@@ -364,8 +371,17 @@ impl OKDSetup02BootstrapInterpret {
|
|||||||
&okd_installation_path,
|
&okd_installation_path,
|
||||||
"--log-level=info",
|
"--log-level=info",
|
||||||
])
|
])
|
||||||
.output()
|
.output();
|
||||||
|
|
||||||
|
let timeout = std::time::Duration::from_secs(timeout_minutes * 60);
|
||||||
|
let output = tokio::time::timeout(timeout, child)
|
||||||
.await
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
InterpretError::new(format!(
|
||||||
|
"[Stage 02/Bootstrap] bootstrap-complete timed out after {timeout_minutes} minutes. \
|
||||||
|
Set HARMONY_OKD_BOOTSTRAP_TIMEOUT_MINUTES to increase the timeout and retry."
|
||||||
|
))
|
||||||
|
})?
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
InterpretError::new(format!(
|
InterpretError::new(format!(
|
||||||
"[Stage 02/Bootstrap] Failed to run openshift-install wait-for bootstrap-complete: {e}"
|
"[Stage 02/Bootstrap] Failed to run openshift-install wait-for bootstrap-complete: {e}"
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ impl OKDBootstrapLoadBalancerScore {
|
|||||||
load_balancer_score: LoadBalancerScore {
|
load_balancer_score: LoadBalancerScore {
|
||||||
public_services: vec![],
|
public_services: vec![],
|
||||||
private_services,
|
private_services,
|
||||||
wan_firewall_ports: vec![80, 443],
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,7 +114,6 @@ impl OKDLoadBalancerScore {
|
|||||||
load_balancer_score: LoadBalancerScore {
|
load_balancer_score: LoadBalancerScore {
|
||||||
public_services,
|
public_services,
|
||||||
private_services,
|
private_services,
|
||||||
wan_firewall_ports: vec![80, 443],
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -339,13 +338,6 @@ mod tests {
|
|||||||
assert_eq!(private_service_22623.backend_servers.len(), 3);
|
assert_eq!(private_service_22623.backend_servers.len(), 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_wan_firewall_ports_include_http_and_https() {
|
|
||||||
let topology = create_test_topology();
|
|
||||||
let score = OKDLoadBalancerScore::new(&topology);
|
|
||||||
assert_eq!(score.load_balancer_score.wan_firewall_ports, vec![80, 443]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_all_backend_servers_have_correct_port() {
|
fn test_all_backend_servers_have_correct_port() {
|
||||||
let topology = create_test_topology();
|
let topology = create_test_topology();
|
||||||
|
|||||||
Reference in New Issue
Block a user