From f1209b38230b8445c4319585cebdb09072002898 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 3 Sep 2025 00:00:35 -0400 Subject: [PATCH] feat: OKD bootstrap automation pretty much complete with a few prompt for manual steps --- examples/okd_installation/src/topology.rs | 2 +- harmony/src/domain/topology/ha_cluster.rs | 12 ++- harmony/src/domain/topology/http.rs | 6 +- harmony/src/infra/opnsense/dns.rs | 81 +++++++------- harmony/src/modules/http.rs | 9 +- .../modules/okd/bootstrap_load_balancer.rs | 9 ++ harmony/src/modules/okd/installation.rs | 101 +++++++++++++----- harmony/src/modules/okd/ipxe.rs | 1 + harmony/src/modules/okd/mod.rs | 2 +- harmony/src/modules/okd/templates.rs | 8 ++ harmony/templates/okd/bootstrap.ipxe.j2 | 7 ++ opnsense-config/src/config/config.rs | 6 +- opnsense-config/src/modules/dns.rs | 4 +- opnsense-config/src/modules/dnsmasq.rs | 6 +- 14 files changed, 174 insertions(+), 80 deletions(-) create mode 100644 harmony/templates/okd/bootstrap.ipxe.j2 diff --git a/examples/okd_installation/src/topology.rs b/examples/okd_installation/src/topology.rs index 8a78c93..79d5068 100644 --- a/examples/okd_installation/src/topology.rs +++ b/examples/okd_installation/src/topology.rs @@ -54,7 +54,7 @@ pub async fn get_topology() -> HAClusterTopology { name: "master".to_string(), }], bootstrap_host: LogicalHost { - ip: ip!("192.168.1.20"), + ip: ip!("192.168.1.10"), name: "bootstrap".to_string(), }, workers: vec![], diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index e277b19..c9f565e 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -237,7 +237,11 @@ impl Router for HAClusterTopology { #[async_trait] impl HttpServer for HAClusterTopology { - async fn serve_files(&self, url: &Url, remote_path: &Option) -> Result<(), ExecutorError> { + async fn serve_files( + &self, + url: &Url, + remote_path: &Option, + ) -> Result<(), ExecutorError> { self.http_server.serve_files(url, remote_path).await } @@ -397,7 +401,11 @@ impl TftpServer for DummyInfra { #[async_trait] impl HttpServer for DummyInfra { - async fn serve_files(&self, _url: &Url, _remote_path: &Option) -> Result<(), ExecutorError> { + async fn serve_files( + &self, + _url: &Url, + _remote_path: &Option, + ) -> Result<(), ExecutorError> { unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA) } async fn serve_file_content(&self, _file: &FileContent) -> Result<(), ExecutorError> { diff --git a/harmony/src/domain/topology/http.rs b/harmony/src/domain/topology/http.rs index d7194c4..2459206 100644 --- a/harmony/src/domain/topology/http.rs +++ b/harmony/src/domain/topology/http.rs @@ -5,7 +5,11 @@ use harmony_types::net::IpAddress; use harmony_types::net::Url; #[async_trait] pub trait HttpServer: Send + Sync { - async fn serve_files(&self, url: &Url, remote_path: &Option) -> Result<(), ExecutorError>; + async fn serve_files( + &self, + url: &Url, + remote_path: &Option, + ) -> Result<(), ExecutorError>; async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError>; fn get_ip(&self) -> IpAddress; diff --git a/harmony/src/infra/opnsense/dns.rs b/harmony/src/infra/opnsense/dns.rs index 7a58b64..0d355e1 100644 --- a/harmony/src/infra/opnsense/dns.rs +++ b/harmony/src/infra/opnsense/dns.rs @@ -12,21 +12,22 @@ use super::OPNSenseFirewall; #[async_trait] impl DnsServer for OPNSenseFirewall { async fn register_hosts(&self, hosts: Vec) -> Result<(), ExecutorError> { - let mut writable_opnsense = self.opnsense_config.write().await; - let mut dns = writable_opnsense.dns(); - let hosts = hosts - .iter() - .map(|h| { - Host::new( - h.host.clone(), - h.domain.clone(), - h.record_type.to_string(), - h.value.to_string(), - ) - }) - .collect(); - dns.register_hosts(hosts); - Ok(()) + todo!("Refactor this to use dnsmasq") + // let mut writable_opnsense = self.opnsense_config.write().await; + // let mut dns = writable_opnsense.dns(); + // let hosts = hosts + // .iter() + // .map(|h| { + // Host::new( + // h.host.clone(), + // h.domain.clone(), + // h.record_type.to_string(), + // h.value.to_string(), + // ) + // }) + // .collect(); + // dns.add_static_mapping(hosts); + // Ok(()) } fn remove_record( @@ -38,25 +39,26 @@ impl DnsServer for OPNSenseFirewall { } async fn list_records(&self) -> Vec { - self.opnsense_config - .write() - .await - .dns() - .get_hosts() - .iter() - .map(|h| DnsRecord { - host: h.hostname.clone(), - domain: h.domain.clone(), - record_type: h - .rr - .parse() - .expect("received invalid record type {h.rr} from opnsense"), - value: h - .server - .parse() - .expect("received invalid ipv4 record from opnsense {h.server}"), - }) - .collect() + todo!("Refactor this to use dnsmasq") + // self.opnsense_config + // .write() + // .await + // .dns() + // .get_hosts() + // .iter() + // .map(|h| DnsRecord { + // host: h.hostname.clone(), + // domain: h.domain.clone(), + // record_type: h + // .rr + // .parse() + // .expect("received invalid record type {h.rr} from opnsense"), + // value: h + // .server + // .parse() + // .expect("received invalid ipv4 record from opnsense {h.server}"), + // }) + // .collect() } fn get_ip(&self) -> IpAddress { @@ -68,11 +70,12 @@ impl DnsServer for OPNSenseFirewall { } async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> { - let mut writable_opnsense = self.opnsense_config.write().await; - let mut dns = writable_opnsense.dns(); - dns.register_dhcp_leases(register); - - Ok(()) + todo!("Refactor this to use dnsmasq") + // let mut writable_opnsense = self.opnsense_config.write().await; + // let mut dns = writable_opnsense.dns(); + // dns.register_dhcp_leases(register); + // + // Ok(()) } async fn commit_config(&self) -> Result<(), ExecutorError> { diff --git a/harmony/src/modules/http.rs b/harmony/src/modules/http.rs index 8d4f8a5..c654e20 100644 --- a/harmony/src/modules/http.rs +++ b/harmony/src/modules/http.rs @@ -24,8 +24,9 @@ use harmony_types::{id::Id, net::MacAddress}; /// } /// ``` #[derive(Debug, new, Clone, Serialize)] -pub struct StaticFilesHttpScore { // TODO this should be split in two scores, one for folder and - // other for files +pub struct StaticFilesHttpScore { + // TODO this should be split in two scores, one for folder and + // other for files pub folder_to_serve: Option, pub files: Vec, pub remote_path: Option, @@ -56,7 +57,9 @@ impl Interpret for StaticFilesHttpInterpret { http_server.ensure_initialized().await?; // http_server.set_ip(topology.router.get_gateway()).await?; if let Some(folder) = self.score.folder_to_serve.as_ref() { - http_server.serve_files(folder, &self.score.remote_path).await?; + http_server + .serve_files(folder, &self.score.remote_path) + .await?; } for f in self.score.files.iter() { diff --git a/harmony/src/modules/okd/bootstrap_load_balancer.rs b/harmony/src/modules/okd/bootstrap_load_balancer.rs index d6cd2f3..52250c6 100644 --- a/harmony/src/modules/okd/bootstrap_load_balancer.rs +++ b/harmony/src/modules/okd/bootstrap_load_balancer.rs @@ -54,6 +54,7 @@ impl OKDBootstrapLoadBalancerScore { }, } } + fn topology_to_backend_server(topology: &HAClusterTopology, port: u16) -> Vec { let mut backend: Vec<_> = topology .control_plane @@ -63,6 +64,14 @@ impl OKDBootstrapLoadBalancerScore { port, }) .collect(); + + topology.workers.iter().for_each(|worker| { + backend.push(BackendServer { + address: worker.ip.to_string(), + port, + }) + }); + backend.push(BackendServer { address: topology.bootstrap_host.ip.to_string(), port, diff --git a/harmony/src/modules/okd/installation.rs b/harmony/src/modules/okd/installation.rs index d245fbe..ae851a2 100644 --- a/harmony/src/modules/okd/installation.rs +++ b/harmony/src/modules/okd/installation.rs @@ -73,7 +73,11 @@ use crate::{ dhcp::DhcpHostBindingScore, http::{IPxeMacBootFileScore, StaticFilesHttpScore}, inventory::LaunchDiscoverInventoryAgentScore, - okd::{dns::OKDDnsScore, templates::InstallConfigYaml}, + okd::{ + bootstrap_load_balancer::OKDBootstrapLoadBalancerScore, + dns::OKDDnsScore, + templates::{BootstrapIpxeTpl, InstallConfigYaml}, + }, }, score::Score, topology::{HAClusterTopology, HostBinding}, @@ -207,7 +211,7 @@ impl Interpret for OKDInstallationInterpret { info!("Starting OKD installation pipeline",); - // self.run_inventory_phase(inventory, topology).await?; + self.run_inventory_phase(inventory, topology).await?; self.run_bootstrap_phase(inventory, topology).await?; @@ -289,9 +293,23 @@ impl Interpret for OKDSetup01InventoryInterpret { topology: &HAClusterTopology, ) -> Result { info!("Setting up base DNS config for OKD"); - OKDDnsScore::new(topology) - .interpret(inventory, topology) - .await?; + let cluster_domain = &topology.domain_name; + let load_balancer_ip = &topology.load_balancer.get_ip(); + inquire::Confirm::new(&format!( + "Set hostnames manually in your opnsense dnsmasq config : +*.apps.{cluster_domain} -> {load_balancer_ip} +api.{cluster_domain} -> {load_balancer_ip} +api-int.{cluster_domain} -> {load_balancer_ip} + +When you can dig them, confirm to continue. +" + )) + .prompt() + .expect("Prompt error"); + // TODO reactivate automatic dns config + // OKDDnsScore::new(topology) + // .interpret(inventory, topology) + // .await?; info!( "Launching discovery agent, make sure that your nodes are successfully PXE booted and running inventory agent. They should answer on `http://:8080/inventory`" @@ -368,7 +386,7 @@ struct OKDSetup02BootstrapScore {} impl Score for OKDSetup02BootstrapScore { fn create_interpret(&self) -> Box> { - Box::new(OKDSetup02BootstrapInterpret::new(self.clone())) + Box::new(OKDSetup02BootstrapInterpret::new()) } fn name(&self) -> String { @@ -378,17 +396,15 @@ impl Score for OKDSetup02BootstrapScore { #[derive(Debug, Clone)] struct OKDSetup02BootstrapInterpret { - score: OKDSetup02BootstrapScore, version: Version, status: InterpretStatus, } impl OKDSetup02BootstrapInterpret { - pub fn new(score: OKDSetup02BootstrapScore) -> Self { + pub fn new() -> Self { let version = Version::from("1.0.0").unwrap(); Self { version, - score, status: InterpretStatus::QUEUED, } } @@ -572,20 +588,30 @@ impl OKDSetup02BootstrapInterpret { Ok(output) }; + info!("Successfully prepared ignition files for OKD installation"); // ignition_files_http_path // = PathBuf::from("okd_ignition_files"); - let scos_http_path = PathBuf::from("scos"); info!( r#"Uploading images, they can be refreshed with a command similar to this one: openshift-install coreos print-stream-json | grep -Eo '"https.*(kernel.|initramfs.|rootfs.)\w+(\.img)?"' | grep x86_64 | xargs -n 1 curl -LO"# ); - StaticFilesHttpScore { - folder_to_serve: Some(Url::LocalFolder(okd_images_path.to_string_lossy().to_string())), - remote_path: Some(scos_http_path.to_string_lossy().to_string()), - files: vec![], - } - .interpret(inventory, topology) - .await?; - todo!("What's up next?") + warn!( + "TODO push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved" + ); + inquire::Confirm::new( + "push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved").prompt().expect("Prompt error"); + + // let scos_http_path = PathBuf::from("scos"); + // StaticFilesHttpScore { + // folder_to_serve: Some(Url::LocalFolder( + // okd_images_path.to_string_lossy().to_string(), + // )), + // remote_path: Some(scos_http_path.to_string_lossy().to_string()), + // files: vec![], + // } + // .interpret(inventory, topology) + // .await?; + + Ok(()) } async fn configure_host_binding( @@ -613,12 +639,25 @@ impl OKDSetup02BootstrapInterpret { inventory: &Inventory, topology: &HAClusterTopology, ) -> Result<(), InterpretError> { - // Placeholder: use Harmony templates to emit {MAC}.ipxe selecting SCOS live + bootstrap ignition. - info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node"); + let content = BootstrapIpxeTpl { + http_ip: &topology.http_server.get_ip().to_string(), + scos_path: "scos", // TODO use some constant + installation_device: "/dev/sda", // TODO do something smart based on the host drives + // topology. Something like use the smallest device + // above 200G that is an ssd + } + .to_string(); + let bootstrap_node = self.get_bootstrap_node().await?; + let mac_address = bootstrap_node.get_mac_address(); + + info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node"); + debug!("bootstrap ipxe content : {content}"); + debug!("bootstrap mac addresses : {mac_address:?}"); + IPxeMacBootFileScore { - mac_address: bootstrap_node.get_mac_address(), - content: todo!("templace for bootstrap node"), + mac_address, + content, } .interpret(inventory, topology) .await?; @@ -630,15 +669,25 @@ impl OKDSetup02BootstrapInterpret { inventory: &Inventory, topology: &HAClusterTopology, ) -> Result<(), InterpretError> { - todo!( - "OKD loadbalancer score already exists, just call it here probably? 6443 22623, 80 and 443 \n\nhttps://docs.okd.io/latest/installing/installing_bare_metal/upi/installing-bare-metal.html#installation-load-balancing-user-infra_installing-bare-metal" - ); + let outcome = OKDBootstrapLoadBalancerScore::new(topology) + .interpret(inventory, topology) + .await?; + info!("Successfully executed OKDBootstrapLoadBalancerScore : {outcome:?}"); + Ok(()) } async fn reboot_target(&self) -> Result<(), InterpretError> { // Placeholder: ssh reboot using the inventory ephemeral key info!("[Bootstrap] Rebooting bootstrap node via SSH"); - todo!("[Bootstrap] Rebooting bootstrap node via SSH") + // TODO reboot programatically, there are some logical checks and refactoring to do such as + // accessing the bootstrap node config (ip address) from the inventory + let confirmation = inquire::Confirm::new( + "Now reboot the bootstrap node so it picks up its pxe boot file. Press enter when ready.", + ) + .with_default(true) + .prompt() + .expect("Unexpected prompt error"); + Ok(()) } async fn wait_for_bootstrap_complete(&self) -> Result<(), InterpretError> { diff --git a/harmony/src/modules/okd/ipxe.rs b/harmony/src/modules/okd/ipxe.rs index 968ec41..743efab 100644 --- a/harmony/src/modules/okd/ipxe.rs +++ b/harmony/src/modules/okd/ipxe.rs @@ -120,6 +120,7 @@ impl Interpret f Err(e) => return Err(e), }; } + inquire::Confirm::new("Execute the copy : `scp -r data/pxe/okd/http_files/* root@192.168.1.1:/usr/local/http/` and confirm when done to continue").prompt().expect("Prompt error"); Ok(Outcome::success("Ipxe installed".to_string())) } diff --git a/harmony/src/modules/okd/mod.rs b/harmony/src/modules/okd/mod.rs index 96e45cd..f255959 100644 --- a/harmony/src/modules/okd/mod.rs +++ b/harmony/src/modules/okd/mod.rs @@ -5,5 +5,5 @@ pub mod dns; pub mod installation; pub mod ipxe; pub mod load_balancer; -pub mod upgrade; pub mod templates; +pub mod upgrade; diff --git a/harmony/src/modules/okd/templates.rs b/harmony/src/modules/okd/templates.rs index da7524f..26a1791 100644 --- a/harmony/src/modules/okd/templates.rs +++ b/harmony/src/modules/okd/templates.rs @@ -8,3 +8,11 @@ pub struct InstallConfigYaml<'a> { pub ssh_public_key: &'a str, pub cluster_name: &'a str, } + +#[derive(Template)] +#[template(path = "okd/bootstrap.ipxe.j2")] +pub struct BootstrapIpxeTpl<'a> { + pub http_ip: &'a str, + pub scos_path: &'a str, + pub installation_device: &'a str, +} diff --git a/harmony/templates/okd/bootstrap.ipxe.j2 b/harmony/templates/okd/bootstrap.ipxe.j2 new file mode 100644 index 0000000..9a5a9b9 --- /dev/null +++ b/harmony/templates/okd/bootstrap.ipxe.j2 @@ -0,0 +1,7 @@ +set base-url http://{{ http_ip }}:8080 +set scos-base-url = ${base-url}/{{ scos_path }} +set installation-device = {{ installation_device }} + +kernel ${scos-base-url}/scos-live-kernel.x86_64 initrd=main coreos.live.rootfs_url=${scos-base-url}/scos-live-rootfs.x86_64.img coreos.inst.install_dev=${installation-device} coreos.inst.ignition_url=${base-url}/bootstrap.ign +initrd --name main ${scos-base-url}/scos-live-initramfs.x86_64.img +boot diff --git a/opnsense-config/src/config/config.rs b/opnsense-config/src/config/config.rs index 55464cf..0a1072c 100644 --- a/opnsense-config/src/config/config.rs +++ b/opnsense-config/src/config/config.rs @@ -4,7 +4,7 @@ use crate::{ config::{SshConfigManager, SshCredentials, SshOPNSenseShell}, error::Error, modules::{ - caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::DnsConfig, + caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::UnboundDnsConfig, dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig, tftp::TftpConfig, }, }; @@ -51,8 +51,8 @@ impl Config { DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone()) } - pub fn dns(&mut self) -> DnsConfig<'_> { - DnsConfig::new(&mut self.opnsense) + pub fn dns(&mut self) -> DhcpConfigDnsMasq<'_> { + DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone()) } pub fn tftp(&mut self) -> TftpConfig<'_> { diff --git a/opnsense-config/src/modules/dns.rs b/opnsense-config/src/modules/dns.rs index 3bf045e..517b5ea 100644 --- a/opnsense-config/src/modules/dns.rs +++ b/opnsense-config/src/modules/dns.rs @@ -1,10 +1,10 @@ use opnsense_config_xml::{Host, OPNsense}; -pub struct DnsConfig<'a> { +pub struct UnboundDnsConfig<'a> { opnsense: &'a mut OPNsense, } -impl<'a> DnsConfig<'a> { +impl<'a> UnboundDnsConfig<'a> { pub fn new(opnsense: &'a mut OPNsense) -> Self { Self { opnsense } } diff --git a/opnsense-config/src/modules/dnsmasq.rs b/opnsense-config/src/modules/dnsmasq.rs index 001f442..dd812ea 100644 --- a/opnsense-config/src/modules/dnsmasq.rs +++ b/opnsense-config/src/modules/dnsmasq.rs @@ -144,14 +144,16 @@ impl<'a> DhcpConfigDnsMasq<'a> { let host_to_modify_ip = host_to_modify.ip.content_string(); if host_to_modify_ip != ip_str { warn!( - "Hostname '{}' already exists with a different IP ({}). Appending MAC {}.", + "Hostname '{}' already exists with a different IP ({}). Setting new IP {ip_str}. Appending MAC {}.", hostname, host_to_modify_ip, mac ); + host_to_modify.ip.content = Some(ip_str); } else if host_to_modify.host != hostname { warn!( - "IP {} already exists with a different hostname ('{}'). Appending MAC {}.", + "IP {} already exists with a different hostname ('{}'). Setting hostname to {hostname}. Appending MAC {}.", ipaddr, host_to_modify.host, mac ); + host_to_modify.host = hostname.to_string(); } if !host_to_modify