feat: OKD bootstrap automation pretty much complete with a few prompt for manual steps

examples/okd_installation/src/topology.rs

@@ -54,7 +54,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "master".to_string(),
         }],
         bootstrap_host: LogicalHost {
-            ip: ip!("192.168.1.20"),
+            ip: ip!("192.168.1.10"),
             name: "bootstrap".to_string(),
         },
         workers: vec![],

harmony/src/domain/topology/ha_cluster.rs

@@ -237,7 +237,11 @@ impl Router for HAClusterTopology {
 
 #[async_trait]
 impl HttpServer for HAClusterTopology {
-    async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError> {
+    async fn serve_files(
+        &self,
+        url: &Url,
+        remote_path: &Option<String>,
+    ) -> Result<(), ExecutorError> {
         self.http_server.serve_files(url, remote_path).await
     }
 
@@ -397,7 +401,11 @@ impl TftpServer for DummyInfra {
 
 #[async_trait]
 impl HttpServer for DummyInfra {
-    async fn serve_files(&self, _url: &Url, _remote_path: &Option<String>) -> Result<(), ExecutorError> {
+    async fn serve_files(
+        &self,
+        _url: &Url,
+        _remote_path: &Option<String>,
+    ) -> Result<(), ExecutorError> {
         unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
     }
     async fn serve_file_content(&self, _file: &FileContent) -> Result<(), ExecutorError> {

harmony/src/domain/topology/http.rs

@@ -5,7 +5,11 @@ use harmony_types::net::IpAddress;
 use harmony_types::net::Url;
 #[async_trait]
 pub trait HttpServer: Send + Sync {
-    async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError>;
+    async fn serve_files(
+        &self,
+        url: &Url,
+        remote_path: &Option<String>,
+    ) -> Result<(), ExecutorError>;
     async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError>;
     fn get_ip(&self) -> IpAddress;
 

harmony/src/infra/opnsense/dns.rs

@@ -12,21 +12,22 @@ use super::OPNSenseFirewall;
 #[async_trait]
 impl DnsServer for OPNSenseFirewall {
     async fn register_hosts(&self, hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
-        let mut writable_opnsense = self.opnsense_config.write().await;
-        let mut dns = writable_opnsense.dns();
-        let hosts = hosts
-            .iter()
-            .map(|h| {
-                Host::new(
-                    h.host.clone(),
-                    h.domain.clone(),
-                    h.record_type.to_string(),
-                    h.value.to_string(),
-                )
-            })
-            .collect();
-        dns.register_hosts(hosts);
-        Ok(())
+        todo!("Refactor this to use dnsmasq")
+        // let mut writable_opnsense = self.opnsense_config.write().await;
+        // let mut dns = writable_opnsense.dns();
+        // let hosts = hosts
+        //     .iter()
+        //     .map(|h| {
+        //         Host::new(
+        //             h.host.clone(),
+        //             h.domain.clone(),
+        //             h.record_type.to_string(),
+        //             h.value.to_string(),
+        //         )
+        //     })
+        //     .collect();
+        // dns.add_static_mapping(hosts);
+        // Ok(())
     }
 
     fn remove_record(
@@ -38,25 +39,26 @@ impl DnsServer for OPNSenseFirewall {
     }
 
     async fn list_records(&self) -> Vec<crate::topology::DnsRecord> {
-        self.opnsense_config
-            .write()
-            .await
-            .dns()
-            .get_hosts()
-            .iter()
-            .map(|h| DnsRecord {
-                host: h.hostname.clone(),
-                domain: h.domain.clone(),
-                record_type: h
-                    .rr
-                    .parse()
-                    .expect("received invalid record type {h.rr} from opnsense"),
-                value: h
-                    .server
-                    .parse()
-                    .expect("received invalid ipv4 record from opnsense {h.server}"),
-            })
-            .collect()
+        todo!("Refactor this to use dnsmasq")
+        // self.opnsense_config
+        //     .write()
+        //     .await
+        //     .dns()
+        //     .get_hosts()
+        //     .iter()
+        //     .map(|h| DnsRecord {
+        //         host: h.hostname.clone(),
+        //         domain: h.domain.clone(),
+        //         record_type: h
+        //             .rr
+        //             .parse()
+        //             .expect("received invalid record type {h.rr} from opnsense"),
+        //         value: h
+        //             .server
+        //             .parse()
+        //             .expect("received invalid ipv4 record from opnsense {h.server}"),
+        //     })
+        //     .collect()
     }
 
     fn get_ip(&self) -> IpAddress {
@@ -68,11 +70,12 @@ impl DnsServer for OPNSenseFirewall {
     }
 
     async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> {
-        let mut writable_opnsense = self.opnsense_config.write().await;
-        let mut dns = writable_opnsense.dns();
-        dns.register_dhcp_leases(register);
-
-        Ok(())
+        todo!("Refactor this to use dnsmasq")
+        // let mut writable_opnsense = self.opnsense_config.write().await;
+        // let mut dns = writable_opnsense.dns();
+        // dns.register_dhcp_leases(register);
+        //
+        // Ok(())
     }
 
     async fn commit_config(&self) -> Result<(), ExecutorError> {

harmony/src/modules/http.rs

@@ -24,8 +24,9 @@ use harmony_types::{id::Id, net::MacAddress};
 /// }
 /// ```
 #[derive(Debug, new, Clone, Serialize)]
-pub struct StaticFilesHttpScore { // TODO this should be split in two scores, one for folder and
-                                  // other for files
+pub struct StaticFilesHttpScore {
+    // TODO this should be split in two scores, one for folder and
+    // other for files
     pub folder_to_serve: Option<Url>,
     pub files: Vec<FileContent>,
     pub remote_path: Option<String>,
@@ -56,7 +57,9 @@ impl<T: Topology + HttpServer> Interpret<T> for StaticFilesHttpInterpret {
         http_server.ensure_initialized().await?;
         // http_server.set_ip(topology.router.get_gateway()).await?;
         if let Some(folder) = self.score.folder_to_serve.as_ref() {
-            http_server.serve_files(folder, &self.score.remote_path).await?;
+            http_server
+                .serve_files(folder, &self.score.remote_path)
+                .await?;
         }
 
         for f in self.score.files.iter() {

harmony/src/modules/okd/bootstrap_load_balancer.rs

@@ -54,6 +54,7 @@ impl OKDBootstrapLoadBalancerScore {
             },
         }
     }
+
     fn topology_to_backend_server(topology: &HAClusterTopology, port: u16) -> Vec<BackendServer> {
         let mut backend: Vec<_> = topology
             .control_plane
@@ -63,6 +64,14 @@ impl OKDBootstrapLoadBalancerScore {
                 port,
             })
             .collect();
+
+        topology.workers.iter().for_each(|worker| {
+            backend.push(BackendServer {
+                address: worker.ip.to_string(),
+                port,
+            })
+        });
+
         backend.push(BackendServer {
             address: topology.bootstrap_host.ip.to_string(),
             port,

harmony/src/modules/okd/installation.rs

@@ -73,7 +73,11 @@ use crate::{
         dhcp::DhcpHostBindingScore,
         http::{IPxeMacBootFileScore, StaticFilesHttpScore},
         inventory::LaunchDiscoverInventoryAgentScore,
-        okd::{dns::OKDDnsScore, templates::InstallConfigYaml},
+        okd::{
+            bootstrap_load_balancer::OKDBootstrapLoadBalancerScore,
+            dns::OKDDnsScore,
+            templates::{BootstrapIpxeTpl, InstallConfigYaml},
+        },
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -207,7 +211,7 @@ impl Interpret<HAClusterTopology> for OKDInstallationInterpret {
 
         info!("Starting OKD installation pipeline",);
 
-        // self.run_inventory_phase(inventory, topology).await?;
+        self.run_inventory_phase(inventory, topology).await?;
 
         self.run_bootstrap_phase(inventory, topology).await?;
 
@@ -289,9 +293,23 @@ impl Interpret<HAClusterTopology> for OKDSetup01InventoryInterpret {
         topology: &HAClusterTopology,
     ) -> Result<Outcome, InterpretError> {
         info!("Setting up base DNS config for OKD");
-        OKDDnsScore::new(topology)
-            .interpret(inventory, topology)
-            .await?;
+        let cluster_domain = &topology.domain_name;
+        let load_balancer_ip = &topology.load_balancer.get_ip();
+        inquire::Confirm::new(&format!(
+            "Set hostnames manually in your opnsense dnsmasq config :
+*.apps.{cluster_domain} -> {load_balancer_ip}
+api.{cluster_domain} -> {load_balancer_ip}
+api-int.{cluster_domain} -> {load_balancer_ip}
+
+When you can dig them, confirm to continue.
+"
+        ))
+        .prompt()
+        .expect("Prompt error");
+        // TODO reactivate automatic dns config
+        // OKDDnsScore::new(topology)
+        //     .interpret(inventory, topology)
+        //     .await?;
 
         info!(
             "Launching discovery agent, make sure that your nodes are successfully PXE booted and running inventory agent. They should answer on `http://<node_ip>:8080/inventory`"
@@ -368,7 +386,7 @@ struct OKDSetup02BootstrapScore {}
 
 impl Score<HAClusterTopology> for OKDSetup02BootstrapScore {
     fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
-        Box::new(OKDSetup02BootstrapInterpret::new(self.clone()))
+        Box::new(OKDSetup02BootstrapInterpret::new())
     }
 
     fn name(&self) -> String {
@@ -378,17 +396,15 @@ impl Score<HAClusterTopology> for OKDSetup02BootstrapScore {
 
 #[derive(Debug, Clone)]
 struct OKDSetup02BootstrapInterpret {
-    score: OKDSetup02BootstrapScore,
     version: Version,
     status: InterpretStatus,
 }
 
 impl OKDSetup02BootstrapInterpret {
-    pub fn new(score: OKDSetup02BootstrapScore) -> Self {
+    pub fn new() -> Self {
         let version = Version::from("1.0.0").unwrap();
         Self {
             version,
-            score,
             status: InterpretStatus::QUEUED,
         }
     }
@@ -572,20 +588,30 @@ impl OKDSetup02BootstrapInterpret {
                 Ok(output)
             };
 
+        info!("Successfully prepared ignition files for OKD installation");
         // ignition_files_http_path // = PathBuf::from("okd_ignition_files");
-        let scos_http_path = PathBuf::from("scos");
         info!(
             r#"Uploading images, they can be refreshed with a command similar to this one: openshift-install coreos print-stream-json | grep -Eo '"https.*(kernel.|initramfs.|rootfs.)\w+(\.img)?"'  | grep x86_64 | xargs -n 1 curl -LO"#
         );
-        StaticFilesHttpScore {
-            folder_to_serve: Some(Url::LocalFolder(okd_images_path.to_string_lossy().to_string())),
-            remote_path: Some(scos_http_path.to_string_lossy().to_string()),
-            files: vec![],
-        }
-        .interpret(inventory, topology)
-        .await?;
 
-        todo!("What's up next?")
+        warn!(
+            "TODO push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved"
+        );
+        inquire::Confirm::new(
+            "push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved").prompt().expect("Prompt error");
+
+        // let scos_http_path = PathBuf::from("scos");
+        // StaticFilesHttpScore {
+        //     folder_to_serve: Some(Url::LocalFolder(
+        //         okd_images_path.to_string_lossy().to_string(),
+        //     )),
+        //     remote_path: Some(scos_http_path.to_string_lossy().to_string()),
+        //     files: vec![],
+        // }
+        // .interpret(inventory, topology)
+        // .await?;
+
+        Ok(())
     }
 
     async fn configure_host_binding(
@@ -613,12 +639,25 @@ impl OKDSetup02BootstrapInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<(), InterpretError> {
-        // Placeholder: use Harmony templates to emit {MAC}.ipxe selecting SCOS live + bootstrap ignition.
-        info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node");
+        let content = BootstrapIpxeTpl {
+            http_ip: &topology.http_server.get_ip().to_string(),
+            scos_path: "scos", // TODO use some constant
+            installation_device: "/dev/sda", // TODO do something smart based on the host drives
+                               // topology. Something like use the smallest device
+                               // above 200G that is an ssd
+        }
+        .to_string();
+
         let bootstrap_node = self.get_bootstrap_node().await?;
+        let mac_address = bootstrap_node.get_mac_address();
+
+        info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node");
+        debug!("bootstrap ipxe content : {content}");
+        debug!("bootstrap mac addresses : {mac_address:?}");
+
         IPxeMacBootFileScore {
-            mac_address: bootstrap_node.get_mac_address(),
-            content: todo!("templace for bootstrap node"),
+            mac_address,
+            content,
         }
         .interpret(inventory, topology)
         .await?;
@@ -630,15 +669,25 @@ impl OKDSetup02BootstrapInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<(), InterpretError> {
-        todo!(
-            "OKD loadbalancer score already exists, just call it here probably? 6443 22623, 80 and 443 \n\nhttps://docs.okd.io/latest/installing/installing_bare_metal/upi/installing-bare-metal.html#installation-load-balancing-user-infra_installing-bare-metal"
-        );
+        let outcome = OKDBootstrapLoadBalancerScore::new(topology)
+            .interpret(inventory, topology)
+            .await?;
+        info!("Successfully executed OKDBootstrapLoadBalancerScore : {outcome:?}");
+        Ok(())
     }
 
     async fn reboot_target(&self) -> Result<(), InterpretError> {
         // Placeholder: ssh reboot using the inventory ephemeral key
         info!("[Bootstrap] Rebooting bootstrap node via SSH");
-        todo!("[Bootstrap] Rebooting bootstrap node via SSH")
+        // TODO reboot programatically, there are some logical checks and refactoring to do such as
+        // accessing the bootstrap node config (ip address) from the inventory
+        let confirmation = inquire::Confirm::new(
+                "Now reboot the bootstrap node so it picks up its pxe boot file. Press enter when ready.",
+        )
+        .with_default(true)
+        .prompt()
+        .expect("Unexpected prompt error");
+        Ok(())
     }
 
     async fn wait_for_bootstrap_complete(&self) -> Result<(), InterpretError> {

harmony/src/modules/okd/ipxe.rs

@@ -120,6 +120,7 @@ impl<T: Topology + DhcpServer + TftpServer + HttpServer + Router> Interpret<T> f
                 Err(e) => return Err(e),
             };
         }
+        inquire::Confirm::new("Execute the copy : `scp -r data/pxe/okd/http_files/* root@192.168.1.1:/usr/local/http/` and confirm when done to continue").prompt().expect("Prompt error");
 
         Ok(Outcome::success("Ipxe installed".to_string()))
     }

harmony/src/modules/okd/mod.rs

@@ -5,5 +5,5 @@ pub mod dns;
 pub mod installation;
 pub mod ipxe;
 pub mod load_balancer;
-pub mod upgrade;
 pub mod templates;
+pub mod upgrade;

harmony/src/modules/okd/templates.rs

@@ -8,3 +8,11 @@ pub struct InstallConfigYaml<'a> {
     pub ssh_public_key: &'a str,
     pub cluster_name: &'a str,
 }
+
+#[derive(Template)]
+#[template(path = "okd/bootstrap.ipxe.j2")]
+pub struct BootstrapIpxeTpl<'a> {
+    pub http_ip: &'a str,
+    pub scos_path: &'a str,
+    pub installation_device: &'a str,
+}

harmony/templates/okd/bootstrap.ipxe.j2

@@ -0,0 +1,7 @@
+set base-url http://{{ http_ip }}:8080
+set scos-base-url = ${base-url}/{{ scos_path }}
+set installation-device = {{ installation_device }}
+
+kernel ${scos-base-url}/scos-live-kernel.x86_64 initrd=main coreos.live.rootfs_url=${scos-base-url}/scos-live-rootfs.x86_64.img coreos.inst.install_dev=${installation-device} coreos.inst.ignition_url=${base-url}/bootstrap.ign  
+initrd --name main ${scos-base-url}/scos-live-initramfs.x86_64.img 
+boot

opnsense-config/src/config/config.rs

@@ -4,7 +4,7 @@ use crate::{
     config::{SshConfigManager, SshCredentials, SshOPNSenseShell},
     error::Error,
     modules::{
-        caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::DnsConfig,
+        caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::UnboundDnsConfig,
         dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig, tftp::TftpConfig,
     },
 };
@@ -51,8 +51,8 @@ impl Config {
         DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone())
     }
 
-    pub fn dns(&mut self) -> DnsConfig<'_> {
-        DnsConfig::new(&mut self.opnsense)
+    pub fn dns(&mut self) -> DhcpConfigDnsMasq<'_> {
+        DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone())
     }
 
     pub fn tftp(&mut self) -> TftpConfig<'_> {

opnsense-config/src/modules/dns.rs

@@ -1,10 +1,10 @@
 use opnsense_config_xml::{Host, OPNsense};
 
-pub struct DnsConfig<'a> {
+pub struct UnboundDnsConfig<'a> {
     opnsense: &'a mut OPNsense,
 }
 
-impl<'a> DnsConfig<'a> {
+impl<'a> UnboundDnsConfig<'a> {
     pub fn new(opnsense: &'a mut OPNsense) -> Self {
         Self { opnsense }
     }

opnsense-config/src/modules/dnsmasq.rs

@@ -144,14 +144,16 @@ impl<'a> DhcpConfigDnsMasq<'a> {
                 let host_to_modify_ip = host_to_modify.ip.content_string();
                 if host_to_modify_ip != ip_str {
                     warn!(
-                        "Hostname '{}' already exists with a different IP ({}). Appending MAC {}.",
+                        "Hostname '{}' already exists with a different IP ({}). Setting new IP {ip_str}. Appending MAC {}.",
                         hostname, host_to_modify_ip, mac
                     );
+                    host_to_modify.ip.content = Some(ip_str);
                 } else if host_to_modify.host != hostname {
                     warn!(
-                        "IP {} already exists with a different hostname ('{}'). Appending MAC {}.",
+                        "IP {} already exists with a different hostname ('{}'). Setting hostname to {hostname}. Appending MAC {}.",
                         ipaddr, host_to_modify.host, mac
                     );
+                    host_to_modify.host = hostname.to_string();
                 }
 
                 if !host_to_modify