feat: OKD Installation now generates ignition files, copies them over, also uploads scos images

.gitattributes

@@ -2,3 +2,5 @@ bootx64.efi filter=lfs diff=lfs merge=lfs -text
 grubx64.efi filter=lfs diff=lfs merge=lfs -text
 initrd filter=lfs diff=lfs merge=lfs -text
 linux filter=lfs diff=lfs merge=lfs -text
+data/okd/bin/* filter=lfs diff=lfs merge=lfs -text
+data/okd/installer_image/* filter=lfs diff=lfs merge=lfs -text

adr/agent_discovery/mdns/src/discover.rs

@@ -1,4 +1,3 @@
-use log::debug;
 use mdns_sd::{ServiceDaemon, ServiceEvent};
 
 use crate::SERVICE_TYPE;
@@ -74,7 +73,7 @@ pub async fn discover() {
     // }
 }
 
-async fn discover_example() {
+async fn _discover_example() {
     use mdns_sd::{ServiceDaemon, ServiceEvent};
 
     // Create a daemon

data/okd/installer_image/scos-live-initramfs.x86_64.img

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-initramfs.x86_64.img

data/okd/installer_image/scos-live-kernel.x86_64

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-kernel.x86_64

data/okd/installer_image/scos-live-rootfs.x86_64.img

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-rootfs.x86_64.img

examples/nanodc/src/main.rs

@@ -129,6 +129,7 @@ async fn main() {
             "./data/watchguard/pxe-http-files".to_string(),
         )),
         files: vec![],
+        remote_path: None,
     };
 
     let kickstart_filename = "inventory.kickstart".to_string();

examples/okd_installation/env.sh

@@ -0,0 +1,4 @@
+export HARMONY_SECRET_NAMESPACE=example-vms
+export HARMONY_SECRET_STORE=file
+export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info 
+export RUST_LOG=info

examples/okd_installation/src/topology.rs

@@ -51,7 +51,7 @@ pub async fn get_topology() -> HAClusterTopology {
         dns_server: opnsense.clone(),
         control_plane: vec![LogicalHost {
             ip: ip!("192.168.1.20"),
-            name: "cp0".to_string(),
+            name: "master".to_string(),
         }],
         bootstrap_host: LogicalHost {
             ip: ip!("192.168.1.20"),

examples/opnsense/src/main.rs

@@ -85,6 +85,7 @@ async fn main() {
             "./data/watchguard/pxe-http-files".to_string(),
         )),
         files: vec![],
+        remote_path: None,
     };
 
     harmony_tui::run(

harmony/src/domain/config/secret.rs

@@ -14,3 +14,7 @@ pub struct SshKeyPair {
     pub public: String,
 }
 
+#[derive(Secret, Serialize, Deserialize, Debug, PartialEq)]
+pub struct RedhatSecret {
+    pub pull_secret: String,
+}

harmony/src/domain/interpret/mod.rs

@@ -142,6 +142,12 @@ impl From<PreparationError> for InterpretError {
     }
 }
 
+impl From<harmony_secret::SecretStoreError> for InterpretError {
+    fn from(value: harmony_secret::SecretStoreError) -> Self {
+        InterpretError::new(format!("Interpret error : {value}"))
+    }
+}
+
 impl From<ExecutorError> for InterpretError {
     fn from(value: ExecutorError) -> Self {
         Self {

harmony/src/domain/topology/ha_cluster.rs

@@ -69,6 +69,26 @@ impl K8sclient for HAClusterTopology {
 }
 
 impl HAClusterTopology {
+    // TODO this is a hack to avoid refactoring
+    pub fn get_cluster_name(&self) -> String {
+        self.domain_name
+            .split(".")
+            .next()
+            .expect("Cluster domain name must not be empty")
+            .to_string()
+    }
+
+    pub fn get_cluster_base_domain(&self) -> String {
+        let base_domain = self
+            .domain_name
+            .strip_prefix(&self.get_cluster_name())
+            .expect("cluster domain must start with cluster name");
+        base_domain
+            .strip_prefix(".")
+            .unwrap_or(base_domain)
+            .to_string()
+    }
+
     pub fn autoload() -> Self {
         let dummy_infra = Arc::new(DummyInfra {});
         let dummy_host = LogicalHost {
@@ -217,8 +237,8 @@ impl Router for HAClusterTopology {
 
 #[async_trait]
 impl HttpServer for HAClusterTopology {
-    async fn serve_files(&self, url: &Url) -> Result<(), ExecutorError> {
-        self.http_server.serve_files(url).await
+    async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError> {
+        self.http_server.serve_files(url, remote_path).await
     }
 
     async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError> {
@@ -377,7 +397,7 @@ impl TftpServer for DummyInfra {
 
 #[async_trait]
 impl HttpServer for DummyInfra {
-    async fn serve_files(&self, _url: &Url) -> Result<(), ExecutorError> {
+    async fn serve_files(&self, _url: &Url, _remote_path: &Option<String>) -> Result<(), ExecutorError> {
         unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
     }
     async fn serve_file_content(&self, _file: &FileContent) -> Result<(), ExecutorError> {

harmony/src/domain/topology/http.rs

@@ -5,7 +5,7 @@ use harmony_types::net::IpAddress;
 use harmony_types::net::Url;
 #[async_trait]
 pub trait HttpServer: Send + Sync {
-    async fn serve_files(&self, url: &Url) -> Result<(), ExecutorError>;
+    async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError>;
     async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError>;
     fn get_ip(&self) -> IpAddress;
 

harmony/src/infra/opnsense/http.rs

@@ -10,13 +10,21 @@ const OPNSENSE_HTTP_ROOT_PATH: &str = "/usr/local/http";
 
 #[async_trait]
 impl HttpServer for OPNSenseFirewall {
-    async fn serve_files(&self, url: &Url) -> Result<(), ExecutorError> {
+    async fn serve_files(
+        &self,
+        url: &Url,
+        remote_path: &Option<String>,
+    ) -> Result<(), ExecutorError> {
         let config = self.opnsense_config.read().await;
         info!("Uploading files from url {url} to {OPNSENSE_HTTP_ROOT_PATH}");
+        let remote_upload_path = remote_path
+            .clone()
+            .map(|r| format!("{OPNSENSE_HTTP_ROOT_PATH}/{r}"))
+            .unwrap_or(OPNSENSE_HTTP_ROOT_PATH.to_string());
         match url {
             Url::LocalFolder(path) => {
                 config
-                    .upload_files(path, OPNSENSE_HTTP_ROOT_PATH)
+                    .upload_files(path, &remote_upload_path)
                     .await
                     .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))?;
             }

harmony/src/modules/http.rs

@@ -24,9 +24,11 @@ use harmony_types::{id::Id, net::MacAddress};
 /// }
 /// ```
 #[derive(Debug, new, Clone, Serialize)]
-pub struct StaticFilesHttpScore {
+pub struct StaticFilesHttpScore { // TODO this should be split in two scores, one for folder and
+                                  // other for files
     pub folder_to_serve: Option<Url>,
     pub files: Vec<FileContent>,
+    pub remote_path: Option<String>,
 }
 
 impl<T: Topology + HttpServer> Score<T> for StaticFilesHttpScore {
@@ -54,7 +56,7 @@ impl<T: Topology + HttpServer> Interpret<T> for StaticFilesHttpInterpret {
         http_server.ensure_initialized().await?;
         // http_server.set_ip(topology.router.get_gateway()).await?;
         if let Some(folder) = self.score.folder_to_serve.as_ref() {
-            http_server.serve_files(folder).await?;
+            http_server.serve_files(folder, &self.score.remote_path).await?;
         }
 
         for f in self.score.files.iter() {
@@ -105,6 +107,7 @@ impl<T: Topology + HttpServer> Score<T> for IPxeMacBootFileScore {
 
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
         StaticFilesHttpScore {
+            remote_path: None,
             folder_to_serve: None,
             files: self
                 .mac_address

harmony/src/modules/okd/installation.rs

@@ -47,14 +47,23 @@
 //! - public_domain: External wildcard/apps domain (e.g., apps.example.com).
 //! - internal_domain: Internal cluster domain (e.g., cluster.local or harmony.mcd).
 
+use std::{fmt::Write, path::PathBuf, process::ExitStatus};
+
 use async_trait::async_trait;
 use derive_new::new;
-use harmony_types::id::Id;
-use log::{error, info, warn};
+use harmony_secret::SecretManager;
+use harmony_types::{id::Id, net::Url};
+use log::{debug, error, info, warn};
 use serde::{Deserialize, Serialize};
+use tokio::{
+    fs::File,
+    io::{AsyncReadExt, AsyncWriteExt},
+    process::Command,
+};
 
 use crate::{
-    data::Version,
+    config::secret::{RedhatSecret, SshKeyPair},
+    data::{FileContent, FilePath, Version},
     hardware::PhysicalHost,
     infra::inventory::InventoryRepositoryFactory,
     instrumentation::{HarmonyEvent, instrument},
@@ -64,7 +73,7 @@ use crate::{
         dhcp::DhcpHostBindingScore,
         http::{IPxeMacBootFileScore, StaticFilesHttpScore},
         inventory::LaunchDiscoverInventoryAgentScore,
-        okd::dns::OKDDnsScore,
+        okd::{dns::OKDDnsScore, templates::InstallConfigYaml},
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -113,12 +122,9 @@ impl OKDInstallationInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<(), InterpretError> {
-        // 1) Prepare DNS and DHCP lease registration (optional)
-
-        // 2) Serve default iPXE + Kickstart and poll discovery
-        let discovery_score = OKDSetup01InventoryScore::new();
-        discovery_score.interpret(inventory, topology).await?;
-
+        OKDSetup01InventoryScore::new()
+            .interpret(inventory, topology)
+            .await?;
         Ok(())
     }
 
@@ -127,9 +133,9 @@ impl OKDInstallationInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<(), InterpretError> {
-        // Select and provision bootstrap
-        let bootstrap_score = OKDSetup02BootstrapScore::new();
-        bootstrap_score.interpret(inventory, topology).await?;
+        OKDSetup02BootstrapScore::new()
+            .interpret(inventory, topology)
+            .await?;
         Ok(())
     }
 
@@ -201,7 +207,7 @@ impl Interpret<HAClusterTopology> for OKDInstallationInterpret {
 
         info!("Starting OKD installation pipeline",);
 
-        self.run_inventory_phase(inventory, topology).await?;
+        // self.run_inventory_phase(inventory, topology).await?;
 
         self.run_bootstrap_phase(inventory, topology).await?;
 
@@ -407,12 +413,179 @@ impl OKDSetup02BootstrapInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<(), InterpretError> {
+        let okd_bin_path = PathBuf::from("./data/okd/bin");
+        let okd_installation_path_str = "./data/okd/installation_files";
+        let okd_images_path = &PathBuf::from("./data/okd/installer_image/");
+        let okd_installation_path = &PathBuf::from(okd_installation_path_str);
+
+        let exit_status = Command::new("mkdir")
+            .arg("-p")
+            .arg(okd_installation_path)
+            .spawn()
+            .expect("Command failed to start")
+            .wait()
+            .await
+            .map_err(|e| {
+                InterpretError::new(format!("Failed to create okd installation directory : {e}"))
+            })?;
+        if !exit_status.success() {
+            return Err(InterpretError::new(format!(
+                "Failed to create okd installation directory"
+            )));
+        } else {
+            info!(
+                "Created OKD installation directory {}",
+                okd_installation_path.to_string_lossy()
+            );
+        }
+
+        let redhat_secret = SecretManager::get::<RedhatSecret>().await?;
+        let ssh_key = SecretManager::get::<SshKeyPair>().await?;
+
+        let install_config_yaml = InstallConfigYaml {
+            cluster_name: &topology.get_cluster_name(),
+            cluster_domain: &topology.get_cluster_base_domain(),
+            pull_secret: &redhat_secret.pull_secret,
+            ssh_public_key: &ssh_key.public,
+        }
+        .to_string();
+
+        let install_config_file_path = &okd_installation_path.join("install-config.yaml");
+
+        self.create_file(install_config_file_path, install_config_yaml.as_bytes())
+            .await?;
+
+        let install_config_backup_extension = install_config_file_path
+            .extension()
+            .map(|e| format!("{}.bak", e.to_string_lossy()))
+            .unwrap_or("bak".to_string());
+
+        let mut install_config_backup = install_config_file_path.clone();
+        install_config_backup.set_extension(install_config_backup_extension);
+
+        self.create_file(&install_config_backup, install_config_yaml.as_bytes())
+            .await?;
+
+        info!("Creating manifest files with openshift-install");
+        let output = Command::new(okd_bin_path.join("openshift-install"))
+            .args([
+                "create",
+                "manifests",
+                "--dir",
+                okd_installation_path.to_str().unwrap(),
+            ])
+            .output()
+            .await
+            .map_err(|e| InterpretError::new(format!("Failed to create okd manifest : {e}")))?;
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        info!("openshift-install stdout :\n\n{}", stdout);
+        let stderr = String::from_utf8(output.stderr).unwrap();
+        info!("openshift-install stderr :\n\n{}", stderr);
+        info!("openshift-install exit status : {}", output.status);
+        if !output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Failed to create okd manifest, exit code {} : {}",
+                output.status, stderr
+            )));
+        }
+
+        info!("Creating ignition files with openshift-install");
+        let output = Command::new(okd_bin_path.join("openshift-install"))
+            .args([
+                "create",
+                "ignition-configs",
+                "--dir",
+                okd_installation_path.to_str().unwrap(),
+            ])
+            .output()
+            .await
+            .map_err(|e| {
+                InterpretError::new(format!("Failed to create okd ignition config : {e}"))
+            })?;
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        info!("openshift-install stdout :\n\n{}", stdout);
+        let stderr = String::from_utf8(output.stderr).unwrap();
+        info!("openshift-install stderr :\n\n{}", stderr);
+        info!("openshift-install exit status : {}", output.status);
+        if !output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Failed to create okd manifest, exit code {} : {}",
+                output.status, stderr
+            )));
+        }
+
+        let ignition_files_http_path = PathBuf::from("okd_ignition_files");
+        let prepare_file_content = async |filename: &str| -> Result<FileContent, InterpretError> {
+            let local_path = okd_installation_path.join(filename);
+            let remote_path = ignition_files_http_path.join(filename);
+
+            info!(
+                "Preparing file content for local file : {} to remote : {}",
+                local_path.to_string_lossy(),
+                remote_path.to_string_lossy()
+            );
+
+            let content = tokio::fs::read_to_string(&local_path).await.map_err(|e| {
+                InterpretError::new(format!(
+                    "Could not read file content {} : {e}",
+                    local_path.to_string_lossy()
+                ))
+            })?;
+
+            Ok(FileContent {
+                path: FilePath::Relative(remote_path.to_string_lossy().to_string()),
+                content,
+            })
+        };
+
         StaticFilesHttpScore {
+            remote_path: None,
             folder_to_serve: None,
-            files: todo!(),
+            files: vec![
+                prepare_file_content("bootstrap.ign").await?,
+                prepare_file_content("master.ign").await?,
+                prepare_file_content("worker.ign").await?,
+                prepare_file_content("metadata.json").await?,
+            ],
         }
         .interpret(inventory, topology)
         .await?;
+
+        let run_command =
+            async |cmd: &str, args: Vec<&str>| -> Result<std::process::Output, InterpretError> {
+                let output = Command::new(cmd).args(&args).output().await.map_err(|e| {
+                    InterpretError::new(format!("Failed to launch command {cmd} : {e}"))
+                })?;
+                let stdout = String::from_utf8(output.stdout.clone()).unwrap();
+                info!("{cmd} stdout :\n\n{}", stdout);
+                let stderr = String::from_utf8(output.stderr.clone()).unwrap();
+                info!("{cmd} stderr :\n\n{}", stderr);
+                info!("{cmd} exit status : {}", output.status);
+                if !output.status.success() {
+                    return Err(InterpretError::new(format!(
+                        "Command execution failed, exit code {} : {} {}",
+                        output.status,
+                        cmd,
+                        args.join(" ")
+                    )));
+                }
+                Ok(output)
+            };
+
+        // ignition_files_http_path // = PathBuf::from("okd_ignition_files");
+        let scos_http_path = PathBuf::from("scos");
+        info!(
+            r#"Uploading images, they can be refreshed with a command similar to this one: openshift-install coreos print-stream-json | grep -Eo '"https.*(kernel.|initramfs.|rootfs.)\w+(\.img)?"'  | grep x86_64 | xargs -n 1 curl -LO"#
+        );
+        StaticFilesHttpScore {
+            folder_to_serve: Some(Url::LocalFolder(okd_images_path.to_string_lossy().to_string())),
+            remote_path: Some(scos_http_path.to_string_lossy().to_string()),
+            files: vec![],
+        }
+        .interpret(inventory, topology)
+        .await?;
+
+        todo!("What's up next?")
     }
 
     async fn configure_host_binding(
@@ -465,12 +638,28 @@ impl OKDSetup02BootstrapInterpret {
     async fn reboot_target(&self) -> Result<(), InterpretError> {
         // Placeholder: ssh reboot using the inventory ephemeral key
         info!("[Bootstrap] Rebooting bootstrap node via SSH");
-        Ok(())
+        todo!("[Bootstrap] Rebooting bootstrap node via SSH")
     }
 
     async fn wait_for_bootstrap_complete(&self) -> Result<(), InterpretError> {
         // Placeholder: wait-for bootstrap-complete
         info!("[Bootstrap] Waiting for bootstrap-complete …");
+        todo!("[Bootstrap] Waiting for bootstrap-complete …")
+    }
+
+    async fn create_file(&self, path: &PathBuf, content: &[u8]) -> Result<(), InterpretError> {
+        let mut install_config_file = File::create(path).await.map_err(|e| {
+            InterpretError::new(format!(
+                "Could not create file {} : {e}",
+                path.to_string_lossy()
+            ))
+        })?;
+        install_config_file.write(content).await.map_err(|e| {
+            InterpretError::new(format!(
+                "Could not write file {} : {e}",
+                path.to_string_lossy()
+            ))
+        })?;
         Ok(())
     }
 }

harmony/src/modules/okd/ipxe.rs

@@ -71,6 +71,7 @@ impl<T: Topology + DhcpServer + TftpServer + HttpServer + Router> Interpret<T> f
                 files_to_serve: Url::LocalFolder("./data/pxe/okd/tftpboot/".to_string()),
             }),
             Box::new(StaticFilesHttpScore {
+                remote_path: None,
                 // TODO The current russh based copy is way too slow, check for a lib update or use scp
                 // when available
                 //

harmony/src/modules/okd/mod.rs

@@ -6,3 +6,4 @@ pub mod installation;
 pub mod ipxe;
 pub mod load_balancer;
 pub mod upgrade;
+pub mod templates;

harmony/src/modules/okd/templates.rs

@@ -0,0 +1,10 @@
+use askama::Template;
+
+#[derive(Template)]
+#[template(path = "okd/install-config.yaml.j2")]
+pub struct InstallConfigYaml<'a> {
+    pub cluster_domain: &'a str,
+    pub pull_secret: &'a str,
+    pub ssh_public_key: &'a str,
+    pub cluster_name: &'a str,
+}

harmony/templates/okd/install-config.yaml.j2

@@ -0,0 +1,24 @@
+# Built from https://docs.okd.io/latest/installing/installing_bare_metal/upi/installing-bare-metal.html#installation-bare-metal-config-yaml_installing-bare-metal
+apiVersion: v1
+baseDomain: {{ cluster_domain }}
+compute: 
+- hyperthreading: Enabled 
+  name: worker
+  replicas: 0 
+controlPlane: 
+  hyperthreading: Enabled 
+  name: master
+  replicas: 3 
+metadata:
+  name: {{ cluster_name }} 
+networking:
+  clusterNetwork:
+  - cidr: 10.128.0.0/14 
+    hostPrefix: 23 
+  networkType: OVNKubernetes 
+  serviceNetwork: 
+  - 172.30.0.0/16
+platform:
+  none: {} 
+pullSecret: '{{ pull_secret|safe }}' 
+sshKey: '{{ ssh_public_key }}' 

harmony_secret/src/store/local_file.rs

@@ -29,12 +29,20 @@ impl SecretStore for LocalFileSecretStore {
             file_path.display()
         );
 
+        let content =
             tokio::fs::read(&file_path)
                 .await
                 .map_err(|_| SecretStoreError::NotFound {
                     namespace: ns.to_string(),
                     key: key.to_string(),
-            })
+                })?;
+        info!(
+            "Sum of all vec get {ns} {key} {:?}",
+            content
+                .iter()
+                .fold(0, |acc: u64, val: &u8| { acc + *val as u64 })
+        );
+        Ok(content)
     }
 
     async fn set_raw(&self, ns: &str, key: &str, val: &[u8]) -> Result<(), SecretStoreError> {
@@ -56,6 +64,12 @@ impl SecretStore for LocalFileSecretStore {
                 .map_err(|e| SecretStoreError::Store(Box::new(e)))?;
         }
 
+        info!(
+            "Sum of all vec set {ns} {key} {:?}",
+            val.iter()
+                .fold(0, |acc: u64, val: &u8| { acc + *val as u64 })
+        );
+
         tokio::fs::write(&file_path, val)
             .await
             .map_err(|e| SecretStoreError::Store(Box::new(e)))