fix(inventory_agent): Inventory agent now fallsback on error messages when it cant find values

The process will setup DHCP dnsmasq on opnsense to boot the correct ipxe file depending on the architecture
Then ipxe will chainload to either a mac-specific ipxe boot file or the fallback inventory boot file
Then a kickstart pre script will setup the cluster ssh key to allow ssh connections to the machine and also setup and start harmony_inventory_agent to allow being scraped

Note: there is a bug with the inventory agent currently, it cannot find lsmod on centos stream 9, will fix this soon

data/pxe/okd/README.md

@@ -1,3 +1,8 @@
 Here lies all the data files required for an OKD cluster PXE boot setup.
 
 This inclues ISO files, binary boot files, ipxe, etc.
+
+TODO as of august 2025 :
+
+- `harmony_inventory_agent` should be downloaded from official releases, this embedded version is practical for now though
+- The cluster ssh key should be generated and handled by harmony with the private key saved in a secret store

data/pxe/okd/http_files/.gitattributes

@@ -0,0 +1,9 @@
+harmony_inventory_agent filter=lfs diff=lfs merge=lfs -text
+os filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9 filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/initrd.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/vmlinuz filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/efiboot.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/install.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/pxeboot filter=lfs diff=lfs merge=lfs -text

data/pxe/okd/http_files/cluster_ssh_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBx6bDylvC68cVpjKfEFtLQJ/dOFi6PVS2vsIOqPDJIc jeangab@liliane2

examples/okd_pxe/ssh_example_key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAcemw8pbwuvHFaYynxBbS0Cf3ThYuj1Utr7CDqjwySHAAAAJikacCNpGnA
+jQAAAAtzc2gtZWQyNTUxOQAAACAcemw8pbwuvHFaYynxBbS0Cf3ThYuj1Utr7CDqjwySHA
+AAAECiiKk4V6Q5cVs6axDM4sjAzZn/QCZLQekmYQXS9XbEYxx6bDylvC68cVpjKfEFtLQJ
+/dOFi6PVS2vsIOqPDJIcAAAAEGplYW5nYWJAbGlsaWFuZTIBAgMEBQ==
+-----END OPENSSH PRIVATE KEY-----

examples/okd_pxe/ssh_example_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBx6bDylvC68cVpjKfEFtLQJ/dOFi6PVS2vsIOqPDJIc jeangab@liliane2

examples/okd_pxe/templates/fallback.ipxe.j2

@@ -30,7 +30,7 @@ echo "Configuring kernel boot arguments..."
 # - inst.ks:                CRITICAL: Points to our Kickstart file for automation.
 # - ip=dhcp:                Ensures the live environment configures its network.
 # - console=...:            Provides boot output on both serial and graphical consoles for debugging.
-imgargs vmlinuz initrd=initrd.img inst.stage2=${os_base_url} inst.ks=${ks_url} ip=dhcp console=ttyS0,115200 console=tty1
+imgargs vmlinuz initrd=initrd.img inst.sshd inst.stage2=${os_base_url} inst.ks=${ks_url} ip=dhcp console=ttyS0,115200 console=tty1
 
 echo "Booting into CentOS Stream 9 live environment..."
 boot || goto failed

examples/okd_pxe/templates/inventory.kickstart.j2

@@ -1,66 +1,38 @@
-# =================================================================
-#  Harmony Discovery Agent - Kickstart File (inventory.kickstart)
-# =================================================================
-#
-# This Kickstart file configures the CentOS Stream 9 live environment.
-# It does NOT install to disk. It sets up SSH for remote access
-# and downloads and runs the harmony-inventory-agent.
-#
-
-# --- System Configuration
-lang en_US.UTF-8
-keyboard --xlayouts='us'
-timezone America/New_York --isUtc
-
-# --- Network Configuration
-# Ensure the network is activated using DHCP.
-network --bootproto=dhcp --device=link --activate
-
-# --- Security Configuration
-# Disable the firewall for this isolated provisioning network.
-firewall --disabled
-# Disable SELinux for simplicity in the live environment.
-selinux --disabled
-# Disable password-based root login for security.
-rootpw --lock
-
-# --- Service Configuration
-# Ensure the SSH daemon is enabled.
-services --enabled="sshd"
-
-# We are running a live environment, so no disk partitioning.
-# The 'liveimg' command would be used here if booting from a squashfs,
-# but since we are booting from kernel/initrd, we just use the %post.
-
-# Do not run the graphical initial setup wizard.
-firstboot --disable
-
-# --- Post-Boot Scripting
+# --- Pre-Boot Scripting (The Main Goal) ---
 # This section runs after the live environment has booted into RAM.
-%post --log=/root/ks-post.log
+# It sets up SSH and downloads/runs the harmony-inventory-agent.
+%pre --log=/root/ks-pre.log
 
-echo "Harmony Kickstart: Post-boot script started."
+echo "Harmony Kickstart: Pre-boot script started."
 
-# 1. Configure SSH Access
+# 1. Configure SSH Access for Root
 # Create the .ssh directory and set correct permissions.
-echo "  - Setting up SSH authorized_keys..."
+echo "  - Setting up SSH authorized_keys for root..."
 mkdir -p /root/.ssh
 chmod 700 /root/.ssh
 
-# Download the public key and place it in authorized_keys.
-curl -sSL "http://{{ gateway_ip }}:8080/{{ cluster_pubkey_filename }}" -o /root/.ssh/authorized_keys
-chmod 600 /root/.ssh/authorized_keys
-
-# SELinux context is handled by 'selinux --disabled' above,
-# but if SELinux were enabled, this would be essential:
-# restorecon -R /root/.ssh
+# Download the public key from the provisioning server.
+# The -sS flags make curl silent but show errors. -L follows redirects.
+curl -vSL "http://{{ gateway_ip }}:8080/{{ cluster_pubkey_filename }}" -o /root/.ssh/authorized_keys
+if [ $? -ne 0 ]; then
+    echo "  - ERROR: Failed to download SSH public key."
+else
+    echo "  - SSH key downloaded successfully."
+    chmod 600 /root/.ssh/authorized_keys
+fi
 
 # 2. Download the Harmony Inventory Agent
 echo "  - Downloading harmony-inventory-agent..."
-curl -sSL "http://{{ gateway_ip }}:8080/{{ harmony_inventory_agent }}" -o /usr/local/bin/harmony-inventory-agent
-chmod +x /usr/local/bin/harmony-inventory-agent
+curl -vSL "http://{{ gateway_ip }}:8080/{{ harmony_inventory_agent }}" -o /usr/bin/harmony-inventory-agent
+if [ $? -ne 0 ]; then
+    echo "  - ERROR: Failed to download harmony_inventory_agent."
+else
+    echo "  - Agent binary downloaded successfully."
+    chmod +x /usr/bin/harmony-inventory-agent
+fi
 
-# 3. Create a systemd service to run the agent persistently
+# 3. Create a systemd service to run the agent persistently.
+# This is the most robust method to ensure the agent stays running.
 echo "  - Creating systemd service for the agent..."
 cat > /etc/systemd/system/harmony-agent.service << EOF
 [Unit]
@@ -69,8 +41,9 @@ After=network-online.target
 Wants=network-online.target
 
 [Service]
-ExecStart=/usr/local/bin/harmony-inventory-agent
-Restart=always
+Type=simple
+ExecStart=/usr/bin/harmony-inventory-agent
+Restart=on-failure
 RestartSec=5
 
 [Install]
@@ -78,15 +51,77 @@ WantedBy=multi-user.target
 EOF
 
 # 4. Enable and start the service
+# The 'systemctl' commands will work correctly within the chroot environment of the %pre script.
 echo "  - Enabling and starting harmony-agent.service..."
 systemctl daemon-reload
 systemctl enable --now harmony-agent.service
 
-echo "Harmony Kickstart: Post-boot script finished. The inventory agent is running."
+# Check if the service started correctly
+systemctl is-active --quiet harmony-agent.service
+if [ $? -eq 0 ]; then
+    echo "  - Harmony Inventory Agent service is now running."
+else
+    echo "  - ERROR: Harmony Inventory Agent service failed to start."
+fi
 
-curl localhost:8080/inventory | tee -a /tmp/harmony_inventory.json
+echo "Harmony Kickstart: Pre-boot script finished. The machine is ready for inventory."
+
+echo "Running cat - to pause system indefinitely"
+cat -
 
 %end
 
-# Do not automatically reboot or poweroff.
-# The machine should remain running for inventory scraping.
+# =================================================================
+#  Harmony Discovery Agent - Kickstart File (NON-INSTALL, LIVE BOOT)
+# =================================================================
+#
+# This file achieves a fully automated, non-interactive boot into a
+# live CentOS environment. It does NOT install to disk.
+#
+
+# --- Automation and Interaction Control ---
+# Perform the installation in command-line mode. This is critical for
+# preventing Anaconda from starting a UI and halting for input.
+cmdline
+
+# Accept the End User License Agreement to prevent a prompt.
+eula --agreed
+
+# --- Core System Configuration (Required by Anaconda) ---
+# Set keyboard and language. These are mandatory.
+keyboard --vckeymap=us --xlayouts='us'
+lang en_US.UTF-8
+
+# Configure networking. This is essential for the %post script to work.
+# The --activate flag ensures this device is brought up in the installer environment.
+network --bootproto=dhcp --device=link --activate
+
+# Set a locked root password. This is a mandatory command.
+rootpw --lock
+
+# Set the timezone. This is a mandatory command.
+timezone UTC
+
+# --- Disable Installation-Specific Features ---
+# CRITICAL: Do not install a bootloader. The --disabled flag prevents
+# this step and avoids errors about where to install it.
+bootloader --disabled
+
+# CRITICAL: Ignore all disks. This prevents Anaconda from stopping at the
+# "Installation Destination" screen asking where to install.
+# ignoredisk --drives /dev/sda
+
+# Do not run the Initial Setup wizard on first boot.
+firstboot --disable
+
+# --- Package Selection ---
+# We are not installing, so this section can be minimal.
+# An empty %packages section is valid and ensures no time is wasted
+# resolving dependencies for an installation that will not happen.
+%packages
+%end
+
+
+# IMPORTANT: Do not include a final action command like 'reboot' or 'poweroff'.
+# The default action is 'halt', which in cmdline mode will leave the system
+# running in the live environment with the agent active, which is the desired state.

harmony_inventory_agent/src/hwinfo.rs

@@ -1,4 +1,4 @@
-use log::debug;
+use log::{debug, warn};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::fs;
@@ -104,48 +104,58 @@ impl PhysicalHost {
 
     fn all_tools_available() -> Result<(), String> {
         let required_tools = [
-            ("lsblk", "--version"),
-            ("lspci", "--version"),
-            ("lsmod", "--version"),
-            ("dmidecode", "--version"),
-            ("smartctl", "--version"),
-            ("ip", "route"), // No version flag available
+            ("lsblk", Some("--version")),
+            ("lspci", Some("--version")),
+            ("lsmod", None),
+            ("dmidecode", Some("--version")),
+            ("smartctl", Some("--version")),
+            ("ip", Some("route")), // No version flag available
         ];
 
         let mut missing_tools = Vec::new();
 
+        debug!("Looking for required_tools {required_tools:?}");
         for (tool, tool_arg) in required_tools.iter() {
             // First check if tool exists in PATH using which(1)
-            let exists = if let Ok(output) = Command::new("which").arg(tool).output() {
+            let mut exists = if let Ok(output) = Command::new("which").arg(tool).output() {
                 output.status.success()
             } else {
-                // Fallback: manual PATH search if which(1) is unavailable
-                if let Ok(path_var) = std::env::var("PATH") {
-                    path_var.split(':').any(|dir| {
-                        let tool_path = std::path::Path::new(dir).join(tool);
-                        tool_path.exists() && Self::is_executable(&tool_path)
-                    })
-                } else {
-                    false
-                }
+                false
             };
 
             if !exists {
+                // Fallback: manual PATH search if which(1) is unavailable
+                debug!("Looking for {tool} in path");
+                if let Ok(path_var) = std::env::var("PATH") {
+                    debug!("PATH is {path_var}");
+                    exists = path_var.split(':').any(|dir| {
+                        let tool_path = std::path::Path::new(dir).join(tool);
+                        tool_path.exists() && Self::is_executable(&tool_path)
+                    })
+                }
+            }
+
+            if !exists {
+                warn!("Unable to find tool {tool} from PATH");
                 missing_tools.push(*tool);
                 continue;
             }
 
             // Verify tool is functional by checking version/help output
             let mut cmd = Command::new(tool);
-            cmd.arg(tool_arg);
+            if let Some(tool_arg) = tool_arg {
+                cmd.arg(tool_arg);
+            }
             cmd.stdout(std::process::Stdio::null());
             cmd.stderr(std::process::Stdio::null());
 
             if let Ok(status) = cmd.status() {
                 if !status.success() {
+                    warn!("Unable to test {tool} status failed");
                     missing_tools.push(*tool);
                 }
             } else {
+                warn!("Unable to test {tool}");
                 missing_tools.push(*tool);
             }
         }
@@ -167,6 +177,7 @@ impl PhysicalHost {
 
     #[cfg(unix)]
     fn is_executable(path: &std::path::Path) -> bool {
+        debug!("Checking if {} is executable", path.to_string_lossy());
         use std::os::unix::fs::PermissionsExt;
 
         match std::fs::metadata(path) {
@@ -285,11 +296,11 @@ impl PhysicalHost {
             if device_path.exists() {
                 if drive.model.is_empty() {
                     drive.model = Self::read_sysfs_string(&device_path.join("device/model"))
-                        .map_err(|e| format!("Failed to read model for {}: {}", name, e))?;
+                        .unwrap_or(format!("Failed to read model for {}", name));
                 }
                 if drive.serial.is_empty() {
                     drive.serial = Self::read_sysfs_string(&device_path.join("device/serial"))
-                        .map_err(|e| format!("Failed to read serial for {}: {}", name, e))?;
+                        .unwrap_or(format!("Failed to read serial for {}", name));
                 }
             }
 
@@ -655,6 +666,10 @@ impl PhysicalHost {
             Ok("IDE".to_string())
         } else if device_name.starts_with("vd") {
             Ok("VirtIO".to_string())
+        } else if device_name.starts_with("sr") {
+            Ok("CDROM".to_string())
+        } else if device_name.starts_with("zram") {
+            Ok("Ramdisk".to_string())
         } else {
             // Try to determine from device path
             let subsystem = Self::read_sysfs_string(&device_path.join("device/subsystem"))?;