feat(inventory-agent): Discover algorithm by scanning a subnet of ips, slower than mdns but more reliable and versatile

Cargo.lock

@@ -1853,6 +1853,18 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-penpot"
+version = "0.1.0"
+dependencies = [
+ "harmony",
+ "harmony_cli",
+ "harmony_macros",
+ "harmony_types",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "example-pxe"
 version = "0.1.0"
@@ -2479,6 +2491,19 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "harmony_inventory_builder"
+version = "0.1.0"
+dependencies = [
+ "cidr",
+ "harmony",
+ "harmony_cli",
+ "harmony_macros",
+ "harmony_types",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "harmony_macros"
 version = "0.1.0"

examples/cli/src/main.rs

@@ -2,7 +2,7 @@ use harmony::{
     inventory::Inventory,
     modules::{
         dummy::{ErrorScore, PanicScore, SuccessScore},
-        inventory::LaunchDiscoverInventoryAgentScore,
+        inventory::{HarmonyDiscoveryStrategy, LaunchDiscoverInventoryAgentScore},
     },
     topology::LocalhostTopology,
 };
@@ -18,6 +18,7 @@ async fn main() {
             Box::new(PanicScore {}),
             Box::new(LaunchDiscoverInventoryAgentScore {
                 discovery_timeout: Some(10),
+                discovery_strategy: HarmonyDiscoveryStrategy::MDNS,
             }),
         ],
         None,

examples/harmony_inventory_builder/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "harmony_inventory_builder"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+harmony = { path = "../../harmony" }
+harmony_cli = { path = "../../harmony_cli" }
+harmony_macros = { path = "../../harmony_macros" }
+harmony_types = { path = "../../harmony_types" }
+tokio.workspace = true
+url.workspace = true
+cidr.workspace = true

examples/harmony_inventory_builder/build_docker.sh

@@ -0,0 +1,11 @@
+cargo build -p harmony_inventory_builder --release --target x86_64-unknown-linux-musl
+
+SCRIPT_DIR="$(dirname ${0})"
+
+cd "${SCRIPT_DIR}/docker/"
+
+cp ../../../target/x86_64-unknown-linux-musl/release/harmony_inventory_builder .
+
+docker build . -t hub.nationtech.io/harmony/harmony_inventory_builder
+
+docker push hub.nationtech.io/harmony/harmony_inventory_builder

examples/harmony_inventory_builder/docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM debian:12-slim
+
+RUN mkdir  /app
+WORKDIR /app/
+
+COPY harmony_inventory_builder /app/
+
+ENV RUST_LOG=info
+
+CMD ["sleep", "infinity"]

examples/harmony_inventory_builder/src/main.rs

@@ -0,0 +1,36 @@
+use harmony::{
+    inventory::{HostRole, Inventory},
+    modules::inventory::{DiscoverHostForRoleScore, HarmonyDiscoveryStrategy},
+    topology::LocalhostTopology,
+};
+use harmony_macros::cidrv4;
+
+#[tokio::main]
+async fn main() {
+    let discover_worker = DiscoverHostForRoleScore {
+        role: HostRole::Worker,
+        number_desired_hosts: 3,
+        discovery_strategy: HarmonyDiscoveryStrategy::SUBNET {
+            cidr: cidrv4!("192.168.0.1/25"),
+            port: 25000,
+        },
+    };
+
+    let discover_control_plane = DiscoverHostForRoleScore {
+        role: HostRole::ControlPlane,
+        number_desired_hosts: 3,
+        discovery_strategy: HarmonyDiscoveryStrategy::SUBNET {
+            cidr: cidrv4!("192.168.0.1/25"),
+            port: 25000,
+        },
+    };
+
+    harmony_cli::run(
+        Inventory::autoload(),
+        LocalhostTopology::new(),
+        vec![Box::new(discover_worker), Box::new(discover_control_plane)],
+        None,
+    )
+    .await
+    .unwrap();
+}

harmony/src/domain/interpret/mod.rs

@@ -4,6 +4,8 @@ use std::error::Error;
 use async_trait::async_trait;
 use derive_new::new;
 
+use crate::inventory::HostRole;
+
 use super::{
     data::Version, executors::ExecutorError, inventory::Inventory, topology::PreparationError,
 };

harmony/src/domain/topology/tenant/k8s.rs

@@ -14,7 +14,7 @@ use k8s_openapi::{
     },
     apimachinery::pkg::util::intstr::IntOrString,
 };
-use kube::Resource;
+use kube::{Resource, api::DynamicObject};
 use log::debug;
 use serde::de::DeserializeOwned;
 use serde_json::json;

harmony/src/modules/inventory/discovery.rs

@@ -5,11 +5,10 @@ use serde::{Deserialize, Serialize};
 
 use crate::{
     data::Version,
-    hardware::PhysicalHost,
     infra::inventory::InventoryRepositoryFactory,
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::{HostRole, Inventory},
-    modules::inventory::LaunchDiscoverInventoryAgentScore,
+    modules::inventory::{HarmonyDiscoveryStrategy, LaunchDiscoverInventoryAgentScore},
     score::Score,
     topology::Topology,
 };
@@ -17,11 +16,13 @@ use crate::{
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DiscoverHostForRoleScore {
     pub role: HostRole,
+    pub number_desired_hosts: i16,
+    pub discovery_strategy : HarmonyDiscoveryStrategy,
 }
 
 impl<T: Topology> Score<T> for DiscoverHostForRoleScore {
     fn name(&self) -> String {
-        "DiscoverInventoryAgentScore".to_string()
+        format!("DiscoverHostForRoleScore({:?})", self.role)
     }
 
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
@@ -48,13 +49,15 @@ impl<T: Topology> Interpret<T> for DiscoverHostForRoleInterpret {
         );
         LaunchDiscoverInventoryAgentScore {
             discovery_timeout: None,
+            discovery_strategy: self.score.discovery_strategy.clone(),
         }
         .interpret(inventory, topology)
         .await?;
 
-        let host: PhysicalHost;
+        let mut chosen_hosts = vec![];
         let host_repo = InventoryRepositoryFactory::build().await?;
 
+        let mut assigned_hosts = 0;
         loop {
             let all_hosts = host_repo.get_all_hosts().await?;
 
@@ -75,15 +78,24 @@ impl<T: Topology> Interpret<T> for DiscoverHostForRoleInterpret {
             match ans {
                 Ok(choice) => {
                     info!(
-                        "Selected {} as the {:?} node.",
+                        "Assigned role {:?} for node {}",
-                        choice.summary(),
+                        self.score.role,
-                        self.score.role
+                        choice.summary()
                     );
                     host_repo
                         .save_role_mapping(&self.score.role, &choice)
                         .await?;
-                    host = choice;
+                    chosen_hosts.push(choice);
-                    break;
+                    assigned_hosts += 1;
+
+                    info!(
+                        "Found {assigned_hosts} hosts for role {:?}",
+                        self.score.role
+                    );
+
+                    if assigned_hosts == self.score.number_desired_hosts {
+                        break;
+                    }
                 }
                 Err(inquire::InquireError::OperationCanceled) => {
                     info!("Refresh requested. Fetching list of discovered hosts again...");
@@ -100,8 +112,13 @@ impl<T: Topology> Interpret<T> for DiscoverHostForRoleInterpret {
         }
 
         Ok(Outcome::success(format!(
-            "Successfully discovered host {} for role {:?}",
+            "Successfully discovered {} hosts {} for role {:?}",
-            host.summary(),
+            self.score.number_desired_hosts,
+            chosen_hosts
+                .iter()
+                .map(|h| h.summary())
+                .collect::<Vec<String>>()
+                .join(", "),
             self.score.role
         )))
     }

harmony/src/modules/inventory/mod.rs

@@ -1,6 +1,10 @@
 mod discovery;
 pub mod inspect;
+use std::net::Ipv4Addr;
+
+use cidr::{Ipv4Cidr, Ipv4Inet};
 pub use discovery::*;
+use tokio::time::{Duration, timeout};
 
 use async_trait::async_trait;
 use harmony_inventory_agent::local_presence::DiscoveryEvent;
@@ -24,6 +28,7 @@ use harmony_types::id::Id;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LaunchDiscoverInventoryAgentScore {
     pub discovery_timeout: Option<u64>,
+    pub discovery_strategy: HarmonyDiscoveryStrategy,
 }
 
 impl<T: Topology> Score<T> for LaunchDiscoverInventoryAgentScore {
@@ -43,6 +48,12 @@ struct DiscoverInventoryAgentInterpret {
     score: LaunchDiscoverInventoryAgentScore,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum HarmonyDiscoveryStrategy {
+    MDNS,
+    SUBNET { cidr: cidr::Ipv4Cidr, port: u16 },
+}
+
 #[async_trait]
 impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
     async fn execute(
@@ -57,6 +68,37 @@ impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
             ),
         };
 
+        match self.score.discovery_strategy {
+            HarmonyDiscoveryStrategy::MDNS => self.launch_mdns_discovery().await,
+            HarmonyDiscoveryStrategy::SUBNET { cidr, port } => {
+                self.launch_cidr_discovery(&cidr, port).await
+            }
+        };
+
+        Ok(Outcome::success(
+            "Discovery process completed successfully".to_string(),
+        ))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        InterpretName::DiscoverInventoryAgent
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl DiscoverInventoryAgentInterpret {
+    async fn launch_mdns_discovery(&self) {
         harmony_inventory_agent::local_presence::discover_agents(
             self.score.discovery_timeout,
             |event: DiscoveryEvent| -> Result<(), String> {
@@ -112,6 +154,8 @@ impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
                                 cpus,
                             };
 
+                            // FIXME only save the host when it is new or something changed in it.
+                            // we currently are saving the host every time it is discovered.
                             let repo = InventoryRepositoryFactory::build()
                                 .await
                                 .map_err(|e| format!("Could not build repository : {e}"))
@@ -132,25 +176,111 @@ impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
                 Ok(())
             },
         )
-        .await;
+        .await
-        Ok(Outcome::success(
-            "Discovery process completed successfully".to_string(),
-        ))
     }
 
-    fn get_name(&self) -> InterpretName {
+    // async fn launch_cidr_discovery(&self, cidr : &Ipv4Cidr, port: u16) {
-        InterpretName::DiscoverInventoryAgent
+    //     todo!("launnch cidr discovery for {cidr} : {port}
-    }
+    //         - Iterate over all possible addresses in cidr
+    //         - make calls in batches of 20 attempting to reach harmony inventory agent on <addr, port> using same as above harmony_inventory_agent::client::get_host_inventory(&address, port)
+    //         - Log warn when response is 404, it means the port was used by something else unexpected
+    //         - Log error when response is 5xx
+    //         - Log debug when no response (timeout 15 seconds)
+    //         - Log info when found and response is 2xx
+    //         ");
+    // }
+    async fn launch_cidr_discovery(&self, cidr: &Ipv4Cidr, port: u16) {
+        let addrs: Vec<Ipv4Inet> = cidr.iter().collect();
+        let total = addrs.len();
+        info!(
+            "Starting CIDR discovery for {} hosts on {}/{} (port {})",
+            total,
+            cidr.network_length(),
+            cidr,
+            port
+        );
 
-    fn get_version(&self) -> Version {
+        let batch_size: usize = 20;
-        todo!()
+        let timeout_secs = 5;
-    }
+        let request_timeout = Duration::from_secs(timeout_secs);
 
-    fn get_status(&self) -> InterpretStatus {
+        let mut current_batch = 0;
-        todo!()
+        let num_batches = addrs.len() / batch_size;
-    }
 
-    fn get_children(&self) -> Vec<Id> {
+        for batch in addrs.chunks(batch_size) {
-        todo!()
+            current_batch += 1;
+            info!("Starting query batch {current_batch} of {num_batches}, timeout {timeout_secs}");
+            let mut tasks = Vec::with_capacity(batch.len());
+
+            for addr in batch {
+                let addr = addr.address().to_string();
+                let port = port;
+
+                let task = tokio::spawn(async move {
+                    match timeout(
+                        request_timeout,
+                        harmony_inventory_agent::client::get_host_inventory(&addr, port),
+                    )
+                    .await
+                    {
+                        Ok(Ok(host)) => {
+                            info!("Found and response is 2xx for {addr}:{port}");
+
+                            // Reuse the same conversion to PhysicalHost as MDNS flow
+                            let harmony_inventory_agent::hwinfo::PhysicalHost {
+                                storage_drives,
+                                storage_controller,
+                                memory_modules,
+                                cpus,
+                                chipset,
+                                network_interfaces,
+                                management_interface,
+                                host_uuid,
+                            } = host;
+
+                            let host = PhysicalHost {
+                                id: Id::from(host_uuid),
+                                category: HostCategory::Server,
+                                network: network_interfaces,
+                                storage: storage_drives,
+                                labels: vec![Label {
+                                    name: "discovered-by".to_string(),
+                                    value: "harmony-inventory-agent".to_string(),
+                                }],
+                                memory_modules,
+                                cpus,
+                            };
+
+                            // Save host to inventory
+                            let repo = InventoryRepositoryFactory::build()
+                                .await
+                                .map_err(|e| format!("Could not build repository : {e}"))
+                                .unwrap();
+                            if let Err(e) = repo.save(&host).await {
+                                log::debug!("Failed to save host {}: {e}", host.id);
+                            } else {
+                                info!("Saved host id {}, summary : {}", host.id, host.summary());
+                            }
+                        }
+                        Ok(Err(e)) => {
+                            log::info!("Error querying inventory agent on {addr}:{port} : {e}");
+                        }
+                        Err(_) => {
+                            // Timeout for this host
+                            log::debug!("No response (timeout) for {addr}:{port}");
+                        }
+                    }
+                });
+
+                tasks.push(task);
+            }
+
+            // Wait for this batch to complete
+            for t in tasks {
+                let _ = t.await;
+            }
+        }
+
+        info!("CIDR discovery completed");
     }
 }

harmony/src/modules/okd/bootstrap_01_prepare.rs

@@ -4,7 +4,7 @@ use crate::{
     infra::inventory::InventoryRepositoryFactory,
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::{HostRole, Inventory},
-    modules::inventory::DiscoverHostForRoleScore,
+    modules::inventory::{DiscoverHostForRoleScore, HarmonyDiscoveryStrategy},
     score::Score,
     topology::HAClusterTopology,
 };
@@ -104,6 +104,8 @@ When you can dig them, confirm to continue.
             bootstrap_host = hosts.into_iter().next().to_owned();
             DiscoverHostForRoleScore {
                 role: HostRole::Bootstrap,
+                number_desired_hosts: 1,
+                discovery_strategy: HarmonyDiscoveryStrategy::MDNS,
             }
             .interpret(inventory, topology)
             .await?;

harmony/src/modules/okd/bootstrap_03_control_plane.rs

@@ -6,7 +6,7 @@ use crate::{
     inventory::{HostRole, Inventory},
     modules::{
         dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
-        inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
+        inventory::{DiscoverHostForRoleScore, HarmonyDiscoveryStrategy}, okd::templates::BootstrapIpxeTpl,
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -58,38 +58,39 @@ impl OKDSetup03ControlPlaneInterpret {
         inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<Vec<PhysicalHost>, InterpretError> {
-        const REQUIRED_HOSTS: usize = 3;
+        const REQUIRED_HOSTS: i16 = 3;
         let repo = InventoryRepositoryFactory::build().await?;
-        let mut control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
+        let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
 
-        while control_plane_hosts.len() < REQUIRED_HOSTS {
+        info!(
-            info!(
+            "Discovery of {} control plane hosts in progress, current number {}",
-                "Discovery of {} control plane hosts in progress, current number {}",
+            REQUIRED_HOSTS,
-                REQUIRED_HOSTS,
+            control_plane_hosts.len()
-                control_plane_hosts.len()
+        );
-            );
+        // This score triggers the discovery agent for a specific role.
-            // This score triggers the discovery agent for a specific role.
+        DiscoverHostForRoleScore {
-            DiscoverHostForRoleScore {
+            role: HostRole::ControlPlane,
-                role: HostRole::ControlPlane,
+            number_desired_hosts: REQUIRED_HOSTS,
-            }
+            discovery_strategy: HarmonyDiscoveryStrategy::MDNS,
-            .interpret(inventory, topology)
-            .await?;
-            control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
         }
+        .interpret(inventory, topology)
+        .await?;
 
-        if control_plane_hosts.len() < REQUIRED_HOSTS {
+        let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
-            Err(InterpretError::new(format!(
+
+        if control_plane_hosts.len() < REQUIRED_HOSTS as usize {
+            return Err(InterpretError::new(format!(
                 "OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.",
                 REQUIRED_HOSTS,
                 control_plane_hosts.len()
-            )))
+            )));
-        } else {
-            // Take exactly the number of required hosts to ensure consistency.
-            Ok(control_plane_hosts
-                .into_iter()
-                .take(REQUIRED_HOSTS)
-                .collect())
         }
+
+        // Take exactly the number of required hosts to ensure consistency.
+        Ok(control_plane_hosts
+            .into_iter()
+            .take(REQUIRED_HOSTS as usize)
+            .collect())
     }
 
     /// Configures DHCP host bindings for all control plane nodes.

harmony_inventory_agent/build_docker.sh

@@ -0,0 +1,11 @@
+cargo build -p harmony_inventory_agent --release --target x86_64-unknown-linux-musl
+
+SCRIPT_DIR="$(dirname ${0})"
+
+cd "${SCRIPT_DIR}/docker/"
+
+cp ../../target/x86_64-unknown-linux-musl/release/harmony_inventory_agent .
+
+docker build . -t hub.nationtech.io/harmony/harmony_inventory_agent
+
+docker push hub.nationtech.io/harmony/harmony_inventory_agent

harmony_inventory_agent/docker/.gitignore

@@ -0,0 +1 @@
+harmony_inventory_agent

harmony_inventory_agent/docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM debian:12-slim
+
+# install packages required to make these commands available : lspci, lsmod, dmidecode, smartctl, ip
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends pciutils kmod dmidecode smartmontools iproute2 && \
+    rm -rf /var/lib/apt/lists/*
+
+
+RUN mkdir  /app
+WORKDIR /app/
+
+COPY harmony_inventory_agent /app/
+
+ENV RUST_LOG=info
+
+CMD [ "/app/harmony_inventory_agent" ]
+

harmony_inventory_agent/harmony-inventory-agent-daemonset.yaml

@@ -0,0 +1,117 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: harmony-inventory-agent
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: harmony-inventory-agent
+  namespace: harmony-inventory-agent
+---
+# Grant the built-in "privileged" SCC to the SA
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: use-privileged-scc
+  namespace: harmony-inventory-agent
+rules:
+- apiGroups: ["security.openshift.io"]
+  resources: ["securitycontextconstraints"]
+  resourceNames: ["privileged"]
+  verbs: ["use"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: use-privileged-scc
+  namespace: harmony-inventory-agent
+subjects:
+- kind: ServiceAccount
+  name: harmony-inventory-agent
+  namespace: harmony-inventory-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: use-privileged-scc
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: harmony-inventory-agent
+  namespace: harmony-inventory-agent
+spec:
+  selector:
+    matchLabels:
+      app: harmony-inventory-agent
+  template:
+    metadata:
+      labels:
+        app: harmony-inventory-agent
+    spec:
+      serviceAccountName: harmony-inventory-agent
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: "Exists"
+        effect: "NoSchedule"
+      containers:
+      - name: inventory-agent
+        image: hub.nationtech.io/harmony/harmony_inventory_agent
+        imagePullPolicy: Always
+        env:
+          - name: RUST_LOG
+            value: "harmony_inventory_agent=trace,info"
+        resources:
+          limits:
+            cpu: 200m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+        securityContext:
+          privileged: true
+          # optional: leave the rest unset since privileged SCC allows it
+          #
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: harmony-inventory-builder
+  namespace: harmony-inventory-agent
+spec:
+  replicas: 1
+  strategy: {}
+  selector:
+    matchLabels:
+      app: harmony-inventory-builder
+  template:
+    metadata:
+      labels:
+        app: harmony-inventory-builder
+    spec:
+      serviceAccountName: harmony-inventory-agent
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: inventory-agent
+        image: hub.nationtech.io/harmony/harmony_inventory_builder
+        imagePullPolicy: Always
+        env:
+          - name: RUST_LOG
+            value: "harmony_inventory_builder=trace,info"
+        resources:
+          limits:
+            cpu: 200m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+        securityContext:
+          privileged: true
+          # optional: leave the rest unset since privileged SCC allows it

harmony_inventory_agent/src/hwinfo.rs

@@ -1,5 +1,5 @@
 use harmony_types::net::MacAddress;
-use log::{debug, warn};
+use log::{debug, trace, warn};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::fs;
@@ -121,20 +121,48 @@ pub struct ManagementInterface {
 
 impl PhysicalHost {
     pub fn gather() -> Result<Self, String> {
+        trace!("Start gathering physical host information");
         let mut sys = System::new_all();
+        trace!("System new_all called");
         sys.refresh_all();
+        trace!("System refresh_all called");
 
         Self::all_tools_available()?;
 
+        trace!("All tools_available success");
+
+        let storage_drives = Self::gather_storage_drives()?;
+        trace!("got storage drives");
+
+        let storage_controller = Self::gather_storage_controller()?;
+        trace!("got storage controller");
+
+        let memory_modules = Self::gather_memory_modules()?;
+        trace!("got memory_modules");
+
+        let cpus = Self::gather_cpus(&sys)?;
+        trace!("got cpus");
+
+        let chipset = Self::gather_chipset()?;
+        trace!("got chipsets");
+
+        let network_interfaces = Self::gather_network_interfaces()?;
+        trace!("got network_interfaces");
+
+        let management_interface = Self::gather_management_interface()?;
+        trace!("got management_interface");
+
+        let host_uuid = Self::get_host_uuid()?;
+
         Ok(Self {
-            storage_drives: Self::gather_storage_drives()?,
+            storage_drives,
-            storage_controller: Self::gather_storage_controller()?,
+            storage_controller,
-            memory_modules: Self::gather_memory_modules()?,
+            memory_modules,
-            cpus: Self::gather_cpus(&sys)?,
+            cpus,
-            chipset: Self::gather_chipset()?,
+            chipset,
-            network_interfaces: Self::gather_network_interfaces()?,
+            network_interfaces,
-            management_interface: Self::gather_management_interface()?,
+            management_interface,
-            host_uuid: Self::get_host_uuid()?,
+            host_uuid,
         })
     }
 
@@ -208,6 +236,8 @@ impl PhysicalHost {
             ));
         }
 
+        debug!("All tools found!");
+
         Ok(())
     }
 
@@ -231,7 +261,10 @@ impl PhysicalHost {
     fn gather_storage_drives() -> Result<Vec<StorageDrive>, String> {
         let mut drives = Vec::new();
 
+        trace!("Starting storage drive discovery using lsblk");
+
         // Use lsblk with JSON output for robust parsing
+        trace!("Executing 'lsblk -d -o NAME,MODEL,SERIAL,SIZE,ROTA,WWN -n -e 7 --json'");
         let output = Command::new("lsblk")
             .args([
                 "-d",
@@ -245,13 +278,18 @@ impl PhysicalHost {
             .output()
             .map_err(|e| format!("Failed to execute lsblk: {}", e))?;
 
+        trace!(
+            "lsblk command executed successfully (status: {:?})",
+            output.status
+        );
+
         if !output.status.success() {
-            return Err(format!(
+            let stderr_str = String::from_utf8_lossy(&output.stderr);
-                "lsblk command failed: {}",
+            debug!("lsblk command failed: {stderr_str}");
-                String::from_utf8_lossy(&output.stderr)
+            return Err(format!("lsblk command failed: {stderr_str}"));
-            ));
         }
 
+        trace!("Parsing lsblk JSON output");
         let json: Value = serde_json::from_slice(&output.stdout)
             .map_err(|e| format!("Failed to parse lsblk JSON output: {}", e))?;
 
@@ -260,6 +298,8 @@ impl PhysicalHost {
             .and_then(|v| v.as_array())
             .ok_or("Invalid lsblk JSON: missing 'blockdevices' array")?;
 
+        trace!("Found {} blockdevices in lsblk output", blockdevices.len());
+
         for device in blockdevices {
             let name = device
                 .get("name")
@@ -268,52 +308,72 @@ impl PhysicalHost {
                 .to_string();
 
             if name.is_empty() {
+                trace!("Skipping unnamed device entry: {:?}", device);
                 continue;
             }
 
+            trace!("Inspecting block device: {name}");
+
+            // Extract metadata fields
             let model = device
                 .get("model")
                 .and_then(|v| v.as_str())
                 .map(|s| s.trim().to_string())
                 .unwrap_or_default();
+            trace!("Model for {name}: '{}'", model);
 
             let serial = device
                 .get("serial")
                 .and_then(|v| v.as_str())
                 .map(|s| s.trim().to_string())
                 .unwrap_or_default();
+            trace!("Serial for {name}: '{}'", serial);
 
             let size_str = device
                 .get("size")
                 .and_then(|v| v.as_str())
                 .ok_or("Missing 'size' in lsblk device")?;
+            trace!("Reported size for {name}: {}", size_str);
             let size_bytes = Self::parse_size(size_str)?;
+            trace!("Parsed size for {name}: {} bytes", size_bytes);
 
             let rotational = device
                 .get("rota")
                 .and_then(|v| v.as_bool())
                 .ok_or("Missing 'rota' in lsblk device")?;
+            trace!("Rotational flag for {name}: {}", rotational);
 
             let wwn = device
                 .get("wwn")
                 .and_then(|v| v.as_str())
                 .map(|s| s.trim().to_string())
                 .filter(|s| !s.is_empty() && s != "null");
+            trace!("WWN for {name}: {:?}", wwn);
 
             let device_path = Path::new("/sys/block").join(&name);
+            trace!("Sysfs path for {name}: {:?}", device_path);
 
+            trace!("Reading logical block size for {name}");
             let logical_block_size = Self::read_sysfs_u32(
                 &device_path.join("queue/logical_block_size"),
             )
             .map_err(|e| format!("Failed to read logical block size for {}: {}", name, e))?;
+            trace!("Logical block size for {name}: {}", logical_block_size);
 
+            trace!("Reading physical block size for {name}");
             let physical_block_size = Self::read_sysfs_u32(
                 &device_path.join("queue/physical_block_size"),
             )
             .map_err(|e| format!("Failed to read physical block size for {}: {}", name, e))?;
+            trace!("Physical block size for {name}: {}", physical_block_size);
 
+            trace!("Determining interface type for {name}");
             let interface_type = Self::get_interface_type(&name, &device_path)?;
+            trace!("Interface type for {name}: {}", interface_type);
+
+            trace!("Getting SMART status for {name}");
             let smart_status = Self::get_smart_status(&name)?;
+            trace!("SMART status for {name}: {:?}", smart_status);
 
             let mut drive = StorageDrive {
                 name: name.clone(),
@@ -330,19 +390,31 @@ impl PhysicalHost {
 
             // Enhance with additional sysfs info if available
             if device_path.exists() {
+                trace!("Enhancing drive {name} with extra sysfs metadata");
                 if drive.model.is_empty() {
+                    trace!("Reading model from sysfs for {name}");
                     drive.model = Self::read_sysfs_string(&device_path.join("device/model"))
-                        .unwrap_or(format!("Failed to read model for {}", name));
+                        .unwrap_or_else(|_| format!("Failed to read model for {}", name));
                 }
                 if drive.serial.is_empty() {
+                    trace!("Reading serial from sysfs for {name}");
                     drive.serial = Self::read_sysfs_string(&device_path.join("device/serial"))
-                        .unwrap_or(format!("Failed to read serial for {}", name));
+                        .unwrap_or_else(|_| format!("Failed to read serial for {}", name));
                 }
+            } else {
+                trace!(
+                    "Sysfs path {:?} not found for drive {name}, skipping extra metadata",
+                    device_path
+                );
             }
 
+            debug!("Discovered storage drive: {drive:?}");
             drives.push(drive);
         }
 
+        debug!("Discovered total {} storage drives", drives.len());
+        trace!("All discovered dives: {drives:?}");
+
         Ok(drives)
     }
 
@@ -418,6 +490,8 @@ impl PhysicalHost {
             }
         }
 
+        debug!("Found storage controller {controller:?}");
+
         Ok(controller)
     }
 
@@ -486,6 +560,7 @@ impl PhysicalHost {
             }
         }
 
+        debug!("Found memory modules {modules:?}");
         Ok(modules)
     }
 
@@ -501,22 +576,30 @@ impl PhysicalHost {
             frequency_mhz: global_cpu.frequency(),
         });
 
+        debug!("Found cpus {cpus:?}");
+
         Ok(cpus)
     }
 
     fn gather_chipset() -> Result<Chipset, String> {
-        Ok(Chipset {
+        let chipset = Chipset {
             name: Self::read_dmi("baseboard-product-name")?,
             vendor: Self::read_dmi("baseboard-manufacturer")?,
-        })
+        };
+
+        debug!("Found chipset {chipset:?}");
+
+        Ok(chipset)
     }
 
     fn gather_network_interfaces() -> Result<Vec<NetworkInterface>, String> {
         let mut interfaces = Vec::new();
         let sys_net_path = Path::new("/sys/class/net");
+        trace!("Reading /sys/class/net");
 
         let entries = fs::read_dir(sys_net_path)
             .map_err(|e| format!("Failed to read /sys/class/net: {}", e))?;
+        trace!("Got entries {entries:?}");
 
         for entry in entries {
             let entry = entry.map_err(|e| format!("Failed to read directory entry: {}", e))?;
@@ -525,6 +608,7 @@ impl PhysicalHost {
                 .into_string()
                 .map_err(|_| "Invalid UTF-8 in interface name")?;
             let iface_path = entry.path();
+            trace!("Inspecting interface {iface_name} path {iface_path:?}");
 
             // Skip virtual interfaces
             if iface_name.starts_with("lo")
@@ -535,70 +619,101 @@ impl PhysicalHost {
                 || iface_name.starts_with("tun")
                 || iface_name.starts_with("wg")
             {
+                trace!(
+                    "Skipping interface {iface_name} because it appears to be virtual/unsupported"
+                );
                 continue;
             }
 
             // Check if it's a physical interface by looking for device directory
             if !iface_path.join("device").exists() {
+                trace!(
+                    "Skipping interface {iface_name} since {iface_path:?}/device does not exist"
+                );
                 continue;
             }
 
+            trace!("Reading MAC address for {iface_name}");
             let mac_address = Self::read_sysfs_string(&iface_path.join("address"))
                 .map_err(|e| format!("Failed to read MAC address for {}: {}", iface_name, e))?;
             let mac_address = MacAddress::try_from(mac_address).map_err(|e| e.to_string())?;
+            trace!("MAC address for {iface_name}: {mac_address}");
 
-            let speed_mbps = if iface_path.join("speed").exists() {
+            let speed_path = iface_path.join("speed");
-                match Self::read_sysfs_u32(&iface_path.join("speed")) {
+            let speed_mbps = if speed_path.exists() {
-                    Ok(speed) => Some(speed),
+                trace!("Reading speed for {iface_name} from {:?}", speed_path);
+                match Self::read_sysfs_u32(&speed_path) {
+                    Ok(speed) => {
+                        trace!("Speed for {iface_name}: {speed} Mbps");
+                        Some(speed)
+                    }
                     Err(e) => {
                         debug!(
-                            "Failed to read speed for {}: {} . This is expected to fail on wifi interfaces.",
+                            "Failed to read speed for {}: {} (this may be expected on Wi‑Fi interfaces)",
                             iface_name, e
                         );
                         None
                     }
                 }
             } else {
+                trace!("Speed file not found for {iface_name}, skipping");
                 None
             };
 
+            trace!("Reading operstate for {iface_name}");
             let operstate = Self::read_sysfs_string(&iface_path.join("operstate"))
                 .map_err(|e| format!("Failed to read operstate for {}: {}", iface_name, e))?;
+            trace!("Operstate for {iface_name}: {operstate}");
 
+            trace!("Reading MTU for {iface_name}");
             let mtu = Self::read_sysfs_u32(&iface_path.join("mtu"))
                 .map_err(|e| format!("Failed to read MTU for {}: {}", iface_name, e))?;
+            trace!("MTU for {iface_name}: {mtu}");
 
+            trace!("Reading driver for {iface_name}");
             let driver =
                 Self::read_sysfs_symlink_basename(&iface_path.join("device/driver/module"))
                     .map_err(|e| format!("Failed to read driver for {}: {}", iface_name, e))?;
+            trace!("Driver for {iface_name}: {driver}");
 
+            trace!("Reading firmware version for {iface_name}");
             let firmware_version = Self::read_sysfs_opt_string(
                 &iface_path.join("device/firmware_version"),
             )
             .map_err(|e| format!("Failed to read firmware version for {}: {}", iface_name, e))?;
+            trace!("Firmware version for {iface_name}: {firmware_version:?}");
 
-            // Get IP addresses using ip command with JSON output
+            trace!("Fetching IP addresses for {iface_name}");
             let (ipv4_addresses, ipv6_addresses) = Self::get_interface_ips_json(&iface_name)
                 .map_err(|e| format!("Failed to get IP addresses for {}: {}", iface_name, e))?;
+            trace!("Interface {iface_name} has IPv4: {ipv4_addresses:?}, IPv6: {ipv6_addresses:?}");
 
-            interfaces.push(NetworkInterface {
+            let is_up = operstate == "up";
-                name: iface_name,
+            trace!("Constructing NetworkInterface for {iface_name} (is_up={is_up})");
+
+            let iface = NetworkInterface {
+                name: iface_name.clone(),
                 mac_address,
                 speed_mbps,
-                is_up: operstate == "up",
+                is_up,
                 mtu,
                 ipv4_addresses,
                 ipv6_addresses,
                 driver,
                 firmware_version,
-            });
+            };
+
+            debug!("Discovered interface: {iface:?}");
+            interfaces.push(iface);
         }
 
+        debug!("Discovered total {} network interfaces", interfaces.len());
+        trace!("Interfaces collected: {interfaces:?}");
         Ok(interfaces)
     }
 
     fn gather_management_interface() -> Result<Option<ManagementInterface>, String> {
-        if Path::new("/dev/ipmi0").exists() {
+        let mgmt = if Path::new("/dev/ipmi0").exists() {
             Ok(Some(ManagementInterface {
                 kind: "IPMI".to_string(),
                 address: None,
@@ -612,11 +727,16 @@ impl PhysicalHost {
             }))
         } else {
             Ok(None)
-        }
+        };
+
+        debug!("Found management interface {mgmt:?}");
+        mgmt
     }
 
     fn get_host_uuid() -> Result<String, String> {
-        Self::read_dmi("system-uuid")
+        let uuid = Self::read_dmi("system-uuid");
+        debug!("Found uuid {uuid:?}");
+        uuid
     }
 
     // Helper methods
@@ -709,7 +829,8 @@ impl PhysicalHost {
             Ok("Ramdisk".to_string())
         } else {
             // Try to determine from device path
-            let subsystem = Self::read_sysfs_string(&device_path.join("device/subsystem"))?;
+            let subsystem = Self::read_sysfs_string(&device_path.join("device/subsystem"))
+                .unwrap_or(String::new());
             Ok(subsystem
                 .split('/')
                 .next_back()
@@ -779,6 +900,8 @@ impl PhysicalHost {
         size.map(|s| s as u64)
     }
 
+    // FIXME when scanning an interface that is part of a bond/bridge we won't get an address on the
+    // interface, we should be looking at the bond/bridge device. For example, br-ex on k8s nodes.
     fn get_interface_ips_json(iface_name: &str) -> Result<(Vec<String>, Vec<String>), String> {
         let mut ipv4 = Vec::new();
         let mut ipv6 = Vec::new();

harmony_inventory_agent/src/local_presence/advertise.rs

@@ -1,4 +1,4 @@
-use log::{debug, error, info, warn};
+use log::{debug, error, info, trace, warn};
 use mdns_sd::{ServiceDaemon, ServiceInfo};
 use std::collections::HashMap;
 
@@ -12,6 +12,7 @@ use crate::{
 /// This function is synchronous and non-blocking. It spawns a background Tokio task
 /// to handle the mDNS advertisement for the lifetime of the application.
 pub fn advertise(service_port: u16) -> Result<(), PresenceError> {
+    trace!("starting advertisement process for port {service_port}");
     let host_id = match PhysicalHost::gather() {
         Ok(host) => Some(host.host_uuid),
         Err(e) => {
@@ -20,11 +21,15 @@ pub fn advertise(service_port: u16) -> Result<(), PresenceError> {
         }
     };
 
+    trace!("Found host id {host_id:?}");
+
     let instance_name = format!(
         "inventory-agent-{}",
         host_id.clone().unwrap_or("unknown".to_string())
     );
 
+    trace!("Found host id {host_id:?}, name : {instance_name}");
+
     let spawned_msg = format!("Spawned local presence advertisement task for '{instance_name}'.");
 
     tokio::spawn(async move {

harmony_inventory_agent/src/main.rs

@@ -28,7 +28,7 @@ async fn inventory() -> impl Responder {
 async fn main() -> std::io::Result<()> {
     env_logger::init();
 
-    let port = env::var("HARMONY_INVENTORY_AGENT_PORT").unwrap_or_else(|_| "8080".to_string());
+    let port = env::var("HARMONY_INVENTORY_AGENT_PORT").unwrap_or_else(|_| "25000".to_string());
     let port = port
         .parse::<u16>()
         .expect(&format!("Invalid port number, cannot parse to u16 {port}"));