feat: automatically discover inventory (#127)
All checks were successful
Run Check Script / check (pull_request) Successful in 1m15s

## Fully automated inventory gathering now works!

Boot up harmony_inventory_agent with `cargo run -p harmony_inventory_agent`
Launch the DiscoverInventoryAgentScore , currently available this way :

`RUST_LOG=info cargo run -p example-cli -- -f Discover -y`

And you will have automatically all hosts saved to the database. Run `cargo sqlx setup` if you have not done it yet.

Co-authored-by: Ian Letourneau <ian@noma.to>
Reviewed-on: #127
Co-authored-by: Jean-Gabriel Gill-Couture <jg@nationtech.io>
Co-committed-by: Jean-Gabriel Gill-Couture <jg@nationtech.io>
This commit is contained in:
2025-08-31 22:45:07 +00:00
committed by Ian Letourneau
parent f9906cb419
commit 701d8cfab9
19 changed files with 442 additions and 304 deletions

View File

@@ -12,4 +12,12 @@ lazy_static! {
std::env::var("HARMONY_REGISTRY_PROJECT").unwrap_or_else(|_| "harmony".to_string());
pub static ref DRY_RUN: bool =
std::env::var("HARMONY_DRY_RUN").is_ok_and(|value| value.parse().unwrap_or(false));
pub static ref DEFAULT_DATABASE_URL: String = "sqlite://harmony.sqlite".to_string();
pub static ref DATABASE_URL: String = std::env::var("HARMONY_DATABASE_URL")
.map(|value| if value.is_empty() {
(*DEFAULT_DATABASE_URL).clone()
} else {
value
})
.unwrap_or((*DEFAULT_DATABASE_URL).clone());
}

View File

@@ -1,24 +1,24 @@
use std::{str::FromStr, sync::Arc};
use std::sync::Arc;
use derive_new::new;
use harmony_inventory_agent::hwinfo::{CPU, MemoryModule, NetworkInterface, StorageDrive};
use harmony_types::net::MacAddress;
use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct};
use serde::{Deserialize, Serialize};
use serde_value::Value;
pub type HostGroup = Vec<PhysicalHost>;
pub type SwitchGroup = Vec<Switch>;
pub type FirewallGroup = Vec<PhysicalHost>;
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize)]
pub struct PhysicalHost {
pub id: Id,
pub category: HostCategory,
pub network: Vec<NetworkInterface>,
pub management: Arc<dyn ManagementInterface>,
pub storage: Vec<Storage>,
pub storage: Vec<StorageDrive>,
pub labels: Vec<Label>,
pub memory_size: Option<u64>,
pub cpu_count: Option<u64>,
pub memory_modules: Vec<MemoryModule>,
pub cpus: Vec<CPU>,
}
impl PhysicalHost {
@@ -29,12 +29,128 @@ impl PhysicalHost {
network: vec![],
storage: vec![],
labels: vec![],
management: Arc::new(ManualManagementInterface {}),
memory_size: None,
cpu_count: None,
memory_modules: vec![],
cpus: vec![],
}
}
pub fn summary(&self) -> String {
let mut parts = Vec::new();
// Part 1: System Model (from labels) or Category as a fallback
let model = self
.labels
.iter()
.find(|l| l.name == "system-product-name" || l.name == "model")
.map(|l| l.value.clone())
.unwrap_or_else(|| self.category.to_string());
parts.push(model);
// Part 2: CPU Information
if !self.cpus.is_empty() {
let cpu_count = self.cpus.len();
let total_cores = self.cpus.iter().map(|c| c.cores).sum::<u32>();
let total_threads = self.cpus.iter().map(|c| c.threads).sum::<u32>();
let model_name = &self.cpus[0].model;
let cpu_summary = if cpu_count > 1 {
format!(
"{}x {} ({}c/{}t)",
cpu_count, model_name, total_cores, total_threads
)
} else {
format!("{} ({}c/{}t)", model_name, total_cores, total_threads)
};
parts.push(cpu_summary);
}
// Part 3: Memory Information
if !self.memory_modules.is_empty() {
let total_mem_bytes = self
.memory_modules
.iter()
.map(|m| m.size_bytes)
.sum::<u64>();
let total_mem_gb = (total_mem_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).round() as u64;
// Find the most common speed among modules
let mut speeds = std::collections::HashMap::new();
for module in &self.memory_modules {
if let Some(speed) = module.speed_mhz {
*speeds.entry(speed).or_insert(0) += 1;
}
}
let common_speed = speeds
.into_iter()
.max_by_key(|&(_, count)| count)
.map(|(speed, _)| speed);
if let Some(speed) = common_speed {
parts.push(format!("{} GB RAM @ {}MHz", total_mem_gb, speed));
} else {
parts.push(format!("{} GB RAM", total_mem_gb));
}
}
// Part 4: Storage Information
if !self.storage.is_empty() {
let total_storage_bytes = self.storage.iter().map(|d| d.size_bytes).sum::<u64>();
let drive_count = self.storage.len();
let first_drive_model = &self.storage[0].model;
// Helper to format bytes into TB or GB
let format_storage = |bytes: u64| {
let tb = bytes as f64 / (1024.0 * 1024.0 * 1024.0 * 1024.0);
if tb >= 1.0 {
format!("{:.2} TB", tb)
} else {
let gb = bytes as f64 / (1024.0 * 1024.0 * 1024.0);
format!("{:.0} GB", gb)
}
};
let storage_summary = if drive_count > 1 {
format!(
"{} Storage ({}x {})",
format_storage(total_storage_bytes),
drive_count,
first_drive_model
)
} else {
format!(
"{} Storage ({})",
format_storage(total_storage_bytes),
first_drive_model
)
};
parts.push(storage_summary);
}
// Part 5: Network Information
// Prioritize an "up" interface with an IPv4 address
let best_nic = self
.network
.iter()
.find(|n| n.is_up && !n.ipv4_addresses.is_empty())
.or_else(|| self.network.first());
if let Some(nic) = best_nic {
let speed = nic
.speed_mbps
.map(|s| format!("{}Gbps", s / 1000))
.unwrap_or_else(|| "N/A".to_string());
let mac = nic.mac_address.to_string();
let nic_summary = if let Some(ip) = nic.ipv4_addresses.first() {
format!("NIC: {} ({}, {})", speed, ip, mac)
} else {
format!("NIC: {} ({})", speed, mac)
};
parts.push(nic_summary);
}
parts.join(" | ")
}
pub fn cluster_mac(&self) -> MacAddress {
self.network
.first()
@@ -42,37 +158,17 @@ impl PhysicalHost {
.mac_address
}
pub fn cpu(mut self, cpu_count: Option<u64>) -> Self {
self.cpu_count = cpu_count;
self
}
pub fn memory_size(mut self, memory_size: Option<u64>) -> Self {
self.memory_size = memory_size;
self
}
pub fn storage(
mut self,
connection: StorageConnectionType,
kind: StorageKind,
size: u64,
serial: String,
) -> Self {
self.storage.push(Storage {
connection,
kind,
size,
serial,
});
self
}
pub fn mac_address(mut self, mac_address: MacAddress) -> Self {
self.network.push(NetworkInterface {
name: None,
name: String::new(),
mac_address,
speed: None,
speed_mbps: None,
is_up: false,
mtu: 0,
ipv4_addresses: vec![],
ipv6_addresses: vec![],
driver: String::new(),
firmware_version: None,
});
self
}
@@ -81,57 +177,52 @@ impl PhysicalHost {
self.labels.push(Label { name, value });
self
}
pub fn management(mut self, management: Arc<dyn ManagementInterface>) -> Self {
self.management = management;
self
}
}
// Custom Serialize implementation for PhysicalHost
impl Serialize for PhysicalHost {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
// Determine the number of fields
let mut num_fields = 5; // category, network, storage, labels, management
if self.memory_size.is_some() {
num_fields += 1;
}
if self.cpu_count.is_some() {
num_fields += 1;
}
// Create a serialization structure
let mut state = serializer.serialize_struct("PhysicalHost", num_fields)?;
// Serialize the standard fields
state.serialize_field("category", &self.category)?;
state.serialize_field("network", &self.network)?;
state.serialize_field("storage", &self.storage)?;
state.serialize_field("labels", &self.labels)?;
// Serialize optional fields
if let Some(memory) = self.memory_size {
state.serialize_field("memory_size", &memory)?;
}
if let Some(cpu) = self.cpu_count {
state.serialize_field("cpu_count", &cpu)?;
}
let mgmt_data = self.management.serialize_management();
// pub management: Arc<dyn ManagementInterface>,
// Handle management interface - either as a field or flattened
state.serialize_field("management", &mgmt_data)?;
state.end()
}
}
// impl Serialize for PhysicalHost {
// fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
// where
// S: Serializer,
// {
// // Determine the number of fields
// let mut num_fields = 5; // category, network, storage, labels, management
// if self.memory_modules.is_some() {
// num_fields += 1;
// }
// if self.cpus.is_some() {
// num_fields += 1;
// }
//
// // Create a serialization structure
// let mut state = serializer.serialize_struct("PhysicalHost", num_fields)?;
//
// // Serialize the standard fields
// state.serialize_field("category", &self.category)?;
// state.serialize_field("network", &self.network)?;
// state.serialize_field("storage", &self.storage)?;
// state.serialize_field("labels", &self.labels)?;
//
// // Serialize optional fields
// if let Some(memory) = self.memory_modules {
// state.serialize_field("memory_size", &memory)?;
// }
// if let Some(cpu) = self.cpus {
// state.serialize_field("cpu_count", &cpu)?;
// }
//
// let mgmt_data = self.management.serialize_management();
// // pub management: Arc<dyn ManagementInterface>,
//
// // Handle management interface - either as a field or flattened
// state.serialize_field("management", &mgmt_data)?;
//
// state.end()
// }
// }
impl<'de> Deserialize<'de> for PhysicalHost {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
@@ -189,61 +280,10 @@ pub enum HostCategory {
Switch,
}
#[derive(Debug, new, Clone, Serialize)]
pub struct NetworkInterface {
pub name: Option<String>,
pub mac_address: MacAddress,
pub speed: Option<u64>,
}
#[cfg(test)]
use harmony_macros::mac_address;
use harmony_types::id::Id;
#[cfg(test)]
impl NetworkInterface {
pub fn dummy() -> Self {
Self {
name: Some(String::new()),
mac_address: mac_address!("00:00:00:00:00:00"),
speed: Some(0),
}
}
}
#[derive(Debug, new, Clone, Serialize)]
pub enum StorageConnectionType {
Sata3g,
Sata6g,
Sas6g,
Sas12g,
PCIE,
}
#[derive(Debug, Clone, Serialize)]
pub enum StorageKind {
SSD,
NVME,
HDD,
}
#[derive(Debug, new, Clone, Serialize)]
pub struct Storage {
pub connection: StorageConnectionType,
pub kind: StorageKind,
pub size: u64,
pub serial: String,
}
#[cfg(test)]
impl Storage {
pub fn dummy() -> Self {
Self {
connection: StorageConnectionType::Sata3g,
kind: StorageKind::SSD,
size: 0,
serial: String::new(),
}
}
}
#[derive(Debug, Clone, Serialize)]
pub struct Switch {
@@ -274,117 +314,43 @@ impl Location {
}
}
impl std::fmt::Display for HostCategory {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HostCategory::Server => write!(f, "Server"),
HostCategory::Firewall => write!(f, "Firewall"),
HostCategory::Switch => write!(f, "Switch"),
}
}
}
impl std::fmt::Display for Label {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: {}", self.name, self.value)
}
}
impl std::fmt::Display for Location {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Address: {}, Name: {}", self.address, self.name)
}
}
impl std::fmt::Display for PhysicalHost {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.summary())
}
}
impl std::fmt::Display for Switch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Switch with {} interfaces", self._interface.len())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
// Mock implementation of ManagementInterface
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MockHPIlo {
ip: String,
username: String,
password: String,
firmware_version: String,
}
impl ManagementInterface for MockHPIlo {
fn boot_to_pxe(&self) {}
fn get_supported_protocol_names(&self) -> String {
String::new()
}
}
// Another mock implementation
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MockDellIdrac {
hostname: String,
port: u16,
api_token: String,
}
impl ManagementInterface for MockDellIdrac {
fn boot_to_pxe(&self) {}
fn get_supported_protocol_names(&self) -> String {
String::new()
}
}
#[test]
fn test_serialize_physical_host_with_hp_ilo() {
// Create a PhysicalHost with HP iLO management
let host = PhysicalHost {
id: Id::empty(),
category: HostCategory::Server,
network: vec![NetworkInterface::dummy()],
management: Arc::new(MockHPIlo {
ip: "192.168.1.100".to_string(),
username: "admin".to_string(),
password: "password123".to_string(),
firmware_version: "2.5.0".to_string(),
}),
storage: vec![Storage::dummy()],
labels: vec![Label::new("datacenter".to_string(), "us-east".to_string())],
memory_size: Some(64_000_000),
cpu_count: Some(16),
};
// Serialize to JSON
let json = serde_json::to_string(&host).expect("Failed to serialize host");
// Check that the serialized JSON contains the HP iLO details
assert!(json.contains("192.168.1.100"));
assert!(json.contains("admin"));
assert!(json.contains("password123"));
assert!(json.contains("firmware_version"));
assert!(json.contains("2.5.0"));
// Parse back to verify structure (not the exact management interface)
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Failed to parse JSON");
// Verify basic structure
assert_eq!(parsed["cpu_count"], 16);
assert_eq!(parsed["memory_size"], 64_000_000);
assert_eq!(parsed["network"][0]["name"], "");
}
#[test]
fn test_serialize_physical_host_with_dell_idrac() {
// Create a PhysicalHost with Dell iDRAC management
let host = PhysicalHost {
id: Id::empty(),
category: HostCategory::Server,
network: vec![NetworkInterface::dummy()],
management: Arc::new(MockDellIdrac {
hostname: "idrac-server01".to_string(),
port: 443,
api_token: "abcdef123456".to_string(),
}),
storage: vec![Storage::dummy()],
labels: vec![Label::new("env".to_string(), "production".to_string())],
memory_size: Some(128_000_000),
cpu_count: Some(32),
};
// Serialize to JSON
let json = serde_json::to_string(&host).expect("Failed to serialize host");
// Check that the serialized JSON contains the Dell iDRAC details
assert!(json.contains("idrac-server01"));
assert!(json.contains("443"));
assert!(json.contains("abcdef123456"));
// Parse back to verify structure
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Failed to parse JSON");
// Verify basic structure
assert_eq!(parsed["cpu_count"], 32);
assert_eq!(parsed["memory_size"], 128_000_000);
assert_eq!(parsed["storage"][0]["path"], serde_json::Value::Null);
}
#[test]
fn test_different_management_implementations_produce_valid_json() {
@@ -393,31 +359,20 @@ mod tests {
id: Id::empty(),
category: HostCategory::Server,
network: vec![],
management: Arc::new(MockHPIlo {
ip: "10.0.0.1".to_string(),
username: "root".to_string(),
password: "secret".to_string(),
firmware_version: "3.0.0".to_string(),
}),
storage: vec![],
labels: vec![],
memory_size: None,
cpu_count: None,
memory_modules: vec![],
cpus: vec![],
};
let host2 = PhysicalHost {
id: Id::empty(),
category: HostCategory::Server,
network: vec![],
management: Arc::new(MockDellIdrac {
hostname: "server02-idrac".to_string(),
port: 8443,
api_token: "token123".to_string(),
}),
storage: vec![],
labels: vec![],
memory_size: None,
cpu_count: None,
memory_modules: vec![],
cpus: vec![],
};
// Both should serialize successfully
@@ -427,8 +382,5 @@ mod tests {
// Both JSONs should be valid and parseable
let _: serde_json::Value = serde_json::from_str(&json1).expect("Invalid JSON for host1");
let _: serde_json::Value = serde_json::from_str(&json2).expect("Invalid JSON for host2");
// The JSONs should be different because they contain different management interfaces
assert_ne!(json1, json2);
}
}

View File

@@ -18,6 +18,8 @@ impl InventoryFilter {
use derive_new::new;
use log::info;
use crate::hardware::{ManagementInterface, ManualManagementInterface};
use super::{
filter::Filter,
hardware::{FirewallGroup, HostGroup, Location, SwitchGroup},
@@ -30,7 +32,7 @@ pub struct Inventory {
// Firewall is really just a host but with somewhat specialized hardware
// I'm not entirely sure it belongs to its own category but it helps make things easier and
// clearer for now so let's try it this way.
pub firewall: FirewallGroup,
pub firewall_mgmt: Box<dyn ManagementInterface>,
pub worker_host: HostGroup,
pub storage_host: HostGroup,
pub control_plane_host: HostGroup,
@@ -41,7 +43,7 @@ impl Inventory {
Self {
location: Location::new("Empty".to_string(), "location".to_string()),
switch: vec![],
firewall: vec![],
firewall_mgmt: Box::new(ManualManagementInterface {}),
worker_host: vec![],
storage_host: vec![],
control_plane_host: vec![],
@@ -52,7 +54,7 @@ impl Inventory {
Self {
location: Location::test_building(),
switch: SwitchGroup::new(),
firewall: FirewallGroup::new(),
firewall_mgmt: Box::new(ManualManagementInterface {}),
worker_host: HostGroup::new(),
storage_host: HostGroup::new(),
control_plane_host: HostGroup::new(),

View File

@@ -1 +1,17 @@
mod sqlite;
use crate::{
config::DATABASE_URL,
infra::inventory::sqlite::SqliteInventoryRepository,
inventory::{InventoryRepository, RepoError},
};
pub mod sqlite;
pub struct InventoryRepositoryFactory;
impl InventoryRepositoryFactory {
pub async fn build() -> Result<Box<dyn InventoryRepository>, RepoError> {
Ok(Box::new(
SqliteInventoryRepository::new(&(*DATABASE_URL)).await?,
))
}
}

View File

@@ -19,11 +19,7 @@ impl SqliteInventoryRepository {
.await
.map_err(|e| RepoError::ConnectionFailed(e.to_string()))?;
todo!("make sure migrations are up to date");
info!(
"SQLite inventory repository initialized at '{}'",
database_url,
);
info!("SQLite inventory repository initialized at '{database_url}'");
Ok(Self { pool })
}
}
@@ -50,7 +46,7 @@ impl InventoryRepository for SqliteInventoryRepository {
}
async fn get_latest_by_id(&self, host_id: &str) -> Result<Option<PhysicalHost>, RepoError> {
let row = sqlx::query_as!(
let _row = sqlx::query_as!(
DbHost,
r#"SELECT id, version_id, data as "data: Json<PhysicalHost>" FROM physical_hosts WHERE id = ? ORDER BY version_id DESC LIMIT 1"#,
host_id

View File

@@ -1,10 +1,12 @@
use async_trait::async_trait;
use harmony_inventory_agent::local_presence::DiscoveryEvent;
use log::{debug, info};
use log::{debug, info, trace};
use serde::{Deserialize, Serialize};
use crate::{
data::Version,
hardware::{HostCategory, Label, PhysicalHost},
infra::inventory::InventoryRepositoryFactory,
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::Inventory,
score::Score,
@@ -41,20 +43,89 @@ struct DiscoverInventoryAgentInterpret {
impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
_inventory: &Inventory,
_topology: &T,
) -> Result<Outcome, InterpretError> {
harmony_inventory_agent::local_presence::discover_agents(
self.score.discovery_timeout,
|event: DiscoveryEvent| {
println!("Discovery event {event:?}");
|event: DiscoveryEvent| -> Result<(), String> {
debug!("Discovery event {event:?}");
match event {
DiscoveryEvent::ServiceResolved(service) => info!("Found instance {service:?}"),
DiscoveryEvent::ServiceResolved(service) => {
let service_name = service.fullname.clone();
info!("Found service {service_name}");
let address = match service.get_addresses().iter().next() {
Some(address) => address,
None => {
return Err(format!(
"Could not find address for service {service_name}"
));
}
};
let address = address.to_string();
let port = service.get_port();
tokio::task::spawn(async move {
info!("Getting inventory for host {address} at port {port}");
let host =
harmony_inventory_agent::client::get_host_inventory(&address, port)
.await
.unwrap();
trace!("Found host information {host:?}");
// TODO its useless to have two distinct host types but requires a bit much
// refactoring to do it now
let harmony_inventory_agent::hwinfo::PhysicalHost {
storage_drives,
storage_controller,
memory_modules,
cpus,
chipset,
network_interfaces,
management_interface,
host_uuid,
} = host;
let host = PhysicalHost {
id: Id::from(host_uuid),
category: HostCategory::Server,
network: network_interfaces,
storage: storage_drives,
labels: vec![Label {
name: "discovered-by".to_string(),
value: "harmony-inventory-agent".to_string(),
}],
memory_modules,
cpus,
};
let repo = InventoryRepositoryFactory::build()
.await
.map_err(|e| format!("Could not build repository : {e}"))
.unwrap();
repo.save(&host)
.await
.map_err(|e| format!("Could not save host : {e}"))
.unwrap();
info!(
"Saved new host id {}, summary : {}",
host.id,
host.summary()
);
});
}
_ => debug!("Unhandled event {event:?}"),
}
};
Ok(())
},
);
todo!()
)
.await;
Ok(Outcome {
status: InterpretStatus::SUCCESS,
message: "Discovery process completed successfully".to_string(),
})
}
fn get_name(&self) -> InterpretName {