From e548bf619a19e5f0d43d39432c1370e859ccc9d0 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 30 Aug 2025 20:01:52 -0400 Subject: [PATCH 1/8] feat: Can now discover inventory agent and download its host definition, next up save it to db --- Cargo.lock | 3 ++ Cargo.toml | 2 +- harmony/src/domain/hardware/mod.rs | 31 ++++--------- harmony/src/infra/inventory/sqlite.rs | 7 ++- harmony/src/modules/inventory/mod.rs | 46 ++++++++++++++++--- harmony_inventory_agent/Cargo.toml | 3 ++ harmony_inventory_agent/src/client.rs | 14 ++++++ harmony_inventory_agent/src/hwinfo.rs | 25 +++++++++- harmony_inventory_agent/src/lib.rs | 3 +- .../src/local_presence/discover.rs | 14 ++++-- harmony_types/src/net.rs | 28 ++++++++++- 11 files changed, 135 insertions(+), 41 deletions(-) create mode 100644 harmony_inventory_agent/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index a787f9e..62d8aee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2366,9 +2366,12 @@ version = "0.1.0" dependencies = [ "actix-web", "env_logger", + "harmony_macros", + "harmony_types", "local-ip-address", "log", "mdns-sd 0.14.1 (git+https://github.com/jggc/mdns-sd.git?branch=patch-1)", + "reqwest 0.12.20", "serde", "serde_json", "sysinfo", diff --git a/Cargo.toml b/Cargo.toml index 6be0aa9..d92c0e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,4 +67,4 @@ serde = { version = "1.0.209", features = ["derive", "rc"] } serde_json = "1.0.127" askama = "0.14" sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite" ] } -reqwest = { version = "0.12", features = ["stream", "rustls-tls", "http2"], default-features = false } +reqwest = { version = "0.12", features = ["blocking", "stream", "rustls-tls", "http2", "json"], default-features = false } diff --git a/harmony/src/domain/hardware/mod.rs b/harmony/src/domain/hardware/mod.rs index 20c3596..eb1e760 100644 --- a/harmony/src/domain/hardware/mod.rs +++ b/harmony/src/domain/hardware/mod.rs @@ -4,6 +4,7 @@ use derive_new::new; use harmony_types::net::MacAddress; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use serde_value::Value; +use harmony_inventory_agent::hwinfo::NetworkInterface; pub type HostGroup = Vec; pub type SwitchGroup = Vec; @@ -70,9 +71,15 @@ impl PhysicalHost { pub fn mac_address(mut self, mac_address: MacAddress) -> Self { self.network.push(NetworkInterface { - name: None, + name: String::new(), mac_address, - speed: None, + speed_mbps: None, + is_up: false, + mtu: 0, + ipv4_addresses: vec![], + ipv6_addresses: vec![], + driver: String::new(), + firmware_version: None, }); self } @@ -131,7 +138,7 @@ impl Serialize for PhysicalHost { } impl<'de> Deserialize<'de> for PhysicalHost { - fn deserialize(deserializer: D) -> Result + fn deserialize(_deserializer: D) -> Result where D: serde::Deserializer<'de>, { @@ -189,28 +196,10 @@ pub enum HostCategory { Switch, } -#[derive(Debug, new, Clone, Serialize)] -pub struct NetworkInterface { - pub name: Option, - pub mac_address: MacAddress, - pub speed: Option, -} - #[cfg(test)] use harmony_macros::mac_address; use harmony_types::id::Id; -#[cfg(test)] -impl NetworkInterface { - pub fn dummy() -> Self { - Self { - name: Some(String::new()), - mac_address: mac_address!("00:00:00:00:00:00"), - speed: Some(0), - } - } -} - #[derive(Debug, new, Clone, Serialize)] pub enum StorageConnectionType { Sata3g, diff --git a/harmony/src/infra/inventory/sqlite.rs b/harmony/src/infra/inventory/sqlite.rs index d079996..073da78 100644 --- a/harmony/src/infra/inventory/sqlite.rs +++ b/harmony/src/infra/inventory/sqlite.rs @@ -15,16 +15,15 @@ pub struct SqliteInventoryRepository { impl SqliteInventoryRepository { pub async fn new(database_url: &str) -> Result { - let pool = SqlitePool::connect(database_url) + let _pool = SqlitePool::connect(database_url) .await .map_err(|e| RepoError::ConnectionFailed(e.to_string()))?; - todo!("make sure migrations are up to date"); info!( "SQLite inventory repository initialized at '{}'", database_url, ); - Ok(Self { pool }) + Ok(Self { pool: _pool }) } } @@ -50,7 +49,7 @@ impl InventoryRepository for SqliteInventoryRepository { } async fn get_latest_by_id(&self, host_id: &str) -> Result, RepoError> { - let row = sqlx::query_as!( + let _row = sqlx::query_as!( DbHost, r#"SELECT id, version_id, data as "data: Json" FROM physical_hosts WHERE id = ? ORDER BY version_id DESC LIMIT 1"#, host_id diff --git a/harmony/src/modules/inventory/mod.rs b/harmony/src/modules/inventory/mod.rs index e8bbd71..4e0b4a0 100644 --- a/harmony/src/modules/inventory/mod.rs +++ b/harmony/src/modules/inventory/mod.rs @@ -1,10 +1,11 @@ use async_trait::async_trait; use harmony_inventory_agent::local_presence::DiscoveryEvent; -use log::{debug, info}; +use log::{debug, info, trace}; use serde::{Deserialize, Serialize}; use crate::{ data::Version, + hardware::{HostCategory, PhysicalHost}, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, @@ -41,17 +42,50 @@ struct DiscoverInventoryAgentInterpret { impl Interpret for DiscoverInventoryAgentInterpret { async fn execute( &self, - inventory: &Inventory, - topology: &T, + _inventory: &Inventory, + _topology: &T, ) -> Result { harmony_inventory_agent::local_presence::discover_agents( self.score.discovery_timeout, - |event: DiscoveryEvent| { + |event: DiscoveryEvent| -> Result<(), String> { println!("Discovery event {event:?}"); match event { - DiscoveryEvent::ServiceResolved(service) => info!("Found instance {service:?}"), + DiscoveryEvent::ServiceResolved(service) => { + debug!("Found instance {service:?}"); + let address = match service.get_addresses().iter().next() { + Some(address) => address, + None => { + return Err( + "Could not find address for service {service:?}".to_string() + ); + } + }; + + let address = &address.to_string(); + let port = service.get_port(); + + debug!("Getting host inventory on service at {address} port {port}"); + + let host = + harmony_inventory_agent::client::get_host_inventory(address, port)?; + + trace!("Found host information {host:?}"); + // TODO its useless to have two distinct host types but requires a bit much + // refactoring to do it now + let host = PhysicalHost { + id: Id::from(host.host_uuid), + category: HostCategory::Server, + network: todo!(), + management: todo!(), + storage: todo!(), + labels: todo!(), + memory_size: todo!(), + cpu_count: todo!(), + }; + } _ => debug!("Unhandled event {event:?}"), - } + }; + Ok(()) }, ); todo!() diff --git a/harmony_inventory_agent/Cargo.toml b/harmony_inventory_agent/Cargo.toml index 9ffe37e..6952925 100644 --- a/harmony_inventory_agent/Cargo.toml +++ b/harmony_inventory_agent/Cargo.toml @@ -12,6 +12,9 @@ log.workspace = true env_logger.workspace = true tokio.workspace = true thiserror.workspace = true +reqwest.workspace = true # mdns-sd = "0.14.1" mdns-sd = { git = "https://github.com/jggc/mdns-sd.git", branch = "patch-1" } local-ip-address = "0.6.5" +harmony_types = { path = "../harmony_types" } +harmony_macros = { path = "../harmony_macros" } diff --git a/harmony_inventory_agent/src/client.rs b/harmony_inventory_agent/src/client.rs new file mode 100644 index 0000000..80e7c5e --- /dev/null +++ b/harmony_inventory_agent/src/client.rs @@ -0,0 +1,14 @@ +use crate::hwinfo::PhysicalHost; + +pub fn get_host_inventory(host: &str, port: u16) -> Result { + let url = format!("http://{host}:{port}/inventory"); + let client = reqwest::blocking::Client::new(); + let response = client + .get(url) + .send() + .map_err(|e| format!("Failed to download file: {e}"))?; + + let host = response.json().map_err(|e| e.to_string())?; + + Ok(host) +} diff --git a/harmony_inventory_agent/src/hwinfo.rs b/harmony_inventory_agent/src/hwinfo.rs index d381a14..b113e96 100644 --- a/harmony_inventory_agent/src/hwinfo.rs +++ b/harmony_inventory_agent/src/hwinfo.rs @@ -1,3 +1,4 @@ +use harmony_types::net::MacAddress; use log::{debug, warn}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -63,10 +64,10 @@ pub struct Chipset { pub vendor: String, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct NetworkInterface { pub name: String, - pub mac_address: String, + pub mac_address: MacAddress, pub speed_mbps: Option, pub is_up: bool, pub mtu: u32, @@ -76,6 +77,25 @@ pub struct NetworkInterface { pub firmware_version: Option, } +#[cfg(test)] +impl NetworkInterface { + pub fn dummy() -> Self { + use harmony_macros::mac_address; + + Self { + name: String::new(), + mac_address: mac_address!("00:00:00:00:00:00"), + speed_mbps: Some(0), + is_up: false, + mtu: 0, + ipv4_addresses: vec![], + ipv6_addresses: vec![], + driver: String::new(), + firmware_version: None, + } + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct ManagementInterface { pub kind: String, @@ -509,6 +529,7 @@ impl PhysicalHost { let mac_address = Self::read_sysfs_string(&iface_path.join("address")) .map_err(|e| format!("Failed to read MAC address for {}: {}", iface_name, e))?; + let mac_address = MacAddress::try_from(mac_address).map_err(|e| e.to_string())?; let speed_mbps = if iface_path.join("speed").exists() { match Self::read_sysfs_u32(&iface_path.join("speed")) { diff --git a/harmony_inventory_agent/src/lib.rs b/harmony_inventory_agent/src/lib.rs index cbf208b..4f4d153 100644 --- a/harmony_inventory_agent/src/lib.rs +++ b/harmony_inventory_agent/src/lib.rs @@ -1,2 +1,3 @@ -mod hwinfo; +pub mod hwinfo; pub mod local_presence; +pub mod client; diff --git a/harmony_inventory_agent/src/local_presence/discover.rs b/harmony_inventory_agent/src/local_presence/discover.rs index a2ae216..246b202 100644 --- a/harmony_inventory_agent/src/local_presence/discover.rs +++ b/harmony_inventory_agent/src/local_presence/discover.rs @@ -1,10 +1,14 @@ +use log::{debug, error}; use mdns_sd::{ServiceDaemon, ServiceEvent}; use crate::local_presence::SERVICE_NAME; pub type DiscoveryEvent = ServiceEvent; -pub fn discover_agents(timeout: Option, on_event: impl Fn(DiscoveryEvent) + Send + 'static) { +pub fn discover_agents( + timeout: Option, + on_event: impl Fn(DiscoveryEvent) -> Result<(), String> + Send + 'static, +) { // Create a new mDNS daemon. let mdns = ServiceDaemon::new().expect("Failed to create mDNS daemon"); @@ -14,13 +18,15 @@ pub fn discover_agents(timeout: Option, on_event: impl Fn(DiscoveryEvent) + std::thread::spawn(move || { while let Ok(event) = receiver.recv() { - on_event(event.clone()); + if let Err(e) = on_event(event.clone()) { + error!("Event callback failed : {e}"); + } match event { ServiceEvent::ServiceResolved(resolved) => { - println!("Resolved a new service: {}", resolved.fullname); + debug!("Resolved a new service: {}", resolved.fullname); } other_event => { - println!("Received other event: {:?}", &other_event); + debug!("Received other event: {:?}", &other_event); } } } diff --git a/harmony_types/src/net.rs b/harmony_types/src/net.rs index e2905a1..caf023f 100644 --- a/harmony_types/src/net.rs +++ b/harmony_types/src/net.rs @@ -1,6 +1,6 @@ -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct MacAddress(pub [u8; 6]); impl MacAddress { @@ -25,6 +25,30 @@ impl std::fmt::Display for MacAddress { } } +impl TryFrom for MacAddress { + type Error = std::io::Error; + + fn try_from(value: String) -> Result { + let parts: Vec<&str> = value.split(':').collect(); + if parts.len() != 6 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Invalid MAC address format: expected 6 colon-separated hex pairs", + )); + } + let mut bytes = [0u8; 6]; + for (i, part) in parts.iter().enumerate() { + bytes[i] = u8::from_str_radix(part, 16).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("Invalid hex value in part {}: '{}'", i, part), + ) + })?; + } + Ok(MacAddress(bytes)) + } +} + pub type IpAddress = std::net::IpAddr; #[derive(Debug, Clone)] -- 2.39.5 From d9c26f43ee7f8c8a4e371b417cb1fe15f8dc20cb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 31 Aug 2025 00:31:55 -0400 Subject: [PATCH 2/8] wip: saving harmony inventory, currently messing with async stuff, properly understanding stuff now so I should fix it soon. The recv in the inventory agent is sync and blocking the whole thread so the request cannot be sent until the recv is killed, which is wrong. Will fix this by isolating on another thread --- examples/nanodc/src/main.rs | 3 +- examples/okd_pxe/src/topology.rs | 3 +- examples/opnsense/src/main.rs | 3 +- harmony/src/domain/config.rs | 8 + harmony/src/domain/hardware/mod.rs | 242 +++++++----------- harmony/src/domain/inventory/mod.rs | 8 +- harmony/src/infra/inventory/mod.rs | 18 +- harmony/src/modules/inventory/mod.rs | 94 +++++-- harmony_inventory_agent/src/client.rs | 7 +- harmony_inventory_agent/src/hwinfo.rs | 24 +- harmony_inventory_agent/src/lib.rs | 2 +- .../src/local_presence/discover.rs | 13 +- 12 files changed, 233 insertions(+), 192 deletions(-) diff --git a/examples/nanodc/src/main.rs b/examples/nanodc/src/main.rs index a8a17e3..a6bb8e4 100644 --- a/examples/nanodc/src/main.rs +++ b/examples/nanodc/src/main.rs @@ -87,8 +87,7 @@ async fn main() { let inventory = Inventory { location: Location::new("I am mobile".to_string(), "earth".to_string()), switch: SwitchGroup::from([]), - firewall: FirewallGroup::from([PhysicalHost::empty(HostCategory::Firewall) - .management(Arc::new(OPNSenseManagementInterface::new()))]), + firewall_mgmt: Box::new(OPNSenseManagementInterface::new()), storage_host: vec![], worker_host: vec![ PhysicalHost::empty(HostCategory::Server) diff --git a/examples/okd_pxe/src/topology.rs b/examples/okd_pxe/src/topology.rs index eb23908..27eb8c0 100644 --- a/examples/okd_pxe/src/topology.rs +++ b/examples/okd_pxe/src/topology.rs @@ -69,8 +69,7 @@ pub fn get_inventory() -> Inventory { "testopnsense".to_string(), ), switch: SwitchGroup::from([]), - firewall: FirewallGroup::from([PhysicalHost::empty(HostCategory::Firewall) - .management(Arc::new(OPNSenseManagementInterface::new()))]), + firewall_mgmt: Box::new(OPNSenseManagementInterface::new()), storage_host: vec![], worker_host: vec![], control_plane_host: vec![], diff --git a/examples/opnsense/src/main.rs b/examples/opnsense/src/main.rs index 3af30cf..465b0fa 100644 --- a/examples/opnsense/src/main.rs +++ b/examples/opnsense/src/main.rs @@ -63,8 +63,7 @@ async fn main() { "wk".to_string(), ), switch: SwitchGroup::from([]), - firewall: FirewallGroup::from([PhysicalHost::empty(HostCategory::Firewall) - .management(Arc::new(OPNSenseManagementInterface::new()))]), + firewall_mgmt: Box::new(OPNSenseManagementInterface::new()), storage_host: vec![], worker_host: vec![], control_plane_host: vec![ diff --git a/harmony/src/domain/config.rs b/harmony/src/domain/config.rs index 62f612f..1a91684 100644 --- a/harmony/src/domain/config.rs +++ b/harmony/src/domain/config.rs @@ -12,4 +12,12 @@ lazy_static! { std::env::var("HARMONY_REGISTRY_PROJECT").unwrap_or_else(|_| "harmony".to_string()); pub static ref DRY_RUN: bool = std::env::var("HARMONY_DRY_RUN").is_ok_and(|value| value.parse().unwrap_or(false)); + pub static ref DEFAULT_DATABASE_URL: String = "sqlite://harmony.sqlite".to_string(); + pub static ref DATABASE_URL: String = std::env::var("HARMONY_DATABASE_URL") + .map(|value| if value.is_empty() { + (*DEFAULT_DATABASE_URL).clone() + } else { + value + }) + .unwrap_or((*DEFAULT_DATABASE_URL).clone()); } diff --git a/harmony/src/domain/hardware/mod.rs b/harmony/src/domain/hardware/mod.rs index eb1e760..20038d2 100644 --- a/harmony/src/domain/hardware/mod.rs +++ b/harmony/src/domain/hardware/mod.rs @@ -1,25 +1,24 @@ -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use derive_new::new; +use harmony_inventory_agent::hwinfo::{CPU, MemoryModule, NetworkInterface, StorageDrive}; use harmony_types::net::MacAddress; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use serde_value::Value; -use harmony_inventory_agent::hwinfo::NetworkInterface; pub type HostGroup = Vec; pub type SwitchGroup = Vec; pub type FirewallGroup = Vec; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct PhysicalHost { pub id: Id, pub category: HostCategory, pub network: Vec, - pub management: Arc, - pub storage: Vec, + pub storage: Vec, pub labels: Vec