feat: Add verification of opnsense package installation, fix opnsense-config tests, add log file to tui

harmony-rs/harmony/src/domain/topology/load_balancer.rs

@@ -20,7 +20,7 @@ pub trait LoadBalancer: Send + Sync {
         &self,
         service: &LoadBalancerService,
     ) -> Result<(), ExecutorError> {
-        debug!("Listing haproxy services {:?}", self.list_services().await);
+        debug!("Listing LoadBalancer services {:?}", self.list_services().await);
         if !self.list_services().await.contains(service) {
             self.add_service(service).await?;
         }
@@ -60,6 +60,7 @@ impl From<String> for HttpMethod {
         Self::from_str(&value).unwrap()
     }
 }
+
 impl FromStr for HttpMethod {
     type Err = String;
 

harmony-rs/harmony/src/modules/load_balancer.rs

@@ -57,7 +57,11 @@ impl Interpret for LoadBalancerInterpret {
         _inventory: &Inventory,
         topology: &HAClusterTopology,
     ) -> Result<Outcome, InterpretError> {
-        topology.load_balancer.ensure_initialized().await?;
+        info!(
+            "Making sure Load Balancer is initialized: {:?}",
+            topology.load_balancer.ensure_initialized().await?
+        );
+
         for service in self.score.public_services.iter() {
             info!("Ensuring service exists {service:?}");
             topology

harmony-rs/harmony_tui/src/lib.rs

@@ -101,6 +101,7 @@ impl HarmonyTUI {
         tui_logger::init_logger(log::LevelFilter::Info).unwrap();
         // Set default level for unknown targets to Trace
         tui_logger::set_default_level(log::LevelFilter::Info);
+        tui_logger::set_log_file("harmony.log").unwrap();
 
         color_eyre::install()?;
         let mut terminal = ratatui::init();

harmony-rs/opnsense-config-xml/src/data/opnsense.rs

@@ -12,7 +12,7 @@ use super::{Interface, Pischem};
 pub struct OPNsense {
     pub theme: String,
     pub sysctl: Sysctl,
-    pub system: RawXml,
+    pub system: System,
     // pub interfaces: RawXml,
     pub interfaces: NamedList<Interface>,
     pub dhcpd: NamedList<DhcpInterface>,
@@ -172,10 +172,8 @@ pub struct SysctlItem {
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct System {
-    #[yaserde(rename = "use_mfs_tmp")]
+    pub use_mfs_tmp: Option<MaybeString>,
-    pub use_mfs_tmp: MaybeString,
+    pub use_mfs_var: Option<MaybeString>,
-    #[yaserde(rename = "use_mfs_var")]
-    pub use_mfs_var: MaybeString,
     pub serialspeed: u32,
     pub primaryconsole: String,
     pub secondaryconsole: String,
@@ -190,6 +188,7 @@ pub struct System {
     pub timeservers: String,
     pub webgui: WebGui,
     pub usevirtualterminal: u8,
+    pub disablenatreflection: String,
     pub disableconsolemenu: u8,
     pub disablevlanhwfilter: u8,
     pub disablechecksumoffloading: u8,
@@ -200,40 +199,34 @@ pub struct System {
     pub powerd_battery_mode: String,
     pub powerd_normal_mode: String,
     pub bogons: Bogons,
-    pub crypto_hardware: String,
+    pub crypto_hardware: Option<String>,
     pub pf_share_forward: u8,
     pub lb_use_sticky: u8,
-    pub kill_states: u8,
+    pub kill_states: Option<u8>,
     pub ssh: Ssh,
+    pub rrdbackup: Option<i8>,
+    pub netflowbackup: Option<i8>,
     pub firmware: Firmware,
-    pub sudo_allow_wheel: u8,
+    pub sudo_allow_wheel: Option<u8>,
-    pub sudo_allow_group: String,
+    pub sudo_allow_group: Option<String>,
-    pub enablenatreflectionhelper: String,
+    pub enablenatreflectionhelper: Option<String>,
-    pub rulesetoptimization: String,
+    pub rulesetoptimization: Option<String>,
-    pub maximumstates: MaybeString,
+    pub maximumstates: Option<MaybeString>,
-    pub maximumfrags: MaybeString,
+    pub maximumfrags: Option<MaybeString>,
-    pub aliasesresolveinterval: MaybeString,
+    pub aliasesresolveinterval: Option<MaybeString>,
-    pub maximumtableentries: MaybeString,
+    pub maximumtableentries: Option<MaybeString>,
     pub language: String,
     pub dnsserver: MaybeString,
-    #[yaserde(rename = "dns1gw")]
+    pub dns1gw: Option<String>,
-    pub dns1gw: String,
+    pub dns2gw: Option<String>,
-    #[yaserde(rename = "dns2gw")]
+    pub dns3gw: Option<String>,
-    pub dns2gw: String,
+    pub dns4gw: Option<String>,
-    #[yaserde(rename = "dns3gw")]
+    pub dns5gw: Option<String>,
-    pub dns3gw: String,
+    pub dns6gw: Option<String>,
-    #[yaserde(rename = "dns4gw")]
+    pub dns7gw: Option<String>,
-    pub dns4gw: String,
+    pub dns8gw: Option<String>,
-    #[yaserde(rename = "dns5gw")]
-    pub dns5gw: String,
-    #[yaserde(rename = "dns6gw")]
-    pub dns6gw: String,
-    #[yaserde(rename = "dns7gw")]
-    pub dns7gw: String,
-    #[yaserde(rename = "dns8gw")]
-    pub dns8gw: String,
     pub dnsallowoverride: u8,
-    pub dnsallowoverride_exclude: MaybeString,
+    pub dnsallowoverride_exclude: Option<MaybeString>,
 }
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
@@ -257,10 +250,11 @@ pub struct Firmware {
     pub version: String,
     pub mirror: MaybeString,
     pub flavour: MaybeString,
-    pub plugins: String,
+    pub plugins: MaybeString,
     #[yaserde(rename = "type")]
     pub firmware_type: MaybeString,
     pub subscription: MaybeString,
+    pub reboot: MaybeString,
 }
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
@@ -284,14 +278,15 @@ pub struct User {
     pub name: String,
     pub descr: MaybeString,
     pub scope: String,
-    pub groupname: MaybeString,
+    pub groupname: Option<MaybeString>,
     pub password: String,
     pub uid: u32,
-    pub expires: MaybeString,
+    pub expires: Option<MaybeString>,
-    pub authorizedkeys: MaybeString,
+    pub authorizedkeys: Option<MaybeString>,
-    pub ipsecpsk: MaybeString,
+    pub dashboard: Option<MaybeString>,
-    pub otp_seed: MaybeString,
+    pub ipsecpsk: Option<MaybeString>,
-    pub shell: MaybeString,
+    pub otp_seed: Option<MaybeString>,
+    pub shell: Option<MaybeString>,
 }
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
@@ -303,8 +298,8 @@ pub struct WebGui {
     #[yaserde(rename = "ssl-ciphers")]
     pub ssl_ciphers: MaybeString,
     pub interfaces: MaybeString,
-
     pub compression: MaybeString,
+    pub nohttpreferercheck: Option<u8>,
 }
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
@@ -876,7 +871,7 @@ pub struct Servers {}
 pub struct Proxy {
     #[yaserde(attribute = true)]
     pub version: String,
-    pub general: ConfigGeneral,
+    pub general: ProxyGeneral,
     pub forward: Forward,
     pub pac: Option<Pac>,
     #[yaserde(rename = "error_pages")]
@@ -884,7 +879,7 @@ pub struct Proxy {
 }
 
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
-pub struct ConfigGeneral {
+pub struct ProxyGeneral {
     pub enabled: i8,
     pub error_pages: String,
     #[yaserde(rename = "icpPort")]
@@ -919,7 +914,7 @@ pub struct ConfigGeneral {
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct Logging {
     pub enable: Enable,
-    #[yaserde(rename = "ifnoreLogACL")]
+    #[yaserde(rename = "ignoreLogACL")]
     pub ignore_log_acl: MaybeString,
     pub target: MaybeString,
 }

harmony-rs/opnsense-config/src/config/config.rs

@@ -8,7 +8,7 @@ use crate::{
         tftp::TftpConfig,
     },
 };
-use log::{info, trace};
+use log::{debug, info, trace, warn};
 use opnsense_config_xml::OPNsense;
 use russh::client;
 
@@ -57,6 +57,30 @@ impl Config {
         self.shell.upload_folder(source, destination).await
     }
 
+    /// Checks in config file if system.firmware.plugins csv field contains the specified package
+    /// name.
+    ///
+    /// Given this
+    /// ```xml
+    /// <opnsense>
+    ///     <system>
+    ///         <firmware>
+    ///             <plugins>os-haproxy,os-iperf,os-cpu-microcode-intel</plugins>
+    ///         </firmware>
+    ///     </system>
+    /// </opnsense>
+    /// ```
+    ///
+    /// is_package_installed("os-cpu"); // false
+    /// is_package_installed("os-haproxy"); // true
+    /// is_package_installed("os-cpu-microcode-intel"); // true
+    pub fn is_package_installed(&self, package_name: &str) -> bool {
+        match &self.opnsense.system.firmware.plugins.content {
+            Some(plugins) => is_package_in_csv(plugins, package_name),
+            None => false,
+        }
+    }
+
     // Here maybe we should take ownership of `mut self` instead of `&mut self`
     // I don't think there can be faulty pointers to previous versions of the config but I have a
     // hard time wrapping my head around it right now :
@@ -70,11 +94,35 @@ impl Config {
     // read-only reference across the &mut call
     pub async fn install_package(&mut self, package_name: &str) -> Result<(), Error> {
         info!("Installing opnsense package {package_name}");
+        self.check_pkg_opnsense_org_connection().await?;
+
         let output = self.shell
             .exec(&format!("/bin/sh -c \"export LOCKFILE=/dev/stdout && /usr/local/opnsense/scripts/firmware/install.sh {package_name}\""))
             .await?;
         info!("Installation output {output}");
+
         self.reload_config().await?;
+        let is_installed = self.is_package_installed(package_name);
+        debug!("Verifying package installed successfully {is_installed}");
+
+        if !is_installed {
+            info!("Installation successful for {package_name}");
+            Ok(())
+        } else {
+            let msg = format!("Package installation failed for {package_name}, see above logs");
+            warn!("{}", msg);
+            Err(Error::Unexpected(msg))
+        }
+    }
+
+    pub async fn check_pkg_opnsense_org_connection(&mut self) -> Result<(), Error> {
+        let pkg_url = "https://pkg.opnsense.org";
+        info!("Verifying connection to {pkg_url}");
+        let output = self
+            .shell
+            .exec(&format!("/bin/sh -c \"curl -v {pkg_url}\""))
+            .await?;
+        info!("{}", output);
         Ok(())
     }
 
@@ -150,8 +198,8 @@ mod tests {
     async fn test_load_config_from_local_file() {
         for path in vec![
             "src/tests/data/config-vm-test.xml",
-            "src/tests/data/config-full-1.xml",
             "src/tests/data/config-structure.xml",
+            "src/tests/data/config-full-1.xml",
         ] {
             let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
             test_file_path.push(path);
@@ -217,3 +265,53 @@ mod tests {
         assert_eq!(expected_config_file_str, serialized);
     }
 }
+
+/// Checks if a given package name exists in a comma-separated list of packages.
+///
+/// # Arguments
+///
+/// * `csv_string` - A string containing comma-separated package names.
+/// * `package_name` - The package name to search for.
+///
+/// # Returns
+///
+/// * `true` if the package name is found in the CSV string, `false` otherwise.
+fn is_package_in_csv(csv_string: &str, package_name: &str) -> bool {
+    package_name.len() > 0 && csv_string.split(',').any(|pkg| pkg.trim() == package_name)
+}
+
+#[cfg(test)]
+mod tests_2 {
+    use super::*;
+
+    #[test]
+    fn test_is_package_in_csv() {
+        let csv_string = "os-haproxy,os-iperf,os-cpu-microcode-intel";
+
+        assert!(is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(is_package_in_csv(csv_string, "os-iperf"));
+        assert!(is_package_in_csv(csv_string, "os-cpu-microcode-intel"));
+
+        assert!(!is_package_in_csv(csv_string, "os-cpu"));
+        assert!(!is_package_in_csv(csv_string, "non-existent-package"));
+    }
+
+    #[test]
+    fn test_is_package_in_csv_empty() {
+        let csv_string = "";
+
+        assert!(!is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(!is_package_in_csv(csv_string, ""));
+    }
+
+    #[test]
+    fn test_is_package_in_csv_whitespace() {
+        let csv_string = " os-haproxy , os-iperf , os-cpu-microcode-intel ";
+
+        assert!(is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(is_package_in_csv(csv_string, "os-iperf"));
+        assert!(is_package_in_csv(csv_string, "os-cpu-microcode-intel"));
+
+        assert!(!is_package_in_csv(csv_string, " os-haproxy "));
+    }
+}

harmony-rs/opnsense-config/src/lib.rs

@@ -35,6 +35,7 @@ mod test {
     async fn initialize_config() -> Config {
         Config::from_credentials(
             std::net::IpAddr::V4(Ipv4Addr::new(192, 168, 5, 229)),
+            None,
             "root",
             "opnsense",
         )

harmony-rs/opnsense-config/src/modules/dhcp.rs

@@ -184,37 +184,3 @@ impl<'a> DhcpConfig<'a> {
     }
 }
 
-#[cfg(test)]
-mod test {
-    use super::*;
-    use pretty_assertions::assert_eq;
-    use std::net::Ipv4Addr;
-
-    #[test]
-    fn test_ip_in_range() {
-        let range = Range {
-            from: "192.168.1.100".to_string(),
-            to: "192.168.1.200".to_string(),
-        };
-
-        // Test IP within range
-        let ip = "192.168.1.150".parse::<Ipv4Addr>().unwrap();
-        assert_eq!(DhcpConfig::is_ip_in_range(&ip, &range), true);
-
-        // Test IP at start of range
-        let ip = "192.168.1.100".parse::<Ipv4Addr>().unwrap();
-        assert_eq!(DhcpConfig::is_ip_in_range(&ip, &range), true);
-
-        // Test IP at end of range
-        let ip = "192.168.1.200".parse::<Ipv4Addr>().unwrap();
-        assert_eq!(DhcpConfig::is_ip_in_range(&ip, &range), true);
-
-        // Test IP before range
-        let ip = "192.168.1.99".parse::<Ipv4Addr>().unwrap();
-        assert_eq!(DhcpConfig::is_ip_in_range(&ip, &range), false);
-
-        // Test IP after range
-        let ip = "192.168.1.201".parse::<Ipv4Addr>().unwrap();
-        assert_eq!(DhcpConfig::is_ip_in_range(&ip, &range), false);
-    }
-}

harmony-rs/opnsense-config/src/tests/data/config-full-1.xml

@@ -257,6 +257,7 @@
     </webgui>
     <usevirtualterminal>1</usevirtualterminal>
     <disableconsolemenu>1</disableconsolemenu>
+    <disablenatreflection>yes</disablenatreflection>
     <disablevlanhwfilter>1</disablevlanhwfilter>
     <disablechecksumoffloading>1</disablechecksumoffloading>
     <disablesegmentationoffloading>1</disablesegmentationoffloading>
@@ -291,6 +292,7 @@
       <plugins>os-ddclient,os-dyndns,os-haproxy,os-wireguard</plugins>
       <type/>
       <subscription/>
+      <reboot/>
     </firmware>
     <sudo_allow_wheel>1</sudo_allow_wheel>
     <sudo_allow_group>admins</sudo_allow_group>
@@ -2113,6 +2115,8 @@
             <tunneladdress>172.12.0.1/24</tunneladdress>
             <disableroutes>0</disableroutes>
             <gateway/>
+            <endpoint/>
+            <peer_dns/>
             <carp_depend_on/>
             <peers>03031aec-2e84-462e-9eab-57762dde667a,98e6ca3d-1de9-449b-be80-77022221b509,67c0ace5-e802-4d2b-a536-f8b7a2db6f99,74b60fff-7844-4097-9966-f1c2b1ad29ff,3de82ad5-bc1b-4b91-9598-f906e58ac937,a95e6b5e-24a4-40b5-bb41-b79e784f6f1c,6c9a12c6-c1ca-4c14-866b-975406a30590,c33b308b-7125-4688-9561-989ace8787b5,e43f004a-23bf-4027-8fb0-953fbb40479f</peers>
           </server>
@@ -2282,10 +2286,14 @@
           <sslServerVerify>ignore</sslServerVerify>
           <maxDHSize>2048</maxDHSize>
           <bufferSize>16384</bufferSize>
+          <resolversPrefer>ipv4</resolversPrefer>
           <spreadChecks>2</spreadChecks>
           <bogusProxyEnabled>0</bogusProxyEnabled>
           <luaMaxMem>0</luaMaxMem>
           <customOptions/>
+          <ocspUpdateEnabled>0</ocspUpdateEnabled>
+          <ocspUpdateMinDelay>300</ocspUpdateMinDelay>
+          <ocspUpdateMaxDelay>3600</ocspUpdateMaxDelay>
           <ssl_defaultsEnabled>0</ssl_defaultsEnabled>
           <ssl_bindOptions>prefer-client-ciphers</ssl_bindOptions>
           <ssl_minVersion>TLSv1.2</ssl_minVersion>

harmony-rs/opnsense-config/src/tests/data/config-structure-with-dhcp-staticmap-entry.xml

@@ -44,16 +44,16 @@
       <otp_seed/>
     </user>
     <user>
-      <password>$2y$11$55555555556D8198uOASIDJaiojdjd1oijdijosaoijdaoidOIASJDoijdoiadOASdoiK</password>
-      <scope>user</scope>
       <name>someuser</name>
       <descr/>
+      <scope>user</scope>
+      <password>$2y$11$55555555556D8198uOASIDJaiojdjd1oijdijosaoijdaoidOIASJDoijdoiadOASdoiK</password>
+      <uid>2000</uid>
       <expires/>
       <authorizedkeys/>
       <ipsecpsk/>
       <otp_seed/>
       <shell>/bin/sh</shell>
-      <uid>2000</uid>
     </user>
     <nextuid>2001</nextuid>
     <nextgid>2000</nextgid>
@@ -68,6 +68,7 @@
       <compression/>
     </webgui>
     <usevirtualterminal>1</usevirtualterminal>
+    <disablenatreflection>yes</disablenatreflection>
     <disableconsolemenu>1</disableconsolemenu>
     <disablevlanhwfilter>1</disablevlanhwfilter>
     <disablechecksumoffloading>1</disablechecksumoffloading>
@@ -103,6 +104,7 @@
       <plugins>os-ddclient,os-dyndns,os-haproxy,os-wireguard</plugins>
       <type/>
       <subscription/>
+      <reboot/>
     </firmware>
     <sudo_allow_wheel>1</sudo_allow_wheel>
     <sudo_allow_group>admins</sudo_allow_group>
@@ -837,6 +839,8 @@
             <gateway/>
             <carp_depend_on/>
             <peers>03031aec-2e84-462e-9eab-57762dde667a,98e6ca3d-1de9-449b-be80-77022221b509,67c0ace5-e802-4d2b-a536-f8b7a2db6f99,74b60fff-7844-4097-9966-f1c2b1ad29ff,3de82ad5-bc1b-4b91-9598-f906e58ac937,a95e6b5e-24a4-40b5-bb41-b79e784f6f1c,6c9a12c6-c1ca-4c14-866b-975406a30590,c33b308b-7125-4688-9561-989ace8787b5,e43f004a-23bf-4027-8fb0-953fbb40479f</peers>
+            <endpoint/>
+            <peer_dns/>
           </server>
         </servers>
       </server>
@@ -941,6 +945,7 @@
           <root>0</root>
           <maxConnections/>
           <nbthread>1</nbthread>
+          <resolversPrefer>ipv4</resolversPrefer>
           <sslServerVerify>ignore</sslServerVerify>
           <maxDHSize>2048</maxDHSize>
           <bufferSize>16384</bufferSize>
@@ -948,6 +953,9 @@
           <bogusProxyEnabled>0</bogusProxyEnabled>
           <luaMaxMem>0</luaMaxMem>
           <customOptions/>
+          <ocspUpdateEnabled>0</ocspUpdateEnabled>
+          <ocspUpdateMinDelay>300</ocspUpdateMinDelay>
+          <ocspUpdateMaxDelay>3600</ocspUpdateMaxDelay>
           <ssl_defaultsEnabled>0</ssl_defaultsEnabled>
           <ssl_bindOptions>prefer-client-ciphers</ssl_bindOptions>
           <ssl_minVersion>TLSv1.2</ssl_minVersion>

harmony-rs/opnsense-config/src/tests/data/config-structure.xml

@@ -68,6 +68,7 @@
       <compression/>
     </webgui>
     <usevirtualterminal>1</usevirtualterminal>
+    <disablenatreflection>yes</disablenatreflection>
     <disableconsolemenu>1</disableconsolemenu>
     <disablevlanhwfilter>1</disablevlanhwfilter>
     <disablechecksumoffloading>1</disablechecksumoffloading>
@@ -103,6 +104,7 @@
       <plugins>os-ddclient,os-dyndns,os-haproxy,os-wireguard</plugins>
       <type/>
       <subscription/>
+      <reboot/>
     </firmware>
     <sudo_allow_wheel>1</sudo_allow_wheel>
     <sudo_allow_group>admins</sudo_allow_group>
@@ -826,6 +828,8 @@
             <dns/>
             <tunneladdress>172.12.0.1/24</tunneladdress>
             <disableroutes>0</disableroutes>
+            <endpoint/>
+            <peer_dns/>
             <gateway/>
             <carp_depend_on/>
             <peers>03031aec-2e84-462e-9eab-57762dde667a,98e6ca3d-1de9-449b-be80-77022221b509,67c0ace5-e802-4d2b-a536-f8b7a2db6f99,74b60fff-7844-4097-9966-f1c2b1ad29ff,3de82ad5-bc1b-4b91-9598-f906e58ac937,a95e6b5e-24a4-40b5-bb41-b79e784f6f1c,6c9a12c6-c1ca-4c14-866b-975406a30590,c33b308b-7125-4688-9561-989ace8787b5,e43f004a-23bf-4027-8fb0-953fbb40479f</peers>
@@ -936,10 +940,14 @@
           <sslServerVerify>ignore</sslServerVerify>
           <maxDHSize>2048</maxDHSize>
           <bufferSize>16384</bufferSize>
+          <resolversPrefer>ipv4</resolversPrefer>
           <spreadChecks>2</spreadChecks>
           <bogusProxyEnabled>0</bogusProxyEnabled>
           <luaMaxMem>0</luaMaxMem>
           <customOptions/>
+          <ocspUpdateEnabled>0</ocspUpdateEnabled>
+          <ocspUpdateMinDelay>300</ocspUpdateMinDelay>
+          <ocspUpdateMaxDelay>3600</ocspUpdateMaxDelay>
           <ssl_defaultsEnabled>0</ssl_defaultsEnabled>
           <ssl_bindOptions>prefer-client-ciphers</ssl_bindOptions>
           <ssl_minVersion>TLSv1.2</ssl_minVersion>