chore: Reorganize file tree for easier onboarding. Rust project now at the root for simple git clone && cargo run

opnsense-config/src/config/config.rs

@@ -0,0 +1,322 @@
+use std::sync::Arc;
+
+use crate::{
+    config::{SshConfigManager, SshCredentials, SshOPNSenseShell},
+    error::Error,
+    modules::{
+        caddy::CaddyConfig, dhcp::DhcpConfig, dns::DnsConfig, load_balancer::LoadBalancerConfig,
+        tftp::TftpConfig,
+    },
+};
+use log::{debug, info, trace, warn};
+use opnsense_config_xml::OPNsense;
+use russh::client;
+
+use super::{ConfigManager, OPNsenseShell};
+
+#[derive(Debug)]
+pub struct Config {
+    opnsense: OPNsense,
+    repository: Arc<dyn ConfigManager>,
+    shell: Arc<dyn OPNsenseShell>,
+}
+
+impl Config {
+    pub async fn new(
+        repository: Arc<dyn ConfigManager>,
+        shell: Arc<dyn OPNsenseShell>,
+    ) -> Result<Self, Error> {
+        Ok(Self {
+            opnsense: Self::get_opnsense_instance(repository.clone()).await?,
+            repository,
+            shell,
+        })
+    }
+
+    pub fn dhcp(&mut self) -> DhcpConfig {
+        DhcpConfig::new(&mut self.opnsense, self.shell.clone())
+    }
+
+    pub fn dns(&mut self) -> DnsConfig {
+        DnsConfig::new(&mut self.opnsense)
+    }
+
+    pub fn tftp(&mut self) -> TftpConfig {
+        TftpConfig::new(&mut self.opnsense, self.shell.clone())
+    }
+
+    pub fn caddy(&mut self) -> CaddyConfig {
+        CaddyConfig::new(&mut self.opnsense, self.shell.clone())
+    }
+
+    pub fn load_balancer(&mut self) -> LoadBalancerConfig {
+        LoadBalancerConfig::new(&mut self.opnsense, self.shell.clone())
+    }
+
+    pub async fn upload_files(&self, source: &str, destination: &str) -> Result<String, Error> {
+        self.shell.upload_folder(source, destination).await
+    }
+
+    /// Checks in config file if system.firmware.plugins csv field contains the specified package
+    /// name.
+    ///
+    /// Given this
+    /// ```xml
+    /// <opnsense>
+    ///     <system>
+    ///         <firmware>
+    ///             <plugins>os-haproxy,os-iperf,os-cpu-microcode-intel</plugins>
+    ///         </firmware>
+    ///     </system>
+    /// </opnsense>
+    /// ```
+    ///
+    /// is_package_installed("os-cpu"); // false
+    /// is_package_installed("os-haproxy"); // true
+    /// is_package_installed("os-cpu-microcode-intel"); // true
+    pub fn is_package_installed(&self, package_name: &str) -> bool {
+        match &self.opnsense.system.firmware.plugins.content {
+            Some(plugins) => is_package_in_csv(plugins, package_name),
+            None => false,
+        }
+    }
+
+    // Here maybe we should take ownership of `mut self` instead of `&mut self`
+    // I don't think there can be faulty pointers to previous versions of the config but I have a
+    // hard time wrapping my head around it right now :
+    // - the caller has a mutable reference to us
+    // - caller gets a reference to a piece of configuration (.haproxy.general.servers[0])
+    // - caller calls install_package wich reloads the config from remote
+    // - haproxy.general.servers[0] does not exist anymore
+    // - broken?
+    //
+    // Although I did not try explicitely the above workflow so maybe rust prevents taking a
+    // read-only reference across the &mut call
+    pub async fn install_package(&mut self, package_name: &str) -> Result<(), Error> {
+        info!("Installing opnsense package {package_name}");
+        self.check_pkg_opnsense_org_connection().await?;
+
+        let output = self.shell
+            .exec(&format!("/bin/sh -c \"export LOCKFILE=/dev/stdout && /usr/local/opnsense/scripts/firmware/install.sh {package_name}\""))
+            .await?;
+        info!("Installation output {output}");
+
+        self.reload_config().await?;
+        let is_installed = self.is_package_installed(package_name);
+        debug!("Verifying package installed successfully {is_installed}");
+
+        if is_installed {
+            info!("Installation successful for {package_name}");
+            Ok(())
+        } else {
+            let msg = format!("Package installation failed for {package_name}, see above logs");
+            warn!("{}", msg);
+            Err(Error::Unexpected(msg))
+        }
+    }
+
+    pub async fn check_pkg_opnsense_org_connection(&mut self) -> Result<(), Error> {
+        let pkg_url = "https://pkg.opnsense.org";
+        info!("Verifying connection to {pkg_url}");
+        let output = self
+            .shell
+            .exec(&format!("/bin/sh -c \"curl -v {pkg_url}\""))
+            .await?;
+        info!("{}", output);
+        Ok(())
+    }
+
+    async fn reload_config(&mut self) -> Result<(), Error> {
+        info!("Reloading opnsense live config");
+        self.opnsense = Self::get_opnsense_instance(self.repository.clone()).await?;
+        Ok(())
+    }
+
+    pub async fn restart_dns(&self) -> Result<(), Error> {
+        self.shell.exec("configctl unbound restart").await?;
+        Ok(())
+    }
+
+    /// Save the config to the repository. This method is meant NOT to reload services, only save
+    /// the config to the live file/database and perhaps take a backup when relevant.
+    pub async fn save(&self) -> Result<(), Error> {
+        self.repository.save_config(&self.opnsense.to_xml()).await
+    }
+
+    /// Save the configuration and reload all services. Be careful with this one as it will cause
+    /// downtime in many cases, such as a PPPoE renegociation
+    pub async fn apply(&self) -> Result<(), Error> {
+        self.repository
+            .apply_new_config(&self.opnsense.to_xml())
+            .await
+    }
+
+    pub async fn from_credentials(
+        ipaddr: std::net::IpAddr,
+        port: Option<u16>,
+        username: &str,
+        password: &str,
+    ) -> Self {
+        let config = Arc::new(client::Config {
+            inactivity_timeout: None,
+            ..<_>::default()
+        });
+
+        let credentials = SshCredentials::Password {
+            username: String::from(username),
+            password: String::from(password),
+        };
+
+        let port = port.unwrap_or(22);
+
+        let shell = Arc::new(SshOPNSenseShell::new((ipaddr, port), credentials, config));
+        let manager = Arc::new(SshConfigManager::new(shell.clone()));
+
+        Config::new(manager, shell).await.unwrap()
+    }
+
+    async fn get_opnsense_instance(repository: Arc<dyn ConfigManager>) -> Result<OPNsense, Error> {
+        let xml = repository.load_as_str().await?;
+        trace!("xml {}", xml);
+
+        Ok(OPNsense::from(xml))
+    }
+
+    pub async fn run_command(&self, command: &str) -> Result<String, Error> {
+        self.shell.exec(command).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::config::{DummyOPNSenseShell, LocalFileConfigManager};
+    use crate::modules::dhcp::DhcpConfig;
+    use std::fs;
+    use std::net::Ipv4Addr;
+
+    use super::*;
+    use pretty_assertions::assert_eq;
+    use std::path::PathBuf;
+
+    #[tokio::test]
+    async fn test_load_config_from_local_file() {
+        for path in vec![
+            "src/tests/data/config-opnsense-25.1.xml",
+            "src/tests/data/config-vm-test.xml",
+            "src/tests/data/config-structure.xml",
+            "src/tests/data/config-full-1.xml",
+        ] {
+            let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+            test_file_path.push(path);
+
+            let config_file_path = test_file_path.to_str().unwrap().to_string();
+            println!("File path {config_file_path}");
+            let repository = Arc::new(LocalFileConfigManager::new(config_file_path));
+            let shell = Arc::new(DummyOPNSenseShell {});
+            let config_file_str = repository.load_as_str().await.unwrap();
+            let config = Config::new(repository, shell)
+                .await
+                .expect("Failed to load config");
+
+            println!("Config {:?}", config);
+
+            let serialized = config.opnsense.to_xml();
+
+            fs::write("/tmp/serialized.xml", &serialized).unwrap();
+
+            // Since the order of all fields is not always the same in opnsense config files
+            // I think it is good enough to have exactly the same amount of the same lines
+            let config_file_str_sorted = vec![config_file_str.lines().collect::<Vec<_>>()].sort();
+            let serialized_sorted = vec![config_file_str.lines().collect::<Vec<_>>()].sort();
+            assert_eq!(config_file_str_sorted, serialized_sorted);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_add_dhcpd_static_entry() {
+        let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        test_file_path.push("src/tests/data/config-structure.xml");
+
+        let config_file_path = test_file_path.to_str().unwrap().to_string();
+        println!("File path {config_file_path}");
+        let repository = Arc::new(LocalFileConfigManager::new(config_file_path));
+        let shell = Arc::new(DummyOPNSenseShell {});
+        let mut config = Config::new(repository, shell.clone())
+            .await
+            .expect("Failed to load config");
+
+        println!("Config {:?}", config);
+
+        let mut dhcp_config = DhcpConfig::new(&mut config.opnsense, shell);
+        dhcp_config
+            .add_static_mapping(
+                "00:00:00:00:00:00",
+                Ipv4Addr::new(192, 168, 20, 100),
+                "hostname",
+            )
+            .expect("Should add static mapping");
+
+        let serialized = config.opnsense.to_xml();
+
+        fs::write("/tmp/serialized.xml", &serialized).unwrap();
+
+        let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        test_file_path.push("src/tests/data/config-structure-with-dhcp-staticmap-entry.xml");
+
+        let config_file_path = test_file_path.to_str().unwrap().to_string();
+        println!("File path {config_file_path}");
+        let repository = Box::new(LocalFileConfigManager::new(config_file_path));
+        let expected_config_file_str = repository.load_as_str().await.unwrap();
+        assert_eq!(expected_config_file_str, serialized);
+    }
+}
+
+/// Checks if a given package name exists in a comma-separated list of packages.
+///
+/// # Arguments
+///
+/// * `csv_string` - A string containing comma-separated package names.
+/// * `package_name` - The package name to search for.
+///
+/// # Returns
+///
+/// * `true` if the package name is found in the CSV string, `false` otherwise.
+fn is_package_in_csv(csv_string: &str, package_name: &str) -> bool {
+    package_name.len() > 0 && csv_string.split(',').any(|pkg| pkg.trim() == package_name)
+}
+
+#[cfg(test)]
+mod tests_2 {
+    use super::*;
+
+    #[test]
+    fn test_is_package_in_csv() {
+        let csv_string = "os-haproxy,os-iperf,os-cpu-microcode-intel";
+
+        assert!(is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(is_package_in_csv(csv_string, "os-iperf"));
+        assert!(is_package_in_csv(csv_string, "os-cpu-microcode-intel"));
+
+        assert!(!is_package_in_csv(csv_string, "os-cpu"));
+        assert!(!is_package_in_csv(csv_string, "non-existent-package"));
+    }
+
+    #[test]
+    fn test_is_package_in_csv_empty() {
+        let csv_string = "";
+
+        assert!(!is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(!is_package_in_csv(csv_string, ""));
+    }
+
+    #[test]
+    fn test_is_package_in_csv_whitespace() {
+        let csv_string = " os-haproxy , os-iperf , os-cpu-microcode-intel ";
+
+        assert!(is_package_in_csv(csv_string, "os-haproxy"));
+        assert!(is_package_in_csv(csv_string, "os-iperf"));
+        assert!(is_package_in_csv(csv_string, "os-cpu-microcode-intel"));
+
+        assert!(!is_package_in_csv(csv_string, " os-haproxy "));
+    }
+}

opnsense-config/src/config/manager/local_file.rs

@@ -0,0 +1,30 @@
+use crate::config::manager::ConfigManager;
+use crate::error::Error;
+use async_trait::async_trait;
+use std::fs;
+
+#[derive(Debug)]
+pub struct LocalFileConfigManager {
+    file_path: String,
+}
+
+impl LocalFileConfigManager {
+    pub fn new(file_path: String) -> Self {
+        Self { file_path }
+    }
+}
+
+#[async_trait]
+impl ConfigManager for LocalFileConfigManager {
+    async fn load_as_str(&self) -> Result<String, Error> {
+        Ok(fs::read_to_string(&self.file_path)?)
+    }
+
+    async fn save_config(&self, content: &str) -> Result<(), Error> {
+        Ok(fs::write(&self.file_path, content)?)
+    }
+
+    async fn apply_new_config(&self, content: &str) -> Result<(), Error> {
+        self.save_config(content).await
+    }
+}

opnsense-config/src/config/manager/mod.rs

@@ -0,0 +1,14 @@
+mod local_file;
+mod ssh;
+use async_trait::async_trait;
+pub use local_file::*;
+pub use ssh::*;
+
+use crate::Error;
+
+#[async_trait]
+pub trait ConfigManager: std::fmt::Debug + Send + Sync {
+    async fn load_as_str(&self) -> Result<String, Error>;
+    async fn save_config(&self, content: &str) -> Result<(), Error>;
+    async fn apply_new_config(&self, content: &str) -> Result<(), Error>;
+}

opnsense-config/src/config/manager/ssh.rs

@@ -0,0 +1,74 @@
+use crate::config::{manager::ConfigManager, OPNsenseShell};
+use crate::error::Error;
+use async_trait::async_trait;
+use log::info;
+use russh_keys::key::KeyPair;
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub enum SshCredentials {
+    SshKey { username: String, key: Arc<KeyPair> },
+    Password { username: String, password: String },
+}
+
+#[derive(Debug)]
+pub struct SshConfigManager {
+    opnsense_shell: Arc<dyn OPNsenseShell>,
+}
+
+impl SshConfigManager {
+    pub fn new(opnsense_shell: Arc<dyn OPNsenseShell>) -> Self {
+        Self { opnsense_shell }
+    }
+}
+
+impl SshConfigManager {
+    async fn backup_config_remote(&self) -> Result<String, Error> {
+        let ts = chrono::Utc::now();
+        let backup_filename = format!("config-{}-harmony.xml", ts.format("%s%.3f"));
+
+        self.opnsense_shell
+            .exec(&format!(
+                "cp /conf/config.xml /conf/backup/{}",
+                backup_filename
+            ))
+            .await
+    }
+
+    async fn move_to_live_config(&self, new_config_path: &str) -> Result<String, Error> {
+        info!("Overwriting OPNSense /conf/config.xml with {new_config_path}");
+        self.opnsense_shell
+            .exec(&format!("mv {new_config_path} /conf/config.xml"))
+            .await
+    }
+
+    async fn reload_all_services(&self) -> Result<String, Error> {
+        info!("Reloading all opnsense services");
+        self.opnsense_shell
+            .exec(&format!("configctl service reload all"))
+            .await
+    }
+}
+
+#[async_trait]
+impl ConfigManager for SshConfigManager {
+    async fn load_as_str(&self) -> Result<String, Error> {
+        self.opnsense_shell.exec("cat /conf/config.xml").await
+    }
+
+    async fn save_config(&self, content: &str) -> Result<(), Error> {
+        let temp_filename = self
+            .opnsense_shell
+            .write_content_to_temp_file(content)
+            .await?;
+        self.backup_config_remote().await?;
+        self.move_to_live_config(&temp_filename).await?;
+        Ok(())
+    }
+
+    async fn apply_new_config(&self, content: &str) -> Result<(), Error> {
+        self.save_config(content).await?;
+        self.reload_all_services().await?;
+        Ok(())
+    }
+}

opnsense-config/src/config/mod.rs

@@ -0,0 +1,6 @@
+mod config;
+mod manager;
+mod shell;
+pub use config::*;
+pub use manager::*;
+pub use shell::*;

opnsense-config/src/config/shell/mod.rs

@@ -0,0 +1,31 @@
+mod ssh;
+pub use ssh::*;
+
+use async_trait::async_trait;
+
+use crate::Error;
+
+#[async_trait]
+pub trait OPNsenseShell: std::fmt::Debug + Send + Sync {
+    async fn exec(&self, command: &str) -> Result<String, Error>;
+    async fn write_content_to_temp_file(&self, content: &str) -> Result<String, Error>;
+    async fn upload_folder(&self, source: &str, destination: &str) -> Result<String, Error>;
+}
+
+#[cfg(test)]
+#[derive(Debug)]
+pub struct DummyOPNSenseShell;
+
+#[cfg(test)]
+#[async_trait]
+impl OPNsenseShell for DummyOPNSenseShell {
+    async fn exec(&self, _command: &str) -> Result<String, Error> {
+        unimplemented!("This is a dummy implementation");
+    }
+    async fn write_content_to_temp_file(&self, _content: &str) -> Result<String, Error> {
+        unimplemented!("This is a dummy implementation");
+    }
+    async fn upload_folder(&self, _source: &str, _destination: &str) -> Result<String, Error> {
+        unimplemented!("This is a dummy implementation");
+    }
+}

opnsense-config/src/config/shell/ssh.rs

@@ -0,0 +1,211 @@
+use std::{
+    net::IpAddr,
+    sync::Arc,
+    time::{SystemTime, UNIX_EPOCH},
+};
+use tokio_stream::StreamExt;
+
+use async_trait::async_trait;
+use log::{debug, info, trace};
+use russh::{
+    client::{Config, Handler, Msg},
+    Channel,
+};
+use russh_keys::key;
+use russh_sftp::client::SftpSession;
+use tokio::io::AsyncWriteExt;
+
+use crate::{config::SshCredentials, Error};
+
+use super::OPNsenseShell;
+use tokio::fs::read_dir;
+use tokio::fs::File;
+use tokio_util::codec::{BytesCodec, FramedRead};
+
+#[derive(Debug)]
+pub struct SshOPNSenseShell {
+    host: (IpAddr, u16),
+    credentials: SshCredentials,
+    ssh_config: Arc<Config>,
+}
+
+#[async_trait]
+impl OPNsenseShell for SshOPNSenseShell {
+    async fn exec(&self, command: &str) -> Result<String, Error> {
+        info!("Executing command on SshOPNSenseShell {command}");
+        self.run_command(command).await
+    }
+
+    async fn write_content_to_temp_file(&self, content: &str) -> Result<String, Error> {
+        let temp_filename = format!(
+            "/tmp/opnsense-config-tmp-config_{}",
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_millis()
+        );
+        let channel = self.get_ssh_channel().await?;
+        channel
+            .request_subsystem(true, "sftp")
+            .await
+            .expect("Should request sftp subsystem");
+        let sftp = SftpSession::new(channel.into_stream())
+            .await
+            .expect("Should acquire sftp subsystem");
+
+        let mut file = sftp.create(&temp_filename).await.unwrap();
+        file.write_all(content.as_bytes()).await?;
+
+        Ok(temp_filename)
+    }
+
+    async fn upload_folder(&self, source: &str, destination: &str) -> Result<String, Error> {
+        let channel = self.get_ssh_channel().await?;
+        channel
+            .request_subsystem(true, "sftp")
+            .await
+            .expect("Should request sftp subsystem");
+        let sftp = SftpSession::new(channel.into_stream())
+            .await
+            .expect("Should acquire sftp subsystem");
+
+        if !sftp.try_exists(destination).await? {
+            info!("Creating remote directory {destination}");
+            sftp.create_dir(destination).await?;
+        }
+
+        info!("Reading local directory {source}");
+        let mut entries = read_dir(source).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            info!(
+                "Checking directory entry {}",
+                entry
+                    .path()
+                    .to_str()
+                    .expect("Directory entry should have a path : {entry:?}")
+            );
+            if entry.file_type().await?.is_file() {
+                debug!("Got a file");
+                let local_path = entry.path();
+                debug!("path {local_path:?}");
+                let file_name = local_path.file_name().unwrap().to_string_lossy();
+                let remote_path = format!("{}/{}", destination, file_name);
+                info!(
+                    "Uploading local file {} to remote {}",
+                    local_path.to_str().unwrap_or_default(),
+                    remote_path
+                );
+
+                debug!("Creating file {remote_path:?}");
+                let mut remote_file = sftp.create(remote_path.as_str()).await?;
+
+                debug!("Writing file {remote_path:?}");
+                let local_file = File::open(&local_path).await?;
+                let mut reader = FramedRead::new(local_file, BytesCodec::new());
+
+                while let Some(result) = reader.next().await {
+                    match result {
+                        Ok(bytes) => {
+                            if !bytes.is_empty() {
+                                remote_file.write(&bytes).await?;
+                            }
+                        }
+                        Err(e) => todo!("Error unhandled {e}"),
+                    };
+                }
+            } else if entry.file_type().await?.is_dir() {
+                let sub_source = entry.path();
+                let sub_destination =
+                    format!("{}/{}", destination, entry.file_name().to_string_lossy());
+                self.upload_folder(sub_source.to_str().unwrap(), &sub_destination)
+                    .await?;
+            }
+        }
+
+        Ok(destination.to_string())
+    }
+}
+
+impl SshOPNSenseShell {
+    pub async fn get_ssh_channel(&self) -> Result<Channel<Msg>, Error> {
+        let mut ssh = russh::client::connect(self.ssh_config.clone(), self.host, Client {}).await?;
+
+        match &self.credentials {
+            SshCredentials::SshKey { username, key } => {
+                ssh.authenticate_publickey(username, key.clone()).await?;
+            }
+            SshCredentials::Password { username, password } => {
+                ssh.authenticate_password(username, password).await?;
+            }
+        }
+
+        Ok(ssh.channel_open_session().await?)
+    }
+
+    async fn run_command(&self, command: &str) -> Result<String, Error> {
+        debug!("Running ssh command {command}");
+        let mut channel = self.get_ssh_channel().await?;
+        channel.exec(true, command).await?;
+        wait_for_completion(&mut channel).await
+    }
+
+    pub fn new(host: (IpAddr, u16), credentials: SshCredentials, ssh_config: Arc<Config>) -> Self {
+        info!("Initializing SshOPNSenseShell on host {host:?}");
+        Self {
+            host,
+            credentials,
+            ssh_config,
+        }
+    }
+}
+
+struct Client {}
+
+#[async_trait]
+impl Handler for Client {
+    type Error = Error;
+
+    async fn check_server_key(
+        &mut self,
+        _server_public_key: &key::PublicKey,
+    ) -> Result<bool, Self::Error> {
+        Ok(true)
+    }
+}
+
+async fn wait_for_completion(channel: &mut Channel<Msg>) -> Result<String, Error> {
+    let mut output = Vec::new();
+
+    loop {
+        let Some(msg) = channel.wait().await else {
+            break;
+        };
+
+        match msg {
+            russh::ChannelMsg::ExtendedData { ref data, .. }
+            | russh::ChannelMsg::Data { ref data } => {
+                output.append(&mut data.to_vec());
+            }
+            russh::ChannelMsg::ExitStatus { exit_status } => {
+                if exit_status != 0 {
+                    return Err(Error::Command(format!(
+                        "Command failed with exit status {exit_status}, output {}",
+                        String::from_utf8(output).unwrap_or_default()
+                    )));
+                }
+            }
+            russh::ChannelMsg::Success { .. }
+            | russh::ChannelMsg::WindowAdjusted { .. }
+            | russh::ChannelMsg::Eof { .. } => {}
+            _ => {
+                return Err(Error::Unexpected(format!(
+                    "Russh got unexpected msg {msg:?}"
+                )))
+            }
+        }
+    }
+
+    let output = String::from_utf8(output).expect("Output should be UTF-8 compatible");
+    trace!("{output}");
+    Ok(output)
+}