enhance hvee environment variables

Provide diagrams for a Virtualized Execution Environment

Propose interfaces for a cli toolkit

Cargo.lock

@@ -1719,6 +1719,24 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-ha-cluster"
+version = "0.1.0"
+dependencies = [
+ "brocade",
+ "cidr",
+ "env_logger",
+ "harmony",
+ "harmony_macros",
+ "harmony_secret",
+ "harmony_tui",
+ "harmony_types",
+ "log",
+ "serde",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "example-kube-rs"
 version = "0.1.0"
@@ -1838,6 +1856,24 @@ dependencies = [
 [[package]]
 name = "example-opnsense"
 version = "0.1.0"
+dependencies = [
+ "brocade",
+ "cidr",
+ "env_logger",
+ "harmony",
+ "harmony_cli",
+ "harmony_macros",
+ "harmony_secret",
+ "harmony_types",
+ "log",
+ "serde",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "example-opnsense-2"
+version = "0.1.0"
 dependencies = [
  "brocade",
  "cidr",

examples/ha_cluster/Cargo.toml

@@ -1,21 +1,21 @@
 [package]
-name = "example-opnsense-node-exporter"
+name = "example-ha-cluster"
 edition = "2024"
 version.workspace = true
 readme.workspace = true
 license.workspace = true
+publish = false
 
 [dependencies]
 harmony = { path = "../../harmony" }
-harmony_cli = { path = "../../harmony_cli" }
+harmony_tui = { path = "../../harmony_tui" }
 harmony_types = { path = "../../harmony_types" }
-harmony_secret = { path = "../../harmony_secret" }
-harmony_secret_derive = { path = "../../harmony_secret_derive" }
 cidr = { workspace = true }
 tokio = { workspace = true }
 harmony_macros = { path = "../../harmony_macros" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
-serde.workspace = true
+harmony_secret = { path = "../../harmony_secret" }
-async-trait.workspace = true
+brocade = { path = "../../brocade" }
+serde = { workspace = true }

examples/ha_cluster/README.md

@@ -0,0 +1,15 @@
+## OPNSense demo
+
+Download the virtualbox snapshot from {{TODO URL}}
+
+Start the virtualbox image
+
+This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
+
+Credentials are opnsense default (root/opnsense)
+
+Run the project with the correct ip address on the command line :
+
+```bash
+cargo run -p example-opnsense -- 192.168.5.229
+```

examples/ha_cluster/src/main.rs

@@ -0,0 +1,141 @@
+use std::{
+    net::{IpAddr, Ipv4Addr},
+    sync::Arc,
+};
+
+use brocade::BrocadeOptions;
+use cidr::Ipv4Cidr;
+use harmony::{
+    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    inventory::Inventory,
+    modules::{
+        dummy::{ErrorScore, PanicScore, SuccessScore},
+        http::StaticFilesHttpScore,
+        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
+        opnsense::OPNsenseShellCommandScore,
+        tftp::TftpScore,
+    },
+    topology::{LogicalHost, UnmanagedRouter},
+};
+use harmony_macros::{ip, mac_address};
+use harmony_secret::{Secret, SecretManager};
+use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
+
+#[tokio::main]
+async fn main() {
+    let firewall = harmony::topology::LogicalHost {
+        ip: ip!("192.168.5.229"),
+        name: String::from("opnsense-1"),
+    };
+
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
+    let opnsense = Arc::new(
+        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
+    );
+    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
+    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
+    let gateway_ip = IpAddr::V4(gateway_ipv4);
+    let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
+        domain_name: "demo.harmony.mcd".to_string(),
+        router: Arc::new(UnmanagedRouter::new(
+            gateway_ip,
+            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
+        )),
+        load_balancer: opnsense.clone(),
+        firewall: opnsense.clone(),
+        tftp_server: opnsense.clone(),
+        http_server: opnsense.clone(),
+        dhcp_server: opnsense.clone(),
+        dns_server: opnsense.clone(),
+        control_plane: vec![LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        }],
+        bootstrap_host: LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        },
+        workers: vec![],
+        switch_client: switch_client.clone(),
+    };
+
+    let inventory = Inventory {
+        location: Location::new(
+            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
+            "wk".to_string(),
+        ),
+        switch: SwitchGroup::from([]),
+        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
+        storage_host: vec![],
+        worker_host: vec![],
+        control_plane_host: vec![
+            PhysicalHost::empty(HostCategory::Server)
+                .mac_address(mac_address!("08:00:27:62:EC:C3")),
+        ],
+    };
+
+    // TODO regroup smaller scores in a larger one such as this
+    // let okd_boostrap_preparation();
+
+    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
+    let dns_score = OKDDnsScore::new(&topology);
+    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
+
+    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
+    let http_score = StaticFilesHttpScore {
+        folder_to_serve: Some(Url::LocalFolder(
+            "./data/watchguard/pxe-http-files".to_string(),
+        )),
+        files: vec![],
+        remote_path: None,
+    };
+
+    harmony_tui::run(
+        inventory,
+        topology,
+        vec![
+            Box::new(dns_score),
+            Box::new(dhcp_score),
+            Box::new(load_balancer_score),
+            Box::new(tftp_score),
+            Box::new(http_score),
+            Box::new(OPNsenseShellCommandScore {
+                opnsense: opnsense.get_opnsense_config(),
+                command: "touch /tmp/helloharmonytouching".to_string(),
+            }),
+            Box::new(SuccessScore {}),
+            Box::new(ErrorScore {}),
+            Box::new(PanicScore {}),
+        ],
+    )
+    .await
+    .unwrap();
+}
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/nanodc/src/main.rs

@@ -106,7 +106,6 @@ async fn main() {
                 name: "wk2".to_string(),
             },
         ],
-        node_exporter: opnsense.clone(),
         switch_client: switch_client.clone(),
     };
 

examples/okd_installation/src/topology.rs

@@ -80,7 +80,6 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "bootstrap".to_string(),
         },
         workers: vec![],
-        node_exporter: opnsense.clone(),
         switch_client: switch_client.clone(),
     }
 }

examples/okd_pxe/src/topology.rs

@@ -75,7 +75,6 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        node_exporter: opnsense.clone(),
         switch_client: switch_client.clone(),
     }
 }

examples/opnsense/Cargo.toml

@@ -8,7 +8,7 @@ publish = false
 
 [dependencies]
 harmony = { path = "../../harmony" }
-harmony_tui = { path = "../../harmony_tui" }
+harmony_cli = { path = "../../harmony_cli" }
 harmony_types = { path = "../../harmony_types" }
 cidr = { workspace = true }
 tokio = { workspace = true }

examples/opnsense/README.md

@@ -1,15 +1,23 @@
-## OPNSense demo
+# OPNSense Demo
 
-Download the virtualbox snapshot from {{TODO URL}}
+This example demonstrate how to manage an Opnsense server with harmony.
 
-Start the virtualbox image
+todo: add more info
 
-This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
+## Demo instructions
 
-Credentials are opnsense default (root/opnsense)
+todo: add detailed instructions
 
-Run the project with the correct ip address on the command line :
+  - setup the example execution environment
+  - setup your system configuration
+    - topology
+    - scores
+    - secrets
+  - build
+  - execute
+  - verify/inspect
+
+## Example execution
+
+See [learning tool documentation](./scripts/README.md)
 
-```bash
-cargo run -p example-opnsense -- 192.168.5.229
-```

examples/opnsense/env.sh

@@ -0,0 +1,4 @@
+export HARMONY_SECRET_NAMESPACE=example-opnsense
+export HARMONY_SECRET_STORE=file
+export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info 
+export RUST_LOG=info

examples/opnsense/scripts/README.md

@@ -0,0 +1,96 @@
+# Virtualized Execution Environment for Harmony
+
+Scripts included in this directory have 3 purposes:
+
+  - automate initial setup of localhost or VM (nested virtualization) for this example
+  - prototype a solution for an 'OpensenseLocalhostTopology'
+  - prototype 
+
+This exprimentation aim to find an approach suitable for using harmony on virtualised execution environment such that:
+
+  - it straights forward for a user with minimal knowledge to start testing harmony
+  - installation and execution have **minimal impact on the user desktop**
+
+## Usage
+
+### Installation
+
+1. download this directory
+2. add this directory in your PATH (example `. setup`)
+
+
+```
+# show help page
+harmony-vee
+
+# show active configurations
+harmony-vee config
+
+# show what will be modified at installation
+harmony-vee install --dry-run
+
+# install
+harmony-vee install
+
+# show what will be modified at unistallation
+harmony-vee uninstall --dry-run
+
+```
+
+### Create and start a new Virtual Execution Environment
+
+
+```
+# Create a HVEE to test opnsense_score
+harmony-vee init opnsense_score
+
+# List existing HVEE
+harmony-vee list
+
+# Show HVEE information including devices ip and vault type/location
+harmony-vee show opnsense_score
+
+# Start/Stop a HVEE
+harmony-vee stop opnsense_score
+
+
+# Destroy a HVEE instance
+harmony-vee destroy opnsense_score
+
+```
+
+
+### Variable d'environnement
+
+```
+## directory containing harmony-ve data
+# HVE_ROOT=~/.harmony-ve
+
+## OPNSENSE SRC
+# main mirror
+# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
+# first alternative mirror
+# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
+# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
+
+
+## Network
+# HVE_NETWORK_LABEL=harmony
+```
+
+
+## Remarks
+
+  - A nested VM setup could be safer
+
+
+
+## Architecture
+
+### Learning environment directly on host
+
+![localhost case](./doc/automate-opnsense-example-localhost.drawio.png)
+
+### Learning environment nested in a "workspace vm"
+
+![localhost case](./doc/automate-opnsense-example-nested-virtualization.drawio.png)

examples/opnsense/scripts/common

@@ -0,0 +1,17 @@
+#! /bin/bash
+
+_warn(){ >&2 echo "WARNING: $*" ; }
+
+_fatal(){
+  >&2 echo "FATAL ERROR: $*"
+  >&2 echo stopping...
+  exit 1
+}
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}

examples/opnsense/scripts/default-env-var

@@ -0,0 +1,31 @@
+# Conventions:
+#   - Namespaced with HVE, short for Harmony Virtualised Execution Environment
+#   - Prefixed values used internally
+#   - Not prefixed may be supercharged by the user
+
+# Root of harmony data
+_HVE_ROOT=${HVE_ROOT:-$HOME/harmony-ve}
+[ -d "$_HVE_ROOT" ] || mkdir -p "${_HVE_ROOT}"
+
+
+_HVE_SRC_IMG=${_HVE_ROOT}/src/images
+[ -d "$_HVE_SRC_IMG" ] || mkdir -p "${_HVE_SRC_IMG}"
+_HVE_IMG=${_HVE_ROOT}/images
+[ -d "$_HVE_IMG" ] || mkdir -p "$_HVE_IMG"
+
+# Opnsense
+_HVE_OPNSENSE_URL=${HVE_OPNSENSE_URL:-https://pkg.opnsense.org/releases}
+# first alternative mirror
+_HVE_OPNSENSE_URL_ALT1=${HVE_OPNSENSE_URL_ALT1:-https://mirror.vraphim.com/opnsense/releases}
+_HVE_OPNSENSE_URL_ALT2=${HVE_OPNSENSE_URL_ALT2:-https://mirror.winsub.kr/opnsense/releases}
+
+_HVE_OPNSENSE_SRC_IMG=${_HVE_SRC_IMG}/opnsense
+[ -d "$_HVE_OPNSENSE_SRC_IMG" ] || mkdir -p "${_HVE_OPNSENSE_SRC_IMG}"
+_HVE_OPNSENSE_IMG=${_HVE_IMG}/opnsense
+[ -d "$_HVE_OPNSENSE_IMG" ] || mkdir -p "${_HVE_OPNSENSE_IMG}"
+
+# Network
+_HVE_NETWORK=${HVE_NETWORK:-harmony}
+_HVE_WAN_BRIDGE=${HVE_WAN_BRIDGE:-${_HVE_NETWORK}-wan-brd}
+_HVE_LAN_BRIDGE=${HVE_LAN_BRIDGE:-${_HVE_NETWORK}-lann-brd}
+

examples/opnsense/scripts/dependencies-management

@@ -0,0 +1,48 @@
+#! /bin/bash
+is_string_empty(){
+    if [ "${*:-}" != "" ]; then
+     return 0
+   else
+     return 1
+   fi
+}
+
+is_debian_family()(
+   is_string_empty "$(apt --version 2> /dev/null )"
+)
+
+has_ip(){
+  is_string_empty "$(ip -V 2> /dev/null)"
+}
+
+has_virsh(){
+  is_string_empty "$(virsh --version 2> /dev/null)"
+}
+
+has_virt_customize(){
+  is_string_empty "$(virt-customize --version 2> /dev/null)"
+}
+
+has_curl(){
+  is_string_empty "$(curl --version 2> /dev/null)"
+}
+has_wget(){
+  is_string_empty "$(wget --version 2> /dev/null)"
+}
+
+install_kvm(){
+  sudo apt install -y --no-install-recommends qemu-system libvirt-clients libvirt-daemon-system
+  sudo usermod -aG libvirt "$USER"
+  # todo: finf how to fix image access out of /var/lib/libvirt/images
+  sudo setfacl -Rm u:libvirt-qemu:rx $_HVE_IMG
+  sudo systemctl restart libvirtd
+}
+
+install_virt_customize(){
+   sudo apt install -y libguestfs-tools
+}
+
+install_wget(){
+  sudo apt install -y wget
+}
+

examples/opnsense/scripts/doc/automate-opnsense-example.drawio

@@ -0,0 +1,321 @@
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" version="28.2.9" pages="2">
+  <diagram name="localhost" id="lK0WmoXmZXwFmV5PC-RW">
+    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="1Ax8jaXdU0J25Zkiwu96-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="300" y="230" width="700" height="550" as="geometry" />
+        </mxCell>
+        <mxCell id="VVCo9gNhF-9Hs0fZa6i8-1" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="380" y="240" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="283" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-2" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="590" y="393" width="310" height="270" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-3" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="620" y="403" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="620" y="250" width="280" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-5" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="610" y="250" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-6" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="294" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-7" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
+          <mxGeometry x="760" y="290" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-8" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-7" target="X0RF5wBsf5JYURxROWwA-9" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-9" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="771" y="342" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-13" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="810" y="290" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-14" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-13" target="X0RF5wBsf5JYURxROWwA-15" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-15" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="820" y="342" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-16" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="760" y="383" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-17" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-16" target="X0RF5wBsf5JYURxROWwA-18" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-18" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="772.5" y="353" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-19" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="809" y="383" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-20" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-19" target="X0RF5wBsf5JYURxROWwA-21" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-21" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="821.5" y="353" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-22" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="440" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-23" value="&lt;b&gt;Example dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="450" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-24" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="510" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-25" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="590" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-26" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="600" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-27" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="361" y="650" width="208" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-28" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="353" width="230" height="77" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-29" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="361.5" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-30" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="620" y="450" width="250" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-31" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="323" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-33" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs and&lt;/i&gt;&lt;/font&gt;&lt;div&gt;&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;runtime info.&lt;/i&gt;&lt;/font&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="395" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="470" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-35" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="626.5" y="264" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-36" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="369" y="620" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-37" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="626.5" y="420" width="200" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+  <diagram id="oOuscOXp9aWETXQepMaW" name="nested-virtualization">
+    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="190" y="60" width="1240" height="660" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-37" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="540" y="210" width="830" height="480" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-2" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="84" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-3" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="223" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="855" y="403.75" width="310" height="266.25" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-5" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="885" y="413.75" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-6" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="810" y="230" width="530" height="120" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-7" value="&lt;b&gt;workspace VM network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="825" y="234" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-8" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="234" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-10" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-9" target="vj3pG7w7rTpS1tnBMssI-11" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-12" target="vj3pG7w7rTpS1tnBMssI-14" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-15" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1025" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-16" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-15" target="vj3pG7w7rTpS1tnBMssI-17" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-17" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1037.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-18" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1074" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-19" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-18" target="vj3pG7w7rTpS1tnBMssI-20" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-20" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1086.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-21" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="380" width="230" height="120" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-22" value="&lt;b&gt;Learning dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="391" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-23" value="&lt;ul&gt;&lt;li&gt;harmony&lt;/li&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="450" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="540" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-25" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="550" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-26" value="&lt;ul&gt;&lt;li&gt;Can be mounted locally&lt;br&gt;for persistence&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="251" y="600" width="189" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-27" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="293" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-28" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="301.5" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-29" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="885" y="460.75" width="250" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-30" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="263" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-31" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="321" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-32" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="410" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="259" y="570" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-35" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="891.5" y="430.75" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-38" value="&lt;b&gt;workspace VM&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="234" width="130" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-39" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="570" y="305" width="230" height="215" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-40" value="&lt;b&gt;workspace VM dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="316" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-41" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="375" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-44" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="570" y="540" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-48" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="830" y="277" width="235" height="45" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-45" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="590" y="550" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-46" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="581" y="600" width="208" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-47" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="589" y="570" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-9" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
+          <mxGeometry x="1025" y="300.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-11" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1036" y="352.75" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-49" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="1069" y="275" width="251" height="45" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1075" y="300.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-14" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1085" y="352.75" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-50" value="&lt;b&gt;wan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="821.5" y="277" width="70" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-51" value="&lt;b&gt;lan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1055" y="275" width="70" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-52" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="1181" y="403.75" width="159" height="116.25" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-53" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-54" target="vj3pG7w7rTpS1tnBMssI-55" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-54" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1211" y="305" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-55" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1221" y="357" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-58" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1210" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-59" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-58" target="vj3pG7w7rTpS1tnBMssI-60" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-60" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1222.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-61" value="&lt;b&gt;other vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1181" y="430" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-63" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="vj3pG7w7rTpS1tnBMssI-24" target="vj3pG7w7rTpS1tnBMssI-44" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="600" y="770" as="sourcePoint" />
+            <mxPoint x="360" y="885" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="799" y="84" width="280" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-2" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="789" y="84" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-6" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="989" y="124" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-7" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-6" target="Za91R7Nqk7Jj5VTRes6Y-8" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-8" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="999" y="176" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="988" y="217" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-12" target="Za91R7Nqk7Jj5VTRes6Y-14" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-14" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1000.5" y="187" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-15" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;No modification required&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="805.5" y="98" width="200" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

examples/opnsense/scripts/env-var

@@ -0,0 +1,13 @@
+## directory containing harmony-ve data
+# HVE_ROOT=~/.harmony-ve
+
+## OPNSENSE SRC
+# main mirror
+# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
+# first alternative mirror
+# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
+# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
+
+
+## Network
+# HVE_NETWORK_LABEL=harmony

examples/opnsense/scripts/harmony-ve

@@ -0,0 +1,105 @@
+
+#! /bin/bash
+
+harmony-ve()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve
+
+  DESCRIPTION
+
+    CLI management toolkit for Harmony Virtualized Execution Environment
+
+  SYNOPSYS
+
+    harmony-ve [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve dependencies      # manage localhost depend
+    harmony-ve network           # manage localhost netork
+
+    harmony-ve opnsense-img-src  # manage opnsense OS source images
+    harmony-ve opnsense-img      # manage opnsense OS images
+
+    harmony-ve vm                # manage vm
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  DESCRIPTION
+
+    Automation CLI to easily provision and manage a Virtualized Execution Environment for testing and learning Harmony.
+
+    This tool allows:
+      - new harmony users to start testing within 15 minutes on their development desktop
+      - automate virtualized test in pipeline
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+
+  }
+
+
+  # Implement functions
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+   deps|dependencies)
+	   harmony-ve-dependencies "${@:2}"
+	   ;;
+   net|network)
+	   harmony-ve-network"${@:2}"
+	   ;;
+   img-src|opnsense-img-src)
+	   harmony-ve-opnsense-img-src "${@:2}"
+	   ;;
+   img|opnsense-img)
+	   harmony-ve-opnsense-img "${@:2}"
+	   ;;
+   vm)
+	   harmony-ve-vm "${@:2}"
+	   ;;
+
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve "${@}"
+

examples/opnsense/scripts/harmony-ve-dependencies

@@ -0,0 +1,125 @@
+#! /bin/bash
+#
+# virt-install <= virtinst
+# quemu-img
+
+
+
+harmony-ve-dependencies()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-dependencies 
+
+  DESCRIPTION
+
+    Manage localhost dependencies needed for Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    devops-dependencies [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    devops check     # Check that dependencies are installed
+    devops install   # Install missing dependencies
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  _check_dependencies(){
+      . "${SCRIPTS_DIR}/dependencies-management"
+      missing=0
+      NEED_IP=false
+      NEED_KVM=false
+      NEED_VIRT_CUSTOMIZE=false
+      NEED_WGET=false
+      is_debian_family || _fatal only debian based version is supported
+      has_ip || { 
+	missing=$(( missing + 1)); 
+	_warn "ip command is missing";
+        NEED_IP=true
+      }
+      has_virsh ||{
+	missing=$(( missing + 1)); 
+	_warn "virsh command is missing";
+        NEED_KVM=true
+      } 
+      has_virt_customize || {
+	missing=$(( missing + 1)); 
+	_warn "virt-customize command is missing";
+        NEED_VIRT_CUSTOMIZE=true
+      }
+      has_wget || has_curl || {
+	missing=$(( missing + 1)); 
+	_warn "wget and curl commands are missing"; NEED_WGET=true
+
+      }
+
+  }
+
+  _install_dependencies(){
+
+    [ "$NEED_KVM" != "true" ] || install_kvm 
+    [ "$NEED_VIRT_CUSTOMIZE" != "true" ] || install_virt_customize 
+    [ "$NEED_WGET" != "true" ] || install_wget
+  }
+
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   cdeps|check-dependencies)
+	   _check_dependencies
+           if [ "$missing" -gt 0 ]; then 
+	      exit 1
+           fi
+	   _warn No missing dependencies
+	   ;;
+   ideps|install-dependencies)
+	   _check_dependencies
+	   _install_dependencies
+	   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-dependencies "${@}"
+

examples/opnsense/scripts/harmony-ve-network

@@ -0,0 +1,232 @@
+
+#! /bin/bash
+# todo: allow wan to switch from ethernet to wifi 
+
+
+harmony-ve-network()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-network
+
+  DESCRIPTION
+
+    Modify localhost network for Harmony Virtual Execution Environment
+  SYNOPSYS
+
+    harmony-ve-network [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-network check
+    harmony-ve-network setup
+    harmony-ve-network cleanup 
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+
+
+  IMPLEMENTATION NOTES
+
+    - use the network manager present to add 2 bridges (wan + lan)
+
+
+EOM
+
+  }
+
+  # dependency management
+
+  _is_service_used(){
+    service=$1
+    sudo systemctl list-unit-files $service || return 1
+    sudo systemctl status --no-pager $service || return 1
+  }
+
+
+
+  # Implement functions
+
+  _list_bridges(){
+
+    ip -o link show type bridge | awk '{print $2}' | sed 's/://g'
+  }
+
+  _is_a_bridge(){
+    bridge=$1
+    matched=$(_list_bridges | grep "$bridge")
+    [ "$matched" = "$bridge" ] || return 1
+  }
+
+  _bridge_is_up(){
+    _fatal Not implemented
+  }
+
+  _rename_nmcli_profile(){
+    device=$1
+    profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$device:" | cut -d':' -f2)
+    [ "$profile" != "" ] || _fatal Failed to find nmcli profile 
+
+    sudo nmcli con mod "$profile" con-name "$device"
+
+  }
+
+  _create_a_bridge_using_networkmanager(){
+    bridge=$1
+
+   profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$PRIMARY_INTERFACE:" | cut -d':' -f2)
+
+   nmcli conn delete "$profile"
+
+   nmcli conn add type bridge ifname $bridge con-name $bridge || _fatal Fail to create a bridge using nmcli
+
+
+   nmcli con add type bridge-slave ifname $PRIMARY_INTERFACE master $bridge || _fatal Fail to create a slave-interface using nmcli
+
+   nmcli con up $bridge || _fatal Fail to set interface up using nmcli
+ 
+   sudo systemctl restart NetworkManager.service
+  # todo: use a check loop until connection with a timeout
+  #sleep 10
+  #ping nationtech.io | _fatal Internet connection lost
+
+  }
+
+  _delete_a_bridge_using_networkmanager(){
+     device=$1
+
+    nmcli conn delete bridge-slave-$PRIMARY_INTERFACE
+    nmcli conn delete $device
+    
+    nmcli con add type ethernet ifname $PRIMARY_INTERFACE con-name $PRIMARY_INTERFACE autoconnect yes ipv4.method auto ipv6.method ignore
+
+   nmcli conn up "$PRIMARY_INTERFACE"
+   sudo systemctl restart NetworkManager.service
+
+  # todo: use a check loop until connection with a timeout
+  #sleep 10
+  #ping nationtech.io | _fatal Internet connection lost
+
+  }
+
+
+
+  _create_a_bridge(){
+
+    bridge=$1
+
+    [ $USE_NETWORKMANAGER = 0 ] | _fatal "Only NetworkManager is implemented"
+
+    _create_a_bridge_using_networkmanager $bridge
+
+  }
+
+  _setup_a_bridge(){
+
+    $bridge
+
+    bridge_exist=1
+    bridge_is_up=1
+    bridge_has_ip=1
+    bridge_has_route=1
+    bridge_is_working=1
+
+    _is_a_bridge $bridge && bridge_exists=0 || _create_a_bridge $bridge || _fatal Fail to create a bridge
+
+
+    }
+  
+  _get_networkmanager_profile_from_device(){
+
+    device=$1
+
+    profile=$(nmcli -t -f DEVICE,NAME c show --active | grep "^$device:" | cut -d':' -f2)
+
+    [ "$profile" != "" ] || _fatal Fail to retreive nmcli profile
+
+    echo "$profile"
+
+  }
+
+  _find_primary_interface(){
+    PRIMARY_INTERFACE=$(ip route | grep '^default' | sed 's/ dev /!/g' | cut -d'!' -f 2 | awk '{ print $1 }' )  
+    [ "$PRIMARY_INTERFACE" != "" ] || _fatal Fail to find the primary interface
+  }
+
+  _find_used_network_manager(){
+
+    _is_service_used NetworkManager.service && USE_NETWORKMANAGER=0 || USE_NETWORKMANAGER=1
+    _is_service_used systemd-networkd.service && USE_SYSTEMD_NETWORKD=0 || USE_SYSTEMD_NETWORKD=1
+    _is_service_used dhcpd.service && USE_DHCPD=0 || USE_DHCPD=1 
+
+    USE_MANUAL=0 && [ $USE_NETWORKMANAGER = 0 ] || [ $USE_SYSTEMD_NETWORKD = 0 ] || [ $USE_DHCPD = 0 ] || USE_MANUAL=0
+
+    }
+
+  _connect(){
+
+    _find_used_network_manager
+    _find_primary_interface
+
+    _setup_a_bridge $_HVE_WAN_BRIDGE
+    _setup_a_bridge $_HVE_LAN_BRIDGE
+  }
+
+
+
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+  connect)
+    _connect "${@:2}"
+    ;;
+  disconnect)
+    _disconnect "${@:2}"
+    ;;
+ 
+  dev) 
+
+    "${@:2}"
+   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-network "${@}"
+

examples/opnsense/scripts/harmony-ve-opnsense-img

@@ -0,0 +1,150 @@
+
+#! /bin/bash
+
+harmony-ve-opnsense-img()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"  
+
+  export PATH=$SCRIPTS_DIR:$PATH
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+  
+    harmony-ve-opnsense-img
+
+  DESCRIPTION
+
+    Manage opnsense images needed by Harmony Virtual Execution Environment
+  SYNOPSYS
+
+    harmony-vee-opnsense-img [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-opnsense-img list
+    harmony-ve-opnsense-img init NAME VERSION
+    harmony-ve-opnsense-img start NAME
+    harmony-ve-opnsense-img update NAME
+    harmony-ve-opnsense-img delete [NAME]
+
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  # assertions
+
+  _assert_image_do_not_exists(){
+    name=$1
+    [ ! -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "An image '$name' already exists"
+  }
+  _assert_image_exists(){
+    name=$1
+    [ -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "Image '$name' do not exists"
+  }
+
+
+  # Implement functions
+  _init(){
+    name=$1
+    version=${2}
+
+    _assert_image_do_not_exists $name
+    mkdir -p "${_HVE_OPNSENSE_IMG}/$name"
+
+    harmony-ve opnsense-img-src download $version
+
+    sudo qemu-img convert -f raw -O qcow2 "$_HVE_OPNSENSE_SRC_IMG/OPNsense-${version}-nano-amd64.img" "/var/lib/libvirt/images/opnsense-$name.qcow2"
+
+
+
+
+    cat <<-EOM > "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  virt-install \
+	--name $name \
+	--os-variant freebsd14.0 \
+	--vcpus=2,sockets=1,cores=2,threads=1 \
+	--memory 4096 \
+	--disk path="/var/lib/libvirt/images/opnsense-$name.qcow2" \
+	--network bridge=${_HVE_WAN_BRIDGE},model=virtio \
+	--network bridge=${_HVE_LAN_BRIDGE},model=virtio \
+	--graphics none \
+	--console pty,target_type=serial \
+	--import \
+	--autostart
+EOM
+    
+  chmod +x "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  }
+
+
+  _start(){
+    name=$1
+    _assert_image_exists $name
+
+    "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  }
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+  init)
+    _init "${@:2}"
+    ;;
+  start)
+    _start "${@:2}"
+     ;;
+  delete)
+    rm -r ${_HVE_OPNSENSE_IMG}/"$2"
+    ;;
+  ls|list)
+    ls ${_HVE_OPNSENSE_IMG} | cat
+    ;;
+  show)
+    ls ${_HVE_OPNSENSE_IMG}/"$2" | cat
+    ;;
+  *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img "${@}"
+

examples/opnsense/scripts/harmony-ve-opnsense-img-src

@@ -0,0 +1,310 @@
+
+#! /bin/bash
+
+harmony-ve-opnsense-img-src()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"
+
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-opnsense-img-src
+
+  DESCRIPTION
+
+    Manage opnsense source images needed by Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    harmony-vee-opnsense-img-src [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-vee-opnsense-img-src list [--remote]
+    harmony-vee-opnsense-img-src download [VERSION]
+    harmony-vee-opnsense-img-src check [VERSION]
+    harmony-vee-opnsense-img-src delete [VERSION]
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+     
+  DETAILS
+
+    - for 'list', show local images available
+    - for 'list --remote', show available upstream images
+    - for 'download', when no VERSION is specified, use the latest 
+    - for 'delete', when no VERSION is specified, delete all image
+    - use the 'nano' flavor
+EOM
+
+  }
+
+
+  # Implement functions
+  _parse_version_from_image_file_string(){
+    #https://pkg.opnsense.org/releases/25.7/OPNsense-25.7-nano-amd64.img.bz2
+    echo "$1" | cut -d '/' -f 5
+}
+
+
+  _list_local_images(){
+    ls "${_HVE_OPNSENSE_SRC_IMG}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
+  }
+
+
+  _list_remote_images(){
+     curl -L -s "${_HVE_OPNSENSE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2  | cut -d '/' -f 1 | sort -r
+  }
+
+  _latest_version(){
+    _list_remote_images | head -n 1
+  }
+
+  _is_downloaded(){
+     version=$1
+     name="${_HVE_OPNSENSE_SRC_IMG}/OPNsense-${version}-nano-amd64.img"
+    [ -f "$name" ] && return 0 || return 1
+  }
+
+  _is_valid_version(){
+   version=${1}
+    matched_version=$(_list_remote_images | grep "$version")
+    [ "$matched_version" != "" ] && return 0 || return 1
+  }
+
+  _download_img(){
+     version=$1
+
+    _download_crypto_files $version
+
+
+     name="OPNsense-${version}-nano-amd64.img"
+     compressed_name=$name.bz2
+
+     _is_downloaded $version && {
+        _warn "Image '$name' is already downloaded"
+     } || {
+       url=$_HVE_OPNSENSE_URL/$version/$compressed_name
+       >&2 echo DOWNLOAD $url 
+       wget -q -c "${url}"
+
+       _verify_image_checksum $version
+
+       >&2 echo DECOMPRESS $url 
+       bzip2 -d $compressed_name
+     }
+  }
+
+
+  _compare_files_checksum(){
+
+    file1=$1
+    file2=$2
+
+    sha256_1=$(openssl sha256 $file1 | cut -d" " -f2)
+    sha256_2=$(openssl sha256 $file2 | cut -d" " -f2)
+
+    [ "$sha256_1" = "$sha256_2" ] || return 1
+
+  }
+
+  _download_crypto_files(){
+    # see: https://docs.opnsense.org/manual/install.html#download-and-verification
+
+     version=$1
+
+     # download multiple pubkeys from different server
+     pubkey="OPNsense-${version}.pub"
+
+     rm -f $pubkey $pubkey.sig $pubkey.alt1 $pubkey.alt2
+
+       url=$_HVE_OPNSENSE_URL/$version/$pubkey
+       wget -q -c "${url}"
+ 
+      # failing: 
+      wget -q -c "${url}.sig" 
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $pubkey.sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $pubkey || _fatal "Can't verify the signature of the public key"
+
+       url_alt1=$_HVE_OPNSENSE_URL_ALT1/$version/$pubkey
+       wget -q -c -O "$pubkey.alt1" "${url_alt1}"
+
+       url_alt2=$_HVE_OPNSENSE_URL_ALT2/$version/$pubkey
+       wget -q -c -O "$pubkey.alt2" "${url_alt2}"
+     
+       _compare_files_checksum $pubkey $pubkey.alt1 || _fatal "Fail to compare pubkeys" ;
+
+       _compare_files_checksum $pubkey $pubkey.alt2 || _fatal "Fail to compare pubkeys" ;
+
+
+    
+     img_sig="OPNsense-${version}-nano-amd64.img.sig"
+
+     sha256_name="OPNsense-${version}-checksums-amd64.sha256"
+     sha256_sig=$sha256_name.sig
+
+     [ ! -f "$img_sig" ]     || rm $img_sig
+     [ ! -f "$sha256_name" ] || rm $sha256_name
+     [ ! -f "$sha256_sig" ]  || rm $sha256_sig
+
+
+     for file in $img_sig $sha256_name $sha256_sig;
+     do
+       url=$_HVE_OPNSENSE_URL/$version/$file
+       wget -q -c "${url}"
+    done
+
+
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $sha256_sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $sha256_name || _fatal "Can't verify the signature of the checksum file"
+
+  }
+
+  _download(){
+
+    version=${1:-}
+    [ "${version:-}" != "" ] || _fatal "Must pass a VERSION for downloading"
+
+    _is_valid_version $version  || _fatal "'$version' is not a valid version number"
+
+
+    _download_img ${version}
+
+  }
+
+  _verify_image_checksum(){
+
+    version=$1
+    name="OPNsense-${version}-nano-amd64.img.bz2"
+    sha256_file="OPNsense-${version}-checksums-amd64.sha256"
+    
+    sha256=$(cat $sha256_file | grep "$name" | cut -d'=' -f 2 | tr -s [:space:])
+
+    echo "$sha256 $name" | sha256sum -c || _fatal "Checksum failed for '$name'"
+
+  }
+
+  _verify_image_signature(){
+     version=$1
+
+     # download multiple pubkeys from different server
+     pubkey="OPNsense-${version}.pub"
+     img_name="OPNsense-${version}-nano-amd64.img"
+     img_sig="${img_name}.sig"
+
+
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $img_sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $img_name || _fatal "Can't verify image signature"
+
+  }
+
+  _check(){
+    version=${1:-}
+    if [ "${version:-}" = "" ] ; then
+      for version in $(_list_local_images);
+      do
+        >&2 echo check $version
+        _download_crypto_files $version
+        _verify_image_signature $version
+      done
+    else
+      _download_crypto_files $version
+      _verify_image_signature $version
+    fi
+  }
+
+
+  _delete(){
+    version=${1:-}
+    if [ -z "${version:-1}" ]; then
+       _clear
+       rm -f *.img
+    else
+       rm -f *$version*.img
+    fi
+
+
+  }
+
+  _clear(){
+     rm -f *.pub *.sig *.bz2 *.alt1 *.alt2 *.sha256
+  }
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+  ls|list)
+     
+    if [ "${2:-}" == "" ]; then
+       _list_local_images
+    elif [ "${2:-}" == "--remote" ]; then
+       _list_remote_images
+    else
+       _warn "Unknown option '$2'"
+    fi
+    ;;
+    download)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _download "${2:-"$(_latest_version)"}"
+      popd 
+      ;;
+    delete)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _delete "${@:2}"
+      popd
+      ;;
+    check)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _check "${@:2}"
+      popd
+      ;;
+    show)
+      ls $_HVE_OPNSENSE_SRC_IMG | cat
+      ;;
+    clear)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _clear "${@:2}"
+      popd
+      ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img-src "${@}"
+

examples/opnsense/scripts/harmony-ve-vm

@@ -0,0 +1,77 @@
+
+#! /bin/bash
+
+harmony-ve-vm()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+	harmony-ve-mv
+
+  DESCRIPTION
+
+    Manage virtalized hosts (VM) dependencies by Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    harmony-ve-vm [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-vm list
+    harmony-ve-vm create
+    harmony-ve-vm start
+    harmony-ve-vm stop
+    harmony-ve-vm login
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  # Implement functions
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-vm "${@}"
+

examples/opnsense/scripts/learn-harmony

@@ -0,0 +1,75 @@
+#! /bin/bash
+
+learn-harmony()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  export PATH=$SCRIPTS_DIR:$PATH
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+    learn-harmony -- Harmony Learning Tool prototype
+    ($(basename ${BASE_DIR}) example)
+
+  SYNOPSYS
+
+    learn-harmony [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    learn-harmony list              # List learning steps
+    learn-harmony show STEP         # Show instruction of step STEP
+    learn-harmony check [STEP]        # Verify that your ready to begin step STEP+1
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+case "${1:-}" in
+
+  -h|--help|"")
+	_short_help
+	_extra_help
+	;;
+   ls|list)
+	   echo "not implemented"
+	   ;;
+   show)
+	   echo "not implemented"
+	   ;;
+   verify)
+	   echo "not implemented"
+	   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || learn-harmony "${@}"
+

examples/opnsense/scripts/setup

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+
+export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")"):"${PATH}"

examples/opnsense/src/main.rs

@@ -1,135 +1,70 @@
-use std::{
-    net::{IpAddr, Ipv4Addr},
-    sync::Arc,
-};
-
-use brocade::BrocadeOptions;
-use cidr::Ipv4Cidr;
 use harmony::{
-    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
+    config::secret::OPNSenseFirewallCredentials,
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    infra::opnsense::OPNSenseFirewall,
     inventory::Inventory,
-    modules::{
+    modules::{dhcp::DhcpScore, opnsense::OPNsenseShellCommandScore},
-        dummy::{ErrorScore, PanicScore, SuccessScore},
+    topology::LogicalHost,
-        http::StaticFilesHttpScore,
-        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
-        opnsense::OPNsenseShellCommandScore,
-        tftp::TftpScore,
-    },
-    topology::{LogicalHost, UnmanagedRouter},
 };
-use harmony_macros::{ip, mac_address};
+use harmony_macros::{ip, ipv4};
 use harmony_secret::{Secret, SecretManager};
-use harmony_types::net::Url;
 use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
-    let firewall = harmony::topology::LogicalHost {
+    let firewall = LogicalHost {
-        ip: ip!("192.168.5.229"),
+        ip: ip!("192.168.1.1"),
         name: String::from("opnsense-1"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+    let opnsense_auth = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>()
         .await
         .expect("Failed to get credentials");
 
-    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
+    let opnsense = OPNSenseFirewall::new(
-    let brocade_options = Some(BrocadeOptions {
+        firewall,
-        dry_run: *harmony::config::DRY_RUN,
+        None,
-        ..Default::default()
+        &opnsense_auth.username,
-    });
+        &opnsense_auth.password,
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
     )
-    .await
+    .await;
-    .expect("Failed to connect to switch");
 
-    let switch_client = Arc::new(switch_client);
+    let dhcp_score = DhcpScore {
-
+        dhcp_range: (
-    let opnsense = Arc::new(
+            ipv4!("192.168.1.100").into(),
-        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
+            ipv4!("192.168.1.150").into(),
-    );
-    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
-    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
-    let gateway_ip = IpAddr::V4(gateway_ipv4);
-    let topology = harmony::topology::HAClusterTopology {
-        kubeconfig: None,
-        domain_name: "demo.harmony.mcd".to_string(),
-        router: Arc::new(UnmanagedRouter::new(
-            gateway_ip,
-            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
-        )),
-        load_balancer: opnsense.clone(),
-        firewall: opnsense.clone(),
-        tftp_server: opnsense.clone(),
-        http_server: opnsense.clone(),
-        dhcp_server: opnsense.clone(),
-        dns_server: opnsense.clone(),
-        control_plane: vec![LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        }],
-        bootstrap_host: LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        },
-        workers: vec![],
-        node_exporter: opnsense.clone(),
-        switch_client: switch_client.clone(),
-    };
-
-    let inventory = Inventory {
-        location: Location::new(
-            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
-            "wk".to_string(),
         ),
-        switch: SwitchGroup::from([]),
+        host_binding: vec![],
-        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
+        next_server: None,
-        storage_host: vec![],
+        boot_filename: None,
-        worker_host: vec![],
+        filename: None,
-        control_plane_host: vec![
+        filename64: None,
-            PhysicalHost::empty(HostCategory::Server)
+        filenameipxe: Some("filename.ipxe".to_string()),
-                .mac_address(mac_address!("08:00:27:62:EC:C3")),
+        domain: None,
-        ],
     };
+    // let dns_score = OKDDnsScore::new(&topology);
+    // let load_balancer_score = OKDLoadBalancerScore::new(&topology);
+    //
+    // let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
+    // let http_score = StaticFilesHttpScore {
+    //     folder_to_serve: Some(Url::LocalFolder(
+    //         "./data/watchguard/pxe-http-files".to_string(),
+    //     )),
+    //     files: vec![],
+    //     remote_path: None,
+    // };
+    let opnsense_config = opnsense.get_opnsense_config();
 
-    // TODO regroup smaller scores in a larger one such as this
+    harmony_cli::run(
-    // let okd_boostrap_preparation();
+        Inventory::autoload(),
-
+        opnsense,
-    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
-    let dns_score = OKDDnsScore::new(&topology);
-    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
-
-    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    let http_score = StaticFilesHttpScore {
-        folder_to_serve: Some(Url::LocalFolder(
-            "./data/watchguard/pxe-http-files".to_string(),
-        )),
-        files: vec![],
-        remote_path: None,
-    };
-
-    harmony_tui::run(
-        inventory,
-        topology,
         vec![
-            Box::new(dns_score),
             Box::new(dhcp_score),
-            Box::new(load_balancer_score),
-            Box::new(tftp_score),
-            Box::new(http_score),
             Box::new(OPNsenseShellCommandScore {
-                opnsense: opnsense.get_opnsense_config(),
+                opnsense: opnsense_config,
-                command: "touch /tmp/helloharmonytouching".to_string(),
+                command: "touch /tmp/helloharmonytouching_2".to_string(),
             }),
-            Box::new(SuccessScore {}),
-            Box::new(ErrorScore {}),
-            Box::new(PanicScore {}),
         ],
+        None,
     )
     .await
     .unwrap();

examples/opnsense_node_exporter/src/main.rs

@@ -1,79 +0,0 @@
-use std::{
-    net::{IpAddr, Ipv4Addr},
-    sync::Arc,
-};
-
-use async_trait::async_trait;
-use cidr::Ipv4Cidr;
-use harmony::{
-    executors::ExecutorError,
-    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
-    inventory::Inventory,
-    modules::opnsense::node_exporter::NodeExporterScore,
-    topology::{
-        HAClusterTopology, LogicalHost, PreparationError, PreparationOutcome, Topology,
-        UnmanagedRouter, node_exporter::NodeExporter,
-    },
-};
-use harmony_macros::{ip, ipv4, mac_address};
-
-struct OpnSenseTopology {
-    node_exporter: Arc<dyn NodeExporter>,
-}
-
-#[async_trait]
-impl Topology for OpnSenseTopology {
-    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
-        Ok(PreparationOutcome::Success {
-            details: "Success".to_string(),
-        })
-    }
-    fn name(&self) -> &str {
-        "OpnsenseTopology"
-    }
-}
-
-#[async_trait]
-impl NodeExporter for OpnSenseTopology {
-    async fn ensure_initialized(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.ensure_initialized().await
-    }
-
-    async fn commit_config(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.commit_config().await
-    }
-
-    async fn reload_restart(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.reload_restart().await
-    }
-}
-
-#[tokio::main]
-async fn main() {
-    let firewall = harmony::topology::LogicalHost {
-        ip: ip!("192.168.1.1"),
-        name: String::from("fw0"),
-    };
-
-    let opnsense = Arc::new(
-        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
-    );
-
-    let topology = OpnSenseTopology {
-        node_exporter: opnsense.clone(),
-    };
-
-    let inventory = Inventory::empty();
-
-    let node_exporter_score = NodeExporterScore {};
-
-    harmony_cli::run(
-        inventory,
-        topology,
-        vec![Box::new(node_exporter_score)],
-        None,
-    )
-    .await
-    .unwrap();
-}

harmony/src/domain/maestro/mod.rs

@@ -67,16 +67,16 @@ impl<T: Topology> Maestro<T> {
         }
     }
 
-    pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
-        let mut score_mut = self.scores.write().expect("Should acquire lock");
-        score_mut.append(&mut scores);
-    }
-
     fn is_topology_initialized(&self) -> bool {
         self.topology_state.status == TopologyStatus::Success
             || self.topology_state.status == TopologyStatus::Noop
     }
 
+    pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
+        let mut score_mut = self.scores.write().expect("Should acquire lock");
+        score_mut.append(&mut scores);
+    }
+
     pub async fn interpret(&self, score: Box<dyn Score<T>>) -> Result<Outcome, InterpretError> {
         if !self.is_topology_initialized() {
             warn!(

harmony/src/domain/topology/ha_cluster.rs

@@ -8,15 +8,12 @@ use kube::api::ObjectMeta;
 use log::debug;
 use log::info;
 
+use crate::modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy};
 use crate::topology::PxeOptions;
 use crate::{data::FileContent, modules::okd::crd::nmstate::NMState};
 use crate::{
     executors::ExecutorError, modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec,
 };
-use crate::{
-    modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy},
-    topology::node_exporter::NodeExporter,
-};
 
 use super::{
     DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
@@ -38,7 +35,6 @@ pub struct HAClusterTopology {
     pub tftp_server: Arc<dyn TftpServer>,
     pub http_server: Arc<dyn HttpServer>,
     pub dns_server: Arc<dyn DnsServer>,
-    pub node_exporter: Arc<dyn NodeExporter>,
     pub switch_client: Arc<dyn SwitchClient>,
     pub bootstrap_host: LogicalHost,
     pub control_plane: Vec<LogicalHost>,
@@ -301,7 +297,6 @@ impl HAClusterTopology {
             tftp_server: dummy_infra.clone(),
             http_server: dummy_infra.clone(),
             dns_server: dummy_infra.clone(),
-            node_exporter: dummy_infra.clone(),
             switch_client: dummy_infra.clone(),
             bootstrap_host: dummy_host,
             control_plane: vec![],
@@ -479,23 +474,6 @@ impl Switch for HAClusterTopology {
         self.configure_bond(config).await?;
         self.configure_port_channel(config).await
     }
-
-    //TODO add snmp here
-}
-
-#[async_trait]
-impl NodeExporter for HAClusterTopology {
-    async fn ensure_initialized(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.ensure_initialized().await
-    }
-
-    async fn commit_config(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.commit_config().await
-    }
-
-    async fn reload_restart(&self) -> Result<(), ExecutorError> {
-        self.node_exporter.reload_restart().await
-    }
 }
 
 #[derive(Debug)]
@@ -685,21 +663,6 @@ impl DnsServer for DummyInfra {
     }
 }
 
-#[async_trait]
-impl NodeExporter for DummyInfra {
-    async fn ensure_initialized(&self) -> Result<(), ExecutorError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-
-    async fn commit_config(&self) -> Result<(), ExecutorError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-
-    async fn reload_restart(&self) -> Result<(), ExecutorError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-}
-
 #[async_trait]
 impl SwitchClient for DummyInfra {
     async fn setup(&self) -> Result<(), SwitchError> {

harmony/src/domain/topology/mod.rs

@@ -1,6 +1,6 @@
 mod ha_cluster;
 pub mod ingress;
-pub mod node_exporter;
+pub mod opnsense;
 use harmony_types::net::IpAddress;
 mod host_binding;
 mod http;

harmony/src/domain/topology/node_exporter.rs

@@ -1,17 +0,0 @@
-use async_trait::async_trait;
-
-use crate::executors::ExecutorError;
-
-#[async_trait]
-pub trait NodeExporter: Send + Sync {
-    async fn ensure_initialized(&self) -> Result<(), ExecutorError>;
-    async fn commit_config(&self) -> Result<(), ExecutorError>;
-    async fn reload_restart(&self) -> Result<(), ExecutorError>;
-}
-
-//TODO complete this impl
-impl std::fmt::Debug for dyn NodeExporter {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_fmt(format_args!("NodeExporter ",))
-    }
-}

harmony/src/domain/topology/opnsense.rs

@@ -0,0 +1,23 @@
+use async_trait::async_trait;
+use log::info;
+
+use crate::{
+    infra::opnsense::OPNSenseFirewall,
+    topology::{PreparationError, PreparationOutcome, Topology},
+};
+
+#[async_trait]
+impl Topology for OPNSenseFirewall {
+    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
+        // FIXME we should be initializing the opnsense config here instead of
+        // OPNSenseFirewall::new as this causes the config to be loaded too early in
+        // harmony initialization process
+        let details = "OPNSenseFirewall topology is ready".to_string();
+        info!("{}", details);
+        Ok(PreparationOutcome::Success { details })
+    }
+
+    fn name(&self) -> &str {
+        "OPNSenseFirewall"
+    }
+}

harmony/src/infra/opnsense/mod.rs

@@ -4,7 +4,6 @@ mod firewall;
 mod http;
 mod load_balancer;
 mod management;
-pub mod node_exporter;
 mod tftp;
 use std::sync::Arc;
 
@@ -26,6 +25,8 @@ impl OPNSenseFirewall {
         self.host.ip
     }
 
+    /// panics : if the opnsense config file cannot be loaded by the underlying opnsense_config
+    /// crate
     pub async fn new(host: LogicalHost, port: Option<u16>, username: &str, password: &str) -> Self {
         Self {
             opnsense_config: Arc::new(RwLock::new(

harmony/src/infra/opnsense/node_exporter.rs

@@ -1,47 +0,0 @@
-use async_trait::async_trait;
-use log::debug;
-
-use crate::{
-    executors::ExecutorError, infra::opnsense::OPNSenseFirewall,
-    topology::node_exporter::NodeExporter,
-};
-
-#[async_trait]
-impl NodeExporter for OPNSenseFirewall {
-    async fn ensure_initialized(&self) -> Result<(), ExecutorError> {
-        let mut config = self.opnsense_config.write().await;
-        let node_exporter = config.node_exporter();
-        if let Some(config) = node_exporter.get_full_config() {
-            debug!(
-                "Node exporter available in opnsense config, assuming it is already installed. {config:?}"
-            );
-        } else {
-            config
-                .install_package("os-node_exporter")
-                .await
-                .map_err(|e| {
-                        ExecutorError::UnexpectedError(format!("Executor failed when trying to install os-node_exporter package with error {e:?}"
-                                ))
-                    })?;
-        }
-
-        config
-            .node_exporter()
-            .enable(true)
-            .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))?;
-        Ok(())
-    }
-    async fn commit_config(&self) -> Result<(), ExecutorError> {
-        OPNSenseFirewall::commit_config(self).await
-    }
-
-    async fn reload_restart(&self) -> Result<(), ExecutorError> {
-        self.opnsense_config
-            .write()
-            .await
-            .node_exporter()
-            .reload_restart()
-            .await
-            .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))
-    }
-}

harmony/src/modules/opnsense/mod.rs

@@ -1,4 +1,3 @@
-pub mod node_exporter;
 mod shell;
 mod upgrade;
 pub use shell::*;

harmony/src/modules/opnsense/node_exporter.rs

@@ -1,70 +0,0 @@
-use async_trait::async_trait;
-use harmony_types::id::Id;
-use log::info;
-use serde::Serialize;
-
-use crate::{
-    data::Version,
-    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
-    inventory::Inventory,
-    score::Score,
-    topology::{Topology, node_exporter::NodeExporter},
-};
-
-#[derive(Debug, Clone, Serialize)]
-pub struct NodeExporterScore {}
-
-impl<T: Topology + NodeExporter> Score<T> for NodeExporterScore {
-    fn name(&self) -> String {
-        "NodeExporterScore".to_string()
-    }
-
-    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(NodeExporterInterpret {})
-    }
-}
-
-#[derive(Debug)]
-pub struct NodeExporterInterpret {}
-
-#[async_trait]
-impl<T: Topology + NodeExporter> Interpret<T> for NodeExporterInterpret {
-    async fn execute(
-        &self,
-        _inventory: &Inventory,
-        node_exporter: &T,
-    ) -> Result<Outcome, InterpretError> {
-        info!(
-            "Making sure node exporter is initiailized: {:?}",
-            node_exporter.ensure_initialized().await?
-        );
-
-        info!("Applying Node Exporter configuration");
-
-        node_exporter.commit_config().await?;
-
-        info!("Reloading and restarting Node Exporter");
-
-        node_exporter.reload_restart().await?;
-
-        Ok(Outcome::success(format!(
-            "NodeExporter successfully configured"
-        )))
-    }
-
-    fn get_name(&self) -> InterpretName {
-        InterpretName::Custom("NodeExporter")
-    }
-
-    fn get_version(&self) -> Version {
-        todo!()
-    }
-
-    fn get_status(&self) -> InterpretStatus {
-        todo!()
-    }
-
-    fn get_children(&self) -> Vec<Id> {
-        todo!()
-    }
-}

opnsense-config-xml/src/data/opnsense.rs

@@ -17,7 +17,7 @@ pub struct OPNsense {
     pub interfaces: NamedList<Interface>,
     pub dhcpd: NamedList<DhcpInterface>,
     pub snmpd: Snmpd,
-    pub syslog: Option<Syslog>,
+    pub syslog: Syslog,
     pub nat: Nat,
     pub filter: Filters,
     pub load_balancer: Option<LoadBalancer>,
@@ -190,7 +190,7 @@ pub struct System {
     pub webgui: WebGui,
     pub usevirtualterminal: u8,
     pub disablenatreflection: Option<String>,
-    pub disableconsolemenu: Option<u8>,
+    pub disableconsolemenu: u8,
     pub disablevlanhwfilter: u8,
     pub disablechecksumoffloading: u8,
     pub disablesegmentationoffloading: u8,
@@ -216,7 +216,7 @@ pub struct System {
     pub maximumfrags: Option<MaybeString>,
     pub aliasesresolveinterval: Option<MaybeString>,
     pub maximumtableentries: Option<MaybeString>,
-    pub language: Option<String>,
+    pub language: Option<MaybeString>,
     pub dnsserver: Option<MaybeString>,
     pub dns1gw: Option<String>,
     pub dns2gw: Option<String>,
@@ -233,16 +233,16 @@ pub struct System {
 #[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct Ssh {
     pub group: String,
-    pub noauto: Option<u8>,
+    pub noauto: u8,
-    pub interfaces: Option<MaybeString>,
+    pub interfaces: MaybeString,
-    pub kex: Option<MaybeString>,
+    pub kex: MaybeString,
-    pub ciphers: Option<MaybeString>,
+    pub ciphers: MaybeString,
-    pub macs: Option<MaybeString>,
+    pub macs: MaybeString,
-    pub keys: Option<MaybeString>,
+    pub keys: MaybeString,
-    pub enabled: Option<String>,
+    pub enabled: String,
-    pub passwordauth: Option<u8>,
+    pub passwordauth: u8,
-    pub keysig: Option<MaybeString>,
+    pub keysig: MaybeString,
-    pub permitrootlogin: Option<u8>,
+    pub permitrootlogin: u8,
     pub rekeylimit: Option<MaybeString>,
 }
 
@@ -306,11 +306,11 @@ pub struct WebGui {
     pub protocol: String,
     #[yaserde(rename = "ssl-certref")]
     pub ssl_certref: String,
-    pub port: Option<MaybeString>,
+    pub port: MaybeString,
     #[yaserde(rename = "ssl-ciphers")]
-    pub ssl_ciphers: Option<MaybeString>,
+    pub ssl_ciphers: MaybeString,
-    pub interfaces: Option<MaybeString>,
+    pub interfaces: MaybeString,
-    pub compression: Option<MaybeString>,
+    pub compression: MaybeString,
     pub nohttpreferercheck: Option<u8>,
 }
 
@@ -433,7 +433,7 @@ pub struct OPNsenseXmlSection {
     #[yaserde(rename = "Interfaces")]
     pub interfaces: Option<ConfigInterfaces>,
     #[yaserde(rename = "NodeExporter")]
-    pub node_exporter: Option<NodeExporter>,
+    pub node_exporter: Option<RawXml>,
     #[yaserde(rename = "Kea")]
     pub kea: Option<RawXml>,
     pub monit: Option<Monit>,
@@ -1595,21 +1595,3 @@ pub struct Ifgroups {
     #[yaserde(attribute = true)]
     pub version: String,
 }
-
-#[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
-pub struct NodeExporter {
-    pub enabled: u8,
-    pub listenaddress: Option<MaybeString>,
-    pub listenport: u16,
-    pub cpu: u8,
-    pub exec: u8,
-    pub filesystem: u8,
-    pub loadavg: u8,
-    pub meminfo: u8,
-    pub netdev: u8,
-    pub time: u8,
-    pub devstat: u8,
-    pub interrupts: u8,
-    pub ntp: u8,
-    pub zfs: u8,
-}

opnsense-config/src/config/config.rs

@@ -5,8 +5,7 @@ use crate::{
     error::Error,
     modules::{
         caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::UnboundDnsConfig,
-        dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig,
+        dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig, tftp::TftpConfig,
-        node_exporter::NodeExporterConfig, tftp::TftpConfig,
     },
 };
 use log::{debug, info, trace, warn};
@@ -14,7 +13,6 @@ use opnsense_config_xml::OPNsense;
 use russh::client;
 use serde::Serialize;
 use sha2::Digest;
-use tokio::time::{sleep, Duration};
 
 use super::{ConfigManager, OPNsenseShell};
 
@@ -73,10 +71,6 @@ impl Config {
         LoadBalancerConfig::new(&mut self.opnsense, self.shell.clone())
     }
 
-    pub fn node_exporter(&mut self) -> NodeExporterConfig<'_> {
-        NodeExporterConfig::new(&mut self.opnsense, self.shell.clone())
-    }
-
     pub async fn upload_files(&self, source: &str, destination: &str) -> Result<String, Error> {
         self.shell.upload_folder(source, destination).await
     }
@@ -156,8 +150,7 @@ impl Config {
 
     async fn reload_config(&mut self) -> Result<(), Error> {
         info!("Reloading opnsense live config");
-        let (opnsense, _sha2) = Self::get_opnsense_instance(self.repository.clone()).await?;
+        let (opnsense, sha2) = Self::get_opnsense_instance(self.repository.clone()).await?;
-        self.opnsense = opnsense;
         Ok(())
     }
 

opnsense-config/src/modules/mod.rs

@@ -4,5 +4,4 @@ pub mod dhcp_legacy;
 pub mod dns;
 pub mod dnsmasq;
 pub mod load_balancer;
-pub mod node_exporter;
 pub mod tftp;

opnsense-config/src/modules/node_exporter.rs

@@ -1,54 +0,0 @@
-use std::sync::Arc;
-
-use opnsense_config_xml::{NodeExporter, OPNsense};
-
-use crate::{config::OPNsenseShell, Error};
-
-pub struct NodeExporterConfig<'a> {
-    opnsense: &'a mut OPNsense,
-    opnsense_shell: Arc<dyn OPNsenseShell>,
-}
-
-impl<'a> NodeExporterConfig<'a> {
-    pub fn new(opnsense: &'a mut OPNsense, opnsense_shell: Arc<dyn OPNsenseShell>) -> Self {
-        Self {
-            opnsense,
-            opnsense_shell,
-        }
-    }
-
-    pub fn get_full_config(&self) -> &Option<NodeExporter> {
-        &self.opnsense.opnsense.node_exporter
-    }
-
-    fn with_node_exporter<F, R>(&mut self, f: F) -> Result<R, &'static str>
-    where
-        F: FnOnce(&mut NodeExporter) -> R,
-    {
-        match &mut self.opnsense.opnsense.node_exporter.as_mut() {
-            Some(node_exporter) => Ok(f(node_exporter)),
-            None => Err("node exporter is not yet installed"),
-        }
-    }
-
-    pub fn enable(&mut self, enabled: bool) -> Result<(), &'static str> {
-        self.with_node_exporter(|node_exporter| node_exporter.enabled = enabled as u8)
-            .map(|_| ())
-    }
-
-    pub async fn reload_restart(&self) -> Result<(), Error> {
-        self.opnsense_shell
-            .exec("configctl node_exporter stop")
-            .await?;
-        self.opnsense_shell
-            .exec("configctl template reload OPNsense/NodeExporter")
-            .await?;
-        self.opnsense_shell
-            .exec("configctl node_exporter configtest")
-            .await?;
-        self.opnsense_shell
-            .exec("configctl node_exporter start")
-            .await?;
-        Ok(())
-    }
-}