feat: provide an unmanaged switch

Also alias `port` to support both `port` and `ports` as per the nmstate spec.

Reviewed-on: NationTech/harmony#186

Cargo.lock

@@ -1719,24 +1719,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "example-ha-cluster"
-version = "0.1.0"
-dependencies = [
- "brocade",
- "cidr",
- "env_logger",
- "harmony",
- "harmony_macros",
- "harmony_secret",
- "harmony_tui",
- "harmony_types",
- "log",
- "serde",
- "tokio",
- "url",
-]
-
 [[package]]
 name = "example-kube-rs"
 version = "0.1.0"
@@ -1856,24 +1838,6 @@ dependencies = [
 [[package]]
 name = "example-opnsense"
 version = "0.1.0"
-dependencies = [
- "brocade",
- "cidr",
- "env_logger",
- "harmony",
- "harmony_cli",
- "harmony_macros",
- "harmony_secret",
- "harmony_types",
- "log",
- "serde",
- "tokio",
- "url",
-]
-
-[[package]]
-name = "example-opnsense-2"
-version = "0.1.0"
 dependencies = [
  "brocade",
  "cidr",

examples/ha_cluster/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "example-ha-cluster"
-edition = "2024"
-version.workspace = true
-readme.workspace = true
-license.workspace = true
-publish = false
-
-[dependencies]
-harmony = { path = "../../harmony" }
-harmony_tui = { path = "../../harmony_tui" }
-harmony_types = { path = "../../harmony_types" }
-cidr = { workspace = true }
-tokio = { workspace = true }
-harmony_macros = { path = "../../harmony_macros" }
-log = { workspace = true }
-env_logger = { workspace = true }
-url = { workspace = true }
-harmony_secret = { path = "../../harmony_secret" }
-brocade = { path = "../../brocade" }
-serde = { workspace = true }

examples/ha_cluster/README.md

@@ -1,15 +0,0 @@
-## OPNSense demo
-
-Download the virtualbox snapshot from {{TODO URL}}
-
-Start the virtualbox image
-
-This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
-
-Credentials are opnsense default (root/opnsense)
-
-Run the project with the correct ip address on the command line :
-
-```bash
-cargo run -p example-opnsense -- 192.168.5.229
-```

examples/ha_cluster/src/main.rs

@@ -1,141 +0,0 @@
-use std::{
-    net::{IpAddr, Ipv4Addr},
-    sync::Arc,
-};
-
-use brocade::BrocadeOptions;
-use cidr::Ipv4Cidr;
-use harmony::{
-    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
-    inventory::Inventory,
-    modules::{
-        dummy::{ErrorScore, PanicScore, SuccessScore},
-        http::StaticFilesHttpScore,
-        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
-        opnsense::OPNsenseShellCommandScore,
-        tftp::TftpScore,
-    },
-    topology::{LogicalHost, UnmanagedRouter},
-};
-use harmony_macros::{ip, mac_address};
-use harmony_secret::{Secret, SecretManager};
-use harmony_types::net::Url;
-use serde::{Deserialize, Serialize};
-
-#[tokio::main]
-async fn main() {
-    let firewall = harmony::topology::LogicalHost {
-        ip: ip!("192.168.5.229"),
-        name: String::from("opnsense-1"),
-    };
-
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .expect("Failed to get credentials");
-
-    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
-    )
-    .await
-    .expect("Failed to connect to switch");
-
-    let switch_client = Arc::new(switch_client);
-
-    let opnsense = Arc::new(
-        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
-    );
-    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
-    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
-    let gateway_ip = IpAddr::V4(gateway_ipv4);
-    let topology = harmony::topology::HAClusterTopology {
-        kubeconfig: None,
-        domain_name: "demo.harmony.mcd".to_string(),
-        router: Arc::new(UnmanagedRouter::new(
-            gateway_ip,
-            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
-        )),
-        load_balancer: opnsense.clone(),
-        firewall: opnsense.clone(),
-        tftp_server: opnsense.clone(),
-        http_server: opnsense.clone(),
-        dhcp_server: opnsense.clone(),
-        dns_server: opnsense.clone(),
-        control_plane: vec![LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        }],
-        bootstrap_host: LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        },
-        workers: vec![],
-        switch_client: switch_client.clone(),
-    };
-
-    let inventory = Inventory {
-        location: Location::new(
-            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
-            "wk".to_string(),
-        ),
-        switch: SwitchGroup::from([]),
-        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
-        storage_host: vec![],
-        worker_host: vec![],
-        control_plane_host: vec![
-            PhysicalHost::empty(HostCategory::Server)
-                .mac_address(mac_address!("08:00:27:62:EC:C3")),
-        ],
-    };
-
-    // TODO regroup smaller scores in a larger one such as this
-    // let okd_boostrap_preparation();
-
-    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
-    let dns_score = OKDDnsScore::new(&topology);
-    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
-
-    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    let http_score = StaticFilesHttpScore {
-        folder_to_serve: Some(Url::LocalFolder(
-            "./data/watchguard/pxe-http-files".to_string(),
-        )),
-        files: vec![],
-        remote_path: None,
-    };
-
-    harmony_tui::run(
-        inventory,
-        topology,
-        vec![
-            Box::new(dns_score),
-            Box::new(dhcp_score),
-            Box::new(load_balancer_score),
-            Box::new(tftp_score),
-            Box::new(http_score),
-            Box::new(OPNsenseShellCommandScore {
-                opnsense: opnsense.get_opnsense_config(),
-                command: "touch /tmp/helloharmonytouching".to_string(),
-            }),
-            Box::new(SuccessScore {}),
-            Box::new(ErrorScore {}),
-            Box::new(PanicScore {}),
-        ],
-    )
-    .await
-    .unwrap();
-}
-
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}

examples/nanodc/src/main.rs

@@ -1,6 +1,6 @@
 use std::{
     net::{IpAddr, Ipv4Addr},
-    sync::Arc,
+    sync::{Arc, OnceLock},
 };
 
 use brocade::BrocadeOptions;
@@ -107,6 +107,7 @@ async fn main() {
             },
         ],
         switch_client: switch_client.clone(),
+        network_manager: OnceLock::new(),
     };
 
     let inventory = Inventory {

examples/okd_installation/src/topology.rs

@@ -9,7 +9,10 @@ use harmony::{
 use harmony_macros::{ip, ipv4};
 use harmony_secret::{Secret, SecretManager};
 use serde::{Deserialize, Serialize};
-use std::{net::IpAddr, sync::Arc};
+use std::{
+    net::IpAddr,
+    sync::{Arc, OnceLock},
+};
 
 #[derive(Secret, Serialize, Deserialize, Debug, PartialEq)]
 struct OPNSenseFirewallConfig {
@@ -81,6 +84,7 @@ pub async fn get_topology() -> HAClusterTopology {
         },
         workers: vec![],
         switch_client: switch_client.clone(),
+        network_manager: OnceLock::new(),
     }
 }
 

examples/okd_pxe/src/topology.rs

@@ -10,7 +10,10 @@ use harmony::{
 use harmony_macros::{ip, ipv4};
 use harmony_secret::{Secret, SecretManager};
 use serde::{Deserialize, Serialize};
-use std::{net::IpAddr, sync::Arc};
+use std::{
+    net::IpAddr,
+    sync::{Arc, OnceLock},
+};
 
 pub async fn get_topology() -> HAClusterTopology {
     let firewall = harmony::topology::LogicalHost {
@@ -76,6 +79,7 @@ pub async fn get_topology() -> HAClusterTopology {
         },
         workers: vec![],
         switch_client: switch_client.clone(),
+        network_manager: OnceLock::new(),
     }
 }
 

examples/opnsense/Cargo.toml

@@ -8,7 +8,7 @@ publish = false
 
 [dependencies]
 harmony = { path = "../../harmony" }
-harmony_cli = { path = "../../harmony_cli" }
+harmony_tui = { path = "../../harmony_tui" }
 harmony_types = { path = "../../harmony_types" }
 cidr = { workspace = true }
 tokio = { workspace = true }

examples/opnsense/README.md

@@ -1,23 +1,15 @@
-# OPNSense Demo
+## OPNSense demo
 
-This example demonstrate how to manage an Opnsense server with harmony.
+Download the virtualbox snapshot from {{TODO URL}}
 
-todo: add more info
+Start the virtualbox image
 
-## Demo instructions
+This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
 
-todo: add detailed instructions
+Credentials are opnsense default (root/opnsense)
 
-  - setup the example execution environment
+Run the project with the correct ip address on the command line :
-  - setup your system configuration
-    - topology
-    - scores
-    - secrets
-  - build
-  - execute
-  - verify/inspect
-
-## Example execution
-
-See [learning tool documentation](./scripts/README.md)
 
+```bash
+cargo run -p example-opnsense -- 192.168.5.229
+```

examples/opnsense/env.sh

@@ -1,4 +0,0 @@
-export HARMONY_SECRET_NAMESPACE=example-opnsense
-export HARMONY_SECRET_STORE=file
-export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info 
-export RUST_LOG=info

examples/opnsense/scripts/DEVELOP.md

@@ -1,38 +0,0 @@
-## réseau
-
-- fonctionne uniquement avec connection filaire, pas wifi
-- on crée des bridges pour le réseau WAN et LAN pour qu'ils soient facilement accessible sur le poste de travail
-
-```
-# paramètres
-PRIMARY_PROFIL=enpXsY
-NETWORK_LABEL=harmony
-
-WAN_BRIDGE=$NETWORL_LABEL-wan-brd   # max 15 char
-LAN_NIC=$NETWORK_LABEL-lan-nic
-LAN_BRIDGE=$NETWORK_LABEL-lan-brd
-
-
-# Setup WAN bridge
-nmcli c down $PRIMARY
-nmcli c add type bridge ifname $WAN_BRIDGE con-name $WAN_BRIDGE
-nmcli c add type bridge-slave ifname $PRIMARY_PROFILE master $WAN_BRIDGE ipv4.method auto
-# nmcli c up $WAN_BRIDGE
-
-
-# Setup LAN nic
-sudo modprobe dummy
-sudo ip link add $LAN_NIC type dummy
-ip tuntap add dev $LAN_NIC mode tap user root  # todo: why user root?
-
-# Setup LAN bridge
-nmcli c add type bridge ifname $LAN_BRIDGE con-name $LAN_BRIDGE
-nmcli c add type bridge-slave ifname $LAN_NIC master $LAN_BRIDGE ipv4.method auto
-nmcli c up $LAN_BRIDGE
-```
-
-LAN bridge do not have an IP address and appear down. But it is successfully used by opnsense and can be accessed from the host network.
-
-## config opnsense minimale
-
-  - ssh enabled

examples/opnsense/scripts/README.md

@@ -1,96 +0,0 @@
-# Virtualized Execution Environment for Harmony
-
-Scripts included in this directory have 3 purposes:
-
-  - automate initial setup of localhost or VM (nested virtualization) for this example
-  - prototype a solution for an 'OpensenseLocalhostTopology'
-  - prototype 
-
-This exprimentation aim to find an approach suitable for using harmony on virtualised execution environment such that:
-
-  - it straights forward for a user with minimal knowledge to start testing harmony
-  - installation and execution have **minimal impact on the user desktop**
-
-## Usage
-
-### Installation
-
-1. download this directory
-2. add this directory in your PATH (example `. setup`)
-
-
-```
-# show help page
-harmony-vee
-
-# show active configurations
-harmony-vee config
-
-# show what will be modified at installation
-harmony-vee install --dry-run
-
-# install
-harmony-vee install
-
-# show what will be modified at unistallation
-harmony-vee uninstall --dry-run
-
-```
-
-### Create and start a new Virtual Execution Environment
-
-
-```
-# Create a HVEE to test opnsense_score
-harmony-vee init opnsense_score
-
-# List existing HVEE
-harmony-vee list
-
-# Show HVEE information including devices ip and vault type/location
-harmony-vee show opnsense_score
-
-# Start/Stop a HVEE
-harmony-vee stop opnsense_score
-
-
-# Destroy a HVEE instance
-harmony-vee destroy opnsense_score
-
-```
-
-
-### Variable d'environnement
-
-```
-## directory containing harmony-ve data
-# HVE_ROOT=~/.harmony-ve
-
-## OPNSENSE SRC
-# main mirror
-# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
-# first alternative mirror
-# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
-# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
-
-
-## Network
-# HVE_NETWORK_LABEL=harmony
-```
-
-
-## Remarks
-
-  - A nested VM setup could be safer
-
-
-
-## Architecture
-
-### Learning environment directly on host
-
-![localhost case](./doc/automate-opnsense-example-localhost.drawio.png)
-
-### Learning environment nested in a "workspace vm"
-
-![localhost case](./doc/automate-opnsense-example-nested-virtualization.drawio.png)

examples/opnsense/scripts/common

@@ -1,17 +0,0 @@
-#! /bin/bash
-
-_warn(){ >&2 echo "WARNING: $*" ; }
-
-_fatal(){
-  >&2 echo "FATAL ERROR: $*"
-  >&2 echo stopping...
-  exit 1
-}
-
-pushd () {
-    command pushd "$@" > /dev/null
-}
-
-popd () {
-    command popd "$@" > /dev/null
-}

examples/opnsense/scripts/default-env-var

@@ -1,31 +0,0 @@
-# Conventions:
-#   - Namespaced with HVE, short for Harmony Virtualised Execution Environment
-#   - Prefixed values used internally
-#   - Not prefixed may be supercharged by the user
-
-# Root of harmony data
-_HVE_ROOT=${HVE_ROOT:-$HOME/harmony-ve}
-[ -d "$_HVE_ROOT" ] || mkdir -p "${_HVE_ROOT}"
-
-
-_HVE_SRC_IMG=${_HVE_ROOT}/src/images
-[ -d "$_HVE_SRC_IMG" ] || mkdir -p "${_HVE_SRC_IMG}"
-_HVE_IMG=${_HVE_ROOT}/images
-[ -d "$_HVE_IMG" ] || mkdir -p "$_HVE_IMG"
-
-# Opnsense
-_HVE_OPNSENSE_URL=${HVE_OPNSENSE_URL:-https://pkg.opnsense.org/releases}
-# first alternative mirror
-_HVE_OPNSENSE_URL_ALT1=${HVE_OPNSENSE_URL_ALT1:-https://mirror.vraphim.com/opnsense/releases}
-_HVE_OPNSENSE_URL_ALT2=${HVE_OPNSENSE_URL_ALT2:-https://mirror.winsub.kr/opnsense/releases}
-
-_HVE_OPNSENSE_SRC_IMG=${_HVE_SRC_IMG}/opnsense
-[ -d "$_HVE_OPNSENSE_SRC_IMG" ] || mkdir -p "${_HVE_OPNSENSE_SRC_IMG}"
-_HVE_OPNSENSE_IMG=${_HVE_IMG}/opnsense
-[ -d "$_HVE_OPNSENSE_IMG" ] || mkdir -p "${_HVE_OPNSENSE_IMG}"
-
-# Network
-_HVE_NETWORK=${HVE_NETWORK:-harmony}
-_HVE_WAN_BRIDGE=${HVE_WAN_BRIDGE:-${_HVE_NETWORK}-wan-brd}
-_HVE_LAN_BRIDGE=${HVE_LAN_BRIDGE:-${_HVE_NETWORK}-lann-brd}
-

examples/opnsense/scripts/dependencies-management

@@ -1,48 +0,0 @@
-#! /bin/bash
-is_string_empty(){
-    if [ "${*:-}" != "" ]; then
-     return 0
-   else
-     return 1
-   fi
-}
-
-is_debian_family()(
-   is_string_empty "$(apt --version 2> /dev/null )"
-)
-
-has_ip(){
-  is_string_empty "$(ip -V 2> /dev/null)"
-}
-
-has_virsh(){
-  is_string_empty "$(virsh --version 2> /dev/null)"
-}
-
-has_virt_customize(){
-  is_string_empty "$(virt-customize --version 2> /dev/null)"
-}
-
-has_curl(){
-  is_string_empty "$(curl --version 2> /dev/null)"
-}
-has_wget(){
-  is_string_empty "$(wget --version 2> /dev/null)"
-}
-
-install_kvm(){
-  sudo apt install -y --no-install-recommends qemu-system libvirt-clients libvirt-daemon-system
-  sudo usermod -aG libvirt "$USER"
-  # todo: finf how to fix image access out of /var/lib/libvirt/images
-  sudo setfacl -Rm u:libvirt-qemu:rx $_HVE_IMG
-  sudo systemctl restart libvirtd
-}
-
-install_virt_customize(){
-   sudo apt install -y libguestfs-tools
-}
-
-install_wget(){
-  sudo apt install -y wget
-}
-

examples/opnsense/scripts/doc/automate-opnsense-example.drawio

@@ -1,321 +0,0 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" version="28.2.9" pages="2">
-  <diagram name="localhost" id="lK0WmoXmZXwFmV5PC-RW">
-    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
-      <root>
-        <mxCell id="0" />
-        <mxCell id="1" parent="0" />
-        <mxCell id="1Ax8jaXdU0J25Zkiwu96-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="300" y="230" width="700" height="550" as="geometry" />
-        </mxCell>
-        <mxCell id="VVCo9gNhF-9Hs0fZa6i8-1" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="380" y="240" width="60" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="350" y="283" width="230" height="67" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-2" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="590" y="393" width="310" height="270" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-3" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="620" y="403" width="90" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="620" y="250" width="280" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-5" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="610" y="250" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-6" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="370" y="294" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-7" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
-          <mxGeometry x="760" y="290" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-8" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-7" target="X0RF5wBsf5JYURxROWwA-9" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-9" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="771" y="342" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-13" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="810" y="290" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-14" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-13" target="X0RF5wBsf5JYURxROWwA-15" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-15" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="820" y="342" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-16" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="760" y="383" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-17" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-16" target="X0RF5wBsf5JYURxROWwA-18" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-18" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="772.5" y="353" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-19" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="809" y="383" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-20" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-19" target="X0RF5wBsf5JYURxROWwA-21" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-21" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="821.5" y="353" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-22" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="350" y="440" width="230" height="130" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-23" value="&lt;b&gt;Example dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="360" y="450" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-24" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="360" y="510" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-25" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="350" y="590" width="230" height="130" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-26" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="370" y="600" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-27" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="361" y="650" width="208" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-28" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="350" y="353" width="230" height="77" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-29" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="370" y="361.5" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-30" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="620" y="450" width="250" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-31" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="370" y="323" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-33" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs and&lt;/i&gt;&lt;/font&gt;&lt;div&gt;&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;runtime info.&lt;/i&gt;&lt;/font&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="370" y="395" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="360" y="470" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-35" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="626.5" y="264" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-36" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="369" y="620" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="X0RF5wBsf5JYURxROWwA-37" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="626.5" y="420" width="200" height="30" as="geometry" />
-        </mxCell>
-      </root>
-    </mxGraphModel>
-  </diagram>
-  <diagram id="oOuscOXp9aWETXQepMaW" name="nested-virtualization">
-    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
-      <root>
-        <mxCell id="0" />
-        <mxCell id="1" parent="0" />
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="190" y="60" width="1240" height="660" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-37" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="540" y="210" width="830" height="480" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-2" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="84" width="60" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-3" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="240" y="223" width="230" height="67" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="855" y="403.75" width="310" height="266.25" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-5" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="885" y="413.75" width="90" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-6" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="810" y="230" width="530" height="120" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-7" value="&lt;b&gt;workspace VM network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="825" y="234" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-8" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="234" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-10" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-9" target="vj3pG7w7rTpS1tnBMssI-11" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-12" target="vj3pG7w7rTpS1tnBMssI-14" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-15" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1025" y="393.75" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-16" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-15" target="vj3pG7w7rTpS1tnBMssI-17" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-17" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="1037.5" y="363.75" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-18" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1074" y="393.75" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-19" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-18" target="vj3pG7w7rTpS1tnBMssI-20" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-20" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="1086.5" y="363.75" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-21" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="240" y="380" width="230" height="120" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-22" value="&lt;b&gt;Learning dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="250" y="391" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-23" value="&lt;ul&gt;&lt;li&gt;harmony&lt;/li&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="250" y="450" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="240" y="540" width="230" height="130" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-25" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="550" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-26" value="&lt;ul&gt;&lt;li&gt;Can be mounted locally&lt;br&gt;for persistence&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="251" y="600" width="189" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-27" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="240" y="293" width="230" height="67" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-28" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="301.5" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-29" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="885" y="460.75" width="250" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-30" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="263" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-31" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="260" y="321" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-32" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="250" y="410" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="259" y="570" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-35" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="891.5" y="430.75" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-38" value="&lt;b&gt;workspace VM&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="580" y="234" width="130" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-39" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="570" y="305" width="230" height="215" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-40" value="&lt;b&gt;workspace VM dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="580" y="316" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-41" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="580" y="375" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-44" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="570" y="540" width="230" height="130" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-48" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="830" y="277" width="235" height="45" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-45" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="590" y="550" width="152" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-46" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="581" y="600" width="208" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-47" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="589" y="570" width="200" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-9" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
-          <mxGeometry x="1025" y="300.75" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-11" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="1036" y="352.75" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-49" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="1069" y="275" width="251" height="45" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1075" y="300.75" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-14" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="1085" y="352.75" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-50" value="&lt;b&gt;wan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="821.5" y="277" width="70" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-51" value="&lt;b&gt;lan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1055" y="275" width="70" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-52" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="1181" y="403.75" width="159" height="116.25" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-53" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-54" target="vj3pG7w7rTpS1tnBMssI-55" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-54" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1211" y="305" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-55" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="1221" y="357" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-58" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1210" y="393.75" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-59" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-58" target="vj3pG7w7rTpS1tnBMssI-60" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-60" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="1222.5" y="363.75" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-61" value="&lt;b&gt;other vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="1181" y="430" width="90" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="vj3pG7w7rTpS1tnBMssI-63" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="vj3pG7w7rTpS1tnBMssI-24" target="vj3pG7w7rTpS1tnBMssI-44" edge="1">
-          <mxGeometry width="50" height="50" relative="1" as="geometry">
-            <mxPoint x="600" y="770" as="sourcePoint" />
-            <mxPoint x="360" y="885" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="799" y="84" width="280" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-2" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="789" y="84" width="150" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-6" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="989" y="124" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-7" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-6" target="Za91R7Nqk7Jj5VTRes6Y-8" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-8" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
-          <mxGeometry x="999" y="176" width="8" height="8" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="988" y="217" width="30" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-12" target="Za91R7Nqk7Jj5VTRes6Y-14" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-14" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
-          <mxGeometry x="1000.5" y="187" width="5" height="10" as="geometry" />
-        </mxCell>
-        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-15" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;No modification required&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
-          <mxGeometry x="805.5" y="98" width="200" height="30" as="geometry" />
-        </mxCell>
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

examples/opnsense/scripts/env-var

@@ -1,13 +0,0 @@
-## directory containing harmony-ve data
-# HVE_ROOT=~/.harmony-ve
-
-## OPNSENSE SRC
-# main mirror
-# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
-# first alternative mirror
-# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
-# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
-
-
-## Network
-# HVE_NETWORK_LABEL=harmony

examples/opnsense/scripts/harmony-ve

@@ -1,105 +0,0 @@
-
-#! /bin/bash
-
-harmony-ve()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-
-    harmony-ve
-
-  DESCRIPTION
-
-    CLI management toolkit for Harmony Virtualized Execution Environment
-
-  SYNOPSYS
-
-    harmony-ve [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    harmony-ve dependencies      # manage localhost depend
-    harmony-ve network           # manage localhost netork
-
-    harmony-ve opnsense-img-src  # manage opnsense OS source images
-    harmony-ve opnsense-img      # manage opnsense OS images
-
-    harmony-ve vm                # manage vm
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  DESCRIPTION
-
-    Automation CLI to easily provision and manage a Virtualized Execution Environment for testing and learning Harmony.
-
-    This tool allows:
-      - new harmony users to start testing within 15 minutes on their development desktop
-      - automate virtualized test in pipeline
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-EOM
-
-
-  }
-
-
-  # Implement functions
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-   # Commands entrypoints
-   deps|dependencies)
-	   harmony-ve-dependencies "${@:2}"
-	   ;;
-   net|network)
-	   harmony-ve-network"${@:2}"
-	   ;;
-   img-src|opnsense-img-src)
-	   harmony-ve-opnsense-img-src "${@:2}"
-	   ;;
-   img|opnsense-img)
-	   harmony-ve-opnsense-img "${@:2}"
-	   ;;
-   vm)
-	   harmony-ve-vm "${@:2}"
-	   ;;
-
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve "${@}"
-

examples/opnsense/scripts/harmony-ve-dependencies

@@ -1,125 +0,0 @@
-#! /bin/bash
-#
-# virt-install <= virtinst
-# quemu-img
-
-
-
-harmony-ve-dependencies()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-
-    harmony-ve-dependencies 
-
-  DESCRIPTION
-
-    Manage localhost dependencies needed for Harmony Virtual Execution Environment
-
-  SYNOPSYS
-
-    devops-dependencies [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    devops check     # Check that dependencies are installed
-    devops install   # Install missing dependencies
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-EOM
-
-  }
-
-
-  _check_dependencies(){
-      . "${SCRIPTS_DIR}/dependencies-management"
-      missing=0
-      NEED_IP=false
-      NEED_KVM=false
-      NEED_VIRT_CUSTOMIZE=false
-      NEED_WGET=false
-      is_debian_family || _fatal only debian based version is supported
-      has_ip || { 
-	missing=$(( missing + 1)); 
-	_warn "ip command is missing";
-        NEED_IP=true
-      }
-      has_virsh ||{
-	missing=$(( missing + 1)); 
-	_warn "virsh command is missing";
-        NEED_KVM=true
-      } 
-      has_virt_customize || {
-	missing=$(( missing + 1)); 
-	_warn "virt-customize command is missing";
-        NEED_VIRT_CUSTOMIZE=true
-      }
-      has_wget || has_curl || {
-	missing=$(( missing + 1)); 
-	_warn "wget and curl commands are missing"; NEED_WGET=true
-
-      }
-
-  }
-
-  _install_dependencies(){
-
-    [ "$NEED_KVM" != "true" ] || install_kvm 
-    [ "$NEED_VIRT_CUSTOMIZE" != "true" ] || install_virt_customize 
-    [ "$NEED_WGET" != "true" ] || install_wget
-  }
-
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-   cdeps|check-dependencies)
-	   _check_dependencies
-           if [ "$missing" -gt 0 ]; then 
-	      exit 1
-           fi
-	   _warn No missing dependencies
-	   ;;
-   ideps|install-dependencies)
-	   _check_dependencies
-	   _install_dependencies
-	   ;;
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-dependencies "${@}"
-

examples/opnsense/scripts/harmony-ve-network

@@ -1,232 +0,0 @@
-
-#! /bin/bash
-# todo: allow wan to switch from ethernet to wifi 
-
-
-harmony-ve-network()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-  . "${SCRIPTS_DIR}/default-env-var"
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-
-    harmony-ve-network
-
-  DESCRIPTION
-
-    Modify localhost network for Harmony Virtual Execution Environment
-  SYNOPSYS
-
-    harmony-ve-network [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    harmony-ve-network check
-    harmony-ve-network setup
-    harmony-ve-network cleanup 
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-
-
-  IMPLEMENTATION NOTES
-
-    - use the network manager present to add 2 bridges (wan + lan)
-
-
-EOM
-
-  }
-
-  # dependency management
-
-  _is_service_used(){
-    service=$1
-    sudo systemctl list-unit-files $service || return 1
-    sudo systemctl status --no-pager $service || return 1
-  }
-
-
-
-  # Implement functions
-
-  _list_bridges(){
-
-    ip -o link show type bridge | awk '{print $2}' | sed 's/://g'
-  }
-
-  _is_a_bridge(){
-    bridge=$1
-    matched=$(_list_bridges | grep "$bridge")
-    [ "$matched" = "$bridge" ] || return 1
-  }
-
-  _bridge_is_up(){
-    _fatal Not implemented
-  }
-
-  _rename_nmcli_profile(){
-    device=$1
-    profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$device:" | cut -d':' -f2)
-    [ "$profile" != "" ] || _fatal Failed to find nmcli profile 
-
-    sudo nmcli con mod "$profile" con-name "$device"
-
-  }
-
-  _create_a_bridge_using_networkmanager(){
-    bridge=$1
-
-   profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$PRIMARY_INTERFACE:" | cut -d':' -f2)
-
-   nmcli conn delete "$profile"
-
-   nmcli conn add type bridge ifname $bridge con-name $bridge || _fatal Fail to create a bridge using nmcli
-
-
-   nmcli con add type bridge-slave ifname $PRIMARY_INTERFACE master $bridge || _fatal Fail to create a slave-interface using nmcli
-
-   nmcli con up $bridge || _fatal Fail to set interface up using nmcli
- 
-   sudo systemctl restart NetworkManager.service
-  # todo: use a check loop until connection with a timeout
-  #sleep 10
-  #ping nationtech.io | _fatal Internet connection lost
-
-  }
-
-  _delete_a_bridge_using_networkmanager(){
-     device=$1
-
-    nmcli conn delete bridge-slave-$PRIMARY_INTERFACE
-    nmcli conn delete $device
-    
-    nmcli con add type ethernet ifname $PRIMARY_INTERFACE con-name $PRIMARY_INTERFACE autoconnect yes ipv4.method auto ipv6.method ignore
-
-   nmcli conn up "$PRIMARY_INTERFACE"
-   sudo systemctl restart NetworkManager.service
-
-  # todo: use a check loop until connection with a timeout
-  #sleep 10
-  #ping nationtech.io | _fatal Internet connection lost
-
-  }
-
-
-
-  _create_a_bridge(){
-
-    bridge=$1
-
-    [ $USE_NETWORKMANAGER = 0 ] | _fatal "Only NetworkManager is implemented"
-
-    _create_a_bridge_using_networkmanager $bridge
-
-  }
-
-  _setup_a_bridge(){
-
-    $bridge
-
-    bridge_exist=1
-    bridge_is_up=1
-    bridge_has_ip=1
-    bridge_has_route=1
-    bridge_is_working=1
-
-    _is_a_bridge $bridge && bridge_exists=0 || _create_a_bridge $bridge || _fatal Fail to create a bridge
-
-
-    }
-  
-  _get_networkmanager_profile_from_device(){
-
-    device=$1
-
-    profile=$(nmcli -t -f DEVICE,NAME c show --active | grep "^$device:" | cut -d':' -f2)
-
-    [ "$profile" != "" ] || _fatal Fail to retreive nmcli profile
-
-    echo "$profile"
-
-  }
-
-  _find_primary_interface(){
-    PRIMARY_INTERFACE=$(ip route | grep '^default' | sed 's/ dev /!/g' | cut -d'!' -f 2 | awk '{ print $1 }' )  
-    [ "$PRIMARY_INTERFACE" != "" ] || _fatal Fail to find the primary interface
-  }
-
-  _find_used_network_manager(){
-
-    _is_service_used NetworkManager.service && USE_NETWORKMANAGER=0 || USE_NETWORKMANAGER=1
-    _is_service_used systemd-networkd.service && USE_SYSTEMD_NETWORKD=0 || USE_SYSTEMD_NETWORKD=1
-    _is_service_used dhcpd.service && USE_DHCPD=0 || USE_DHCPD=1 
-
-    USE_MANUAL=0 && [ $USE_NETWORKMANAGER = 0 ] || [ $USE_SYSTEMD_NETWORKD = 0 ] || [ $USE_DHCPD = 0 ] || USE_MANUAL=0
-
-    }
-
-  _connect(){
-
-    _find_used_network_manager
-    _find_primary_interface
-
-    _setup_a_bridge $_HVE_WAN_BRIDGE
-    _setup_a_bridge $_HVE_LAN_BRIDGE
-  }
-
-
-
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-  connect)
-    _connect "${@:2}"
-    ;;
-  disconnect)
-    _disconnect "${@:2}"
-    ;;
- 
-  dev) 
-
-    "${@:2}"
-   ;;
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-network "${@}"
-

examples/opnsense/scripts/harmony-ve-opnsense-img

@@ -1,150 +0,0 @@
-
-#! /bin/bash
-
-harmony-ve-opnsense-img()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-  . "${SCRIPTS_DIR}/default-env-var"  
-
-  export PATH=$SCRIPTS_DIR:$PATH
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-  
-    harmony-ve-opnsense-img
-
-  DESCRIPTION
-
-    Manage opnsense images needed by Harmony Virtual Execution Environment
-  SYNOPSYS
-
-    harmony-vee-opnsense-img [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    harmony-ve-opnsense-img list
-    harmony-ve-opnsense-img init NAME VERSION
-    harmony-ve-opnsense-img start NAME
-    harmony-ve-opnsense-img update NAME
-    harmony-ve-opnsense-img delete [NAME]
-
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-EOM
-
-  }
-
-
-  # assertions
-
-  _assert_image_do_not_exists(){
-    name=$1
-    [ ! -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "An image '$name' already exists"
-  }
-  _assert_image_exists(){
-    name=$1
-    [ -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "Image '$name' do not exists"
-  }
-
-
-  # Implement functions
-  _init(){
-    name=$1
-    version=${2}
-
-    _assert_image_do_not_exists $name
-    mkdir -p "${_HVE_OPNSENSE_IMG}/$name"
-
-    harmony-ve opnsense-img-src download $version
-
-    sudo qemu-img convert -f raw -O qcow2 "$_HVE_OPNSENSE_SRC_IMG/OPNsense-${version}-nano-amd64.img" "/var/lib/libvirt/images/opnsense-$name.qcow2"
-
-
-
-
-    cat <<-EOM > "$_HVE_OPNSENSE_IMG/$name/$name.sh"
-
-  virt-install \
-	--name $name \
-	--os-variant freebsd14.0 \
-	--vcpus=2,sockets=1,cores=2,threads=1 \
-	--memory 4096 \
-	--disk path="/var/lib/libvirt/images/opnsense-$name.qcow2" \
-	--network bridge=${_HVE_WAN_BRIDGE},model=virtio \
-	--network bridge=${_HVE_LAN_BRIDGE},model=virtio \
-	--graphics none \
-	--console pty,target_type=serial \
-	--import \
-	--autostart
-EOM
-    
-  chmod +x "$_HVE_OPNSENSE_IMG/$name/$name.sh"
-
-  }
-
-
-  _start(){
-    name=$1
-    _assert_image_exists $name
-
-    "$_HVE_OPNSENSE_IMG/$name/$name.sh"
-
-  }
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-   # Commands entrypoints
-  init)
-    _init "${@:2}"
-    ;;
-  start)
-    _start "${@:2}"
-     ;;
-  delete)
-    rm -r ${_HVE_OPNSENSE_IMG}/"$2"
-    ;;
-  ls|list)
-    ls ${_HVE_OPNSENSE_IMG} | cat
-    ;;
-  show)
-    ls ${_HVE_OPNSENSE_IMG}/"$2" | cat
-    ;;
-  *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img "${@}"
-

examples/opnsense/scripts/harmony-ve-opnsense-img-src

@@ -1,310 +0,0 @@
-
-#! /bin/bash
-
-harmony-ve-opnsense-img-src()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-  . "${SCRIPTS_DIR}/default-env-var"
-
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-
-    harmony-ve-opnsense-img-src
-
-  DESCRIPTION
-
-    Manage opnsense source images needed by Harmony Virtual Execution Environment
-
-  SYNOPSYS
-
-    harmony-vee-opnsense-img-src [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    harmony-vee-opnsense-img-src list [--remote]
-    harmony-vee-opnsense-img-src download [VERSION]
-    harmony-vee-opnsense-img-src check [VERSION]
-    harmony-vee-opnsense-img-src delete [VERSION]
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-     
-  DETAILS
-
-    - for 'list', show local images available
-    - for 'list --remote', show available upstream images
-    - for 'download', when no VERSION is specified, use the latest 
-    - for 'delete', when no VERSION is specified, delete all image
-    - use the 'nano' flavor
-EOM
-
-  }
-
-
-  # Implement functions
-  _parse_version_from_image_file_string(){
-    #https://pkg.opnsense.org/releases/25.7/OPNsense-25.7-nano-amd64.img.bz2
-    echo "$1" | cut -d '/' -f 5
-}
-
-
-  _list_local_images(){
-    ls "${_HVE_OPNSENSE_SRC_IMG}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
-  }
-
-
-  _list_remote_images(){
-     curl -L -s "${_HVE_OPNSENSE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2  | cut -d '/' -f 1 | sort -r
-  }
-
-  _latest_version(){
-    _list_remote_images | head -n 1
-  }
-
-  _is_downloaded(){
-     version=$1
-     name="${_HVE_OPNSENSE_SRC_IMG}/OPNsense-${version}-nano-amd64.img"
-    [ -f "$name" ] && return 0 || return 1
-  }
-
-  _is_valid_version(){
-   version=${1}
-    matched_version=$(_list_remote_images | grep "$version")
-    [ "$matched_version" != "" ] && return 0 || return 1
-  }
-
-  _download_img(){
-     version=$1
-
-    _download_crypto_files $version
-
-
-     name="OPNsense-${version}-nano-amd64.img"
-     compressed_name=$name.bz2
-
-     _is_downloaded $version && {
-        _warn "Image '$name' is already downloaded"
-     } || {
-       url=$_HVE_OPNSENSE_URL/$version/$compressed_name
-       >&2 echo DOWNLOAD $url 
-       wget -q -c "${url}"
-
-       _verify_image_checksum $version
-
-       >&2 echo DECOMPRESS $url 
-       bzip2 -d $compressed_name
-     }
-  }
-
-
-  _compare_files_checksum(){
-
-    file1=$1
-    file2=$2
-
-    sha256_1=$(openssl sha256 $file1 | cut -d" " -f2)
-    sha256_2=$(openssl sha256 $file2 | cut -d" " -f2)
-
-    [ "$sha256_1" = "$sha256_2" ] || return 1
-
-  }
-
-  _download_crypto_files(){
-    # see: https://docs.opnsense.org/manual/install.html#download-and-verification
-
-     version=$1
-
-     # download multiple pubkeys from different server
-     pubkey="OPNsense-${version}.pub"
-
-     rm -f $pubkey $pubkey.sig $pubkey.alt1 $pubkey.alt2
-
-       url=$_HVE_OPNSENSE_URL/$version/$pubkey
-       wget -q -c "${url}"
- 
-      # failing: 
-      wget -q -c "${url}.sig" 
-      rm -f /tmp/file.sig
-      openssl base64 -d -in $pubkey.sig -out /tmp/file.sig
-      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $pubkey || _fatal "Can't verify the signature of the public key"
-
-       url_alt1=$_HVE_OPNSENSE_URL_ALT1/$version/$pubkey
-       wget -q -c -O "$pubkey.alt1" "${url_alt1}"
-
-       url_alt2=$_HVE_OPNSENSE_URL_ALT2/$version/$pubkey
-       wget -q -c -O "$pubkey.alt2" "${url_alt2}"
-     
-       _compare_files_checksum $pubkey $pubkey.alt1 || _fatal "Fail to compare pubkeys" ;
-
-       _compare_files_checksum $pubkey $pubkey.alt2 || _fatal "Fail to compare pubkeys" ;
-
-
-    
-     img_sig="OPNsense-${version}-nano-amd64.img.sig"
-
-     sha256_name="OPNsense-${version}-checksums-amd64.sha256"
-     sha256_sig=$sha256_name.sig
-
-     [ ! -f "$img_sig" ]     || rm $img_sig
-     [ ! -f "$sha256_name" ] || rm $sha256_name
-     [ ! -f "$sha256_sig" ]  || rm $sha256_sig
-
-
-     for file in $img_sig $sha256_name $sha256_sig;
-     do
-       url=$_HVE_OPNSENSE_URL/$version/$file
-       wget -q -c "${url}"
-    done
-
-
-      rm -f /tmp/file.sig
-      openssl base64 -d -in $sha256_sig -out /tmp/file.sig
-      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $sha256_name || _fatal "Can't verify the signature of the checksum file"
-
-  }
-
-  _download(){
-
-    version=${1:-}
-    [ "${version:-}" != "" ] || _fatal "Must pass a VERSION for downloading"
-
-    _is_valid_version $version  || _fatal "'$version' is not a valid version number"
-
-
-    _download_img ${version}
-
-  }
-
-  _verify_image_checksum(){
-
-    version=$1
-    name="OPNsense-${version}-nano-amd64.img.bz2"
-    sha256_file="OPNsense-${version}-checksums-amd64.sha256"
-    
-    sha256=$(cat $sha256_file | grep "$name" | cut -d'=' -f 2 | tr -s [:space:])
-
-    echo "$sha256 $name" | sha256sum -c || _fatal "Checksum failed for '$name'"
-
-  }
-
-  _verify_image_signature(){
-     version=$1
-
-     # download multiple pubkeys from different server
-     pubkey="OPNsense-${version}.pub"
-     img_name="OPNsense-${version}-nano-amd64.img"
-     img_sig="${img_name}.sig"
-
-
-      rm -f /tmp/file.sig
-      openssl base64 -d -in $img_sig -out /tmp/file.sig
-      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $img_name || _fatal "Can't verify image signature"
-
-  }
-
-  _check(){
-    version=${1:-}
-    if [ "${version:-}" = "" ] ; then
-      for version in $(_list_local_images);
-      do
-        >&2 echo check $version
-        _download_crypto_files $version
-        _verify_image_signature $version
-      done
-    else
-      _download_crypto_files $version
-      _verify_image_signature $version
-    fi
-  }
-
-
-  _delete(){
-    version=${1:-}
-    if [ -z "${version:-1}" ]; then
-       _clear
-       rm -f *.img
-    else
-       rm -f *$version*.img
-    fi
-
-
-  }
-
-  _clear(){
-     rm -f *.pub *.sig *.bz2 *.alt1 *.alt2 *.sha256
-  }
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-  ls|list)
-     
-    if [ "${2:-}" == "" ]; then
-       _list_local_images
-    elif [ "${2:-}" == "--remote" ]; then
-       _list_remote_images
-    else
-       _warn "Unknown option '$2'"
-    fi
-    ;;
-    download)
-      pushd "${_HVE_OPNSENSE_SRC_IMG}"
-      _download "${2:-"$(_latest_version)"}"
-      popd 
-      ;;
-    delete)
-      pushd "${_HVE_OPNSENSE_SRC_IMG}"
-      _delete "${@:2}"
-      popd
-      ;;
-    check)
-      pushd "${_HVE_OPNSENSE_SRC_IMG}"
-      _check "${@:2}"
-      popd
-      ;;
-    show)
-      ls $_HVE_OPNSENSE_SRC_IMG | cat
-      ;;
-    clear)
-      pushd "${_HVE_OPNSENSE_SRC_IMG}"
-      _clear "${@:2}"
-      popd
-      ;;
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img-src "${@}"
-

examples/opnsense/scripts/harmony-ve-vm

@@ -1,77 +0,0 @@
-
-#! /bin/bash
-
-harmony-ve-vm()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-
-	harmony-ve-mv
-
-  DESCRIPTION
-
-    Manage virtalized hosts (VM) dependencies by Harmony Virtual Execution Environment
-
-  SYNOPSYS
-
-    harmony-ve-vm [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    harmony-ve-vm list
-    harmony-ve-vm create
-    harmony-ve-vm start
-    harmony-ve-vm stop
-    harmony-ve-vm login
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-EOM
-
-  }
-
-
-  # Implement functions
-
-case "${1:-}" in
-
-  "")
-	_short_help
-	;;
-  -h|--help)
-	_short_help
-	_extra_help
-	;;
-   # Commands entrypoints
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-vm "${@}"
-

examples/opnsense/scripts/learn-harmony

@@ -1,75 +0,0 @@
-#! /bin/bash
-
-learn-harmony()(
-
-  set -eu
-
-  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
-  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
-
-  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
-  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
-  . "${SCRIPTS_DIR}/common"
-  export PATH=$SCRIPTS_DIR:$PATH
-
-  _short_help(){
-
-    cat <<-EOM
-  
-  NAME
-    learn-harmony -- Harmony Learning Tool prototype
-    ($(basename ${BASE_DIR}) example)
-
-  SYNOPSYS
-
-    learn-harmony [GLOBAL_OPTIONS] COMMAND [OPTIONS]
-
-    learn-harmony list              # List learning steps
-    learn-harmony show STEP         # Show instruction of step STEP
-    learn-harmony check [STEP]        # Verify that your ready to begin step STEP+1
-
-EOM
-
-  }
-
-  _extra_help(){
-
-    cat <<-EOM
-
-  GLOBAL_OPTIONS
-
-    -d     Debug mode.
-
-  WARNINGS
-
-    This script is experimetal. Use with caution.
-EOM
-
-  }
-
-case "${1:-}" in
-
-  -h|--help|"")
-	_short_help
-	_extra_help
-	;;
-   ls|list)
-	   echo "not implemented"
-	   ;;
-   show)
-	   echo "not implemented"
-	   ;;
-   verify)
-	   echo "not implemented"
-	   ;;
-   *)
-        _warn "Unknown COMMAND '$1'"
-	exit 1
-	;;
-esac 
-
-)
-
-
-[ "$0" != "${BASH_SOURCE}" ] || learn-harmony "${@}"
-

examples/opnsense/scripts/setup

@@ -1,4 +0,0 @@
-#! /bin/bash
-
-
-export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")"):"${PATH}"

examples/opnsense/src/main.rs

@@ -1,70 +1,135 @@
-use harmony::{
+use std::{
-    config::secret::OPNSenseFirewallCredentials,
+    net::{IpAddr, Ipv4Addr},
-    infra::opnsense::OPNSenseFirewall,
+    sync::{Arc, OnceLock},
-    inventory::Inventory,
-    modules::{dhcp::DhcpScore, opnsense::OPNsenseShellCommandScore},
-    topology::LogicalHost,
 };
-use harmony_macros::{ip, ipv4};
+
+use brocade::BrocadeOptions;
+use cidr::Ipv4Cidr;
+use harmony::{
+    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    inventory::Inventory,
+    modules::{
+        dummy::{ErrorScore, PanicScore, SuccessScore},
+        http::StaticFilesHttpScore,
+        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
+        opnsense::OPNsenseShellCommandScore,
+        tftp::TftpScore,
+    },
+    topology::{LogicalHost, UnmanagedRouter},
+};
+use harmony_macros::{ip, mac_address};
 use harmony_secret::{Secret, SecretManager};
+use harmony_types::net::Url;
 use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
-    let firewall = LogicalHost {
+    let firewall = harmony::topology::LogicalHost {
-        ip: ip!("192.168.1.1"),
+        ip: ip!("192.168.5.229"),
         name: String::from("opnsense-1"),
     };
 
-    let opnsense_auth = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>()
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
         .await
         .expect("Failed to get credentials");
 
-    let opnsense = OPNSenseFirewall::new(
+    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
-        firewall,
+    let brocade_options = Some(BrocadeOptions {
-        None,
+        dry_run: *harmony::config::DRY_RUN,
-        &opnsense_auth.username,
+        ..Default::default()
-        &opnsense_auth.password,
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
     )
-    .await;
+    .await
+    .expect("Failed to connect to switch");
 
-    let dhcp_score = DhcpScore {
+    let switch_client = Arc::new(switch_client);
-        dhcp_range: (
+
-            ipv4!("192.168.1.100").into(),
+    let opnsense = Arc::new(
-            ipv4!("192.168.1.150").into(),
+        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
-        ),
+    );
-        host_binding: vec![],
+    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
-        next_server: None,
+    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
-        boot_filename: None,
+    let gateway_ip = IpAddr::V4(gateway_ipv4);
-        filename: None,
+    let topology = harmony::topology::HAClusterTopology {
-        filename64: None,
+        kubeconfig: None,
-        filenameipxe: Some("filename.ipxe".to_string()),
+        domain_name: "demo.harmony.mcd".to_string(),
-        domain: None,
+        router: Arc::new(UnmanagedRouter::new(
+            gateway_ip,
+            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
+        )),
+        load_balancer: opnsense.clone(),
+        firewall: opnsense.clone(),
+        tftp_server: opnsense.clone(),
+        http_server: opnsense.clone(),
+        dhcp_server: opnsense.clone(),
+        dns_server: opnsense.clone(),
+        control_plane: vec![LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        }],
+        bootstrap_host: LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        },
+        workers: vec![],
+        switch_client: switch_client.clone(),
+        network_manager: OnceLock::new(),
     };
-    // let dns_score = OKDDnsScore::new(&topology);
-    // let load_balancer_score = OKDLoadBalancerScore::new(&topology);
-    //
-    // let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    // let http_score = StaticFilesHttpScore {
-    //     folder_to_serve: Some(Url::LocalFolder(
-    //         "./data/watchguard/pxe-http-files".to_string(),
-    //     )),
-    //     files: vec![],
-    //     remote_path: None,
-    // };
-    let opnsense_config = opnsense.get_opnsense_config();
 
-    harmony_cli::run(
+    let inventory = Inventory {
-        Inventory::autoload(),
+        location: Location::new(
-        opnsense,
+            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
-        vec![
+            "wk".to_string(),
-            Box::new(dhcp_score),
+        ),
-            Box::new(OPNsenseShellCommandScore {
+        switch: SwitchGroup::from([]),
-                opnsense: opnsense_config,
+        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
-                command: "touch /tmp/helloharmonytouching_2".to_string(),
+        storage_host: vec![],
-            }),
+        worker_host: vec![],
+        control_plane_host: vec![
+            PhysicalHost::empty(HostCategory::Server)
+                .mac_address(mac_address!("08:00:27:62:EC:C3")),
+        ],
+    };
+
+    // TODO regroup smaller scores in a larger one such as this
+    // let okd_boostrap_preparation();
+
+    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
+    let dns_score = OKDDnsScore::new(&topology);
+    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
+
+    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
+    let http_score = StaticFilesHttpScore {
+        folder_to_serve: Some(Url::LocalFolder(
+            "./data/watchguard/pxe-http-files".to_string(),
+        )),
+        files: vec![],
+        remote_path: None,
+    };
+
+    harmony_tui::run(
+        inventory,
+        topology,
+        vec![
+            Box::new(dns_score),
+            Box::new(dhcp_score),
+            Box::new(load_balancer_score),
+            Box::new(tftp_score),
+            Box::new(http_score),
+            Box::new(OPNsenseShellCommandScore {
+                opnsense: opnsense.get_opnsense_config(),
+                command: "touch /tmp/helloharmonytouching".to_string(),
+            }),
+            Box::new(SuccessScore {}),
+            Box::new(ErrorScore {}),
+            Box::new(PanicScore {}),
         ],
-        None,
     )
     .await
     .unwrap();

harmony/src/domain/maestro/mod.rs

@@ -67,16 +67,16 @@ impl<T: Topology> Maestro<T> {
         }
     }
 
-    fn is_topology_initialized(&self) -> bool {
-        self.topology_state.status == TopologyStatus::Success
-            || self.topology_state.status == TopologyStatus::Noop
-    }
-
     pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
         let mut score_mut = self.scores.write().expect("Should acquire lock");
         score_mut.append(&mut scores);
     }
 
+    fn is_topology_initialized(&self) -> bool {
+        self.topology_state.status == TopologyStatus::Success
+            || self.topology_state.status == TopologyStatus::Noop
+    }
+
     pub async fn interpret(&self, score: Box<dyn Score<T>>) -> Result<Outcome, InterpretError> {
         if !self.is_topology_initialized() {
             warn!(

harmony/src/domain/topology/ha_cluster.rs

@@ -1,29 +1,25 @@
 use async_trait::async_trait;
 use harmony_macros::ip;
 use harmony_types::{
+    id::Id,
     net::{MacAddress, Url},
     switch::PortLocation,
 };
-use kube::api::ObjectMeta;
 use log::debug;
 use log::info;
 
-use crate::modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy};
+use crate::infra::network_manager::OpenShiftNmStateNetworkManager;
 use crate::topology::PxeOptions;
-use crate::{data::FileContent, modules::okd::crd::nmstate::NMState};
+use crate::{data::FileContent, executors::ExecutorError};
-use crate::{
-    executors::ExecutorError, modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec,
-};
 
 use super::{
     DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
-    HttpServer, IpAddress, K8sclient, LoadBalancer, LoadBalancerService, LogicalHost,
+    HttpServer, IpAddress, K8sclient, LoadBalancer, LoadBalancerService, LogicalHost, NetworkError,
-    PreparationError, PreparationOutcome, Router, Switch, SwitchClient, SwitchError, TftpServer,
+    NetworkManager, PreparationError, PreparationOutcome, Router, Switch, SwitchClient,
-    Topology, k8s::K8sClient,
+    SwitchError, TftpServer, Topology, k8s::K8sClient,
 };
 
-use std::collections::BTreeMap;
+use std::sync::{Arc, OnceLock};
-use std::sync::Arc;
 
 #[derive(Debug, Clone)]
 pub struct HAClusterTopology {
@@ -40,6 +36,7 @@ pub struct HAClusterTopology {
     pub control_plane: Vec<LogicalHost>,
     pub workers: Vec<LogicalHost>,
     pub kubeconfig: Option<String>,
+    pub network_manager: OnceLock<Arc<dyn NetworkManager>>,
 }
 
 #[async_trait]
@@ -63,7 +60,7 @@ impl K8sclient for HAClusterTopology {
                 K8sClient::try_default().await.map_err(|e| e.to_string())?,
             )),
             Some(kubeconfig) => {
-                let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else {
+                let Some(client) = K8sClient::from_kubeconfig(kubeconfig).await else {
                     return Err("Failed to create k8s client".to_string());
                 };
                 Ok(Arc::new(client))
@@ -93,191 +90,12 @@ impl HAClusterTopology {
             .to_string()
     }
 
-    async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
+    pub async fn network_manager(&self) -> &dyn NetworkManager {
-        let k8s_client = self.k8s_client().await?;
+        let k8s_client = self.k8s_client().await.unwrap();
 
-        debug!("Installing NMState controller...");
+        self.network_manager
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml
+            .get_or_init(|| Arc::new(OpenShiftNmStateNetworkManager::new(k8s_client.clone())))
-").unwrap(), Some("nmstate"))
+            .as_ref()
-            .await
-            .map_err(|e| e.to_string())?;
-
-        debug!("Creating NMState namespace...");
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml
-").unwrap(), Some("nmstate"))
-            .await
-            .map_err(|e| e.to_string())?;
-
-        debug!("Creating NMState service account...");
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml
-").unwrap(), Some("nmstate"))
-            .await
-            .map_err(|e| e.to_string())?;
-
-        debug!("Creating NMState role...");
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml
-").unwrap(), Some("nmstate"))
-            .await
-            .map_err(|e| e.to_string())?;
-
-        debug!("Creating NMState role binding...");
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml
-").unwrap(), Some("nmstate"))
-            .await
-            .map_err(|e| e.to_string())?;
-
-        debug!("Creating NMState operator...");
-        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml
-").unwrap(), Some("nmstate"))
-            .await
-            .map_err(|e| e.to_string())?;
-
-        k8s_client
-            .wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None)
-            .await?;
-
-        let nmstate = NMState {
-            metadata: ObjectMeta {
-                name: Some("nmstate".to_string()),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-        debug!("Creating NMState: {nmstate:#?}");
-        k8s_client
-            .apply(&nmstate, None)
-            .await
-            .map_err(|e| e.to_string())?;
-
-        Ok(())
-    }
-
-    fn get_next_bond_id(&self) -> u8 {
-        42 // FIXME: Find a better way to declare the bond id
-    }
-
-    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
-        self.ensure_nmstate_operator_installed()
-            .await
-            .map_err(|e| {
-                SwitchError::new(format!(
-                    "Can't configure bond, NMState operator not available: {e}"
-                ))
-            })?;
-
-        let bond_config = self.create_bond_configuration(config);
-        debug!(
-            "Applying NMState bond config for host {}: {bond_config:#?}",
-            config.host_id
-        );
-        self.k8s_client()
-            .await
-            .unwrap()
-            .apply(&bond_config, None)
-            .await
-            .map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?;
-
-        Ok(())
-    }
-
-    fn create_bond_configuration(
-        &self,
-        config: &HostNetworkConfig,
-    ) -> NodeNetworkConfigurationPolicy {
-        let host_name = &config.host_id;
-        let bond_id = self.get_next_bond_id();
-        let bond_name = format!("bond{bond_id}");
-
-        info!("Configuring bond '{bond_name}' for host '{host_name}'...");
-
-        let mut bond_mtu: Option<u32> = None;
-        let mut copy_mac_from: Option<String> = None;
-        let mut bond_ports = Vec::new();
-        let mut interfaces: Vec<nmstate::InterfaceSpec> = Vec::new();
-
-        for switch_port in &config.switch_ports {
-            let interface_name = switch_port.interface.name.clone();
-
-            interfaces.push(nmstate::InterfaceSpec {
-                name: interface_name.clone(),
-                description: Some(format!("Member of bond {bond_name}")),
-                r#type: "ethernet".to_string(),
-                state: "up".to_string(),
-                mtu: Some(switch_port.interface.mtu),
-                mac_address: Some(switch_port.interface.mac_address.to_string()),
-                ipv4: Some(nmstate::IpStackSpec {
-                    enabled: Some(false),
-                    ..Default::default()
-                }),
-                ipv6: Some(nmstate::IpStackSpec {
-                    enabled: Some(false),
-                    ..Default::default()
-                }),
-                link_aggregation: None,
-                ..Default::default()
-            });
-
-            bond_ports.push(interface_name.clone());
-
-            // Use the first port's details for the bond mtu and mac address
-            if bond_mtu.is_none() {
-                bond_mtu = Some(switch_port.interface.mtu);
-            }
-            if copy_mac_from.is_none() {
-                copy_mac_from = Some(interface_name);
-            }
-        }
-
-        interfaces.push(nmstate::InterfaceSpec {
-            name: bond_name.clone(),
-            description: Some(format!("Network bond for host {host_name}")),
-            r#type: "bond".to_string(),
-            state: "up".to_string(),
-            copy_mac_from,
-            ipv4: Some(nmstate::IpStackSpec {
-                dhcp: Some(true),
-                enabled: Some(true),
-                ..Default::default()
-            }),
-            ipv6: Some(nmstate::IpStackSpec {
-                dhcp: Some(true),
-                autoconf: Some(true),
-                enabled: Some(true),
-                ..Default::default()
-            }),
-            link_aggregation: Some(nmstate::BondSpec {
-                mode: "802.3ad".to_string(),
-                ports: bond_ports,
-                ..Default::default()
-            }),
-            ..Default::default()
-        });
-
-        NodeNetworkConfigurationPolicy {
-            metadata: ObjectMeta {
-                name: Some(format!("{host_name}-bond-config")),
-                ..Default::default()
-            },
-            spec: NodeNetworkConfigurationPolicySpec {
-                node_selector: Some(BTreeMap::from([(
-                    "kubernetes.io/hostname".to_string(),
-                    host_name.to_string(),
-                )])),
-                desired_state: nmstate::DesiredStateSpec { interfaces },
-            },
-        }
-    }
-
-    async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
-        debug!("Configuring port channel: {config:#?}");
-        let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
-
-        self.switch_client
-            .configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports)
-            .await
-            .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
-
-        Ok(())
     }
 
     pub fn autoload() -> Self {
@@ -301,6 +119,7 @@ impl HAClusterTopology {
             bootstrap_host: dummy_host,
             control_plane: vec![],
             workers: vec![],
+            network_manager: OnceLock::new(),
         }
     }
 }
@@ -458,21 +277,40 @@ impl HttpServer for HAClusterTopology {
 #[async_trait]
 impl Switch for HAClusterTopology {
     async fn setup_switch(&self) -> Result<(), SwitchError> {
-        self.switch_client.setup().await?;
+        self.switch_client.setup().await.map(|_| ())
-        Ok(())
     }
 
     async fn get_port_for_mac_address(
         &self,
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError> {
-        let port = self.switch_client.find_port(mac_address).await?;
+        self.switch_client.find_port(mac_address).await
-        Ok(port)
     }
 
-    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
+    async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
-        self.configure_bond(config).await?;
+        debug!("Configuring port channel: {config:#?}");
-        self.configure_port_channel(config).await
+        let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
+
+        self.switch_client
+            .configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports)
+            .await
+            .map_err(|e| SwitchError::new(format!("Failed to configure port-channel: {e}")))?;
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl NetworkManager for HAClusterTopology {
+    async fn ensure_network_manager_installed(&self) -> Result<(), NetworkError> {
+        self.network_manager()
+            .await
+            .ensure_network_manager_installed()
+            .await
+    }
+
+    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), NetworkError> {
+        self.network_manager().await.configure_bond(config).await
     }
 }
 

harmony/src/domain/topology/k8s.rs

@@ -5,13 +5,15 @@ use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
     api::{
         apps::v1::Deployment,
-        core::v1::{Pod, ServiceAccount},
+        core::v1::{Node, Pod, ServiceAccount},
     },
     apimachinery::pkg::version::Info,
 };
 use kube::{
     Client, Config, Discovery, Error, Resource,
-    api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
+    api::{
+        Api, AttachParams, DeleteParams, ListParams, ObjectList, Patch, PatchParams, ResourceExt,
+    },
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
     discovery::{ApiCapabilities, Scope},
@@ -23,7 +25,7 @@ use kube::{
     api::{ApiResource, GroupVersionKind},
     runtime::wait::await_condition,
 };
-use log::{debug, error, info, trace, warn};
+use log::{debug, error, trace, warn};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::json;
 use similar::TextDiff;
@@ -564,7 +566,58 @@ impl K8sClient {
         Ok(())
     }
 
-    pub(crate) async fn from_kubeconfig(path: &str) -> Option<K8sClient> {
+    /// Gets a single named resource of a specific type `K`.
+    ///
+    /// This function uses the `ApplyStrategy` trait to correctly determine
+    /// whether to look in a specific namespace or in the entire cluster.
+    ///
+    /// Returns `Ok(None)` if the resource is not found (404).
+    pub async fn get_resource<K>(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+    ) -> Result<Option<K>, Error>
+    where
+        K: Resource + Clone + std::fmt::Debug + DeserializeOwned,
+        <K as Resource>::Scope: ApplyStrategy<K>,
+        <K as kube::Resource>::DynamicType: Default,
+    {
+        let api: Api<K> =
+            <<K as Resource>::Scope as ApplyStrategy<K>>::get_api(&self.client, namespace);
+
+        api.get_opt(name).await
+    }
+
+    /// Lists all resources of a specific type `K`.
+    ///
+    /// This function uses the `ApplyStrategy` trait to correctly determine
+    /// whether to list from a specific namespace or from the entire cluster.
+    pub async fn list_resources<K>(
+        &self,
+        namespace: Option<&str>,
+        list_params: Option<ListParams>,
+    ) -> Result<ObjectList<K>, Error>
+    where
+        K: Resource + Clone + std::fmt::Debug + DeserializeOwned,
+        <K as Resource>::Scope: ApplyStrategy<K>,
+        <K as kube::Resource>::DynamicType: Default,
+    {
+        let api: Api<K> =
+            <<K as Resource>::Scope as ApplyStrategy<K>>::get_api(&self.client, namespace);
+
+        let list_params = list_params.unwrap_or_default();
+        api.list(&list_params).await
+    }
+
+    /// Fetches a list of all Nodes in the cluster.
+    pub async fn get_nodes(
+        &self,
+        list_params: Option<ListParams>,
+    ) -> Result<ObjectList<Node>, Error> {
+        self.list_resources(None, list_params).await
+    }
+
+    pub async fn from_kubeconfig(path: &str) -> Option<K8sClient> {
         let k = match Kubeconfig::read_from(path) {
             Ok(k) => k,
             Err(e) => {

harmony/src/domain/topology/mod.rs

@@ -1,6 +1,5 @@
 mod ha_cluster;
 pub mod ingress;
-pub mod opnsense;
 use harmony_types::net::IpAddress;
 mod host_binding;
 mod http;

harmony/src/domain/topology/network.rs

@@ -15,7 +15,7 @@ use harmony_types::{
 };
 use serde::Serialize;
 
-use crate::{executors::ExecutorError, hardware::PhysicalHost};
+use crate::executors::ExecutorError;
 
 use super::{LogicalHost, k8s::K8sClient};
 
@@ -183,6 +183,37 @@ impl FromStr for DnsRecordType {
     }
 }
 
+#[async_trait]
+pub trait NetworkManager: Debug + Send + Sync {
+    async fn ensure_network_manager_installed(&self) -> Result<(), NetworkError>;
+    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), NetworkError>;
+}
+
+#[derive(Debug, Clone, new)]
+pub struct NetworkError {
+    msg: String,
+}
+
+impl fmt::Display for NetworkError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.msg)
+    }
+}
+
+impl Error for NetworkError {}
+
+impl From<kube::Error> for NetworkError {
+    fn from(value: kube::Error) -> Self {
+        NetworkError::new(value.to_string())
+    }
+}
+
+impl From<String> for NetworkError {
+    fn from(value: String) -> Self {
+        NetworkError::new(value)
+    }
+}
+
 #[async_trait]
 pub trait Switch: Send + Sync {
     async fn setup_switch(&self) -> Result<(), SwitchError>;
@@ -192,7 +223,7 @@ pub trait Switch: Send + Sync {
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError>;
 
-    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>;
+    async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>;
 }
 
 #[derive(Clone, Debug, PartialEq)]

harmony/src/domain/topology/opnsense.rs

@@ -1,23 +0,0 @@
-use async_trait::async_trait;
-use log::info;
-
-use crate::{
-    infra::opnsense::OPNSenseFirewall,
-    topology::{PreparationError, PreparationOutcome, Topology},
-};
-
-#[async_trait]
-impl Topology for OPNSenseFirewall {
-    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
-        // FIXME we should be initializing the opnsense config here instead of
-        // OPNSenseFirewall::new as this causes the config to be loaded too early in
-        // harmony initialization process
-        let details = "OPNSenseFirewall topology is ready".to_string();
-        info!("{}", details);
-        Ok(PreparationOutcome::Success { details })
-    }
-
-    fn name(&self) -> &str {
-        "OPNSenseFirewall"
-    }
-}

harmony/src/infra/brocade.rs

@@ -113,6 +113,37 @@ impl SwitchClient for BrocadeSwitchClient {
     }
 }
 
+#[derive(Debug)]
+pub struct UnmanagedSwitch;
+
+impl UnmanagedSwitch {
+    pub async fn init( ) -> Result<Self, ()> {
+        Ok(Self)
+    }
+}
+
+#[async_trait]
+impl SwitchClient for UnmanagedSwitch {
+    async fn setup(&self) -> Result<(), SwitchError> {
+        todo!("unmanaged switch. Nothing to do.")
+    }
+
+    async fn find_port(
+        &self,
+        mac_address: &MacAddress,
+    ) -> Result<Option<PortLocation>, SwitchError> {
+        todo!("unmanaged switch. Nothing to do.")
+    }
+
+    async fn configure_port_channel(
+        &self,
+        channel_name: &str,
+        switch_ports: Vec<PortLocation>,
+    ) -> Result<u8, SwitchError> {
+        todo!("unmanaged switch. Nothing to do.") 
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};

harmony/src/infra/kube.rs

@@ -0,0 +1,182 @@
+use k8s_openapi::Resource as K8sResource;
+use kube::api::{ApiResource, DynamicObject, GroupVersionKind};
+use kube::core::TypeMeta;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+
+/// Convert a typed Kubernetes resource `K` into a `DynamicObject`.
+///
+/// Requirements:
+/// - `K` must be a k8s_openapi resource (provides static GVK via `Resource`).
+/// - `K` must have standard Kubernetes shape (metadata + payload fields).
+///
+/// Notes:
+/// - We set `types` (apiVersion/kind) and copy `metadata`.
+/// - We place the remaining top-level fields into `obj.data` as JSON.
+/// - Scope is not encoded on the object itself; you still need the corresponding
+///   `DynamicResource` (derived from K::group/version/kind) when constructing an Api.
+///
+/// Example usage:
+///   let dyn_obj = kube_resource_to_dynamic(secret)?;
+///   let api: Api<DynamicObject> = Api::namespaced_with(client, "ns", &dr);
+///   api.patch(&dyn_obj.name_any(), &PatchParams::apply("mgr"), &Patch::Apply(dyn_obj)).await?;
+pub fn kube_resource_to_dynamic<K>(res: &K) -> Result<DynamicObject, String>
+where
+    K: K8sResource + Serialize + DeserializeOwned,
+{
+    // Serialize the typed resource to JSON so we can split metadata and payload
+    let mut v = serde_json::to_value(res).map_err(|e| format!("Failed to serialize : {e}"))?;
+    let obj = v
+        .as_object_mut()
+        .ok_or_else(|| "expected object JSON".to_string())?;
+
+    // Extract and parse metadata into kube::core::ObjectMeta
+    let metadata_value = obj
+        .remove("metadata")
+        .ok_or_else(|| "missing metadata".to_string())?;
+    let metadata: kube::core::ObjectMeta = serde_json::from_value(metadata_value)
+        .map_err(|e| format!("Failed to deserialize : {e}"))?;
+
+    // Name is required for DynamicObject::new; prefer metadata.name
+    let name = metadata
+        .name
+        .clone()
+        .ok_or_else(|| "metadata.name is required".to_string())?;
+
+    // Remaining fields (spec/status/data/etc.) become the dynamic payload
+    let payload = Value::Object(obj.clone());
+
+    // Construct the DynamicObject
+    let mut dyn_obj = DynamicObject::new(
+        &name,
+        &ApiResource::from_gvk(&GroupVersionKind::gvk(K::GROUP, K::VERSION, K::KIND)),
+    );
+    dyn_obj.types = Some(TypeMeta {
+        api_version: api_version_for::<K>(),
+        kind: K::KIND.into(),
+    });
+
+    // Preserve namespace/labels/annotations/etc.
+    dyn_obj.metadata = metadata;
+
+    // Attach payload
+    dyn_obj.data = payload;
+
+    Ok(dyn_obj)
+}
+
+/// Helper: compute apiVersion string ("group/version" or "v1" for core).
+fn api_version_for<K>() -> String
+where
+    K: K8sResource,
+{
+    let group = K::GROUP;
+    let version = K::VERSION;
+    if group.is_empty() {
+        version.to_string() // core/v1 => "v1"
+    } else {
+        format!("{}/{}", group, version)
+    }
+}
+#[cfg(test)]
+mod test {
+    use super::*;
+    use k8s_openapi::api::{
+        apps::v1::{Deployment, DeploymentSpec},
+        core::v1::{PodTemplateSpec, Secret},
+    };
+    use kube::api::ObjectMeta;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn secret_to_dynamic_roundtrip() {
+        // Create a sample Secret resource
+        let mut secret = Secret {
+            metadata: ObjectMeta {
+                name: Some("my-secret".to_string()),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        };
+
+        // Convert to DynamicResource
+        let dynamic: DynamicObject =
+            kube_resource_to_dynamic(&secret).expect("Failed to convert Secret to DynamicResource");
+
+        // Serialize both the original and dynamic resources to Value
+        let original_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        let dynamic_value =
+            serde_json::to_value(&dynamic).expect("Failed to serialize DynamicResource");
+
+        // Assert that they are identical
+        assert_eq!(original_value, dynamic_value);
+
+        secret.metadata.namespace = Some("false".to_string());
+        let modified_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        assert_ne!(modified_value, dynamic_value);
+    }
+
+    #[test]
+    fn deployment_to_dynamic_roundtrip() {
+        // Create a sample Deployment with nested structures
+        let mut deployment = Deployment {
+            metadata: ObjectMeta {
+                name: Some("my-deployment".to_string()),
+                labels: Some({
+                    let mut map = std::collections::BTreeMap::new();
+                    map.insert("app".to_string(), "nginx".to_string());
+                    map
+                }),
+                ..Default::default()
+            },
+            spec: Some(DeploymentSpec {
+                replicas: Some(3),
+                selector: Default::default(),
+                template: PodTemplateSpec {
+                    metadata: Some(ObjectMeta {
+                        labels: Some({
+                            let mut map = std::collections::BTreeMap::new();
+                            map.insert("app".to_string(), "nginx".to_string());
+                            map
+                        }),
+                        ..Default::default()
+                    }),
+                    spec: Some(Default::default()), // PodSpec with empty containers for simplicity
+                },
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let dynamic = kube_resource_to_dynamic(&deployment).expect("Failed to convert Deployment");
+
+        let original_value = serde_json::to_value(&deployment).unwrap();
+        let dynamic_value = serde_json::to_value(&dynamic).unwrap();
+
+        assert_eq!(original_value, dynamic_value);
+
+        assert_eq!(
+            dynamic.data.get("spec").unwrap().get("replicas").unwrap(),
+            3
+        );
+        assert_eq!(
+            dynamic
+                .data
+                .get("spec")
+                .unwrap()
+                .get("template")
+                .unwrap()
+                .get("metadata")
+                .unwrap()
+                .get("labels")
+                .unwrap()
+                .get("app")
+                .unwrap()
+                .as_str()
+                .unwrap(),
+            "nginx".to_string()
+        );
+    }
+}

harmony/src/infra/mod.rs

@@ -3,5 +3,7 @@ pub mod executors;
 pub mod hp_ilo;
 pub mod intel_amt;
 pub mod inventory;
+pub mod kube;
+pub mod network_manager;
 pub mod opnsense;
 mod sqlx;

harmony/src/infra/network_manager.rs

@@ -0,0 +1,257 @@
+use std::{
+    collections::{BTreeMap, HashSet},
+    sync::Arc,
+};
+
+use async_trait::async_trait;
+use harmony_types::id::Id;
+use k8s_openapi::api::core::v1::Node;
+use kube::{
+    ResourceExt,
+    api::{ObjectList, ObjectMeta},
+};
+use log::{debug, info};
+
+use crate::{
+    modules::okd::crd::nmstate,
+    topology::{HostNetworkConfig, NetworkError, NetworkManager, k8s::K8sClient},
+};
+
+pub struct OpenShiftNmStateNetworkManager {
+    k8s_client: Arc<K8sClient>,
+}
+
+impl std::fmt::Debug for OpenShiftNmStateNetworkManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("OpenShiftNmStateNetworkManager").finish()
+    }
+}
+
+#[async_trait]
+impl NetworkManager for OpenShiftNmStateNetworkManager {
+    async fn ensure_network_manager_installed(&self) -> Result<(), NetworkError> {
+        debug!("Installing NMState controller...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        debug!("Creating NMState namespace...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        debug!("Creating NMState service account...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        debug!("Creating NMState role...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        debug!("Creating NMState role binding...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        debug!("Creating NMState operator...");
+        self.k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml
+").unwrap(), Some("nmstate"))
+            .await?;
+
+        self.k8s_client
+            .wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None)
+            .await?;
+
+        let nmstate = nmstate::NMState {
+            metadata: ObjectMeta {
+                name: Some("nmstate".to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        debug!(
+            "Creating NMState:\n{}",
+            serde_yaml::to_string(&nmstate).unwrap()
+        );
+        self.k8s_client.apply(&nmstate, None).await?;
+
+        Ok(())
+    }
+
+    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), NetworkError> {
+        let hostname = self.get_hostname(&config.host_id).await.map_err(|e| {
+            NetworkError::new(format!(
+                "Can't configure bond, can't get hostname for host '{}': {e}",
+                config.host_id
+            ))
+        })?;
+        let bond_id = self.get_next_bond_id(&hostname).await.map_err(|e| {
+            NetworkError::new(format!(
+                "Can't configure bond, can't get an available bond id for host '{}': {e}",
+                config.host_id
+            ))
+        })?;
+        let bond_config = self.create_bond_configuration(&hostname, &bond_id, config);
+
+        debug!(
+            "Applying NMState bond config for host {}:\n{}",
+            config.host_id,
+            serde_yaml::to_string(&bond_config).unwrap(),
+        );
+        self.k8s_client
+            .apply(&bond_config, None)
+            .await
+            .map_err(|e| NetworkError::new(format!("Failed to configure bond: {e}")))?;
+
+        Ok(())
+    }
+}
+
+impl OpenShiftNmStateNetworkManager {
+    pub fn new(k8s_client: Arc<K8sClient>) -> Self {
+        Self { k8s_client }
+    }
+
+    fn create_bond_configuration(
+        &self,
+        host: &str,
+        bond_name: &str,
+        config: &HostNetworkConfig,
+    ) -> nmstate::NodeNetworkConfigurationPolicy {
+        info!("Configuring bond '{bond_name}' for host '{host}'...");
+
+        let mut bond_mtu: Option<u32> = None;
+        let mut copy_mac_from: Option<String> = None;
+        let mut bond_ports = Vec::new();
+        let mut interfaces: Vec<nmstate::Interface> = Vec::new();
+
+        for switch_port in &config.switch_ports {
+            let interface_name = switch_port.interface.name.clone();
+
+            interfaces.push(nmstate::Interface {
+                name: interface_name.clone(),
+                description: Some(format!("Member of bond {bond_name}")),
+                r#type: nmstate::InterfaceType::Ethernet,
+                state: "up".to_string(),
+                ipv4: Some(nmstate::IpStackSpec {
+                    enabled: Some(false),
+                    ..Default::default()
+                }),
+                ipv6: Some(nmstate::IpStackSpec {
+                    enabled: Some(false),
+                    ..Default::default()
+                }),
+                link_aggregation: None,
+                ..Default::default()
+            });
+
+            bond_ports.push(interface_name.clone());
+
+            // Use the first port's details for the bond mtu and mac address
+            if bond_mtu.is_none() {
+                bond_mtu = Some(switch_port.interface.mtu);
+            }
+            if copy_mac_from.is_none() {
+                copy_mac_from = Some(interface_name);
+            }
+        }
+
+        interfaces.push(nmstate::Interface {
+            name: bond_name.to_string(),
+            description: Some(format!("HARMONY - Network bond for host {host}")),
+            r#type: nmstate::InterfaceType::Bond,
+            state: "up".to_string(),
+            copy_mac_from,
+            ipv4: Some(nmstate::IpStackSpec {
+                dhcp: Some(true),
+                enabled: Some(true),
+                ..Default::default()
+            }),
+            ipv6: Some(nmstate::IpStackSpec {
+                dhcp: Some(true),
+                autoconf: Some(true),
+                enabled: Some(true),
+                ..Default::default()
+            }),
+            link_aggregation: Some(nmstate::BondSpec {
+                mode: "802.3ad".to_string(),
+                ports: bond_ports,
+                ..Default::default()
+            }),
+            ..Default::default()
+        });
+
+        nmstate::NodeNetworkConfigurationPolicy {
+            metadata: ObjectMeta {
+                name: Some(format!("{host}-bond-config")),
+                ..Default::default()
+            },
+            spec: nmstate::NodeNetworkConfigurationPolicySpec {
+                node_selector: Some(BTreeMap::from([(
+                    "kubernetes.io/hostname".to_string(),
+                    host.to_string(),
+                )])),
+                desired_state: nmstate::NetworkState {
+                    interfaces,
+                    ..Default::default()
+                },
+            },
+        }
+    }
+
+    async fn get_hostname(&self, host_id: &Id) -> Result<String, String> {
+        let nodes: ObjectList<Node> = self
+            .k8s_client
+            .list_resources(None, None)
+            .await
+            .map_err(|e| format!("Failed to list nodes: {e}"))?;
+
+        let Some(node) = nodes.iter().find(|n| {
+            n.status
+                .as_ref()
+                .and_then(|s| s.node_info.as_ref())
+                .map(|i| i.system_uuid == host_id.to_string())
+                .unwrap_or(false)
+        }) else {
+            return Err(format!("No node found for host '{host_id}'"));
+        };
+
+        node.labels()
+            .get("kubernetes.io/hostname")
+            .ok_or(format!(
+                "Node '{host_id}' has no kubernetes.io/hostname label"
+            ))
+            .cloned()
+    }
+
+    async fn get_next_bond_id(&self, hostname: &str) -> Result<String, String> {
+        let network_state: Option<nmstate::NodeNetworkState> = self
+            .k8s_client
+            .get_resource(hostname, None)
+            .await
+            .map_err(|e| format!("Failed to list nodes: {e}"))?;
+
+        let interfaces = vec![];
+        let existing_bonds: Vec<&nmstate::Interface> = network_state
+            .as_ref()
+            .and_then(|network_state| network_state.status.current_state.as_ref())
+            .map_or(&interfaces, |current_state| &current_state.interfaces)
+            .iter()
+            .filter(|i| i.r#type == nmstate::InterfaceType::Bond)
+            .collect();
+
+        let used_ids: HashSet<u32> = existing_bonds
+            .iter()
+            .filter_map(|i| {
+                i.name
+                    .strip_prefix("bond")
+                    .and_then(|id| id.parse::<u32>().ok())
+            })
+            .collect();
+
+        let next_id = (0..).find(|id| !used_ids.contains(id)).unwrap();
+        Ok(format!("bond{next_id}"))
+    }
+}

harmony/src/infra/opnsense/mod.rs

@@ -25,8 +25,6 @@ impl OPNSenseFirewall {
         self.host.ip
     }
 
-    /// panics : if the opnsense config file cannot be loaded by the underlying opnsense_config
-    /// crate
     pub async fn new(host: LogicalHost, port: Option<u16>, username: &str, password: &str) -> Self {
         Self {
             opnsense_config: Arc::new(RwLock::new(

harmony/src/modules/inventory/discovery.rs

@@ -74,7 +74,11 @@ impl<T: Topology> Interpret<T> for DiscoverHostForRoleInterpret {
 
             match ans {
                 Ok(choice) => {
-                    info!("Selected {} as the bootstrap node.", choice.summary());
+                    info!(
+                        "Selected {} as the {:?} node.",
+                        choice.summary(),
+                        self.score.role
+                    );
                     host_repo
                         .save_role_mapping(&self.score.role, &choice)
                         .await?;
@@ -90,10 +94,7 @@ impl<T: Topology> Interpret<T> for DiscoverHostForRoleInterpret {
                         "Failed to select node for role {:?} : {}",
                         self.score.role, e
                     );
-                    return Err(InterpretError::new(format!(
+                    return Err(InterpretError::new(format!("Could not select host : {e}")));
-                        "Could not select host : {}",
-                        e.to_string()
-                    )));
                 }
             }
         }

harmony/src/modules/okd/crd/nmstate.rs

@@ -1,6 +1,7 @@
 use std::collections::BTreeMap;
 
-use kube::CustomResource;
+use k8s_openapi::{ClusterResourceScope, Resource};
+use kube::{CustomResource, api::ObjectMeta};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -47,28 +48,223 @@ pub struct ProbeDns {
     group = "nmstate.io",
     version = "v1",
     kind = "NodeNetworkConfigurationPolicy",
-    namespaced
+    namespaced = false
 )]
 #[serde(rename_all = "camelCase")]
 pub struct NodeNetworkConfigurationPolicySpec {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub node_selector: Option<BTreeMap<String, String>>,
-    pub desired_state: DesiredStateSpec,
+    pub desired_state: NetworkState,
+}
+
+// Currently, kube-rs derive doesn't support resources without a `spec` field, so we have
+// to implement it ourselves.
+//
+// Ref:
+// - https://github.com/kube-rs/kube/issues/1763
+// - https://github.com/kube-rs/kube/discussions/1762
+#[derive(Deserialize, Serialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeNetworkState {
+    metadata: ObjectMeta,
+    pub status: NodeNetworkStateStatus,
+}
+
+impl Resource for NodeNetworkState {
+    const API_VERSION: &'static str = "nmstate.io/v1beta1";
+    const GROUP: &'static str = "nmstate.io";
+    const VERSION: &'static str = "v1beta1";
+    const KIND: &'static str = "NodeNetworkState";
+    const URL_PATH_SEGMENT: &'static str = "nodenetworkstates";
+    type Scope = ClusterResourceScope;
+}
+
+impl k8s_openapi::Metadata for NodeNetworkState {
+    type Ty = ObjectMeta;
+
+    fn metadata(&self) -> &Self::Ty {
+        &self.metadata
+    }
+
+    fn metadata_mut(&mut self) -> &mut Self::Ty {
+        &mut self.metadata
+    }
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeNetworkStateStatus {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub current_state: Option<NetworkState>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub handler_nmstate_version: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub host_network_manager_version: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_successful_update_time: Option<String>,
+}
+
+/// The NetworkState is the top-level struct, representing the entire
+/// desired or current network state.
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
-pub struct DesiredStateSpec {
+#[serde(deny_unknown_fields)]
-    pub interfaces: Vec<InterfaceSpec>,
+pub struct NetworkState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hostname: Option<HostNameState>,
+    #[serde(rename = "dns-resolver", skip_serializing_if = "Option::is_none")]
+    pub dns: Option<DnsState>,
+    #[serde(rename = "route-rules", skip_serializing_if = "Option::is_none")]
+    pub rules: Option<RouteRuleState>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub routes: Option<RouteState>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub interfaces: Vec<Interface>,
+    #[serde(rename = "ovs-db", skip_serializing_if = "Option::is_none")]
+    pub ovsdb: Option<OvsDbGlobalConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ovn: Option<OvnConfiguration>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
-pub struct InterfaceSpec {
+pub struct HostNameState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub running: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<String>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct DnsState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub running: Option<DnsResolverConfig>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<DnsResolverConfig>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct DnsResolverConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub search: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub server: Option<Vec<String>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct RouteRuleState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<Vec<RouteRule>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub running: Option<Vec<RouteRule>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct RouteState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config: Option<Vec<Route>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub running: Option<Vec<Route>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct RouteRule {
+    #[serde(rename = "ip-from", skip_serializing_if = "Option::is_none")]
+    pub ip_from: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub priority: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub route_table: Option<u32>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct Route {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub destination: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub metric: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub next_hop_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub next_hop_interface: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub table_id: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mtu: Option<u32>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct OvsDbGlobalConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub external_ids: Option<BTreeMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub other_config: Option<BTreeMap<String, String>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct OvnConfiguration {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bridge_mappings: Option<Vec<OvnBridgeMapping>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct OvnBridgeMapping {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub localnet: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bridge: Option<String>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(untagged)]
+#[serde(rename_all = "kebab-case")]
+pub enum StpSpec {
+    Bool(bool),
+    Options(StpOptions),
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct LldpState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<bool>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct OvsDb {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub external_ids: Option<BTreeMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub other_config: Option<BTreeMap<String, String>>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct PatchState {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub peer: Option<String>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub struct Interface {
     pub name: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
-    pub r#type: String,
+    pub r#type: InterfaceType,
     pub state: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_address: Option<String>,
@@ -99,9 +295,81 @@ pub struct InterfaceSpec {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub linux_bridge: Option<LinuxBridgeSpec>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(alias = "bridge")]
     pub ovs_bridge: Option<OvsBridgeSpec>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub ethtool: Option<EthtoolSpec>,
+    pub ethtool: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub accept_all_mac_addresses: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub identifier: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub lldp: Option<LldpState>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub permanent_mac_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_mtu: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub min_mtu: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mptcp: Option<Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub profile_name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub wait_ip: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ovs_db: Option<OvsDb>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub driver: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub patch: Option<PatchState>,
+}
+
+#[derive(Deserialize, Serialize, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, JsonSchema)]
+#[serde(rename_all = "kebab-case")]
+pub enum InterfaceType {
+    #[serde(rename = "unknown")]
+    Unknown,
+    #[serde(rename = "dummy")]
+    Dummy,
+    #[serde(rename = "loopback")]
+    Loopback,
+    #[serde(rename = "linux-bridge")]
+    LinuxBridge,
+    #[serde(rename = "ovs-bridge")]
+    OvsBridge,
+    #[serde(rename = "ovs-interface")]
+    OvsInterface,
+    #[serde(rename = "bond")]
+    Bond,
+    #[serde(rename = "ipvlan")]
+    IpVlan,
+    #[serde(rename = "vlan")]
+    Vlan,
+    #[serde(rename = "vxlan")]
+    Vxlan,
+    #[serde(rename = "mac-vlan")]
+    Macvlan,
+    #[serde(rename = "mac-vtap")]
+    Macvtap,
+    #[serde(rename = "ethernet")]
+    Ethernet,
+    #[serde(rename = "infiniband")]
+    Infiniband,
+    #[serde(rename = "vrf")]
+    Vrf,
+    #[serde(rename = "veth")]
+    Veth,
+    #[serde(rename = "ipsec")]
+    Ipsec,
+    #[serde(rename = "hsr")]
+    Hrs,
+}
+
+impl Default for InterfaceType {
+    fn default() -> Self {
+        Self::Loopback
+    }
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
@@ -149,6 +417,7 @@ pub struct EthernetSpec {
 #[serde(rename_all = "kebab-case")]
 pub struct BondSpec {
     pub mode: String,
+    #[serde(alias = "port")]
     pub ports: Vec<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<BTreeMap<String, Value>>,
@@ -287,11 +556,15 @@ pub struct OvsBridgeSpec {
 #[serde(rename_all = "kebab-case")]
 pub struct OvsBridgeOptions {
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub stp: Option<bool>,
+    pub stp: Option<StpSpec>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub rstp: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mcast_snooping_enable: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub datapath: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fail_mode: Option<String>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
@@ -305,18 +578,3 @@ pub struct OvsPortSpec {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub r#type: Option<String>,
 }
-
-#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
-#[serde(rename_all = "kebab-case")]
-pub struct EthtoolSpec {
-    // TODO: Properly describe this spec (https://nmstate.io/devel/yaml_api.html#ethtool)
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
-#[serde(rename_all = "kebab-case")]
-pub struct EthtoolFecSpec {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub auto: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mode: Option<String>,
-}

harmony/src/modules/okd/host_network.rs

@@ -1,6 +1,6 @@
 use async_trait::async_trait;
 use harmony_types::id::Id;
-use log::{debug, info};
+use log::{info, warn};
 use serde::Serialize;
 
 use crate::{
@@ -9,7 +9,7 @@ use crate::{
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
     score::Score,
-    topology::{HostNetworkConfig, NetworkInterface, Switch, SwitchPort, Topology},
+    topology::{HostNetworkConfig, NetworkInterface, NetworkManager, Switch, SwitchPort, Topology},
 };
 
 #[derive(Debug, Clone, Serialize)]
@@ -17,7 +17,7 @@ pub struct HostNetworkConfigurationScore {
     pub hosts: Vec<PhysicalHost>,
 }
 
-impl<T: Topology + Switch> Score<T> for HostNetworkConfigurationScore {
+impl<T: Topology + NetworkManager + Switch> Score<T> for HostNetworkConfigurationScore {
     fn name(&self) -> String {
         "HostNetworkConfigurationScore".into()
     }
@@ -35,7 +35,7 @@ pub struct HostNetworkConfigurationInterpret {
 }
 
 impl HostNetworkConfigurationInterpret {
-    async fn configure_network_for_host<T: Topology + Switch>(
+    async fn configure_network_for_host<T: Topology + NetworkManager + Switch>(
         &self,
         topology: &T,
         host: &PhysicalHost,
@@ -49,6 +49,13 @@ impl HostNetworkConfigurationInterpret {
                 switch_ports: vec![],
             });
         }
+        if host.network.len() == 1 {
+            info!("[Host {current_host}/{total_hosts}] Only one interface to configure, skipping");
+            return Ok(HostNetworkConfig {
+                host_id: host.id.clone(),
+                switch_ports: vec![],
+            });
+        }
 
         let switch_ports = self
             .collect_switch_ports_for_host(topology, host, current_host, total_hosts)
@@ -59,7 +66,7 @@ impl HostNetworkConfigurationInterpret {
             switch_ports,
         };
 
-        if !config.switch_ports.is_empty() {
+        if config.switch_ports.len() > 1 {
             info!(
                 "[Host {current_host}/{total_hosts}] Found {} ports for {} interfaces",
                 config.switch_ports.len(),
@@ -67,15 +74,25 @@ impl HostNetworkConfigurationInterpret {
             );
 
             info!("[Host {current_host}/{total_hosts}] Configuring host network...");
+            topology.configure_bond(&config).await.map_err(|e| {
+                InterpretError::new(format!("Failed to configure host network: {e}"))
+            })?;
             topology
-                .configure_host_network(&config)
+                .configure_port_channel(&config)
                 .await
-                .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
+                .map_err(|e| {
-        } else {
+                    InterpretError::new(format!("Failed to configure host network: {e}"))
+                })?;
+        } else if config.switch_ports.is_empty() {
             info!(
                 "[Host {current_host}/{total_hosts}] No ports found for {} interfaces, skipping",
                 host.network.len()
             );
+        } else {
+            warn!(
+                "[Host {current_host}/{total_hosts}] Found a single port for {} interfaces, skipping",
+                host.network.len()
+            );
         }
 
         Ok(config)
@@ -113,7 +130,7 @@ impl HostNetworkConfigurationInterpret {
                         port,
                     });
                 }
-                Ok(None) => debug!("No port found for '{mac_address}', skipping"),
+                Ok(None) => {}
                 Err(e) => {
                     return Err(InterpretError::new(format!(
                         "Failed to get port for host '{}': {}",
@@ -133,15 +150,6 @@ impl HostNetworkConfigurationInterpret {
         ];
 
         for config in configs {
-            let host = self
-                .score
-                .hosts
-                .iter()
-                .find(|h| h.id == config.host_id)
-                .unwrap();
-
-            println!("[Host] {host}");
-
             if config.switch_ports.is_empty() {
                 report.push(format!(
                     "⏭️ Host {}: SKIPPED (No matching switch ports found)",
@@ -169,7 +177,7 @@ impl HostNetworkConfigurationInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
+impl<T: Topology + NetworkManager + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
     fn get_name(&self) -> InterpretName {
         InterpretName::Custom("HostNetworkConfigurationInterpret")
     }
@@ -198,6 +206,12 @@ impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
         let host_count = self.score.hosts.len();
         info!("Started network configuration for {host_count} host(s)...",);
 
+        info!("Setting up NetworkManager...",);
+        topology
+            .ensure_network_manager_installed()
+            .await
+            .map_err(|e| InterpretError::new(format!("NetworkManager setup failed: {e}")))?;
+
         info!("Setting up switch with sane defaults...");
         topology
             .setup_switch()
@@ -216,6 +230,7 @@ impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
             host_configurations.push(host_configuration);
             current_host += 1;
         }
+
         if current_host > 1 {
             let details = self.format_host_configuration(host_configurations);
 
@@ -242,7 +257,8 @@ mod tests {
     use crate::{
         hardware::HostCategory,
         topology::{
-            HostNetworkConfig, PreparationError, PreparationOutcome, SwitchError, SwitchPort,
+            HostNetworkConfig, NetworkError, PreparationError, PreparationOutcome, SwitchError,
+            SwitchPort,
         },
     };
     use std::{
@@ -267,6 +283,18 @@ mod tests {
             speed_mbps: None,
             mtu: 1,
         };
+        pub static ref YET_ANOTHER_EXISTING_INTERFACE: NetworkInterface = NetworkInterface {
+            mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F3".to_string()).unwrap(),
+            name: "interface-3".into(),
+            speed_mbps: None,
+            mtu: 1,
+        };
+        pub static ref LAST_EXISTING_INTERFACE: NetworkInterface = NetworkInterface {
+            mac_address: MacAddress::try_from("AA:BB:CC:DD:EE:F4".to_string()).unwrap(),
+            name: "interface-4".into(),
+            speed_mbps: None,
+            mtu: 1,
+        };
         pub static ref UNKNOWN_INTERFACE: NetworkInterface = NetworkInterface {
             mac_address: MacAddress::try_from("11:22:33:44:55:61".to_string()).unwrap(),
             name: "unknown-interface".into(),
@@ -275,6 +303,8 @@ mod tests {
         };
         pub static ref PORT: PortLocation = PortLocation(1, 0, 42);
         pub static ref ANOTHER_PORT: PortLocation = PortLocation(2, 0, 42);
+        pub static ref YET_ANOTHER_PORT: PortLocation = PortLocation(1, 0, 45);
+        pub static ref LAST_PORT: PortLocation = PortLocation(2, 0, 45);
     }
 
     #[tokio::test]
@@ -290,28 +320,33 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn host_with_one_mac_address_should_create_bond_with_one_interface() {
+    async fn should_setup_network_manager() {
         let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]);
         let score = given_score(vec![host]);
         let topology = TopologyWithSwitch::new();
 
         let _ = score.interpret(&Inventory::empty(), &topology).await;
 
-        let configured_host_networks = topology.configured_host_networks.lock().unwrap();
+        let network_manager_setup = topology.network_manager_setup.lock().unwrap();
-        assert_that!(*configured_host_networks).contains_exactly(vec![(
+        assert_that!(*network_manager_setup).is_true();
-            HOST_ID.clone(),
-            HostNetworkConfig {
-                host_id: HOST_ID.clone(),
-                switch_ports: vec![SwitchPort {
-                    interface: EXISTING_INTERFACE.clone(),
-                    port: PORT.clone(),
-                }],
-            },
-        )]);
     }
 
     #[tokio::test]
-    async fn host_with_multiple_mac_addresses_should_create_one_bond_with_all_interfaces() {
+    async fn host_with_one_mac_address_should_skip_host_configuration() {
+        let host = given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]);
+        let score = given_score(vec![host]);
+        let topology = TopologyWithSwitch::new();
+
+        let _ = score.interpret(&Inventory::empty(), &topology).await;
+
+        let config = topology.configured_bonds.lock().unwrap();
+        assert_that!(*config).is_empty();
+        let config = topology.configured_port_channels.lock().unwrap();
+        assert_that!(*config).is_empty();
+    }
+
+    #[tokio::test]
+    async fn host_with_multiple_mac_addresses_should_configure_one_bond_with_all_interfaces() {
         let score = given_score(vec![given_host(
             &HOST_ID,
             vec![
@@ -323,8 +358,8 @@ mod tests {
 
         let _ = score.interpret(&Inventory::empty(), &topology).await;
 
-        let configured_host_networks = topology.configured_host_networks.lock().unwrap();
+        let config = topology.configured_bonds.lock().unwrap();
-        assert_that!(*configured_host_networks).contains_exactly(vec![(
+        assert_that!(*config).contains_exactly(vec![(
             HOST_ID.clone(),
             HostNetworkConfig {
                 host_id: HOST_ID.clone(),
@@ -343,49 +378,183 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn multiple_hosts_should_create_one_bond_per_host() {
+    async fn host_with_multiple_mac_addresses_should_configure_one_port_channel_with_all_interfaces()
+     {
+        let score = given_score(vec![given_host(
+            &HOST_ID,
+            vec![
+                EXISTING_INTERFACE.clone(),
+                ANOTHER_EXISTING_INTERFACE.clone(),
+            ],
+        )]);
+        let topology = TopologyWithSwitch::new();
+
+        let _ = score.interpret(&Inventory::empty(), &topology).await;
+
+        let config = topology.configured_port_channels.lock().unwrap();
+        assert_that!(*config).contains_exactly(vec![(
+            HOST_ID.clone(),
+            HostNetworkConfig {
+                host_id: HOST_ID.clone(),
+                switch_ports: vec![
+                    SwitchPort {
+                        interface: EXISTING_INTERFACE.clone(),
+                        port: PORT.clone(),
+                    },
+                    SwitchPort {
+                        interface: ANOTHER_EXISTING_INTERFACE.clone(),
+                        port: ANOTHER_PORT.clone(),
+                    },
+                ],
+            },
+        )]);
+    }
+
+    #[tokio::test]
+    async fn multiple_hosts_should_configure_one_bond_per_host() {
         let score = given_score(vec![
-            given_host(&HOST_ID, vec![EXISTING_INTERFACE.clone()]),
+            given_host(
-            given_host(&ANOTHER_HOST_ID, vec![ANOTHER_EXISTING_INTERFACE.clone()]),
+                &HOST_ID,
+                vec![
+                    EXISTING_INTERFACE.clone(),
+                    ANOTHER_EXISTING_INTERFACE.clone(),
+                ],
+            ),
+            given_host(
+                &ANOTHER_HOST_ID,
+                vec![
+                    YET_ANOTHER_EXISTING_INTERFACE.clone(),
+                    LAST_EXISTING_INTERFACE.clone(),
+                ],
+            ),
         ]);
         let topology = TopologyWithSwitch::new();
 
         let _ = score.interpret(&Inventory::empty(), &topology).await;
 
-        let configured_host_networks = topology.configured_host_networks.lock().unwrap();
+        let config = topology.configured_bonds.lock().unwrap();
-        assert_that!(*configured_host_networks).contains_exactly(vec![
+        assert_that!(*config).contains_exactly(vec![
             (
                 HOST_ID.clone(),
                 HostNetworkConfig {
                     host_id: HOST_ID.clone(),
-                    switch_ports: vec![SwitchPort {
+                    switch_ports: vec![
-                        interface: EXISTING_INTERFACE.clone(),
+                        SwitchPort {
-                        port: PORT.clone(),
+                            interface: EXISTING_INTERFACE.clone(),
-                    }],
+                            port: PORT.clone(),
+                        },
+                        SwitchPort {
+                            interface: ANOTHER_EXISTING_INTERFACE.clone(),
+                            port: ANOTHER_PORT.clone(),
+                        },
+                    ],
                 },
             ),
             (
                 ANOTHER_HOST_ID.clone(),
                 HostNetworkConfig {
                     host_id: ANOTHER_HOST_ID.clone(),
-                    switch_ports: vec![SwitchPort {
+                    switch_ports: vec![
-                        interface: ANOTHER_EXISTING_INTERFACE.clone(),
+                        SwitchPort {
-                        port: ANOTHER_PORT.clone(),
+                            interface: YET_ANOTHER_EXISTING_INTERFACE.clone(),
-                    }],
+                            port: YET_ANOTHER_PORT.clone(),
+                        },
+                        SwitchPort {
+                            interface: LAST_EXISTING_INTERFACE.clone(),
+                            port: LAST_PORT.clone(),
+                        },
+                    ],
                 },
             ),
         ]);
     }
 
     #[tokio::test]
-    async fn port_not_found_for_mac_address_should_not_configure_interface() {
+    async fn multiple_hosts_should_configure_one_port_channel_per_host() {
+        let score = given_score(vec![
+            given_host(
+                &HOST_ID,
+                vec![
+                    EXISTING_INTERFACE.clone(),
+                    ANOTHER_EXISTING_INTERFACE.clone(),
+                ],
+            ),
+            given_host(
+                &ANOTHER_HOST_ID,
+                vec![
+                    YET_ANOTHER_EXISTING_INTERFACE.clone(),
+                    LAST_EXISTING_INTERFACE.clone(),
+                ],
+            ),
+        ]);
+        let topology = TopologyWithSwitch::new();
+
+        let _ = score.interpret(&Inventory::empty(), &topology).await;
+
+        let config = topology.configured_port_channels.lock().unwrap();
+        assert_that!(*config).contains_exactly(vec![
+            (
+                HOST_ID.clone(),
+                HostNetworkConfig {
+                    host_id: HOST_ID.clone(),
+                    switch_ports: vec![
+                        SwitchPort {
+                            interface: EXISTING_INTERFACE.clone(),
+                            port: PORT.clone(),
+                        },
+                        SwitchPort {
+                            interface: ANOTHER_EXISTING_INTERFACE.clone(),
+                            port: ANOTHER_PORT.clone(),
+                        },
+                    ],
+                },
+            ),
+            (
+                ANOTHER_HOST_ID.clone(),
+                HostNetworkConfig {
+                    host_id: ANOTHER_HOST_ID.clone(),
+                    switch_ports: vec![
+                        SwitchPort {
+                            interface: YET_ANOTHER_EXISTING_INTERFACE.clone(),
+                            port: YET_ANOTHER_PORT.clone(),
+                        },
+                        SwitchPort {
+                            interface: LAST_EXISTING_INTERFACE.clone(),
+                            port: LAST_PORT.clone(),
+                        },
+                    ],
+                },
+            ),
+        ]);
+    }
+
+    #[tokio::test]
+    async fn port_not_found_for_mac_address_should_not_configure_host() {
         let score = given_score(vec![given_host(&HOST_ID, vec![UNKNOWN_INTERFACE.clone()])]);
         let topology = TopologyWithSwitch::new_port_not_found();
 
         let _ = score.interpret(&Inventory::empty(), &topology).await;
 
-        let configured_host_networks = topology.configured_host_networks.lock().unwrap();
+        let config = topology.configured_port_channels.lock().unwrap();
-        assert_that!(*configured_host_networks).is_empty();
+        assert_that!(*config).is_empty();
+        let config = topology.configured_bonds.lock().unwrap();
+        assert_that!(*config).is_empty();
+    }
+
+    #[tokio::test]
+    async fn only_one_port_found_for_multiple_mac_addresses_should_not_configure_host() {
+        let score = given_score(vec![given_host(
+            &HOST_ID,
+            vec![EXISTING_INTERFACE.clone(), UNKNOWN_INTERFACE.clone()],
+        )]);
+        let topology = TopologyWithSwitch::new_single_port_found();
+
+        let _ = score.interpret(&Inventory::empty(), &topology).await;
+
+        let config = topology.configured_port_channels.lock().unwrap();
+        assert_that!(*config).is_empty();
+        let config = topology.configured_bonds.lock().unwrap();
+        assert_that!(*config).is_empty();
     }
 
     fn given_score(hosts: Vec<PhysicalHost>) -> HostNetworkConfigurationScore {
@@ -422,26 +591,48 @@ mod tests {
         }
     }
 
+    #[derive(Debug)]
     struct TopologyWithSwitch {
         available_ports: Arc<Mutex<Vec<PortLocation>>>,
-        configured_host_networks: Arc<Mutex<Vec<(Id, HostNetworkConfig)>>>,
+        configured_port_channels: Arc<Mutex<Vec<(Id, HostNetworkConfig)>>>,
         switch_setup: Arc<Mutex<bool>>,
+        network_manager_setup: Arc<Mutex<bool>>,
+        configured_bonds: Arc<Mutex<Vec<(Id, HostNetworkConfig)>>>,
     }
 
     impl TopologyWithSwitch {
         fn new() -> Self {
             Self {
-                available_ports: Arc::new(Mutex::new(vec![PORT.clone(), ANOTHER_PORT.clone()])),
+                available_ports: Arc::new(Mutex::new(vec![
-                configured_host_networks: Arc::new(Mutex::new(vec![])),
+                    PORT.clone(),
+                    ANOTHER_PORT.clone(),
+                    YET_ANOTHER_PORT.clone(),
+                    LAST_PORT.clone(),
+                ])),
+                configured_port_channels: Arc::new(Mutex::new(vec![])),
                 switch_setup: Arc::new(Mutex::new(false)),
+                network_manager_setup: Arc::new(Mutex::new(false)),
+                configured_bonds: Arc::new(Mutex::new(vec![])),
             }
         }
 
         fn new_port_not_found() -> Self {
             Self {
                 available_ports: Arc::new(Mutex::new(vec![])),
-                configured_host_networks: Arc::new(Mutex::new(vec![])),
+                configured_port_channels: Arc::new(Mutex::new(vec![])),
                 switch_setup: Arc::new(Mutex::new(false)),
+                network_manager_setup: Arc::new(Mutex::new(false)),
+                configured_bonds: Arc::new(Mutex::new(vec![])),
+            }
+        }
+
+        fn new_single_port_found() -> Self {
+            Self {
+                available_ports: Arc::new(Mutex::new(vec![PORT.clone()])),
+                configured_port_channels: Arc::new(Mutex::new(vec![])),
+                switch_setup: Arc::new(Mutex::new(false)),
+                network_manager_setup: Arc::new(Mutex::new(false)),
+                configured_bonds: Arc::new(Mutex::new(vec![])),
             }
         }
     }
@@ -457,6 +648,22 @@ mod tests {
         }
     }
 
+    #[async_trait]
+    impl NetworkManager for TopologyWithSwitch {
+        async fn ensure_network_manager_installed(&self) -> Result<(), NetworkError> {
+            let mut network_manager_installed = self.network_manager_setup.lock().unwrap();
+            *network_manager_installed = true;
+            Ok(())
+        }
+
+        async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), NetworkError> {
+            let mut configured_bonds = self.configured_bonds.lock().unwrap();
+            configured_bonds.push((config.host_id.clone(), config.clone()));
+
+            Ok(())
+        }
+    }
+
     #[async_trait]
     impl Switch for TopologyWithSwitch {
         async fn setup_switch(&self) -> Result<(), SwitchError> {
@@ -476,12 +683,12 @@ mod tests {
             Ok(Some(ports.remove(0)))
         }
 
-        async fn configure_host_network(
+        async fn configure_port_channel(
             &self,
             config: &HostNetworkConfig,
         ) -> Result<(), SwitchError> {
-            let mut configured_host_networks = self.configured_host_networks.lock().unwrap();
+            let mut configured_port_channels = self.configured_port_channels.lock().unwrap();
-            configured_host_networks.push((config.host_id.clone(), config.clone()));
+            configured_port_channels.push((config.host_id.clone(), config.clone()));
 
             Ok(())
         }

harmony_types/src/net.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
 pub struct MacAddress(pub [u8; 6]);
 
 impl MacAddress {
@@ -19,6 +19,14 @@ impl From<&MacAddress> for String {
     }
 }
 
+impl std::fmt::Debug for MacAddress {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("MacAddress")
+            .field(&String::from(self))
+            .finish()
+    }
+}
+
 impl std::fmt::Display for MacAddress {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str(&String::from(self))

opnsense-config-xml/src/data/interfaces.rs

@@ -9,7 +9,7 @@ pub struct Interface {
     pub physical_interface_name: String,
     pub descr: Option<MaybeString>,
     pub mtu: Option<MaybeString>,
-    pub enable: MaybeString,
+    pub enable: Option<MaybeString>,
     pub lock: Option<MaybeString>,
     #[yaserde(rename = "spoofmac")]
     pub spoof_mac: Option<MaybeString>,
@@ -134,19 +134,15 @@ mod test {
   <interfaces>
     <paul>
       <if></if>
-      <enable/>
     </paul>
     <anotherpaul>
       <if></if>
-      <enable/>
     </anotherpaul>
     <thirdone>
       <if></if>
-      <enable/>
     </thirdone>
     <andgofor4>
       <if></if>
-      <enable/>
     </andgofor4>
   </interfaces>
   <bar>foo</bar>

opnsense-config-xml/src/data/opnsense.rs

@@ -216,7 +216,7 @@ pub struct System {
     pub maximumfrags: Option<MaybeString>,
     pub aliasesresolveinterval: Option<MaybeString>,
     pub maximumtableentries: Option<MaybeString>,
-    pub language: Option<MaybeString>,
+    pub language: String,
     pub dnsserver: Option<MaybeString>,
     pub dns1gw: Option<String>,
     pub dns2gw: Option<String>,