save: document the network setup

Provide diagrams for a Virtualized Execution Environment

Propose interfaces for a cli toolkit

Cargo.lock

@@ -1719,6 +1719,24 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-ha-cluster"
+version = "0.1.0"
+dependencies = [
+ "brocade",
+ "cidr",
+ "env_logger",
+ "harmony",
+ "harmony_macros",
+ "harmony_secret",
+ "harmony_tui",
+ "harmony_types",
+ "log",
+ "serde",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "example-kube-rs"
 version = "0.1.0"
@@ -1838,6 +1856,24 @@ dependencies = [
 [[package]]
 name = "example-opnsense"
 version = "0.1.0"
+dependencies = [
+ "brocade",
+ "cidr",
+ "env_logger",
+ "harmony",
+ "harmony_cli",
+ "harmony_macros",
+ "harmony_secret",
+ "harmony_types",
+ "log",
+ "serde",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "example-opnsense-2"
+version = "0.1.0"
 dependencies = [
  "brocade",
  "cidr",

brocade/src/lib.rs

@@ -31,6 +31,7 @@ pub struct BrocadeOptions {
 pub struct TimeoutConfig {
     pub shell_ready: Duration,
     pub command_execution: Duration,
+    pub command_output: Duration,
     pub cleanup: Duration,
     pub message_wait: Duration,
 }
@@ -40,6 +41,7 @@ impl Default for TimeoutConfig {
         Self {
             shell_ready: Duration::from_secs(10),
             command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while
+            command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output"
             cleanup: Duration::from_secs(10),
             message_wait: Duration::from_millis(500),
         }

brocade/src/network_operating_system.rs

@@ -3,6 +3,7 @@ use std::str::FromStr;
 use async_trait::async_trait;
 use harmony_types::switch::{PortDeclaration, PortLocation};
 use log::{debug, info};
+use regex::Regex;
 
 use crate::{
     BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo,
@@ -103,13 +104,37 @@ impl NetworkOperatingSystemClient {
         };
 
         Some(Ok(InterfaceInfo {
-            name: format!("{} {}", interface_type, port_location),
+            name: format!("{interface_type} {port_location}"),
             port_location,
             interface_type,
             operating_mode,
             status,
         }))
     }
+
+    fn map_configure_interfaces_error(&self, err: Error) -> Error {
+        debug!("[Brocade] {err}");
+
+        if let Error::CommandError(message) = &err {
+            if message.contains("switchport")
+                && message.contains("Cannot configure aggregator member")
+            {
+                let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap();
+
+                if let Some(caps) = re.captures(message) {
+                    let interface_type = &caps[1];
+                    let port_location = &caps[2];
+                    let interface = format!("{interface_type} {port_location}");
+
+                    return Error::CommandError(format!(
+                        "Cannot configure interface '{interface}', it is a member of a port-channel (LAG)"
+                    ));
+                }
+            }
+        }
+
+        err
+    }
 }
 
 #[async_trait]
@@ -197,11 +222,10 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             commands.push("exit".into());
         }
 
-        commands.push("write memory".into());
-
         self.shell
             .run_commands(commands, ExecutionMode::Regular)
-            .await?;
+            .await
+            .map_err(|err| self.map_configure_interfaces_error(err))?;
 
         info!("[Brocade] Interfaces configured.");
 
@@ -213,7 +237,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
 
         let output = self
             .shell
-            .run_command("show port-channel", ExecutionMode::Regular)
+            .run_command("show port-channel summary", ExecutionMode::Regular)
             .await?;
 
         let used_ids: Vec<u8> = output
@@ -248,7 +272,12 @@ impl BrocadeClient for NetworkOperatingSystemClient {
         ports: &[PortLocation],
     ) -> Result<(), Error> {
         info!(
-            "[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}"
+            "[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}",
+            ports
+                .iter()
+                .map(|p| format!("{p}"))
+                .collect::<Vec<String>>()
+                .join(", ")
         );
 
         let interfaces = self.get_interfaces().await?;
@@ -276,8 +305,6 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             commands.push("exit".into());
         }
 
-        commands.push("write memory".into());
-
         self.shell
             .run_commands(commands, ExecutionMode::Regular)
             .await?;
@@ -294,7 +321,6 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             "configure terminal".into(),
             format!("no interface port-channel {}", channel_name),
             "exit".into(),
-            "write memory".into(),
         ];
 
         self.shell

brocade/src/shell.rs

@@ -211,7 +211,7 @@ impl BrocadeSession {
         let mut output = Vec::new();
         let start = Instant::now();
         let read_timeout = Duration::from_millis(500);
-        let log_interval = Duration::from_secs(3);
+        let log_interval = Duration::from_secs(5);
         let mut last_log = Instant::now();
 
         loop {
@@ -221,7 +221,9 @@ impl BrocadeSession {
                 ));
             }
 
-            if start.elapsed() > Duration::from_secs(5) && last_log.elapsed() > log_interval {
+            if start.elapsed() > self.options.timeouts.command_output
+                && last_log.elapsed() > log_interval
+            {
                 info!("[Brocade] Waiting for command output...");
                 last_log = Instant::now();
             }
@@ -276,7 +278,7 @@ impl BrocadeSession {
         let output_lower = output.to_lowercase();
         if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) {
             return Err(Error::CommandError(format!(
-                "Command '{command}' failed: {}",
+                "Command error: {}",
                 output.trim()
             )));
         }

examples/application_monitoring_with_tenant/src/main.rs

@@ -27,7 +27,6 @@ async fn main() {
     };
     let application = Arc::new(RustWebapp {
         name: "example-monitoring".to_string(),
-        dns: "example-monitoring.harmony.mcd".to_string(),
         project_root: PathBuf::from("./examples/rust/webapp"),
         framework: Some(RustWebFramework::Leptos),
         service_port: 3000,

examples/ha_cluster/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "example-ha-cluster"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+harmony = { path = "../../harmony" }
+harmony_tui = { path = "../../harmony_tui" }
+harmony_types = { path = "../../harmony_types" }
+cidr = { workspace = true }
+tokio = { workspace = true }
+harmony_macros = { path = "../../harmony_macros" }
+log = { workspace = true }
+env_logger = { workspace = true }
+url = { workspace = true }
+harmony_secret = { path = "../../harmony_secret" }
+brocade = { path = "../../brocade" }
+serde = { workspace = true }

examples/ha_cluster/README.md

@@ -0,0 +1,15 @@
+## OPNSense demo
+
+Download the virtualbox snapshot from {{TODO URL}}
+
+Start the virtualbox image
+
+This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
+
+Credentials are opnsense default (root/opnsense)
+
+Run the project with the correct ip address on the command line :
+
+```bash
+cargo run -p example-opnsense -- 192.168.5.229
+```

examples/ha_cluster/src/main.rs

@@ -0,0 +1,141 @@
+use std::{
+    net::{IpAddr, Ipv4Addr},
+    sync::Arc,
+};
+
+use brocade::BrocadeOptions;
+use cidr::Ipv4Cidr;
+use harmony::{
+    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    inventory::Inventory,
+    modules::{
+        dummy::{ErrorScore, PanicScore, SuccessScore},
+        http::StaticFilesHttpScore,
+        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
+        opnsense::OPNsenseShellCommandScore,
+        tftp::TftpScore,
+    },
+    topology::{LogicalHost, UnmanagedRouter},
+};
+use harmony_macros::{ip, mac_address};
+use harmony_secret::{Secret, SecretManager};
+use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
+
+#[tokio::main]
+async fn main() {
+    let firewall = harmony::topology::LogicalHost {
+        ip: ip!("192.168.5.229"),
+        name: String::from("opnsense-1"),
+    };
+
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
+    let opnsense = Arc::new(
+        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
+    );
+    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
+    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
+    let gateway_ip = IpAddr::V4(gateway_ipv4);
+    let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
+        domain_name: "demo.harmony.mcd".to_string(),
+        router: Arc::new(UnmanagedRouter::new(
+            gateway_ip,
+            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
+        )),
+        load_balancer: opnsense.clone(),
+        firewall: opnsense.clone(),
+        tftp_server: opnsense.clone(),
+        http_server: opnsense.clone(),
+        dhcp_server: opnsense.clone(),
+        dns_server: opnsense.clone(),
+        control_plane: vec![LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        }],
+        bootstrap_host: LogicalHost {
+            ip: ip!("10.100.8.20"),
+            name: "cp0".to_string(),
+        },
+        workers: vec![],
+        switch_client: switch_client.clone(),
+    };
+
+    let inventory = Inventory {
+        location: Location::new(
+            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
+            "wk".to_string(),
+        ),
+        switch: SwitchGroup::from([]),
+        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
+        storage_host: vec![],
+        worker_host: vec![],
+        control_plane_host: vec![
+            PhysicalHost::empty(HostCategory::Server)
+                .mac_address(mac_address!("08:00:27:62:EC:C3")),
+        ],
+    };
+
+    // TODO regroup smaller scores in a larger one such as this
+    // let okd_boostrap_preparation();
+
+    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
+    let dns_score = OKDDnsScore::new(&topology);
+    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
+
+    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
+    let http_score = StaticFilesHttpScore {
+        folder_to_serve: Some(Url::LocalFolder(
+            "./data/watchguard/pxe-http-files".to_string(),
+        )),
+        files: vec![],
+        remote_path: None,
+    };
+
+    harmony_tui::run(
+        inventory,
+        topology,
+        vec![
+            Box::new(dns_score),
+            Box::new(dhcp_score),
+            Box::new(load_balancer_score),
+            Box::new(tftp_score),
+            Box::new(http_score),
+            Box::new(OPNsenseShellCommandScore {
+                opnsense: opnsense.get_opnsense_config(),
+                command: "touch /tmp/helloharmonytouching".to_string(),
+            }),
+            Box::new(SuccessScore {}),
+            Box::new(ErrorScore {}),
+            Box::new(PanicScore {}),
+        ],
+    )
+    .await
+    .unwrap();
+}
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/nanodc/src/main.rs

@@ -61,6 +61,7 @@ async fn main() {
     let gateway_ipv4 = Ipv4Addr::new(192, 168, 33, 1);
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "ncd0.harmony.mcd".to_string(), // TODO this must be set manually correctly
         // when setting up the opnsense firewall
         router: Arc::new(UnmanagedRouter::new(

examples/okd_installation/src/topology.rs

@@ -59,6 +59,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,

examples/okd_pxe/src/topology.rs

@@ -54,6 +54,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,

examples/opnsense/Cargo.toml

@@ -8,7 +8,7 @@ publish = false
 
 [dependencies]
 harmony = { path = "../../harmony" }
-harmony_tui = { path = "../../harmony_tui" }
+harmony_cli = { path = "../../harmony_cli" }
 harmony_types = { path = "../../harmony_types" }
 cidr = { workspace = true }
 tokio = { workspace = true }

examples/opnsense/README.md

@@ -1,15 +1,23 @@
-## OPNSense demo
+# OPNSense Demo
 
-Download the virtualbox snapshot from {{TODO URL}}
+This example demonstrate how to manage an Opnsense server with harmony.
 
-Start the virtualbox image
+todo: add more info
 
-This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
+## Demo instructions
 
-Credentials are opnsense default (root/opnsense)
+todo: add detailed instructions
 
-Run the project with the correct ip address on the command line :
+  - setup the example execution environment
+  - setup your system configuration
+    - topology
+    - scores
+    - secrets
+  - build
+  - execute
+  - verify/inspect
+
+## Example execution
+
+See [learning tool documentation](./scripts/README.md)
 
-```bash
-cargo run -p example-opnsense -- 192.168.5.229
-```

examples/opnsense/env.sh

@@ -0,0 +1,4 @@
+export HARMONY_SECRET_NAMESPACE=example-opnsense
+export HARMONY_SECRET_STORE=file
+export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info 
+export RUST_LOG=info

examples/opnsense/scripts/DEVELOP.md

@@ -0,0 +1,38 @@
+## réseau
+
+- fonctionne uniquement avec connection filaire, pas wifi
+- on crée des bridges pour le réseau WAN et LAN pour qu'ils soient facilement accessible sur le poste de travail
+
+```
+# paramètres
+PRIMARY_PROFIL=enpXsY
+NETWORK_LABEL=harmony
+
+WAN_BRIDGE=$NETWORL_LABEL-wan-brd   # max 15 char
+LAN_NIC=$NETWORK_LABEL-lan-nic
+LAN_BRIDGE=$NETWORK_LABEL-lan-brd
+
+
+# Setup WAN bridge
+nmcli c down $PRIMARY
+nmcli c add type bridge ifname $WAN_BRIDGE con-name $WAN_BRIDGE
+nmcli c add type bridge-slave ifname $PRIMARY_PROFILE master $WAN_BRIDGE ipv4.method auto
+# nmcli c up $WAN_BRIDGE
+
+
+# Setup LAN nic
+sudo modprobe dummy
+sudo ip link add $LAN_NIC type dummy
+ip tuntap add dev $LAN_NIC mode tap user root  # todo: why user root?
+
+# Setup LAN bridge
+nmcli c add type bridge ifname $LAN_BRIDGE con-name $LAN_BRIDGE
+nmcli c add type bridge-slave ifname $LAN_NIC master $LAN_BRIDGE ipv4.method auto
+nmcli c up $LAN_BRIDGE
+```
+
+LAN bridge do not have an IP address and appear down. But it is successfully used by opnsense and can be accessed from the host network.
+
+## config opnsense minimale
+
+  - ssh enabled

examples/opnsense/scripts/README.md

@@ -0,0 +1,96 @@
+# Virtualized Execution Environment for Harmony
+
+Scripts included in this directory have 3 purposes:
+
+  - automate initial setup of localhost or VM (nested virtualization) for this example
+  - prototype a solution for an 'OpensenseLocalhostTopology'
+  - prototype 
+
+This exprimentation aim to find an approach suitable for using harmony on virtualised execution environment such that:
+
+  - it straights forward for a user with minimal knowledge to start testing harmony
+  - installation and execution have **minimal impact on the user desktop**
+
+## Usage
+
+### Installation
+
+1. download this directory
+2. add this directory in your PATH (example `. setup`)
+
+
+```
+# show help page
+harmony-vee
+
+# show active configurations
+harmony-vee config
+
+# show what will be modified at installation
+harmony-vee install --dry-run
+
+# install
+harmony-vee install
+
+# show what will be modified at unistallation
+harmony-vee uninstall --dry-run
+
+```
+
+### Create and start a new Virtual Execution Environment
+
+
+```
+# Create a HVEE to test opnsense_score
+harmony-vee init opnsense_score
+
+# List existing HVEE
+harmony-vee list
+
+# Show HVEE information including devices ip and vault type/location
+harmony-vee show opnsense_score
+
+# Start/Stop a HVEE
+harmony-vee stop opnsense_score
+
+
+# Destroy a HVEE instance
+harmony-vee destroy opnsense_score
+
+```
+
+
+### Variable d'environnement
+
+```
+## directory containing harmony-ve data
+# HVE_ROOT=~/.harmony-ve
+
+## OPNSENSE SRC
+# main mirror
+# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
+# first alternative mirror
+# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
+# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
+
+
+## Network
+# HVE_NETWORK_LABEL=harmony
+```
+
+
+## Remarks
+
+  - A nested VM setup could be safer
+
+
+
+## Architecture
+
+### Learning environment directly on host
+
+![localhost case](./doc/automate-opnsense-example-localhost.drawio.png)
+
+### Learning environment nested in a "workspace vm"
+
+![localhost case](./doc/automate-opnsense-example-nested-virtualization.drawio.png)

examples/opnsense/scripts/common

@@ -0,0 +1,17 @@
+#! /bin/bash
+
+_warn(){ >&2 echo "WARNING: $*" ; }
+
+_fatal(){
+  >&2 echo "FATAL ERROR: $*"
+  >&2 echo stopping...
+  exit 1
+}
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}

examples/opnsense/scripts/default-env-var

@@ -0,0 +1,31 @@
+# Conventions:
+#   - Namespaced with HVE, short for Harmony Virtualised Execution Environment
+#   - Prefixed values used internally
+#   - Not prefixed may be supercharged by the user
+
+# Root of harmony data
+_HVE_ROOT=${HVE_ROOT:-$HOME/harmony-ve}
+[ -d "$_HVE_ROOT" ] || mkdir -p "${_HVE_ROOT}"
+
+
+_HVE_SRC_IMG=${_HVE_ROOT}/src/images
+[ -d "$_HVE_SRC_IMG" ] || mkdir -p "${_HVE_SRC_IMG}"
+_HVE_IMG=${_HVE_ROOT}/images
+[ -d "$_HVE_IMG" ] || mkdir -p "$_HVE_IMG"
+
+# Opnsense
+_HVE_OPNSENSE_URL=${HVE_OPNSENSE_URL:-https://pkg.opnsense.org/releases}
+# first alternative mirror
+_HVE_OPNSENSE_URL_ALT1=${HVE_OPNSENSE_URL_ALT1:-https://mirror.vraphim.com/opnsense/releases}
+_HVE_OPNSENSE_URL_ALT2=${HVE_OPNSENSE_URL_ALT2:-https://mirror.winsub.kr/opnsense/releases}
+
+_HVE_OPNSENSE_SRC_IMG=${_HVE_SRC_IMG}/opnsense
+[ -d "$_HVE_OPNSENSE_SRC_IMG" ] || mkdir -p "${_HVE_OPNSENSE_SRC_IMG}"
+_HVE_OPNSENSE_IMG=${_HVE_IMG}/opnsense
+[ -d "$_HVE_OPNSENSE_IMG" ] || mkdir -p "${_HVE_OPNSENSE_IMG}"
+
+# Network
+_HVE_NETWORK=${HVE_NETWORK:-harmony}
+_HVE_WAN_BRIDGE=${HVE_WAN_BRIDGE:-${_HVE_NETWORK}-wan-brd}
+_HVE_LAN_BRIDGE=${HVE_LAN_BRIDGE:-${_HVE_NETWORK}-lann-brd}
+

examples/opnsense/scripts/dependencies-management

@@ -0,0 +1,48 @@
+#! /bin/bash
+is_string_empty(){
+    if [ "${*:-}" != "" ]; then
+     return 0
+   else
+     return 1
+   fi
+}
+
+is_debian_family()(
+   is_string_empty "$(apt --version 2> /dev/null )"
+)
+
+has_ip(){
+  is_string_empty "$(ip -V 2> /dev/null)"
+}
+
+has_virsh(){
+  is_string_empty "$(virsh --version 2> /dev/null)"
+}
+
+has_virt_customize(){
+  is_string_empty "$(virt-customize --version 2> /dev/null)"
+}
+
+has_curl(){
+  is_string_empty "$(curl --version 2> /dev/null)"
+}
+has_wget(){
+  is_string_empty "$(wget --version 2> /dev/null)"
+}
+
+install_kvm(){
+  sudo apt install -y --no-install-recommends qemu-system libvirt-clients libvirt-daemon-system
+  sudo usermod -aG libvirt "$USER"
+  # todo: finf how to fix image access out of /var/lib/libvirt/images
+  sudo setfacl -Rm u:libvirt-qemu:rx $_HVE_IMG
+  sudo systemctl restart libvirtd
+}
+
+install_virt_customize(){
+   sudo apt install -y libguestfs-tools
+}
+
+install_wget(){
+  sudo apt install -y wget
+}
+

examples/opnsense/scripts/doc/automate-opnsense-example.drawio

@@ -0,0 +1,321 @@
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" version="28.2.9" pages="2">
+  <diagram name="localhost" id="lK0WmoXmZXwFmV5PC-RW">
+    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="1Ax8jaXdU0J25Zkiwu96-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="300" y="230" width="700" height="550" as="geometry" />
+        </mxCell>
+        <mxCell id="VVCo9gNhF-9Hs0fZa6i8-1" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="380" y="240" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="283" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-2" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="590" y="393" width="310" height="270" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-3" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="620" y="403" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="620" y="250" width="280" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-5" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="610" y="250" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-6" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="294" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-7" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
+          <mxGeometry x="760" y="290" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-8" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-7" target="X0RF5wBsf5JYURxROWwA-9" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-9" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="771" y="342" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-13" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="810" y="290" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-14" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-13" target="X0RF5wBsf5JYURxROWwA-15" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-15" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="820" y="342" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-16" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="760" y="383" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-17" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-16" target="X0RF5wBsf5JYURxROWwA-18" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-18" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="772.5" y="353" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-19" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="809" y="383" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-20" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-19" target="X0RF5wBsf5JYURxROWwA-21" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-21" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="821.5" y="353" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-22" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="440" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-23" value="&lt;b&gt;Example dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="450" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-24" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="510" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-25" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="590" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-26" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="600" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-27" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="361" y="650" width="208" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-28" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="350" y="353" width="230" height="77" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-29" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="361.5" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-30" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="620" y="450" width="250" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-31" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="323" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-33" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs and&lt;/i&gt;&lt;/font&gt;&lt;div&gt;&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;runtime info.&lt;/i&gt;&lt;/font&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="370" y="395" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="360" y="470" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-35" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="626.5" y="264" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-36" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="369" y="620" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="X0RF5wBsf5JYURxROWwA-37" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="626.5" y="420" width="200" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+  <diagram id="oOuscOXp9aWETXQepMaW" name="nested-virtualization">
+    <mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="190" y="60" width="1240" height="660" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-37" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="540" y="210" width="830" height="480" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-2" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="84" width="60" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-3" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="223" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="855" y="403.75" width="310" height="266.25" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-5" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="885" y="413.75" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-6" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="810" y="230" width="530" height="120" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-7" value="&lt;b&gt;workspace VM network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="825" y="234" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-8" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="234" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-10" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-9" target="vj3pG7w7rTpS1tnBMssI-11" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-12" target="vj3pG7w7rTpS1tnBMssI-14" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-15" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1025" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-16" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-15" target="vj3pG7w7rTpS1tnBMssI-17" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-17" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1037.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-18" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1074" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-19" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-18" target="vj3pG7w7rTpS1tnBMssI-20" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-20" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1086.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-21" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="380" width="230" height="120" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-22" value="&lt;b&gt;Learning dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="391" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-23" value="&lt;ul&gt;&lt;li&gt;harmony&lt;/li&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="450" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="540" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-25" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="550" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-26" value="&lt;ul&gt;&lt;li&gt;Can be mounted locally&lt;br&gt;for persistence&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="251" y="600" width="189" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-27" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="240" y="293" width="230" height="67" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-28" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="301.5" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-29" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="885" y="460.75" width="250" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-30" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="263" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-31" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="260" y="321" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-32" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="250" y="410" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="259" y="570" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-35" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="891.5" y="430.75" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-38" value="&lt;b&gt;workspace VM&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="234" width="130" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-39" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="570" y="305" width="230" height="215" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-40" value="&lt;b&gt;workspace VM dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="316" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-41" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="580" y="375" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-44" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="570" y="540" width="230" height="130" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-48" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="830" y="277" width="235" height="45" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-45" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="590" y="550" width="152" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-46" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="581" y="600" width="208" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-47" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="589" y="570" width="200" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-9" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
+          <mxGeometry x="1025" y="300.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-11" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1036" y="352.75" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-49" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="1069" y="275" width="251" height="45" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1075" y="300.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-14" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1085" y="352.75" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-50" value="&lt;b&gt;wan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="821.5" y="277" width="70" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-51" value="&lt;b&gt;lan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1055" y="275" width="70" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-52" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="1181" y="403.75" width="159" height="116.25" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-53" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-54" target="vj3pG7w7rTpS1tnBMssI-55" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-54" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1211" y="305" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-55" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="1221" y="357" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-58" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1210" y="393.75" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-59" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-58" target="vj3pG7w7rTpS1tnBMssI-60" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-60" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1222.5" y="363.75" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-61" value="&lt;b&gt;other vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="1181" y="430" width="90" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="vj3pG7w7rTpS1tnBMssI-63" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="vj3pG7w7rTpS1tnBMssI-24" target="vj3pG7w7rTpS1tnBMssI-44" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="600" y="770" as="sourcePoint" />
+            <mxPoint x="360" y="885" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+          <mxGeometry x="799" y="84" width="280" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-2" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="789" y="84" width="150" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-6" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="989" y="124" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-7" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-6" target="Za91R7Nqk7Jj5VTRes6Y-8" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-8" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
+          <mxGeometry x="999" y="176" width="8" height="8" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="988" y="217" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-12" target="Za91R7Nqk7Jj5VTRes6Y-14" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-14" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
+          <mxGeometry x="1000.5" y="187" width="5" height="10" as="geometry" />
+        </mxCell>
+        <mxCell id="Za91R7Nqk7Jj5VTRes6Y-15" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;No modification required&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="805.5" y="98" width="200" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

examples/opnsense/scripts/env-var

@@ -0,0 +1,13 @@
+## directory containing harmony-ve data
+# HVE_ROOT=~/.harmony-ve
+
+## OPNSENSE SRC
+# main mirror
+# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
+# first alternative mirror
+# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
+# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
+
+
+## Network
+# HVE_NETWORK_LABEL=harmony

examples/opnsense/scripts/harmony-ve

@@ -0,0 +1,105 @@
+
+#! /bin/bash
+
+harmony-ve()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve
+
+  DESCRIPTION
+
+    CLI management toolkit for Harmony Virtualized Execution Environment
+
+  SYNOPSYS
+
+    harmony-ve [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve dependencies      # manage localhost depend
+    harmony-ve network           # manage localhost netork
+
+    harmony-ve opnsense-img-src  # manage opnsense OS source images
+    harmony-ve opnsense-img      # manage opnsense OS images
+
+    harmony-ve vm                # manage vm
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  DESCRIPTION
+
+    Automation CLI to easily provision and manage a Virtualized Execution Environment for testing and learning Harmony.
+
+    This tool allows:
+      - new harmony users to start testing within 15 minutes on their development desktop
+      - automate virtualized test in pipeline
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+
+  }
+
+
+  # Implement functions
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+   deps|dependencies)
+	   harmony-ve-dependencies "${@:2}"
+	   ;;
+   net|network)
+	   harmony-ve-network"${@:2}"
+	   ;;
+   img-src|opnsense-img-src)
+	   harmony-ve-opnsense-img-src "${@:2}"
+	   ;;
+   img|opnsense-img)
+	   harmony-ve-opnsense-img "${@:2}"
+	   ;;
+   vm)
+	   harmony-ve-vm "${@:2}"
+	   ;;
+
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve "${@}"
+

examples/opnsense/scripts/harmony-ve-dependencies

@@ -0,0 +1,125 @@
+#! /bin/bash
+#
+# virt-install <= virtinst
+# quemu-img
+
+
+
+harmony-ve-dependencies()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-dependencies 
+
+  DESCRIPTION
+
+    Manage localhost dependencies needed for Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    devops-dependencies [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    devops check     # Check that dependencies are installed
+    devops install   # Install missing dependencies
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  _check_dependencies(){
+      . "${SCRIPTS_DIR}/dependencies-management"
+      missing=0
+      NEED_IP=false
+      NEED_KVM=false
+      NEED_VIRT_CUSTOMIZE=false
+      NEED_WGET=false
+      is_debian_family || _fatal only debian based version is supported
+      has_ip || { 
+	missing=$(( missing + 1)); 
+	_warn "ip command is missing";
+        NEED_IP=true
+      }
+      has_virsh ||{
+	missing=$(( missing + 1)); 
+	_warn "virsh command is missing";
+        NEED_KVM=true
+      } 
+      has_virt_customize || {
+	missing=$(( missing + 1)); 
+	_warn "virt-customize command is missing";
+        NEED_VIRT_CUSTOMIZE=true
+      }
+      has_wget || has_curl || {
+	missing=$(( missing + 1)); 
+	_warn "wget and curl commands are missing"; NEED_WGET=true
+
+      }
+
+  }
+
+  _install_dependencies(){
+
+    [ "$NEED_KVM" != "true" ] || install_kvm 
+    [ "$NEED_VIRT_CUSTOMIZE" != "true" ] || install_virt_customize 
+    [ "$NEED_WGET" != "true" ] || install_wget
+  }
+
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   cdeps|check-dependencies)
+	   _check_dependencies
+           if [ "$missing" -gt 0 ]; then 
+	      exit 1
+           fi
+	   _warn No missing dependencies
+	   ;;
+   ideps|install-dependencies)
+	   _check_dependencies
+	   _install_dependencies
+	   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-dependencies "${@}"
+

examples/opnsense/scripts/harmony-ve-network

@@ -0,0 +1,232 @@
+
+#! /bin/bash
+# todo: allow wan to switch from ethernet to wifi 
+
+
+harmony-ve-network()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-network
+
+  DESCRIPTION
+
+    Modify localhost network for Harmony Virtual Execution Environment
+  SYNOPSYS
+
+    harmony-ve-network [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-network check
+    harmony-ve-network setup
+    harmony-ve-network cleanup 
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+
+
+  IMPLEMENTATION NOTES
+
+    - use the network manager present to add 2 bridges (wan + lan)
+
+
+EOM
+
+  }
+
+  # dependency management
+
+  _is_service_used(){
+    service=$1
+    sudo systemctl list-unit-files $service || return 1
+    sudo systemctl status --no-pager $service || return 1
+  }
+
+
+
+  # Implement functions
+
+  _list_bridges(){
+
+    ip -o link show type bridge | awk '{print $2}' | sed 's/://g'
+  }
+
+  _is_a_bridge(){
+    bridge=$1
+    matched=$(_list_bridges | grep "$bridge")
+    [ "$matched" = "$bridge" ] || return 1
+  }
+
+  _bridge_is_up(){
+    _fatal Not implemented
+  }
+
+  _rename_nmcli_profile(){
+    device=$1
+    profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$device:" | cut -d':' -f2)
+    [ "$profile" != "" ] || _fatal Failed to find nmcli profile 
+
+    sudo nmcli con mod "$profile" con-name "$device"
+
+  }
+
+  _create_a_bridge_using_networkmanager(){
+    bridge=$1
+
+   profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$PRIMARY_INTERFACE:" | cut -d':' -f2)
+
+   nmcli conn delete "$profile"
+
+   nmcli conn add type bridge ifname $bridge con-name $bridge || _fatal Fail to create a bridge using nmcli
+
+
+   nmcli con add type bridge-slave ifname $PRIMARY_INTERFACE master $bridge || _fatal Fail to create a slave-interface using nmcli
+
+   nmcli con up $bridge || _fatal Fail to set interface up using nmcli
+ 
+   sudo systemctl restart NetworkManager.service
+  # todo: use a check loop until connection with a timeout
+  #sleep 10
+  #ping nationtech.io | _fatal Internet connection lost
+
+  }
+
+  _delete_a_bridge_using_networkmanager(){
+     device=$1
+
+    nmcli conn delete bridge-slave-$PRIMARY_INTERFACE
+    nmcli conn delete $device
+    
+    nmcli con add type ethernet ifname $PRIMARY_INTERFACE con-name $PRIMARY_INTERFACE autoconnect yes ipv4.method auto ipv6.method ignore
+
+   nmcli conn up "$PRIMARY_INTERFACE"
+   sudo systemctl restart NetworkManager.service
+
+  # todo: use a check loop until connection with a timeout
+  #sleep 10
+  #ping nationtech.io | _fatal Internet connection lost
+
+  }
+
+
+
+  _create_a_bridge(){
+
+    bridge=$1
+
+    [ $USE_NETWORKMANAGER = 0 ] | _fatal "Only NetworkManager is implemented"
+
+    _create_a_bridge_using_networkmanager $bridge
+
+  }
+
+  _setup_a_bridge(){
+
+    $bridge
+
+    bridge_exist=1
+    bridge_is_up=1
+    bridge_has_ip=1
+    bridge_has_route=1
+    bridge_is_working=1
+
+    _is_a_bridge $bridge && bridge_exists=0 || _create_a_bridge $bridge || _fatal Fail to create a bridge
+
+
+    }
+  
+  _get_networkmanager_profile_from_device(){
+
+    device=$1
+
+    profile=$(nmcli -t -f DEVICE,NAME c show --active | grep "^$device:" | cut -d':' -f2)
+
+    [ "$profile" != "" ] || _fatal Fail to retreive nmcli profile
+
+    echo "$profile"
+
+  }
+
+  _find_primary_interface(){
+    PRIMARY_INTERFACE=$(ip route | grep '^default' | sed 's/ dev /!/g' | cut -d'!' -f 2 | awk '{ print $1 }' )  
+    [ "$PRIMARY_INTERFACE" != "" ] || _fatal Fail to find the primary interface
+  }
+
+  _find_used_network_manager(){
+
+    _is_service_used NetworkManager.service && USE_NETWORKMANAGER=0 || USE_NETWORKMANAGER=1
+    _is_service_used systemd-networkd.service && USE_SYSTEMD_NETWORKD=0 || USE_SYSTEMD_NETWORKD=1
+    _is_service_used dhcpd.service && USE_DHCPD=0 || USE_DHCPD=1 
+
+    USE_MANUAL=0 && [ $USE_NETWORKMANAGER = 0 ] || [ $USE_SYSTEMD_NETWORKD = 0 ] || [ $USE_DHCPD = 0 ] || USE_MANUAL=0
+
+    }
+
+  _connect(){
+
+    _find_used_network_manager
+    _find_primary_interface
+
+    _setup_a_bridge $_HVE_WAN_BRIDGE
+    _setup_a_bridge $_HVE_LAN_BRIDGE
+  }
+
+
+
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+  connect)
+    _connect "${@:2}"
+    ;;
+  disconnect)
+    _disconnect "${@:2}"
+    ;;
+ 
+  dev) 
+
+    "${@:2}"
+   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-network "${@}"
+

examples/opnsense/scripts/harmony-ve-opnsense-img

@@ -0,0 +1,150 @@
+
+#! /bin/bash
+
+harmony-ve-opnsense-img()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"  
+
+  export PATH=$SCRIPTS_DIR:$PATH
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+  
+    harmony-ve-opnsense-img
+
+  DESCRIPTION
+
+    Manage opnsense images needed by Harmony Virtual Execution Environment
+  SYNOPSYS
+
+    harmony-vee-opnsense-img [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-opnsense-img list
+    harmony-ve-opnsense-img init NAME VERSION
+    harmony-ve-opnsense-img start NAME
+    harmony-ve-opnsense-img update NAME
+    harmony-ve-opnsense-img delete [NAME]
+
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  # assertions
+
+  _assert_image_do_not_exists(){
+    name=$1
+    [ ! -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "An image '$name' already exists"
+  }
+  _assert_image_exists(){
+    name=$1
+    [ -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "Image '$name' do not exists"
+  }
+
+
+  # Implement functions
+  _init(){
+    name=$1
+    version=${2}
+
+    _assert_image_do_not_exists $name
+    mkdir -p "${_HVE_OPNSENSE_IMG}/$name"
+
+    harmony-ve opnsense-img-src download $version
+
+    sudo qemu-img convert -f raw -O qcow2 "$_HVE_OPNSENSE_SRC_IMG/OPNsense-${version}-nano-amd64.img" "/var/lib/libvirt/images/opnsense-$name.qcow2"
+
+
+
+
+    cat <<-EOM > "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  virt-install \
+	--name $name \
+	--os-variant freebsd14.0 \
+	--vcpus=2,sockets=1,cores=2,threads=1 \
+	--memory 4096 \
+	--disk path="/var/lib/libvirt/images/opnsense-$name.qcow2" \
+	--network bridge=${_HVE_WAN_BRIDGE},model=virtio \
+	--network bridge=${_HVE_LAN_BRIDGE},model=virtio \
+	--graphics none \
+	--console pty,target_type=serial \
+	--import \
+	--autostart
+EOM
+    
+  chmod +x "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  }
+
+
+  _start(){
+    name=$1
+    _assert_image_exists $name
+
+    "$_HVE_OPNSENSE_IMG/$name/$name.sh"
+
+  }
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+  init)
+    _init "${@:2}"
+    ;;
+  start)
+    _start "${@:2}"
+     ;;
+  delete)
+    rm -r ${_HVE_OPNSENSE_IMG}/"$2"
+    ;;
+  ls|list)
+    ls ${_HVE_OPNSENSE_IMG} | cat
+    ;;
+  show)
+    ls ${_HVE_OPNSENSE_IMG}/"$2" | cat
+    ;;
+  *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img "${@}"
+

examples/opnsense/scripts/harmony-ve-opnsense-img-src

@@ -0,0 +1,310 @@
+
+#! /bin/bash
+
+harmony-ve-opnsense-img-src()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  . "${SCRIPTS_DIR}/default-env-var"
+
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+    harmony-ve-opnsense-img-src
+
+  DESCRIPTION
+
+    Manage opnsense source images needed by Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    harmony-vee-opnsense-img-src [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-vee-opnsense-img-src list [--remote]
+    harmony-vee-opnsense-img-src download [VERSION]
+    harmony-vee-opnsense-img-src check [VERSION]
+    harmony-vee-opnsense-img-src delete [VERSION]
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+     
+  DETAILS
+
+    - for 'list', show local images available
+    - for 'list --remote', show available upstream images
+    - for 'download', when no VERSION is specified, use the latest 
+    - for 'delete', when no VERSION is specified, delete all image
+    - use the 'nano' flavor
+EOM
+
+  }
+
+
+  # Implement functions
+  _parse_version_from_image_file_string(){
+    #https://pkg.opnsense.org/releases/25.7/OPNsense-25.7-nano-amd64.img.bz2
+    echo "$1" | cut -d '/' -f 5
+}
+
+
+  _list_local_images(){
+    ls "${_HVE_OPNSENSE_SRC_IMG}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
+  }
+
+
+  _list_remote_images(){
+     curl -L -s "${_HVE_OPNSENSE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2  | cut -d '/' -f 1 | sort -r
+  }
+
+  _latest_version(){
+    _list_remote_images | head -n 1
+  }
+
+  _is_downloaded(){
+     version=$1
+     name="${_HVE_OPNSENSE_SRC_IMG}/OPNsense-${version}-nano-amd64.img"
+    [ -f "$name" ] && return 0 || return 1
+  }
+
+  _is_valid_version(){
+   version=${1}
+    matched_version=$(_list_remote_images | grep "$version")
+    [ "$matched_version" != "" ] && return 0 || return 1
+  }
+
+  _download_img(){
+     version=$1
+
+    _download_crypto_files $version
+
+
+     name="OPNsense-${version}-nano-amd64.img"
+     compressed_name=$name.bz2
+
+     _is_downloaded $version && {
+        _warn "Image '$name' is already downloaded"
+     } || {
+       url=$_HVE_OPNSENSE_URL/$version/$compressed_name
+       >&2 echo DOWNLOAD $url 
+       wget -q -c "${url}"
+
+       _verify_image_checksum $version
+
+       >&2 echo DECOMPRESS $url 
+       bzip2 -d $compressed_name
+     }
+  }
+
+
+  _compare_files_checksum(){
+
+    file1=$1
+    file2=$2
+
+    sha256_1=$(openssl sha256 $file1 | cut -d" " -f2)
+    sha256_2=$(openssl sha256 $file2 | cut -d" " -f2)
+
+    [ "$sha256_1" = "$sha256_2" ] || return 1
+
+  }
+
+  _download_crypto_files(){
+    # see: https://docs.opnsense.org/manual/install.html#download-and-verification
+
+     version=$1
+
+     # download multiple pubkeys from different server
+     pubkey="OPNsense-${version}.pub"
+
+     rm -f $pubkey $pubkey.sig $pubkey.alt1 $pubkey.alt2
+
+       url=$_HVE_OPNSENSE_URL/$version/$pubkey
+       wget -q -c "${url}"
+ 
+      # failing: 
+      wget -q -c "${url}.sig" 
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $pubkey.sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $pubkey || _fatal "Can't verify the signature of the public key"
+
+       url_alt1=$_HVE_OPNSENSE_URL_ALT1/$version/$pubkey
+       wget -q -c -O "$pubkey.alt1" "${url_alt1}"
+
+       url_alt2=$_HVE_OPNSENSE_URL_ALT2/$version/$pubkey
+       wget -q -c -O "$pubkey.alt2" "${url_alt2}"
+     
+       _compare_files_checksum $pubkey $pubkey.alt1 || _fatal "Fail to compare pubkeys" ;
+
+       _compare_files_checksum $pubkey $pubkey.alt2 || _fatal "Fail to compare pubkeys" ;
+
+
+    
+     img_sig="OPNsense-${version}-nano-amd64.img.sig"
+
+     sha256_name="OPNsense-${version}-checksums-amd64.sha256"
+     sha256_sig=$sha256_name.sig
+
+     [ ! -f "$img_sig" ]     || rm $img_sig
+     [ ! -f "$sha256_name" ] || rm $sha256_name
+     [ ! -f "$sha256_sig" ]  || rm $sha256_sig
+
+
+     for file in $img_sig $sha256_name $sha256_sig;
+     do
+       url=$_HVE_OPNSENSE_URL/$version/$file
+       wget -q -c "${url}"
+    done
+
+
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $sha256_sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $sha256_name || _fatal "Can't verify the signature of the checksum file"
+
+  }
+
+  _download(){
+
+    version=${1:-}
+    [ "${version:-}" != "" ] || _fatal "Must pass a VERSION for downloading"
+
+    _is_valid_version $version  || _fatal "'$version' is not a valid version number"
+
+
+    _download_img ${version}
+
+  }
+
+  _verify_image_checksum(){
+
+    version=$1
+    name="OPNsense-${version}-nano-amd64.img.bz2"
+    sha256_file="OPNsense-${version}-checksums-amd64.sha256"
+    
+    sha256=$(cat $sha256_file | grep "$name" | cut -d'=' -f 2 | tr -s [:space:])
+
+    echo "$sha256 $name" | sha256sum -c || _fatal "Checksum failed for '$name'"
+
+  }
+
+  _verify_image_signature(){
+     version=$1
+
+     # download multiple pubkeys from different server
+     pubkey="OPNsense-${version}.pub"
+     img_name="OPNsense-${version}-nano-amd64.img"
+     img_sig="${img_name}.sig"
+
+
+      rm -f /tmp/file.sig
+      openssl base64 -d -in $img_sig -out /tmp/file.sig
+      openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $img_name || _fatal "Can't verify image signature"
+
+  }
+
+  _check(){
+    version=${1:-}
+    if [ "${version:-}" = "" ] ; then
+      for version in $(_list_local_images);
+      do
+        >&2 echo check $version
+        _download_crypto_files $version
+        _verify_image_signature $version
+      done
+    else
+      _download_crypto_files $version
+      _verify_image_signature $version
+    fi
+  }
+
+
+  _delete(){
+    version=${1:-}
+    if [ -z "${version:-1}" ]; then
+       _clear
+       rm -f *.img
+    else
+       rm -f *$version*.img
+    fi
+
+
+  }
+
+  _clear(){
+     rm -f *.pub *.sig *.bz2 *.alt1 *.alt2 *.sha256
+  }
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+  ls|list)
+     
+    if [ "${2:-}" == "" ]; then
+       _list_local_images
+    elif [ "${2:-}" == "--remote" ]; then
+       _list_remote_images
+    else
+       _warn "Unknown option '$2'"
+    fi
+    ;;
+    download)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _download "${2:-"$(_latest_version)"}"
+      popd 
+      ;;
+    delete)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _delete "${@:2}"
+      popd
+      ;;
+    check)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _check "${@:2}"
+      popd
+      ;;
+    show)
+      ls $_HVE_OPNSENSE_SRC_IMG | cat
+      ;;
+    clear)
+      pushd "${_HVE_OPNSENSE_SRC_IMG}"
+      _clear "${@:2}"
+      popd
+      ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img-src "${@}"
+

examples/opnsense/scripts/harmony-ve-vm

@@ -0,0 +1,77 @@
+
+#! /bin/bash
+
+harmony-ve-vm()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+
+	harmony-ve-mv
+
+  DESCRIPTION
+
+    Manage virtalized hosts (VM) dependencies by Harmony Virtual Execution Environment
+
+  SYNOPSYS
+
+    harmony-ve-vm [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    harmony-ve-vm list
+    harmony-ve-vm create
+    harmony-ve-vm start
+    harmony-ve-vm stop
+    harmony-ve-vm login
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+
+  # Implement functions
+
+case "${1:-}" in
+
+  "")
+	_short_help
+	;;
+  -h|--help)
+	_short_help
+	_extra_help
+	;;
+   # Commands entrypoints
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-vm "${@}"
+

examples/opnsense/scripts/learn-harmony

@@ -0,0 +1,75 @@
+#! /bin/bash
+
+learn-harmony()(
+
+  set -eu
+
+  [ "${1:-}" != "-d" ]  || { set -x ; shift ; }
+  trap '[ "$?" = "0" ] || >&2 echo  ABNORMAL TERMINATION' EXIT
+
+  BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
+  SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
+  . "${SCRIPTS_DIR}/common"
+  export PATH=$SCRIPTS_DIR:$PATH
+
+  _short_help(){
+
+    cat <<-EOM
+  
+  NAME
+    learn-harmony -- Harmony Learning Tool prototype
+    ($(basename ${BASE_DIR}) example)
+
+  SYNOPSYS
+
+    learn-harmony [GLOBAL_OPTIONS] COMMAND [OPTIONS]
+
+    learn-harmony list              # List learning steps
+    learn-harmony show STEP         # Show instruction of step STEP
+    learn-harmony check [STEP]        # Verify that your ready to begin step STEP+1
+
+EOM
+
+  }
+
+  _extra_help(){
+
+    cat <<-EOM
+
+  GLOBAL_OPTIONS
+
+    -d     Debug mode.
+
+  WARNINGS
+
+    This script is experimetal. Use with caution.
+EOM
+
+  }
+
+case "${1:-}" in
+
+  -h|--help|"")
+	_short_help
+	_extra_help
+	;;
+   ls|list)
+	   echo "not implemented"
+	   ;;
+   show)
+	   echo "not implemented"
+	   ;;
+   verify)
+	   echo "not implemented"
+	   ;;
+   *)
+        _warn "Unknown COMMAND '$1'"
+	exit 1
+	;;
+esac 
+
+)
+
+
+[ "$0" != "${BASH_SOURCE}" ] || learn-harmony "${@}"
+

examples/opnsense/scripts/setup

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+
+export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")"):"${PATH}"

examples/opnsense/src/main.rs

@@ -1,133 +1,70 @@
-use std::{
-    net::{IpAddr, Ipv4Addr},
-    sync::Arc,
-};
-
-use brocade::BrocadeOptions;
-use cidr::Ipv4Cidr;
 use harmony::{
-    hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    config::secret::OPNSenseFirewallCredentials,
+    infra::opnsense::OPNSenseFirewall,
     inventory::Inventory,
-    modules::{
-        dummy::{ErrorScore, PanicScore, SuccessScore},
-        http::StaticFilesHttpScore,
-        okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
-        opnsense::OPNsenseShellCommandScore,
-        tftp::TftpScore,
-    },
-    topology::{LogicalHost, UnmanagedRouter},
+    modules::{dhcp::DhcpScore, opnsense::OPNsenseShellCommandScore},
+    topology::LogicalHost,
 };
-use harmony_macros::{ip, mac_address};
+use harmony_macros::{ip, ipv4};
 use harmony_secret::{Secret, SecretManager};
-use harmony_types::net::Url;
 use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
-    let firewall = harmony::topology::LogicalHost {
-        ip: ip!("192.168.5.229"),
+    let firewall = LogicalHost {
+        ip: ip!("192.168.1.1"),
         name: String::from("opnsense-1"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+    let opnsense_auth = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>()
         .await
         .expect("Failed to get credentials");
 
-    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
+    let opnsense = OPNSenseFirewall::new(
+        firewall,
+        None,
+        &opnsense_auth.username,
+        &opnsense_auth.password,
     )
-    .await
-    .expect("Failed to connect to switch");
+    .await;
 
-    let switch_client = Arc::new(switch_client);
-
-    let opnsense = Arc::new(
-        harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
-    );
-    let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
-    let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
-    let gateway_ip = IpAddr::V4(gateway_ipv4);
-    let topology = harmony::topology::HAClusterTopology {
-        domain_name: "demo.harmony.mcd".to_string(),
-        router: Arc::new(UnmanagedRouter::new(
-            gateway_ip,
-            Ipv4Cidr::new(lan_subnet, 24).unwrap(),
-        )),
-        load_balancer: opnsense.clone(),
-        firewall: opnsense.clone(),
-        tftp_server: opnsense.clone(),
-        http_server: opnsense.clone(),
-        dhcp_server: opnsense.clone(),
-        dns_server: opnsense.clone(),
-        control_plane: vec![LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        }],
-        bootstrap_host: LogicalHost {
-            ip: ip!("10.100.8.20"),
-            name: "cp0".to_string(),
-        },
-        workers: vec![],
-        switch_client: switch_client.clone(),
-    };
-
-    let inventory = Inventory {
-        location: Location::new(
-            "232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
-            "wk".to_string(),
+    let dhcp_score = DhcpScore {
+        dhcp_range: (
+            ipv4!("192.168.1.100").into(),
+            ipv4!("192.168.1.150").into(),
         ),
-        switch: SwitchGroup::from([]),
-        firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
-        storage_host: vec![],
-        worker_host: vec![],
-        control_plane_host: vec![
-            PhysicalHost::empty(HostCategory::Server)
-                .mac_address(mac_address!("08:00:27:62:EC:C3")),
-        ],
+        host_binding: vec![],
+        next_server: None,
+        boot_filename: None,
+        filename: None,
+        filename64: None,
+        filenameipxe: Some("filename.ipxe".to_string()),
+        domain: None,
     };
+    // let dns_score = OKDDnsScore::new(&topology);
+    // let load_balancer_score = OKDLoadBalancerScore::new(&topology);
+    //
+    // let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
+    // let http_score = StaticFilesHttpScore {
+    //     folder_to_serve: Some(Url::LocalFolder(
+    //         "./data/watchguard/pxe-http-files".to_string(),
+    //     )),
+    //     files: vec![],
+    //     remote_path: None,
+    // };
+    let opnsense_config = opnsense.get_opnsense_config();
 
-    // TODO regroup smaller scores in a larger one such as this
-    // let okd_boostrap_preparation();
-
-    let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
-    let dns_score = OKDDnsScore::new(&topology);
-    let load_balancer_score = OKDLoadBalancerScore::new(&topology);
-
-    let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    let http_score = StaticFilesHttpScore {
-        folder_to_serve: Some(Url::LocalFolder(
-            "./data/watchguard/pxe-http-files".to_string(),
-        )),
-        files: vec![],
-        remote_path: None,
-    };
-
-    harmony_tui::run(
-        inventory,
-        topology,
+    harmony_cli::run(
+        Inventory::autoload(),
+        opnsense,
         vec![
-            Box::new(dns_score),
             Box::new(dhcp_score),
-            Box::new(load_balancer_score),
-            Box::new(tftp_score),
-            Box::new(http_score),
             Box::new(OPNsenseShellCommandScore {
-                opnsense: opnsense.get_opnsense_config(),
-                command: "touch /tmp/helloharmonytouching".to_string(),
+                opnsense: opnsense_config,
+                command: "touch /tmp/helloharmonytouching_2".to_string(),
             }),
-            Box::new(SuccessScore {}),
-            Box::new(ErrorScore {}),
-            Box::new(PanicScore {}),
         ],
+        None,
     )
     .await
     .unwrap();

examples/rhob_application_monitoring/src/main.rs

@@ -16,7 +16,6 @@ use harmony_types::net::Url;
 async fn main() {
     let application = Arc::new(RustWebapp {
         name: "test-rhob-monitoring".to_string(),
-        dns: "test-rhob-monitoring.harmony.mcd".to_string(),
         project_root: PathBuf::from("./webapp"), // Relative from 'harmony-path' param
         framework: Some(RustWebFramework::Leptos),
         service_port: 3000,

examples/rust/src/main.rs

@@ -19,7 +19,6 @@ use harmony_macros::hurl;
 async fn main() {
     let application = Arc::new(RustWebapp {
         name: "harmony-example-rust-webapp".to_string(),
-        dns: "harmony-example-rust-webapp.harmony.mcd".to_string(),
         project_root: PathBuf::from("./webapp"),
         framework: Some(RustWebFramework::Leptos),
         service_port: 3000,

examples/try_rust_webapp/files_to_add/main.rs

@@ -2,11 +2,12 @@ use harmony::{
     inventory::Inventory,
     modules::{
         application::{
-            features::{rhob_monitoring::Monitoring, PackagingDeployment}, ApplicationScore, RustWebFramework, RustWebapp
+            ApplicationScore, RustWebFramework, RustWebapp,
+            features::{PackagingDeployment, rhob_monitoring::Monitoring},
         },
         monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
     },
-    topology::{K8sAnywhereTopology, LocalhostTopology},
+    topology::K8sAnywhereTopology,
 };
 use harmony_macros::hurl;
 use std::{path::PathBuf, sync::Arc};
@@ -21,8 +22,8 @@ async fn main() {
     });
 
     let discord_webhook = DiscordWebhook {
-        name: "harmony-demo".to_string(),
-        url: hurl!("https://discord.com/api/webhooks/1415391405681021050/V6KzV41vQ7yvbn7BchejRu9C8OANxy0i2ESZOz2nvCxG8xAY3-2i3s5MS38k568JKTzH"),
+        name: "harmony_demo".to_string(),
+        url: hurl!("http://not_a_url.com"),
     };
 
     let app = ApplicationScore {

examples/try_rust_webapp/src/main.rs

@@ -3,7 +3,7 @@ use harmony::{
     modules::{
         application::{
             ApplicationScore, RustWebFramework, RustWebapp,
-            features::{PackagingDeployment, rhob_monitoring::Monitoring},
+            features::{Monitoring, PackagingDeployment},
         },
         monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
     },
@@ -16,7 +16,6 @@ use std::{path::PathBuf, sync::Arc};
 async fn main() {
     let application = Arc::new(RustWebapp {
         name: "harmony-example-tryrust".to_string(),
-        dns: "tryrust.example.harmony.mcd".to_string(),
         project_root: PathBuf::from("./tryrust.org"), // <== Project root, in this case it is a
         // submodule
         framework: Some(RustWebFramework::Leptos),

harmony/src/domain/maestro/mod.rs

@@ -67,16 +67,16 @@ impl<T: Topology> Maestro<T> {
         }
     }
 
-    pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
-        let mut score_mut = self.scores.write().expect("Should acquire lock");
-        score_mut.append(&mut scores);
-    }
-
     fn is_topology_initialized(&self) -> bool {
         self.topology_state.status == TopologyStatus::Success
             || self.topology_state.status == TopologyStatus::Noop
     }
 
+    pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
+        let mut score_mut = self.scores.write().expect("Should acquire lock");
+        score_mut.append(&mut scores);
+    }
+
     pub async fn interpret(&self, score: Box<dyn Score<T>>) -> Result<Outcome, InterpretError> {
         if !self.is_topology_initialized() {
             warn!(

harmony/src/domain/topology/ha_cluster.rs

@@ -4,19 +4,16 @@ use harmony_types::{
     net::{MacAddress, Url},
     switch::PortLocation,
 };
-use k8s_openapi::api::core::v1::Namespace;
 use kube::api::ObjectMeta;
 use log::debug;
 use log::info;
 
-use crate::data::FileContent;
-use crate::executors::ExecutorError;
-use crate::hardware::PhysicalHost;
-use crate::modules::okd::crd::{
-    InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
-    nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
-};
+use crate::modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy};
 use crate::topology::PxeOptions;
+use crate::{data::FileContent, modules::okd::crd::nmstate::NMState};
+use crate::{
+    executors::ExecutorError, modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec,
+};
 
 use super::{
     DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
@@ -42,6 +39,7 @@ pub struct HAClusterTopology {
     pub bootstrap_host: LogicalHost,
     pub control_plane: Vec<LogicalHost>,
     pub workers: Vec<LogicalHost>,
+    pub kubeconfig: Option<String>,
 }
 
 #[async_trait]
@@ -60,9 +58,17 @@ impl Topology for HAClusterTopology {
 #[async_trait]
 impl K8sclient for HAClusterTopology {
     async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        Ok(Arc::new(
-            K8sClient::try_default().await.map_err(|e| e.to_string())?,
-        ))
+        match &self.kubeconfig {
+            None => Ok(Arc::new(
+                K8sClient::try_default().await.map_err(|e| e.to_string())?,
+            )),
+            Some(kubeconfig) => {
+                let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else {
+                    return Err("Failed to create k8s client".to_string());
+                };
+                Ok(Arc::new(client))
+            }
+        }
     }
 }
 
@@ -88,60 +94,48 @@ impl HAClusterTopology {
     }
 
     async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
-        // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate)
-        debug!("Installing NMState operator...");
         let k8s_client = self.k8s_client().await?;
 
-        let nmstate_namespace = Namespace {
-            metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
-                finalizers: Some(vec!["kubernetes".to_string()]),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-        debug!("Creating NMState namespace: {nmstate_namespace:#?}");
-        k8s_client
-            .apply(&nmstate_namespace, None)
+        debug!("Installing NMState controller...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
-        let nmstate_operator_group = OperatorGroup {
-            metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
-                ..Default::default()
-            },
-            spec: OperatorGroupSpec {
-                target_namespaces: vec!["openshift-nmstate".to_string()],
-            },
-        };
-        debug!("Creating NMState operator group: {nmstate_operator_group:#?}");
-        k8s_client
-            .apply(&nmstate_operator_group, None)
+        debug!("Creating NMState namespace...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
-        let nmstate_subscription = Subscription {
-            metadata: ObjectMeta {
-                name: Some("kubernetes-nmstate-operator".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
-                ..Default::default()
-            },
-            spec: SubscriptionSpec {
-                channel: Some("stable".to_string()),
-                install_plan_approval: Some(InstallPlanApproval::Automatic),
-                name: "kubernetes-nmstate-operator".to_string(),
-                source: "redhat-operators".to_string(),
-                source_namespace: "openshift-marketplace".to_string(),
-            },
-        };
-        debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}");
-        k8s_client
-            .apply(&nmstate_subscription, None)
+        debug!("Creating NMState service account...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
+        debug!("Creating NMState role...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        debug!("Creating NMState role binding...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        debug!("Creating NMState operator...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        k8s_client
+            .wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None)
+            .await?;
+
         let nmstate = NMState {
             metadata: ObjectMeta {
                 name: Some("nmstate".to_string()),
@@ -162,11 +156,7 @@ impl HAClusterTopology {
         42 // FIXME: Find a better way to declare the bond id
     }
 
-    async fn configure_bond(
-        &self,
-        host: &PhysicalHost,
-        config: &HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
+    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
         self.ensure_nmstate_operator_installed()
             .await
             .map_err(|e| {
@@ -175,29 +165,33 @@ impl HAClusterTopology {
                 ))
             })?;
 
-        let bond_config = self.create_bond_configuration(host, config);
-        debug!("Configuring bond for host {host:?}: {bond_config:#?}");
+        let bond_config = self.create_bond_configuration(config);
+        debug!(
+            "Applying NMState bond config for host {}: {bond_config:#?}",
+            config.host_id
+        );
         self.k8s_client()
             .await
             .unwrap()
             .apply(&bond_config, None)
             .await
-            .unwrap();
+            .map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?;
 
-        todo!()
+        Ok(())
     }
 
     fn create_bond_configuration(
         &self,
-        host: &PhysicalHost,
         config: &HostNetworkConfig,
     ) -> NodeNetworkConfigurationPolicy {
-        let host_name = host.id.clone();
-
+        let host_name = &config.host_id;
         let bond_id = self.get_next_bond_id();
         let bond_name = format!("bond{bond_id}");
+
+        info!("Configuring bond '{bond_name}' for host '{host_name}'...");
+
         let mut bond_mtu: Option<u32> = None;
-        let mut bond_mac_address: Option<String> = None;
+        let mut copy_mac_from: Option<String> = None;
         let mut bond_ports = Vec::new();
         let mut interfaces: Vec<nmstate::InterfaceSpec> = Vec::new();
 
@@ -223,14 +217,14 @@ impl HAClusterTopology {
                 ..Default::default()
             });
 
-            bond_ports.push(interface_name);
+            bond_ports.push(interface_name.clone());
 
             // Use the first port's details for the bond mtu and mac address
             if bond_mtu.is_none() {
                 bond_mtu = Some(switch_port.interface.mtu);
             }
-            if bond_mac_address.is_none() {
-                bond_mac_address = Some(switch_port.interface.mac_address.to_string());
+            if copy_mac_from.is_none() {
+                copy_mac_from = Some(interface_name);
             }
         }
 
@@ -239,8 +233,7 @@ impl HAClusterTopology {
             description: Some(format!("Network bond for host {host_name}")),
             r#type: "bond".to_string(),
             state: "up".to_string(),
-            mtu: bond_mtu,
-            mac_address: bond_mac_address,
+            copy_mac_from,
             ipv4: Some(nmstate::IpStackSpec {
                 dhcp: Some(true),
                 enabled: Some(true),
@@ -275,16 +268,12 @@ impl HAClusterTopology {
         }
     }
 
-    async fn configure_port_channel(
-        &self,
-        host: &PhysicalHost,
-        config: &HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
+    async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
         debug!("Configuring port channel: {config:#?}");
         let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
 
         self.switch_client
-            .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
+            .configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports)
             .await
             .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
 
@@ -299,6 +288,7 @@ impl HAClusterTopology {
         };
 
         Self {
+            kubeconfig: None,
             domain_name: "DummyTopology".to_string(),
             router: dummy_infra.clone(),
             load_balancer: dummy_infra.clone(),
@@ -480,13 +470,9 @@ impl Switch for HAClusterTopology {
         Ok(port)
     }
 
-    async fn configure_host_network(
-        &self,
-        host: &PhysicalHost,
-        config: HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
-        self.configure_bond(host, &config).await?;
-        self.configure_port_channel(host, &config).await
+    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
+        self.configure_bond(config).await?;
+        self.configure_port_channel(config).await
     }
 }
 

harmony/src/domain/topology/k8s.rs

@@ -1,10 +1,12 @@
-use std::{collections::HashMap, time::Duration};
+use std::time::Duration;
 
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
-    api::{apps::v1::Deployment, core::v1::Pod},
-    apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition,
+    api::{
+        apps::v1::Deployment,
+        core::v1::{Pod, ServiceAccount},
+    },
     apimachinery::pkg::version::Info,
 };
 use kube::{
@@ -12,6 +14,7 @@ use kube::{
     api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
+    discovery::{ApiCapabilities, Scope},
     error::DiscoveryError,
     runtime::reflector::Lookup,
 };
@@ -20,11 +23,12 @@ use kube::{
     api::{ApiResource, GroupVersionKind},
     runtime::wait::await_condition,
 };
-use log::{debug, error, info, trace};
+use log::{debug, error, info, trace, warn};
 use serde::{Serialize, de::DeserializeOwned};
-use serde_json::{json, Value};
+use serde_json::json;
 use similar::TextDiff;
 use tokio::{io::AsyncReadExt, time::sleep};
+use url::Url;
 
 #[derive(new, Clone)]
 pub struct K8sClient {
@@ -58,146 +62,9 @@ impl K8sClient {
         })
     }
 
-     // Returns true if any deployment in the given namespace matching the label selector
-    // has status.availableReplicas > 0 (or condition Available=True).
-    pub async fn has_healthy_deployment_with_label(
-        &self,
-        namespace: &str,
-        label_selector: &str,
-    ) -> Result<bool, Error> {
-        let api: Api<Deployment> = Api::namespaced(self.client.clone(), namespace);
-        let lp = ListParams::default().labels(label_selector);
-        let list = api.list(&lp).await?;
-        for d in list.items {
-            // Check AvailableReplicas > 0 or Available condition
-            let available = d
-                .status
-                .as_ref()
-                .and_then(|s| s.available_replicas)
-                .unwrap_or(0);
-            if available > 0 {
-                return Ok(true);
-            }
-            // Fallback: scan conditions
-            if let Some(conds) = d.status.as_ref().and_then(|s| s.conditions.as_ref()) {
-                if conds.iter().any(|c| {
-                    c.type_ == "Available"
-                        && c.status == "True"
-                }) {
-                    return Ok(true);
-                }
-            }
-        }
-        Ok(false)
-    }
-
-    // Cluster-wide: returns namespaces that have at least one healthy deployment
-    // matching the label selector (equivalent to kubectl -A -l ...).
-    pub async fn list_namespaces_with_healthy_deployments(
-        &self,
-        label_selector: &str,
-    ) -> Result<Vec<String>, Error> {
-        let api: Api<Deployment> = Api::all(self.client.clone());
-        let lp = ListParams::default().labels(label_selector);
-        let list = api.list(&lp).await?;
-
-        let mut healthy_ns: HashMap<String, bool> = HashMap::new();
-        for d in list.items {
-            let ns = match d.metadata.namespace.clone() {
-                Some(n) => n,
-                None => continue,
-            };
-            let available = d
-                .status
-                .as_ref()
-                .and_then(|s| s.available_replicas)
-                .unwrap_or(0);
-            let is_healthy = if available > 0 {
-                true
-            } else {
-                d.status
-                    .as_ref()
-                    .and_then(|s| s.conditions.as_ref())
-                    .map(|conds| {
-                        conds.iter().any(|c| {
-                            c.type_ == "Available"
-                                && c.status == "True"
-                        })
-                    })
-                    .unwrap_or(false)
-            };
-            if is_healthy {
-                healthy_ns.insert(ns, true);
-            }
-        }
-
-        Ok(healthy_ns.into_keys().collect())
-    }
-
-    // Get the application-controller ServiceAccount name (fallback to default)
-    pub async fn get_argocd_controller_sa_name(&self, ns: &str) -> Result<String, Error> {
-        let api: Api<Deployment> = Api::namespaced(self.client.clone(), ns);
-        let lp = ListParams::default().labels("app.kubernetes.io/component=controller");
-        let list = api.list(&lp).await?;
-        if let Some(dep) = list.items.get(0) {
-            if let Some(sa) = dep
-                .spec
-                .as_ref()
-                .and_then(|ds| ds.template.spec.as_ref())
-                .and_then(|ps| ps.service_account_name.clone())
-            {
-                return Ok(sa);
-            }
-        }
-        Ok("argocd-application-controller".to_string())
-    }
-
-    // List ClusterRoleBindings dynamically and return as JSON values
-    pub async fn list_clusterrolebindings_json(&self) -> Result<Vec<Value>, Error> {
-        let gvk = kube::api::GroupVersionKind::gvk(
-            "rbac.authorization.k8s.io",
-            "v1",
-            "ClusterRoleBinding",
-        );
-        let ar = kube::api::ApiResource::from_gvk(&gvk);
-        let api: Api<kube::api::DynamicObject> = Api::all_with(self.client.clone(), &ar);
-        let crbs = api.list(&ListParams::default()).await?;
-        let mut out = Vec::new();
-        for o in crbs {
-            let v = serde_json::to_value(&o).unwrap_or(Value::Null);
-            out.push(v);
-        }
-        Ok(out)
-    }
-
-    // Determine if Argo controller in ns has cluster-wide permissions via CRBs
-    // TODO This does not belong in the generic k8s client, should be refactored at some point
-    pub async fn is_argocd_cluster_wide(&self, ns: &str) -> Result<bool, Error> {
-        let sa = self.get_argocd_controller_sa_name(ns).await?;
-        let crbs = self.list_clusterrolebindings_json().await?;
-        let sa_user = format!("system:serviceaccount:{}:{}", ns, sa);
-        for crb in crbs {
-            if let Some(subjects) = crb.get("subjects").and_then(|s| s.as_array()) {
-                for subj in subjects {
-                    let kind = subj.get("kind").and_then(|v| v.as_str()).unwrap_or("");
-                    let name = subj.get("name").and_then(|v| v.as_str()).unwrap_or("");
-                    let subj_ns = subj.get("namespace").and_then(|v| v.as_str()).unwrap_or("");
-                    if (kind == "ServiceAccount" && name == sa && subj_ns == ns)
-                        || (kind == "User" && name == sa_user)
-                    {
-                        return Ok(true);
-                    }
-                }
-            }
-        }
-        Ok(false)
-    }
-
-    pub async fn has_crd(&self, name: &str) -> Result<bool, Error> {
-        let api: Api<CustomResourceDefinition> = Api::all(self.client.clone());
-        let lp = ListParams::default().fields(&format!("metadata.name={}", name));
-        let crds = api.list(&lp).await?;
-        Ok(!crds.items.is_empty())
+    pub async fn service_account_api(&self, namespace: &str) -> Api<ServiceAccount> {
+        let api: Api<ServiceAccount> = Api::namespaced(self.client.clone(), namespace);
+        api
     }
 
     pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
@@ -223,7 +90,8 @@ impl K8sClient {
         } else {
             Api::default_namespaced_with(self.client.clone(), &gvk)
         };
-        Ok(resource.get(name).await?)
+
+        resource.get(name).await
     }
 
     pub async fn get_deployment(
@@ -238,8 +106,9 @@ impl K8sClient {
             debug!("getting default namespace deployment");
             Api::default_namespaced(self.client.clone())
         };
+
         debug!("getting deployment {} in ns {}", name, namespace.unwrap());
-        Ok(deps.get_opt(name).await?)
+        deps.get_opt(name).await
     }
 
     pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result<Option<Pod>, Error> {
@@ -248,7 +117,8 @@ impl K8sClient {
         } else {
             Api::default_namespaced(self.client.clone())
         };
-        Ok(pods.get_opt(name).await?)
+
+        pods.get_opt(name).await
     }
 
     pub async fn scale_deployment(
@@ -291,9 +161,9 @@ impl K8sClient {
 
     pub async fn wait_until_deployment_ready(
         &self,
-        name: String,
+        name: &str,
         namespace: Option<&str>,
-        timeout: Option<u64>,
+        timeout: Option<Duration>,
     ) -> Result<(), String> {
         let api: Api<Deployment>;
 
@@ -303,9 +173,9 @@ impl K8sClient {
             api = Api::default_namespaced(self.client.clone());
         }
 
-        let establish = await_condition(api, name.as_str(), conditions::is_deployment_completed());
-        let t = timeout.unwrap_or(300);
-        let res = tokio::time::timeout(std::time::Duration::from_secs(t), establish).await;
+        let establish = await_condition(api, name, conditions::is_deployment_completed());
+        let timeout = timeout.unwrap_or(Duration::from_secs(120));
+        let res = tokio::time::timeout(timeout, establish).await;
 
         if res.is_ok() {
             Ok(())
@@ -395,7 +265,7 @@ impl K8sClient {
 
                 if let Some(s) = status.status {
                     let mut stdout_buf = String::new();
-                    if let Some(mut stdout) = process.stdout().take() {
+                    if let Some(mut stdout) = process.stdout() {
                         stdout
                             .read_to_string(&mut stdout_buf)
                             .await
@@ -501,14 +371,14 @@ impl K8sClient {
                 Ok(current) => {
                     trace!("Received current value {current:#?}");
                     // The resource exists, so we calculate and display a diff.
-                    println!("\nPerforming dry-run for resource: '{}'", name);
+                    println!("\nPerforming dry-run for resource: '{name}'");
                     let mut current_yaml = serde_yaml::to_value(&current).unwrap_or_else(|_| {
                         panic!("Could not serialize current value : {current:#?}")
                     });
                     if current_yaml.is_mapping() && current_yaml.get("status").is_some() {
                         let map = current_yaml.as_mapping_mut().unwrap();
                         let removed = map.remove_entry("status");
-                        trace!("Removed status {:?}", removed);
+                        trace!("Removed status {removed:?}");
                     } else {
                         trace!(
                             "Did not find status entry for current object {}/{}",
@@ -537,14 +407,14 @@ impl K8sClient {
                             similar::ChangeTag::Insert => "+",
                             similar::ChangeTag::Equal => " ",
                         };
-                        print!("{}{}", sign, change);
+                        print!("{sign}{change}");
                     }
                     // In a dry run, we return the new resource state that would have been applied.
                     Ok(resource.clone())
                 }
                 Err(Error::Api(ErrorResponse { code: 404, .. })) => {
                     // The resource does not exist, so the "diff" is the entire new resource.
-                    println!("\nPerforming dry-run for new resource: '{}'", name);
+                    println!("\nPerforming dry-run for new resource: '{name}'");
                     println!(
                         "Resource does not exist. It would be created with the following content:"
                     );
@@ -553,14 +423,14 @@ impl K8sClient {
 
                     // Print each line of the new resource with a '+' prefix.
                     for line in new_yaml.lines() {
-                        println!("+{}", line);
+                        println!("+{line}");
                     }
                     // In a dry run, we return the new resource state that would have been created.
                     Ok(resource.clone())
                 }
                 Err(e) => {
                     // Another API error occurred.
-                    error!("Failed to get resource '{}': {}", name, e);
+                    error!("Failed to get resource '{name}': {e}");
                     Err(e)
                 }
             }
@@ -575,7 +445,7 @@ impl K8sClient {
     where
         K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize,
         <K as Resource>::Scope: ApplyStrategy<K>,
-        <K as kube::Resource>::DynamicType: Default,
+        <K as Resource>::DynamicType: Default,
     {
         let mut result = Vec::new();
         for r in resource.iter() {
@@ -640,10 +510,7 @@ impl K8sClient {
 
         // 6. Apply the object to the cluster using Server-Side Apply.
         //    This will create the resource if it doesn't exist, or update it if it does.
-        println!(
-            "Applying Argo Application '{}' in namespace '{}'...",
-            name, namespace
-        );
+        println!("Applying '{name}' in namespace '{namespace}'...",);
         let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name
         let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?;
 
@@ -652,6 +519,51 @@ impl K8sClient {
         Ok(())
     }
 
+    /// Apply a resource from a URL
+    ///
+    /// It is the equivalent of `kubectl apply -f <url>`
+    pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> {
+        let patch_params = PatchParams::apply("harmony");
+        let discovery = kube::Discovery::new(self.client.clone()).run().await?;
+
+        let yaml = reqwest::get(url)
+            .await
+            .expect("Could not get URL")
+            .text()
+            .await
+            .expect("Could not get content from URL");
+
+        for doc in multidoc_deserialize(&yaml).expect("failed to parse YAML from file") {
+            let obj: DynamicObject =
+                serde_yaml::from_value(doc).expect("cannot apply without valid YAML");
+            let namespace = obj.metadata.namespace.as_deref().or(ns);
+            let type_meta = obj
+                .types
+                .as_ref()
+                .expect("cannot apply object without valid TypeMeta");
+            let gvk = GroupVersionKind::try_from(type_meta)
+                .expect("cannot apply object without valid GroupVersionKind");
+            let name = obj.name_any();
+
+            if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) {
+                let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false);
+                trace!(
+                    "Applying {}: \n{}",
+                    gvk.kind,
+                    serde_yaml::to_string(&obj).expect("Failed to serialize YAML")
+                );
+                let data: serde_json::Value =
+                    serde_json::to_value(&obj).expect("Failed to serialize JSON");
+                let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?;
+                debug!("applied {} {}", gvk.kind, name);
+            } else {
+                warn!("Cannot apply document for unknown {gvk:?}");
+            }
+        }
+
+        Ok(())
+    }
+
     pub(crate) async fn from_kubeconfig(path: &str) -> Option<K8sClient> {
         let k = match Kubeconfig::read_from(path) {
             Ok(k) => k,
@@ -671,6 +583,31 @@ impl K8sClient {
     }
 }
 
+fn get_dynamic_api(
+    resource: ApiResource,
+    capabilities: ApiCapabilities,
+    client: Client,
+    ns: Option<&str>,
+    all: bool,
+) -> Api<DynamicObject> {
+    if capabilities.scope == Scope::Cluster || all {
+        Api::all_with(client, &resource)
+    } else if let Some(namespace) = ns {
+        Api::namespaced_with(client, namespace, &resource)
+    } else {
+        Api::default_namespaced_with(client, &resource)
+    }
+}
+
+fn multidoc_deserialize(data: &str) -> Result<Vec<serde_yaml::Value>, serde_yaml::Error> {
+    use serde::Deserialize;
+    let mut docs = vec![];
+    for de in serde_yaml::Deserializer::from_str(data) {
+        docs.push(serde_yaml::Value::deserialize(de)?);
+    }
+    Ok(docs)
+}
+
 pub trait ApplyStrategy<K: Resource> {
     fn get_api(client: &Client, ns: Option<&str>) -> Api<K>;
 }

harmony/src/domain/topology/k8s_anywhere.rs

@@ -1,8 +1,13 @@
-use std::{process::Command, sync::Arc};
+use std::{collections::BTreeMap, process::Command, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use kube::api::GroupVersionKind;
-use log::{debug, info, trace, warn};
+use base64::{Engine, engine::general_purpose};
+use k8s_openapi::api::{
+    core::v1::Secret,
+    rbac::v1::{ClusterRoleBinding, RoleRef, Subject},
+};
+use kube::api::{DynamicObject, GroupVersionKind, ObjectMeta};
+use log::{debug, info, warn};
 use serde::Serialize;
 use tokio::sync::OnceCell;
 
@@ -12,14 +17,26 @@ use crate::{
     inventory::Inventory,
     modules::{
         k3d::K3DInstallationScore,
-        monitoring::kube_prometheus::crd::{
-            crd_alertmanager_config::CRDPrometheus,
-            prometheus_operator::prometheus_operator_helm_chart_score,
-            rhob_alertmanager_config::RHOBObservability,
+        k8s::ingress::{K8sIngressScore, PathType},
+        monitoring::{
+            grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score},
+            kube_prometheus::crd::{
+                crd_alertmanager_config::CRDPrometheus,
+                crd_grafana::{
+                    Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard,
+                    GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource,
+                    GrafanaDatasourceConfig, GrafanaDatasourceJsonData,
+                    GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec,
+                },
+                crd_prometheuses::LabelSelector,
+                prometheus_operator::prometheus_operator_helm_chart_score,
+                rhob_alertmanager_config::RHOBObservability,
+                service_monitor::ServiceMonitor,
+            },
         },
         prometheus::{
             k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
-            prometheus::PrometheusApplicationMonitoring, rhob_alerting_score::RHOBAlertingScore,
+            prometheus::PrometheusMonitoring, rhob_alerting_score::RHOBAlertingScore,
         },
     },
     score::Score,
@@ -71,7 +88,6 @@ pub struct K8sAnywhereTopology {
 #[async_trait]
 impl K8sclient for K8sAnywhereTopology {
     async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        trace!("getting k8s client");
         let state = match self.k8s_state.get() {
             Some(state) => state,
             None => return Err("K8s state not initialized yet".to_string()),
@@ -87,41 +103,172 @@ impl K8sclient for K8sAnywhereTopology {
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+impl Grafana for K8sAnywhereTopology {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        debug!("ensure grafana operator");
+        let client = self.k8s_client().await.unwrap();
+        let grafana_gvk = GroupVersionKind {
+            group: "grafana.integreatly.org".to_string(),
+            version: "v1beta1".to_string(),
+            kind: "Grafana".to_string(),
+        };
+        let name = "grafanas.grafana.integreatly.org";
+        let ns = "grafana";
+
+        let grafana_crd = client
+            .get_resource_json_value(name, Some(ns), &grafana_gvk)
+            .await;
+        match grafana_crd {
+            Ok(_) => {
+                return Ok(PreparationOutcome::Success {
+                    details: "Found grafana CRDs in cluster".to_string(),
+                });
+            }
+
+            Err(_) => {
+                return self
+                    .install_grafana_operator(inventory, Some("grafana"))
+                    .await;
+            }
+        };
+    }
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError> {
+        let ns = "grafana";
+
+        let mut label = BTreeMap::new();
+
+        label.insert("dashboards".to_string(), "grafana".to_string());
+
+        let label_selector = LabelSelector {
+            match_labels: label.clone(),
+            match_expressions: vec![],
+        };
+
+        let client = self.k8s_client().await?;
+
+        let grafana = self.build_grafana(ns, &label);
+
+        client.apply(&grafana, Some(ns)).await?;
+        //TODO change this to a ensure ready or something better than just a timeout
+        client
+            .wait_until_deployment_ready(
+                "grafana-grafana-deployment",
+                Some("grafana"),
+                Some(Duration::from_secs(30)),
+            )
+            .await?;
+
+        let sa_name = "grafana-grafana-sa";
+        let token_secret_name = "grafana-sa-token-secret";
+
+        let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns);
+
+        client.apply(&sa_token_secret, Some(ns)).await?;
+        let secret_gvk = GroupVersionKind {
+            group: "".to_string(),
+            version: "v1".to_string(),
+            kind: "Secret".to_string(),
+        };
+
+        let secret = client
+            .get_resource_json_value(token_secret_name, Some(ns), &secret_gvk)
+            .await?;
+
+        let token = format!(
+            "Bearer {}",
+            self.extract_and_normalize_token(&secret).unwrap()
+        );
+
+        debug!("creating grafana clusterrole binding");
+
+        let clusterrolebinding =
+            self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns);
+
+        client.apply(&clusterrolebinding, Some(ns)).await?;
+
+        debug!("creating grafana datasource crd");
+
+        let thanos_url = format!(
+            "https://{}",
+            self.get_domain("thanos-querier-openshift-monitoring")
+                .await
+                .unwrap()
+        );
+
+        let thanos_openshift_datasource = self.build_grafana_datasource(
+            "thanos-openshift-monitoring",
+            ns,
+            &label_selector,
+            &thanos_url,
+            &token,
+        );
+
+        client.apply(&thanos_openshift_datasource, Some(ns)).await?;
+
+        debug!("creating grafana dashboard crd");
+        let dashboard = self.build_grafana_dashboard(ns, &label_selector);
+
+        client.apply(&dashboard, Some(ns)).await?;
+        debug!("creating grafana ingress");
+        let grafana_ingress = self.build_grafana_ingress(ns).await;
+
+        grafana_ingress
+            .interpret(&Inventory::empty(), self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()))?;
+
+        Ok(PreparationOutcome::Success {
+            details: "Installed grafana composants".to_string(),
+        })
+    }
+}
+
+#[async_trait]
+impl PrometheusMonitoring<CRDPrometheus> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &CRDPrometheus,
-        inventory: &Inventory,
-        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        _inventory: &Inventory,
+        _receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let client = self.k8s_client().await?;
+
+        for monitor in sender.service_monitor.iter() {
+            client
+                .apply(monitor, Some(&sender.namespace))
+                .await
+                .map_err(|e| PreparationError::new(e.to_string()))?;
+        }
+        Ok(PreparationOutcome::Success {
+            details: "successfuly installed prometheus components".to_string(),
+        })
+    }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &CRDPrometheus,
+        _inventory: &Inventory,
     ) -> Result<PreparationOutcome, PreparationError> {
         let po_result = self.ensure_prometheus_operator(sender).await?;
 
-        if po_result == PreparationOutcome::Noop {
-            debug!("Skipping Prometheus CR installation due to missing operator.");
-            return Ok(po_result);
-        }
-
-        let result = self
-            .get_k8s_prometheus_application_score(sender.clone(), receivers)
-            .await
-            .interpret(inventory, self)
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome.status {
-                InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success {
-                    details: outcome.message,
-                }),
-                InterpretStatus::NOOP => Ok(PreparationOutcome::Noop),
-                _ => Err(PreparationError::new(outcome.message)),
-            },
-            Err(err) => Err(PreparationError::new(err.to_string())),
+        match po_result {
+            PreparationOutcome::Success { details: _ } => {
+                debug!("Detected prometheus crds operator present in cluster.");
+                return Ok(po_result);
+            }
+            PreparationOutcome::Noop => {
+                debug!("Skipping Prometheus CR installation due to missing operator.");
+                return Ok(po_result);
+            }
         }
     }
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology {
+impl PrometheusMonitoring<RHOBObservability> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &RHOBObservability,
@@ -155,6 +302,14 @@ impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology
             Err(err) => Err(PreparationError::new(err.to_string())),
         }
     }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &RHOBObservability,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        todo!()
+    }
 }
 
 impl Serialize for K8sAnywhereTopology {
@@ -216,6 +371,180 @@ impl K8sAnywhereTopology {
             .await
     }
 
+    fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option<String> {
+        let token_b64 = secret
+            .data
+            .get("token")
+            .or_else(|| secret.data.get("data").and_then(|d| d.get("token")))
+            .and_then(|v| v.as_str())?;
+
+        let bytes = general_purpose::STANDARD.decode(token_b64).ok()?;
+
+        let s = String::from_utf8(bytes).ok()?;
+
+        let cleaned = s
+            .trim_matches(|c: char| c.is_whitespace() || c == '\0')
+            .to_string();
+        Some(cleaned)
+    }
+
+    pub fn build_cluster_rolebinding(
+        &self,
+        service_account_name: &str,
+        clusterrole_name: &str,
+        ns: &str,
+    ) -> ClusterRoleBinding {
+        ClusterRoleBinding {
+            metadata: ObjectMeta {
+                name: Some(format!("{}-view-binding", service_account_name)),
+                ..Default::default()
+            },
+            role_ref: RoleRef {
+                api_group: "rbac.authorization.k8s.io".into(),
+                kind: "ClusterRole".into(),
+                name: clusterrole_name.into(),
+            },
+            subjects: Some(vec![Subject {
+                kind: "ServiceAccount".into(),
+                name: service_account_name.into(),
+                namespace: Some(ns.into()),
+                ..Default::default()
+            }]),
+        }
+    }
+
+    pub fn build_sa_token_secret(
+        &self,
+        secret_name: &str,
+        service_account_name: &str,
+        ns: &str,
+    ) -> Secret {
+        let mut annotations = BTreeMap::new();
+        annotations.insert(
+            "kubernetes.io/service-account.name".to_string(),
+            service_account_name.to_string(),
+        );
+
+        Secret {
+            metadata: ObjectMeta {
+                name: Some(secret_name.into()),
+                namespace: Some(ns.into()),
+                annotations: Some(annotations),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        }
+    }
+
+    fn build_grafana_datasource(
+        &self,
+        name: &str,
+        ns: &str,
+        label_selector: &LabelSelector,
+        url: &str,
+        token: &str,
+    ) -> GrafanaDatasource {
+        let mut json_data = BTreeMap::new();
+        json_data.insert("timeInterval".to_string(), "5s".to_string());
+
+        GrafanaDatasource {
+            metadata: ObjectMeta {
+                name: Some(name.to_string()),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDatasourceSpec {
+                instance_selector: label_selector.clone(),
+                allow_cross_namespace_import: Some(true),
+                values_from: None,
+                datasource: GrafanaDatasourceConfig {
+                    access: "proxy".to_string(),
+                    name: name.to_string(),
+                    r#type: "prometheus".to_string(),
+                    url: url.to_string(),
+                    database: None,
+                    json_data: Some(GrafanaDatasourceJsonData {
+                        time_interval: Some("60s".to_string()),
+                        http_header_name1: Some("Authorization".to_string()),
+                        tls_skip_verify: Some(true),
+                        oauth_pass_thru: Some(true),
+                    }),
+                    secure_json_data: Some(GrafanaDatasourceSecureJsonData {
+                        http_header_value1: Some(format!("Bearer {token}")),
+                    }),
+                    is_default: Some(false),
+                    editable: Some(true),
+                },
+            },
+        }
+    }
+
+    fn build_grafana_dashboard(
+        &self,
+        ns: &str,
+        label_selector: &LabelSelector,
+    ) -> GrafanaDashboard {
+        let graf_dashboard = GrafanaDashboard {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-dashboard-{}", ns)),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDashboardSpec {
+                resync_period: Some("30s".to_string()),
+                instance_selector: label_selector.clone(),
+                datasources: Some(vec![GrafanaDashboardDatasource {
+                    input_name: "DS_PROMETHEUS".to_string(),
+                    datasource_name: "thanos-openshift-monitoring".to_string(),
+                }]),
+                json: None,
+                grafana_com: Some(GrafanaCom {
+                    id: 17406,
+                    revision: None,
+                }),
+            },
+        };
+        graf_dashboard
+    }
+
+    fn build_grafana(&self, ns: &str, labels: &BTreeMap<String, String>) -> GrafanaCRD {
+        let grafana = GrafanaCRD {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-{}", ns)),
+                namespace: Some(ns.to_string()),
+                labels: Some(labels.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaSpec {
+                config: None,
+                admin_user: None,
+                admin_password: None,
+                ingress: None,
+                persistence: None,
+                resources: None,
+            },
+        };
+        grafana
+    }
+
+    async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore {
+        let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap();
+        let name = format!("{}-grafana", ns);
+        let backend_service = format!("grafana-{}-service", ns);
+
+        K8sIngressScore {
+            name: fqdn::fqdn!(&name),
+            host: fqdn::fqdn!(&domain),
+            backend_service: fqdn::fqdn!(&backend_service),
+            port: 3000,
+            path: Some("/".to_string()),
+            path_type: Some(PathType::Prefix),
+            namespace: Some(fqdn::fqdn!(&ns)),
+            ingress_class_name: Some("openshift-default".to_string()),
+        }
+    }
+
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,
@@ -233,13 +562,14 @@ impl K8sAnywhereTopology {
         &self,
         sender: CRDPrometheus,
         receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        service_monitors: Option<Vec<ServiceMonitor>>,
     ) -> K8sPrometheusCRDAlertingScore {
-        K8sPrometheusCRDAlertingScore {
+        return K8sPrometheusCRDAlertingScore {
             sender,
             receivers: receivers.unwrap_or_default(),
-            service_monitors: vec![],
+            service_monitors: service_monitors.unwrap_or_default(),
             prometheus_rules: vec![],
-        }
+        };
     }
 
     async fn openshift_ingress_operator_available(&self) -> Result<(), PreparationError> {
@@ -507,6 +837,30 @@ impl K8sAnywhereTopology {
             details: "prometheus operator present in cluster".into(),
         })
     }
+
+    async fn install_grafana_operator(
+        &self,
+        inventory: &Inventory,
+        ns: Option<&str>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let namespace = ns.unwrap_or("grafana");
+        info!("installing grafana operator in ns {namespace}");
+        let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await;
+        let mut namespace_scope = false;
+        if tenant.is_some() {
+            namespace_scope = true;
+        }
+        let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope)
+            .interpret(inventory, self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()));
+        Ok(PreparationOutcome::Success {
+            details: format!(
+                "Successfully installed grafana operator in ns {}",
+                ns.unwrap()
+            ),
+        })
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -621,56 +975,36 @@ impl TenantManager for K8sAnywhereTopology {
 
 #[async_trait]
 impl Ingress for K8sAnywhereTopology {
+    //TODO this is specifically for openshift/okd which violates the k8sanywhere idea
     async fn get_domain(&self, service: &str) -> Result<String, PreparationError> {
-        use log::{trace, debug, warn};
-
         let client = self.k8s_client().await?;
 
         if let Some(Some(k8s_state)) = self.k8s_state.get() {
             match k8s_state.source {
-                K8sSource::LocalK3d => {
-                    // Local developer UX
-                    return Ok(format!("{service}.local.k3d"));
-                }
+                K8sSource::LocalK3d => Ok(format!("{service}.local.k3d")),
                 K8sSource::Kubeconfig => {
-                    trace!("K8sSource is kubeconfig; attempting to detect domain");
+                    self.openshift_ingress_operator_available().await?;
 
-                    // 1) Try OpenShift IngressController domain (backward compatible)
-                    if self.openshift_ingress_operator_available().await.is_ok() {
-                        trace!("OpenShift ingress operator detected; using IngressController");
-                        let gvk = GroupVersionKind {
-                            group: "operator.openshift.io".into(),
-                            version: "v1".into(),
-                            kind: "IngressController".into(),
-                        };
-                        let ic = client
-                            .get_resource_json_value("default", Some("openshift-ingress-operator"), &gvk)
-                            .await
-                            .map_err(|_| PreparationError::new("Failed to fetch IngressController".to_string()))?;
+                    let gvk = GroupVersionKind {
+                        group: "operator.openshift.io".into(),
+                        version: "v1".into(),
+                        kind: "IngressController".into(),
+                    };
+                    let ic = client
+                        .get_resource_json_value(
+                            "default",
+                            Some("openshift-ingress-operator"),
+                            &gvk,
+                        )
+                        .await
+                        .map_err(|_| {
+                            PreparationError::new("Failed to fetch IngressController".to_string())
+                        })?;
 
-                        if let Some(domain) = ic.data["status"]["domain"].as_str() {
-                            return Ok(format!("{service}.{domain}"));
-                        } else {
-                            warn!("OpenShift IngressController present but no status.domain set");
-                        }
-                    } else {
-                        trace!("OpenShift ingress operator not detected; trying generic Kubernetes");
+                    match ic.data["status"]["domain"].as_str() {
+                        Some(domain) => Ok(format!("{service}.{domain}")),
+                        None => Err(PreparationError::new("Could not find domain".to_string())),
                     }
-
-                    // 2) Try NGINX Ingress Controller common setups
-                    // 2.a) Well-known namespace/name for the controller Service
-                    //      - upstream default: namespace "ingress-nginx", service "ingress-nginx-controller"
-                    //      - some distros: "ingress-nginx-controller" svc in "ingress-nginx" ns
-                    // If found with LoadBalancer ingress hostname, use its base domain.
-                    if let Some(domain) = try_nginx_lb_domain(&client).await? {
-                        return Ok(format!("{service}.{domain}"));
-                    }
-
-                    // 3) Fallback: internal cluster DNS suffix (service.namespace.svc.cluster.local)
-                    // We don't have tenant namespace here, so we fallback to 'default' with a warning.
-                    warn!("Could not determine external ingress domain; falling back to internal-only DNS");
-                    let internal = format!("{service}.default.svc.cluster.local");
-                    Ok(internal)
                 }
             }
         } else {
@@ -680,57 +1014,3 @@ impl Ingress for K8sAnywhereTopology {
         }
     }
 }
-
-async fn try_nginx_lb_domain(client: &K8sClient) -> Result<Option<String>, PreparationError> {
-    use log::{trace, debug};
-
-    // Try common service path: svc/ingress-nginx-controller in ns/ingress-nginx
-    let svc_gvk = GroupVersionKind {
-        group: "".into(), // core
-        version: "v1".into(),
-        kind: "Service".into(),
-    };
-
-    let candidates = [
-        ("ingress-nginx", "ingress-nginx-controller"),
-        ("ingress-nginx", "ingress-nginx-controller-internal"),
-        ("ingress-nginx", "ingress-nginx"), // some charts name the svc like this
-        ("kube-system", "ingress-nginx-controller"), // less common but seen
-    ];
-
-    for (ns, name) in candidates {
-        trace!("Checking NGINX Service {ns}/{name} for LoadBalancer hostname");
-        if let Ok(svc) = client.get_resource_json_value(ns, Some(name), &svc_gvk).await {
-            let lb_hosts = svc.data["status"]["loadBalancer"]["ingress"].as_array().cloned().unwrap_or_default();
-            for entry in lb_hosts {
-                if let Some(host) = entry.get("hostname").and_then(|v| v.as_str()) {
-                    debug!("Found NGINX LB hostname: {host}");
-                    if let Some(domain) = extract_base_domain(host) {
-                        return Ok(Some(domain.to_string()));
-                    } else {
-                        return Ok(Some(host.to_string())); // already a domain
-                    }
-                }
-                if let Some(ip) = entry.get("ip").and_then(|v| v.as_str()) {
-                    // If only an IP is exposed, we can't create a hostname; return None to keep searching
-                    debug!("NGINX LB exposes IP {ip} (no hostname); skipping");
-                }
-            }
-        }
-    }
-
-    Ok(None)
-}
-
-fn extract_base_domain(host: &str) -> Option<String> {
-    // For a host like a1b2c3d4e5f6abcdef.elb.amazonaws.com -> base domain elb.amazonaws.com
-    // For a managed DNS like xyz.example.com -> base domain example.com (keep 2+ labels)
-    // Heuristic: keep last 2 labels by default; special-case known multi-label TLDs if needed.
-    let parts: Vec<&str> = host.split('.').collect();
-    if parts.len() >= 2 {
-        // Very conservative: last 2 labels
-        Some(parts[parts.len() - 2..].join("."))
-    } else {
-        None
-    }
-}

harmony/src/domain/topology/mod.rs

@@ -1,5 +1,6 @@
 mod ha_cluster;
 pub mod ingress;
+pub mod opnsense;
 use harmony_types::net::IpAddress;
 mod host_binding;
 mod http;
@@ -186,7 +187,7 @@ impl TopologyState {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum DeploymentTarget {
     LocalDev,
     Staging,

harmony/src/domain/topology/network.rs

@@ -9,6 +9,7 @@ use std::{
 use async_trait::async_trait;
 use derive_new::new;
 use harmony_types::{
+    id::Id,
     net::{IpAddress, MacAddress},
     switch::PortLocation,
 };
@@ -191,15 +192,12 @@ pub trait Switch: Send + Sync {
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError>;
 
-    async fn configure_host_network(
-        &self,
-        host: &PhysicalHost,
-        config: HostNetworkConfig,
-    ) -> Result<(), SwitchError>;
+    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>;
 }
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct HostNetworkConfig {
+    pub host_id: Id,
     pub switch_ports: Vec<SwitchPort>,
 }
 

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -31,6 +31,7 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        debug!("hit sender configure for AlertingInterpret");
         self.sender.configure(inventory, topology).await?;
         for receiver in self.receivers.iter() {
             receiver.install(&self.sender).await?;
@@ -86,4 +87,5 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 #[async_trait]
 pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
     async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<S>>;
 }

harmony/src/domain/topology/opnsense.rs

@@ -0,0 +1,23 @@
+use async_trait::async_trait;
+use log::info;
+
+use crate::{
+    infra::opnsense::OPNSenseFirewall,
+    topology::{PreparationError, PreparationOutcome, Topology},
+};
+
+#[async_trait]
+impl Topology for OPNSenseFirewall {
+    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
+        // FIXME we should be initializing the opnsense config here instead of
+        // OPNSenseFirewall::new as this causes the config to be loaded too early in
+        // harmony initialization process
+        let details = "OPNSenseFirewall topology is ready".to_string();
+        info!("{}", details);
+        Ok(PreparationOutcome::Success { details })
+    }
+
+    fn name(&self) -> &str {
+        "OPNSenseFirewall"
+    }
+}

harmony/src/infra/inventory/mod.rs

@@ -11,7 +11,7 @@ pub struct InventoryRepositoryFactory;
 impl InventoryRepositoryFactory {
     pub async fn build() -> Result<Box<dyn InventoryRepository>, RepoError> {
         Ok(Box::new(
-            SqliteInventoryRepository::new(&(*DATABASE_URL)).await?,
+            SqliteInventoryRepository::new(&DATABASE_URL).await?,
         ))
     }
 }

harmony/src/infra/opnsense/dns.rs

@@ -10,7 +10,7 @@ use super::OPNSenseFirewall;
 
 #[async_trait]
 impl DnsServer for OPNSenseFirewall {
-    async fn register_hosts(&self, _hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
+    async fn register_hosts(&self, hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
         todo!("Refactor this to use dnsmasq")
         // let mut writable_opnsense = self.opnsense_config.write().await;
         // let mut dns = writable_opnsense.dns();
@@ -68,7 +68,7 @@ impl DnsServer for OPNSenseFirewall {
         self.host.clone()
     }
 
-    async fn register_dhcp_leases(&self, _register: bool) -> Result<(), ExecutorError> {
+    async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> {
         todo!("Refactor this to use dnsmasq")
         // let mut writable_opnsense = self.opnsense_config.write().await;
         // let mut dns = writable_opnsense.dns();

harmony/src/infra/opnsense/mod.rs

@@ -25,6 +25,8 @@ impl OPNSenseFirewall {
         self.host.ip
     }
 
+    /// panics : if the opnsense config file cannot be loaded by the underlying opnsense_config
+    /// crate
     pub async fn new(host: LogicalHost, port: Option<u16>, username: &str, password: &str) -> Self {
         Self {
             opnsense_config: Arc::new(RwLock::new(

harmony/src/modules/application/features/argo_types.rs

@@ -21,7 +21,7 @@ pub struct Helm {
     pub skip_schema_validation: Option<bool>,
     pub version: Option<String>,
     pub kube_version: Option<String>,
-    // pub api_versions: Vec<String>,
+    pub api_versions: Vec<String>,
     pub namespace: Option<String>,
 }
 
@@ -105,7 +105,7 @@ impl Default for ArgoApplication {
                     skip_schema_validation: None,
                     version: None,
                     kube_version: None,
-                    // api_versions: vec![],
+                    api_versions: vec![],
                     namespace: None,
                 },
                 path: "".to_string(),
@@ -155,7 +155,7 @@ impl From<CDApplicationConfig> for ArgoApplication {
                     skip_schema_validation: None,
                     version: None,
                     kube_version: None,
-                    // api_versions: vec![],
+                    api_versions: vec![],
                     namespace: None,
                 },
             },
@@ -181,11 +181,13 @@ impl From<CDApplicationConfig> for ArgoApplication {
 }
 
 impl ArgoApplication {
-    pub fn to_yaml(&self, target_namespace: Option<&str>) -> serde_yaml::Value {
+    pub fn to_yaml(&self) -> serde_yaml::Value {
         let name = &self.name;
-        let default_ns = "argocd".to_string();
-        let namespace: &str =
-            target_namespace.unwrap_or(self.namespace.as_ref().unwrap_or(&default_ns));
+        let namespace = if let Some(ns) = self.namespace.as_ref() {
+            ns
+        } else {
+            "argocd"
+        };
         let project = &self.project;
 
         let yaml_str = format!(
@@ -283,7 +285,7 @@ mod tests {
                     skip_schema_validation: None,
                     version: None,
                     kube_version: None,
-                    // api_versions: vec![],
+                    api_versions: vec![],
                     namespace: None,
                 },
                 path: "".to_string(),
@@ -343,7 +345,7 @@ spec:
 
         assert_eq!(
             expected_yaml_output.trim(),
-            serde_yaml::to_string(&app.clone().to_yaml(None))
+            serde_yaml::to_string(&app.clone().to_yaml())
                 .unwrap()
                 .trim()
         );

harmony/src/modules/application/features/helm_argocd_score.rs

@@ -1,21 +1,22 @@
 use async_trait::async_trait;
 use harmony_macros::hurl;
 use kube::{Api, api::GroupVersionKind};
-use log::{debug, info, trace, warn};
+use log::{debug, warn};
 use non_blank_string_rs::NonBlankString;
 use serde::Serialize;
-use std::{str::FromStr, sync::Arc};
+use serde::de::DeserializeOwned;
+use std::{process::Command, str::FromStr, sync::Arc};
 
 use crate::{
     data::Version,
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
-    modules::{
-        argocd::{ArgoDeploymentType, detect_argo_deployment_type},
-        helm::chart::{HelmChartScore, HelmRepository},
-    },
+    modules::helm::chart::{HelmChartScore, HelmRepository},
     score::Score,
-    topology::{HelmCommand, K8sclient, Topology, ingress::Ingress, k8s::K8sClient},
+    topology::{
+        HelmCommand, K8sclient, PreparationError, PreparationOutcome, Topology, ingress::Ingress,
+        k8s::K8sClient,
+    },
 };
 use harmony_types::id::Id;
 
@@ -24,7 +25,6 @@ use super::ArgoApplication;
 #[derive(Debug, Serialize, Clone)]
 pub struct ArgoHelmScore {
     pub namespace: String,
-    // TODO: remove and rely on topology (it now knows the flavor)
     pub openshift: bool,
     pub argo_apps: Vec<ArgoApplication>,
 }
@@ -55,101 +55,29 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
-        trace!("Starting ArgoInterpret execution {self:?}");
-        let k8s_client: Arc<K8sClient> = topology.k8s_client().await?;
-        trace!("Got k8s client");
-        let desired_ns = self.score.namespace.clone();
-
-        debug!("ArgoInterpret detecting cluster configuration");
-        let svc = format!("argo-{}", desired_ns);
+        let k8s_client = topology.k8s_client().await?;
+        let svc = format!("argo-{}", self.score.namespace.clone());
         let domain = topology.get_domain(&svc).await?;
-        debug!("Resolved Argo service domain for '{}': {}", svc, domain);
+        let helm_score =
+            argo_helm_chart_score(&self.score.namespace, self.score.openshift, &domain);
 
-        // Detect current Argo deployment type
-        let current = detect_argo_deployment_type(&k8s_client, &desired_ns).await?;
-        info!("Detected Argo deployment type: {:?}", current);
+        helm_score.interpret(inventory, topology).await?;
 
-        // Decide control namespace and whether we must install
-        let (control_ns, must_install) = match current.clone() {
-            ArgoDeploymentType::NotInstalled => {
-                info!(
-                    "Argo CD not installed. Will install via Helm into namespace '{}'.",
-                    desired_ns
-                );
-                (desired_ns.clone(), true)
-            }
-            ArgoDeploymentType::AvailableInDesiredNamespace(ns) => {
-                info!(
-                    "Argo CD already installed by Harmony in '{}'. Skipping install.",
-                    ns
-                );
-                (ns, false)
-            }
-            ArgoDeploymentType::InstalledClusterWide(ns) => {
-                info!(
-                    "Argo CD installed cluster-wide in namespace '{}'.",
-                    ns
-                );
-                (ns, false)
-            }
-            ArgoDeploymentType::InstalledNamespaceScoped(ns) => {
-                // TODO we could support this use case by installing a new argo instance. But that
-                // means handling a few cases that are out of scope for now :
-                // - Wether argo operator is installed
-                // - Managing CRD versions compatibility
-                // - Potentially handling the various k8s flavors and setups we might encounter
-                //
-                // There is a possibility that the helm chart already handles most or even all of these use cases but they are out of scope for now.
-                let msg = format!(
-                    "Argo CD found in '{}' but it is namespace-scoped and not supported for attachment yet.",
-                    ns
-                );
-                warn!("{}", msg);
-                return Err(InterpretError::new(msg));
-            }
-        };
-
-        info!("ArgoCD will be installed : {must_install} . Current argocd status : {current:?} ");
-
-        if must_install {
-            let helm_score = argo_helm_chart_score(&desired_ns, self.score.openshift, &domain);
-            info!(
-                "Installing Argo CD via Helm into namespace '{}' ...",
-                desired_ns
-            );
-            helm_score.interpret(inventory, topology).await?;
-            info!("Argo CD install complete in '{}'.", desired_ns);
-        }
-
-        let yamls: Vec<serde_yaml::Value> = self
-            .argo_apps
-            .iter()
-            .map(|a| a.to_yaml(Some(&control_ns)))
-            .collect();
-        info!(
-            "Applying {} Argo application object(s) into control namespace '{}'.",
-            yamls.len(),
-            control_ns
-        );
         k8s_client
-            .apply_yaml_many(&yamls, Some(control_ns.as_str()))
+            .apply_yaml_many(&self.argo_apps.iter().map(|a| a.to_yaml()).collect(), None)
             .await
-            .map_err(|e| InterpretError::new(format!("Failed applying Argo CRs: {e}")))?;
+            .unwrap();
 
         Ok(Outcome::success_with_details(
             format!(
                 "ArgoCD {} {}",
                 self.argo_apps.len(),
-                if self.argo_apps.len() == 1 {
-                    "application"
-                } else {
-                    "applications"
+                match self.argo_apps.len() {
+                    1 => "application",
+                    _ => "applications",
                 }
             ),
-            vec![
-                format!("control_namespace={}", control_ns),
-                format!("argo ui: http://{}", domain),
-            ],
+            vec![format!("argo application: http://{}", domain)],
         ))
     }
 
@@ -158,7 +86,7 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
     }
 
     fn get_version(&self) -> Version {
-        Version::from("0.1.0").unwrap()
+        todo!()
     }
 
     fn get_status(&self) -> InterpretStatus {
@@ -166,7 +94,39 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
     }
 
     fn get_children(&self) -> Vec<Id> {
-        vec![]
+        todo!()
+    }
+}
+
+impl ArgoInterpret {
+    pub async fn get_host_domain(
+        &self,
+        client: Arc<K8sClient>,
+        openshift: bool,
+    ) -> Result<String, InterpretError> {
+        //This should be the job of the topology to determine if we are in
+        //openshift, potentially we need on openshift topology the same way we create a
+        //localhosttopology
+        match openshift {
+            true => {
+                let gvk = GroupVersionKind {
+                    group: "operator.openshift.io".into(),
+                    version: "v1".into(),
+                    kind: "IngressController".into(),
+                };
+                let ic = client
+                    .get_resource_json_value("default", Some("openshift-ingress-operator"), &gvk)
+                    .await?;
+
+                match ic.data["status"]["domain"].as_str() {
+                    Some(domain) => return Ok(domain.to_string()),
+                    None => return Err(InterpretError::new("Could not find domain".to_string())),
+                }
+            }
+            false => {
+                todo!()
+            }
+        };
     }
 }
 

harmony/src/modules/application/features/monitoring.rs

@@ -2,7 +2,11 @@ use crate::modules::application::{
     Application, ApplicationFeature, InstallationError, InstallationOutcome,
 };
 use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
+use crate::modules::monitoring::grafana::grafana::Grafana;
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
 use crate::topology::MultiTargetTopology;
 use crate::topology::ingress::Ingress;
 use crate::{
@@ -14,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -22,6 +26,7 @@ use base64::{Engine as _, engine::general_purpose};
 use harmony_secret::SecretManager;
 use harmony_secret_derive::Secret;
 use harmony_types::net::Url;
+use kube::api::ObjectMeta;
 use log::{debug, info};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
@@ -40,7 +45,8 @@ impl<
         + TenantManager
         + K8sclient
         + MultiTargetTopology
-        + PrometheusApplicationMonitoring<CRDPrometheus>
+        + PrometheusMonitoring<CRDPrometheus>
+        + Grafana
         + Ingress
         + std::fmt::Debug,
 > ApplicationFeature<T> for Monitoring
@@ -57,10 +63,20 @@ impl<
             .unwrap_or_else(|| self.application.name());
         let domain = topology.get_domain("ntfy").await.unwrap();
 
+        let app_service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.application.name()),
+                namespace: Some(namespace.clone()),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
+
         let mut alerting_score = ApplicationMonitoringScore {
             sender: CRDPrometheus {
                 namespace: namespace.clone(),
                 client: topology.k8s_client().await.unwrap(),
+                service_monitor: vec![app_service_monitor],
             },
             application: self.application.clone(),
             receivers: self.alert_receiver.clone(),

harmony/src/modules/application/features/packaging_deployment.rs

@@ -10,11 +10,12 @@ use crate::{
     data::Version,
     inventory::Inventory,
     modules::application::{
-        features::{ArgoApplication, ArgoHelmScore}, webapp::Webapp, ApplicationFeature, HelmPackage, InstallationError, InstallationOutcome, OCICompliant
+        ApplicationFeature, HelmPackage, InstallationError, InstallationOutcome, OCICompliant,
+        features::{ArgoApplication, ArgoHelmScore},
     },
     score::Score,
     topology::{
-        ingress::Ingress, DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology
+        DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology, ingress::Ingress,
     },
 };
 
@@ -46,11 +47,11 @@ use crate::{
 /// - ArgoCD to install/upgrade/rollback/inspect k8s resources
 /// - Kubernetes for runtime orchestration
 #[derive(Debug, Default, Clone)]
-pub struct PackagingDeployment<A: OCICompliant + HelmPackage + Webapp> {
+pub struct PackagingDeployment<A: OCICompliant + HelmPackage> {
     pub application: Arc<A>,
 }
 
-impl<A: OCICompliant + HelmPackage + Webapp> PackagingDeployment<A> {
+impl<A: OCICompliant + HelmPackage> PackagingDeployment<A> {
     async fn deploy_to_local_k3d(
         &self,
         app_name: String,
@@ -136,7 +137,7 @@ impl<A: OCICompliant + HelmPackage + Webapp> PackagingDeployment<A> {
 
 #[async_trait]
 impl<
-    A: OCICompliant + HelmPackage + Webapp + Clone + 'static,
+    A: OCICompliant + HelmPackage + Clone + 'static,
     T: Topology + HelmCommand + MultiTargetTopology + K8sclient + Ingress + 'static,
 > ApplicationFeature<T> for PackagingDeployment<A>
 {
@@ -145,15 +146,10 @@ impl<
         topology: &T,
     ) -> Result<InstallationOutcome, InstallationError> {
         let image = self.application.image_name();
-
-        let domain = if topology.current_target() == DeploymentTarget::Production {
-            self.application.dns()
-        } else {
- topology
+        let domain = topology
             .get_domain(&self.application.name())
             .await
-            .map_err(|e| e.to_string())?
-        };
+            .map_err(|e| e.to_string())?;
 
         // TODO Write CI/CD workflow files
         // we can autotedect the CI type using the remote url (default to github action for github
@@ -198,7 +194,7 @@ impl<
                     openshift: true,
                     argo_apps: vec![ArgoApplication::from(CDApplicationConfig {
                         // helm pull oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart --version 0.1.0
-                        version: Version::from("0.2.1").unwrap(),
+                        version: Version::from("0.1.0").unwrap(),
                         helm_chart_repo_url: "hub.nationtech.io/harmony".to_string(),
                         helm_chart_name: format!("{}-chart", self.application.name()),
                         values_overrides: None,

harmony/src/modules/application/features/rhob_monitoring.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 use crate::modules::application::{
     Application, ApplicationFeature, InstallationError, InstallationOutcome,
 };
+use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
 use crate::modules::monitoring::application_monitoring::rhobs_application_monitoring_score::ApplicationRHOBMonitoringScore;
 
 use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability;
@@ -17,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -41,7 +42,7 @@ impl<
         + MultiTargetTopology
         + Ingress
         + std::fmt::Debug
-        + PrometheusApplicationMonitoring<RHOBObservability>,
+        + PrometheusMonitoring<RHOBObservability>,
 > ApplicationFeature<T> for Monitoring
 {
     async fn ensure_installed(

harmony/src/modules/application/mod.rs

@@ -2,7 +2,6 @@ mod feature;
 pub mod features;
 pub mod oci;
 mod rust;
-mod webapp;
 use std::sync::Arc;
 
 pub use feature::*;

harmony/src/modules/application/rust.rs

@@ -16,7 +16,6 @@ use tar::{Builder, Header};
 use walkdir::WalkDir;
 
 use crate::config::{REGISTRY_PROJECT, REGISTRY_URL};
-use crate::modules::application::webapp::Webapp;
 use crate::{score::Score, topology::Topology};
 
 use super::{Application, ApplicationFeature, ApplicationInterpret, HelmPackage, OCICompliant};
@@ -61,10 +60,6 @@ pub struct RustWebapp {
     pub project_root: PathBuf,
     pub service_port: u32,
     pub framework: Option<RustWebFramework>,
-    /// Host name that will be used in production environment.
-    ///
-    /// This is the place to put the public host name if this is a public facing webapp.
-    pub dns: String,
 }
 
 impl Application for RustWebapp {
@@ -73,12 +68,6 @@ impl Application for RustWebapp {
     }
 }
 
-impl Webapp for RustWebapp {
-    fn dns(&self) -> String {
-        self.dns.clone()
-    }
-}
-
 #[async_trait]
 impl HelmPackage for RustWebapp {
     async fn build_push_helm_package(
@@ -205,10 +194,10 @@ impl RustWebapp {
                     Some(body_full(tar_data.into())),
                 );
 
-                while let Some(msg) = image_build_stream.next().await {
+                while let Some(mut msg) = image_build_stream.next().await {
                     trace!("Got bollard msg {msg:?}");
                     match msg {
-                        Ok(msg) => {
+                        Ok(mut msg) => {
                             if let Some(progress) = msg.progress_detail {
                                 info!(
                                     "Build progress {}/{}",
@@ -268,6 +257,7 @@ impl RustWebapp {
                 ".harmony_generated",
                 "harmony",
                 "node_modules",
+                "Dockerfile.harmony",
             ];
             let mut entries: Vec<_> = WalkDir::new(project_root)
                 .into_iter()
@@ -471,53 +461,52 @@ impl RustWebapp {
 
         let (image_repo, image_tag) = image_url.rsplit_once(':').unwrap_or((image_url, "latest"));
 
-        let app_name = &self.name;
-        let service_port = self.service_port;
         // Create Chart.yaml
         let chart_yaml = format!(
             r#"
 apiVersion: v2
-name: {chart_name}
-description: A Helm chart for the {app_name} web application.
+name: {}
+description: A Helm chart for the {} web application.
 type: application
-version: 0.2.1
-appVersion: "{image_tag}"
+version: 0.1.0
+appVersion: "{}"
 "#,
+            chart_name, self.name, image_tag
         );
         fs::write(chart_dir.join("Chart.yaml"), chart_yaml)?;
 
         // Create values.yaml
         let values_yaml = format!(
             r#"
-# Default values for {chart_name}.
+# Default values for {}.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
 replicaCount: 1
 
 image:
-  repository: {image_repo}
+  repository: {}
   pullPolicy: IfNotPresent
   # Overridden by the chart's appVersion
-  tag: "{image_tag}"
+  tag: "{}"
 
 service:
   type: ClusterIP
-  port: {service_port}
+  port: {}
 
 ingress:
   enabled: true
-  tls: true
   # Annotations for cert-manager to handle SSL.
   annotations:
     # Add other annotations like nginx ingress class if needed
     # kubernetes.io/ingress.class: nginx
   hosts:
-    - host: {domain}
+    - host: {}
       paths:
         - path: /
           pathType: ImplementationSpecific
 "#,
+            chart_name, image_repo, image_tag, self.service_port, domain,
         );
         fs::write(chart_dir.join("values.yaml"), values_yaml)?;
 
@@ -594,11 +583,7 @@ spec:
         );
         fs::write(templates_dir.join("deployment.yaml"), deployment_yaml)?;
 
-        let service_port = self.service_port;
-
         // Create templates/ingress.yaml
-        // TODO get issuer name and tls config from topology as it may be different from one
-        // cluster to another, also from one version to another
         let ingress_yaml = format!(
             r#"
 {{{{- if $.Values.ingress.enabled -}}}}
@@ -611,11 +596,13 @@ metadata:
 spec:
   {{{{- if $.Values.ingress.tls }}}}
   tls:
-    - secretName: {{{{ include "chart.fullname" . }}}}-tls
-      hosts:
-        {{{{- range $.Values.ingress.hosts }}}}
-        - {{{{ .host | quote }}}}
+    {{{{- range $.Values.ingress.tls }}}}
+    - hosts:
+        {{{{- range .hosts }}}}
+        - {{{{ . | quote }}}}
         {{{{- end }}}}
+      secretName: {{{{ .secretName }}}}
+    {{{{- end }}}}
   {{{{- end }}}}
   rules:
     {{{{- range $.Values.ingress.hosts }}}}
@@ -629,11 +616,12 @@ spec:
               service:
                 name: {{{{ include "chart.fullname" $ }}}}
                 port:
-                  number: {{{{ $.Values.service.port | default {service_port} }}}}
+                  number: {{{{ $.Values.service.port | default {} }}}}
           {{{{- end }}}}
     {{{{- end }}}}
 {{{{- end }}}}
 "#,
+            self.service_port
         );
         fs::write(templates_dir.join("ingress.yaml"), ingress_yaml)?;
 

harmony/src/modules/application/webapp.rs

@@ -1,7 +0,0 @@
-use super::Application;
-use async_trait::async_trait;
-
-#[async_trait]
-pub trait Webapp: Application {
-    fn dns(&self) -> String;
-}

harmony/src/modules/argocd/mod.rs

@@ -1,203 +0,0 @@
-use std::sync::Arc;
-
-use log::{debug, info};
-
-use crate::{interpret::InterpretError, topology::k8s::K8sClient};
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum ArgoScope {
-    ClusterWide(String),
-    NamespaceScoped(String),
-}
-
-#[derive(Clone, Debug)]
-pub struct DiscoveredArgo {
-    pub control_namespace: String,
-    pub scope: ArgoScope,
-    pub has_crds: bool,
-    pub has_applicationset: bool,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum ArgoDeploymentType {
-    NotInstalled,
-    AvailableInDesiredNamespace(String),
-    InstalledClusterWide(String),
-    InstalledNamespaceScoped(String),
-}
-
-pub async fn discover_argo_all(
-    k8s: &Arc<K8sClient>,
-) -> Result<Vec<DiscoveredArgo>, InterpretError> {
-    use log::{debug, info, trace, warn};
-
-    trace!("Starting Argo discovery");
-
-    // CRDs
-    let mut has_crds = true;
-    let required_crds = vec!["applications.argoproj.io", "appprojects.argoproj.io"];
-    trace!("Checking required Argo CRDs: {:?}", required_crds);
-
-    for crd in required_crds {
-        trace!("Verifying CRD presence: {crd}");
-        let crd_exists = k8s.has_crd(crd).await.map_err(|e| {
-            InterpretError::new(format!("Failed to verify existence of CRD {crd}: {e}"))
-        })?;
-
-        debug!("CRD {crd} exists: {crd_exists}");
-        if !crd_exists {
-            info!(
-                "Missing Argo CRD {crd}, looks like Argo CD is not installed (or partially installed)"
-            );
-            has_crds = false;
-            break;
-        }
-    }
-
-    trace!(
-        "Listing namespaces with healthy Argo CD deployments using selector app.kubernetes.io/part-of=argocd"
-    );
-    let mut candidate_namespaces = k8s
-        .list_namespaces_with_healthy_deployments("app.kubernetes.io/part-of=argocd")
-        .await
-        .map_err(|e| InterpretError::new(format!("List healthy argocd deployments: {e}")))?;
-    trace!(
-        "Listing namespaces with healthy Argo CD deployments using selector app.kubernetes.io/name=argo-cd"
-    );
-    candidate_namespaces.append(
-        &mut k8s
-            .list_namespaces_with_healthy_deployments("app.kubernetes.io/name=argo-cd")
-            .await
-            .map_err(|e| InterpretError::new(format!("List healthy argocd deployments: {e}")))?,
-    );
-
-    debug!(
-        "Discovered {} candidate namespace(s) for Argo CD: {:?}",
-        candidate_namespaces.len(),
-        candidate_namespaces
-    );
-
-    let mut found = Vec::new();
-    for ns in candidate_namespaces {
-        trace!("Evaluating namespace '{ns}' for Argo CD instance");
-
-        // Require the application-controller to be healthy (sanity check)
-        trace!(
-            "Checking healthy deployment with label app.kubernetes.io/name=argocd-application-controller in namespace '{ns}'"
-        );
-        let controller_ok = k8s
-            .has_healthy_deployment_with_label(
-                &ns,
-                "app.kubernetes.io/name=argocd-application-controller",
-            )
-            .await
-            .unwrap_or_else(|e| {
-                warn!(
-                    "Error while checking application-controller health in namespace '{ns}': {e}"
-                );
-                false
-            }) || k8s
-            .has_healthy_deployment_with_label(
-                &ns,
-                "app.kubernetes.io/component=controller",
-            )
-            .await
-            .unwrap_or_else(|e| {
-                warn!(
-                    "Error while checking application-controller health in namespace '{ns}': {e}"
-                );
-                false
-            });
-        debug!("Namespace '{ns}': application-controller healthy = {controller_ok}");
-
-        if !controller_ok {
-            trace!("Skipping namespace '{ns}' because application-controller is not healthy");
-            continue;
-        }
-
-        trace!("Determining Argo CD scope for namespace '{ns}' (cluster-wide vs namespace-scoped)");
-        let scope = match k8s.is_argocd_cluster_wide(&ns).await {
-            Ok(true) => {
-                debug!("Namespace '{ns}' identified as cluster-wide Argo CD control plane");
-                ArgoScope::ClusterWide(ns.to_string())
-            }
-            Ok(false) => {
-                debug!("Namespace '{ns}' identified as namespace-scoped Argo CD control plane");
-                ArgoScope::NamespaceScoped(ns.to_string())
-            }
-            Err(e) => {
-                warn!(
-                    "Failed to determine Argo CD scope for namespace '{ns}': {e}. Assuming namespace-scoped."
-                );
-                ArgoScope::NamespaceScoped(ns.to_string())
-            }
-        };
-
-        trace!("Checking optional ApplicationSet CRD (applicationsets.argoproj.io)");
-        let has_applicationset = match k8s.has_crd("applicationsets.argoproj.io").await {
-            Ok(v) => {
-                debug!("applicationsets.argoproj.io present: {v}");
-                v
-            }
-            Err(e) => {
-                warn!("Failed to check applicationsets.argoproj.io CRD: {e}. Assuming absent.");
-                false
-            }
-        };
-
-        let argo = DiscoveredArgo {
-            control_namespace: ns.clone(),
-            scope,
-            has_crds,
-            has_applicationset,
-        };
-
-        debug!("Discovered Argo instance in '{ns}': {argo:?}");
-        found.push(argo);
-    }
-
-    if found.is_empty() {
-        info!("No Argo CD installations discovered");
-    } else {
-        info!(
-            "Argo CD discovery complete: {} instance(s) found",
-            found.len()
-        );
-    }
-
-    Ok(found)
-}
-
-pub async fn detect_argo_deployment_type(
-    k8s: &Arc<K8sClient>,
-    desired_namespace: &str,
-) -> Result<ArgoDeploymentType, InterpretError> {
-    let discovered = discover_argo_all(k8s).await?;
-    debug!("Discovered argo instances {discovered:?}");
-
-    if discovered.is_empty() {
-        return Ok(ArgoDeploymentType::NotInstalled);
-    }
-
-    if let Some(d) = discovered
-        .iter()
-        .find(|d| d.control_namespace == desired_namespace)
-    {
-        return Ok(ArgoDeploymentType::AvailableInDesiredNamespace(
-            d.control_namespace.clone(),
-        ));
-    }
-
-    if let Some(d) = discovered
-        .iter()
-        .find(|d| matches!(d.scope, ArgoScope::ClusterWide(_)))
-    {
-        return Ok(ArgoDeploymentType::InstalledClusterWide(
-            d.control_namespace.clone(),
-        ));
-    }
-
-    Ok(ArgoDeploymentType::InstalledNamespaceScoped(
-        discovered[0].control_namespace.clone(),
-    ))
-}

harmony/src/modules/inventory/mod.rs

@@ -90,12 +90,12 @@ impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
                             // refactoring to do it now
                             let harmony_inventory_agent::hwinfo::PhysicalHost {
                                 storage_drives,
-                                storage_controller: _,
+                                storage_controller,
                                 memory_modules,
                                 cpus,
-                                chipset: _,
+                                chipset,
                                 network_interfaces,
-                                management_interface: _,
+                                management_interface,
                                 host_uuid,
                             } = host;
 

harmony/src/modules/k8s/resource.rs

@@ -38,13 +38,15 @@ impl<
         + 'static
         + Send
         + Clone,
-    T: Topology,
+    T: Topology + K8sclient,
 > Score<T> for K8sResourceScore<K>
 where
     <K as kube::Resource>::DynamicType: Default,
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        todo!()
+        Box::new(K8sResourceInterpret {
+            score: self.clone(),
+        })
     }
 
     fn name(&self) -> String {

harmony/src/modules/mod.rs

@@ -17,4 +17,3 @@ pub mod prometheus;
 pub mod storage;
 pub mod tenant;
 pub mod tftp;
-pub mod argocd;

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -1,21 +1,23 @@
 use std::sync::Arc;
 
-use async_trait::async_trait;
+use log::debug;
 use serde::Serialize;
 
 use crate::{
-    data::Version,
-    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
-    inventory::Inventory,
+    interpret::Interpret,
     modules::{
         application::Application,
-        monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
-    topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
+    topology::{
+        K8sclient, Topology,
+        oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget},
+    },
 };
-use harmony_types::id::Id;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct ApplicationMonitoringScore {
@@ -24,12 +26,16 @@ pub struct ApplicationMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + PrometheusMonitoring<CRDPrometheus> + K8sclient + Grafana> Score<T>
     for ApplicationMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(ApplicationMonitoringInterpret {
-            score: self.clone(),
+        debug!("creating alerting interpret");
+        Box::new(AlertingInterpret {
+            sender: self.sender.clone(),
+            receivers: self.receivers.clone(),
+            rules: vec![],
+            scrape_targets: None,
         })
     }
 
@@ -40,55 +46,3 @@ impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
         )
     }
 }
-
-#[derive(Debug)]
-pub struct ApplicationMonitoringInterpret {
-    score: ApplicationMonitoringScore,
-}
-
-#[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
-    for ApplicationMonitoringInterpret
-{
-    async fn execute(
-        &self,
-        inventory: &Inventory,
-        topology: &T,
-    ) -> Result<Outcome, InterpretError> {
-        let result = topology
-            .install_prometheus(
-                &self.score.sender,
-                inventory,
-                Some(self.score.receivers.clone()),
-            )
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome {
-                PreparationOutcome::Success { details: _ } => {
-                    Ok(Outcome::success("Prometheus installed".into()))
-                }
-                PreparationOutcome::Noop => {
-                    Ok(Outcome::noop("Prometheus installation skipped".into()))
-                }
-            },
-            Err(err) => Err(InterpretError::from(err)),
-        }
-    }
-
-    fn get_name(&self) -> InterpretName {
-        InterpretName::ApplicationMonitoring
-    }
-
-    fn get_version(&self) -> Version {
-        todo!()
-    }
-
-    fn get_status(&self) -> InterpretStatus {
-        todo!()
-    }
-
-    fn get_children(&self) -> Vec<Id> {
-        todo!()
-    }
-}

harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs

@@ -12,7 +12,7 @@ use crate::{
         monitoring::kube_prometheus::crd::{
             crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability,
         },
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
     topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
@@ -26,7 +26,7 @@ pub struct ApplicationRHOBMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<RHOBObservability>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Score<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Score<T>
     for ApplicationRHOBMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
@@ -49,7 +49,7 @@ pub struct ApplicationRHOBMonitoringInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Interpret<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Interpret<T>
     for ApplicationRHOBMonitoringInterpret
 {
     async fn execute(

harmony/src/modules/monitoring/grafana/grafana.rs

@@ -0,0 +1,17 @@
+use async_trait::async_trait;
+use k8s_openapi::Resource;
+
+use crate::{
+    inventory::Inventory,
+    topology::{PreparationError, PreparationOutcome},
+};
+
+#[async_trait]
+pub trait Grafana {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError>;
+}

harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs

@@ -1,27 +1,28 @@
+use harmony_macros::hurl;
 use non_blank_string_rs::NonBlankString;
-use std::str::FromStr;
+use std::{collections::HashMap, str::FromStr};
 
-use crate::modules::helm::chart::HelmChartScore;
-
-pub fn grafana_helm_chart_score(ns: &str) -> HelmChartScore {
-    let values = r#"
-rbac:
-  namespaced: true
-sidecar:
-  dashboards:
-    enabled: true
-        "#
-    .to_string();
+use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
 
+pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore {
+    let mut values_overrides = HashMap::new();
+    values_overrides.insert(
+        NonBlankString::from_str("namespaceScope").unwrap(),
+        namespace_scope.to_string(),
+    );
     HelmChartScore {
         namespace: Some(NonBlankString::from_str(ns).unwrap()),
-        release_name: NonBlankString::from_str("grafana").unwrap(),
-        chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana").unwrap(),
+        release_name: NonBlankString::from_str("grafana-operator").unwrap(),
+        chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(),
         chart_version: None,
-        values_overrides: None,
-        values_yaml: Some(values.to_string()),
+        values_overrides: Some(values_overrides),
+        values_yaml: None,
         create_namespace: true,
         install_only: true,
-        repository: None,
+        repository: Some(HelmRepository::new(
+            "grafana".to_string(),
+            hurl!("https://grafana.github.io/helm-charts"),
+            true,
+        )),
     }
 }

harmony/src/modules/monitoring/grafana/mod.rs

@@ -1 +1,2 @@
+pub mod grafana;
 pub mod helm;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs

@@ -1,12 +1,25 @@
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::topology::{
-    k8s::K8sClient,
-    oberservability::monitoring::{AlertReceiver, AlertSender},
+use crate::{
+    interpret::{InterpretError, Outcome},
+    inventory::Inventory,
+    modules::{
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
+    },
+    topology::{
+        K8sclient, Topology,
+        installable::Installable,
+        k8s::K8sClient,
+        oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget},
+    },
 };
 
 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
@@ -26,6 +39,7 @@ pub struct AlertmanagerConfigSpec {
 pub struct CRDPrometheus {
     pub namespace: String,
     pub client: Arc<K8sClient>,
+    pub service_monitor: Vec<ServiceMonitor>,
 }
 
 impl AlertSender for CRDPrometheus {
@@ -40,6 +54,12 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
     }
 }
 
+impl Clone for Box<dyn ScrapeTarget<CRDPrometheus>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
 impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
     fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -48,3 +68,24 @@ impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
         todo!()
     }
 }
+
+#[async_trait]
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus> + Grafana> Installable<T>
+    for CRDPrometheus
+{
+    async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> {
+        topology.ensure_grafana_operator(inventory).await?;
+        topology.ensure_prometheus_operator(self, inventory).await?;
+        Ok(())
+    }
+
+    async fn ensure_installed(
+        &self,
+        inventory: &Inventory,
+        topology: &T,
+    ) -> Result<(), InterpretError> {
+        topology.install_grafana().await?;
+        topology.install_prometheus(&self, inventory, None).await?;
+        Ok(())
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs

@@ -103,9 +103,34 @@ pub struct GrafanaDashboardSpec {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub resync_period: Option<String>,
 
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub datasources: Option<Vec<GrafanaDashboardDatasource>>,
+
     pub instance_selector: LabelSelector,
 
-    pub json: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub grafana_com: Option<GrafanaCom>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDashboardDatasource {
+    pub input_name: String,
+    pub datasource_name: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaCom {
+    pub id: u32,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub revision: Option<u32>,
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -126,20 +151,79 @@ pub struct GrafanaDatasourceSpec {
     pub allow_cross_namespace_import: Option<bool>,
 
     pub datasource: GrafanaDatasourceConfig,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub values_from: Option<Vec<GrafanaValueFrom>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueFrom {
+    pub target_path: String,
+    pub value_from: GrafanaValueSource,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueSource {
+    pub secret_key_ref: GrafanaSecretKeyRef,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSecretKeyRef {
+    pub name: String,
+    pub key: String,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct GrafanaDatasourceConfig {
     pub access: String,
-    pub database: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub json_data: Option<BTreeMap<String, String>>,
+    pub database: Option<String>,
     pub name: String,
     pub r#type: String,
     pub url: String,
+    /// Represents jsonData in the GrafanaDatasource spec
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json_data: Option<GrafanaDatasourceJsonData>,
+
+    /// Represents secureJsonData (secrets)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub secure_json_data: Option<GrafanaDatasourceSecureJsonData>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub is_default: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub editable: Option<bool>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub time_interval: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_name1: Option<String>,
+
+    /// Disable TLS skip verification (false = verify)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tls_skip_verify: Option<bool>,
+
+    /// Auth type - set to "forward" for OpenShift OAuth identity
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub oauth_pass_thru: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceSecureJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_value1: Option<String>,
+}
 // ------------------------------------------------------------------------------------------------
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]

harmony/src/modules/monitoring/kube_prometheus/crd/rhob_monitoring_stack.rs

@@ -1,8 +1,12 @@
+use std::collections::BTreeMap;
+
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector;
+use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::{
+    LabelSelector, PrometheusSpec,
+};
 
 /// MonitoringStack CRD for monitoring.rhobs/v1alpha1
 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]

harmony/src/modules/monitoring/ntfy/ntfy.rs

@@ -100,11 +100,7 @@ impl<T: Topology + HelmCommand + K8sclient + MultiTargetTopology> Interpret<T> f
 
         info!("deploying ntfy...");
         client
-            .wait_until_deployment_ready(
-                "ntfy".to_string(),
-                Some(self.score.namespace.as_str()),
-                None,
-            )
+            .wait_until_deployment_ready("ntfy", Some(self.score.namespace.as_str()), None)
             .await?;
         info!("ntfy deployed");
 

harmony/src/modules/monitoring/prometheus/prometheus.rs

@@ -114,7 +114,7 @@ impl Prometheus {
         };
 
         if let Some(ns) = namespace.as_deref() {
-            grafana_helm_chart_score(ns)
+            grafana_helm_chart_score(ns, false)
                 .interpret(inventory, topology)
                 .await
         } else {

harmony/src/modules/monitoring/scrape_target/server.rs

@@ -73,4 +73,8 @@ impl ScrapeTarget<CRDPrometheus> for Server {
             self.name.clone()
         )))
     }
+
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<CRDPrometheus>> {
+        Box::new(self.clone())
+    }
 }

harmony/src/modules/okd/bootstrap_03_control_plane.rs

@@ -5,10 +5,8 @@ use crate::{
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::{HostRole, Inventory},
     modules::{
-        dhcp::DhcpHostBindingScore,
-        http::IPxeMacBootFileScore,
-        inventory::DiscoverHostForRoleScore,
-        okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl},
+        dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
+        inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -205,28 +203,6 @@ impl OKDSetup03ControlPlaneInterpret {
 
         Ok(())
     }
-
-    /// Placeholder for automating network bonding configuration.
-    async fn persist_network_bond(
-        &self,
-        inventory: &Inventory,
-        topology: &HAClusterTopology,
-        hosts: &Vec<PhysicalHost>,
-    ) -> Result<(), InterpretError> {
-        info!("[ControlPlane] Ensuring persistent bonding");
-        let score = HostNetworkConfigurationScore {
-            hosts: hosts.clone(),
-        };
-        score.interpret(inventory, topology).await?;
-
-        inquire::Confirm::new(
-            "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
-        )
-        .prompt()
-        .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
-
-        Ok(())
-    }
 }
 
 #[async_trait]
@@ -265,10 +241,6 @@ impl Interpret<HAClusterTopology> for OKDSetup03ControlPlaneInterpret {
         // 4. Reboot the nodes to start the OS installation.
         self.reboot_targets(&nodes).await?;
 
-        // 5. Placeholder for post-boot network configuration (e.g., bonding).
-        self.persist_network_bond(inventory, topology, &nodes)
-            .await?;
-
         // TODO: Implement a step to wait for the control plane nodes to join the cluster
         // and for the cluster operators to become available. This would be similar to
         // the `wait-for bootstrap-complete` command.

harmony/src/modules/okd/bootstrap_persist_network_bond.rs

@@ -0,0 +1,130 @@
+use crate::{
+    data::Version,
+    hardware::PhysicalHost,
+    infra::inventory::InventoryRepositoryFactory,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::{HostRole, Inventory},
+    modules::okd::host_network::HostNetworkConfigurationScore,
+    score::Score,
+    topology::HAClusterTopology,
+};
+use async_trait::async_trait;
+use derive_new::new;
+use harmony_types::id::Id;
+use log::info;
+use serde::Serialize;
+
+// -------------------------------------------------------------------------------------------------
+// Persist Network Bond
+// - Persist bonding via NMState
+// - Persist port channels on the Switch
+// -------------------------------------------------------------------------------------------------
+
+#[derive(Debug, Clone, Serialize, new)]
+pub struct OKDSetupPersistNetworkBondScore {}
+
+impl Score<HAClusterTopology> for OKDSetupPersistNetworkBondScore {
+    fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
+        Box::new(OKDSetupPersistNetworkBondInterpet::new())
+    }
+
+    fn name(&self) -> String {
+        "OKDSetupPersistNetworkBondScore".to_string()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct OKDSetupPersistNetworkBondInterpet {
+    version: Version,
+    status: InterpretStatus,
+}
+
+impl OKDSetupPersistNetworkBondInterpet {
+    pub fn new() -> Self {
+        let version = Version::from("1.0.0").unwrap();
+        Self {
+            version,
+            status: InterpretStatus::QUEUED,
+        }
+    }
+
+    /// Ensures that three physical hosts are discovered and available for the ControlPlane role.
+    /// It will trigger discovery if not enough hosts are found.
+    async fn get_nodes(
+        &self,
+        _inventory: &Inventory,
+        _topology: &HAClusterTopology,
+    ) -> Result<Vec<PhysicalHost>, InterpretError> {
+        const REQUIRED_HOSTS: usize = 3;
+        let repo = InventoryRepositoryFactory::build().await?;
+        let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
+
+        if control_plane_hosts.len() < REQUIRED_HOSTS {
+            Err(InterpretError::new(format!(
+                "OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.",
+                REQUIRED_HOSTS,
+                control_plane_hosts.len()
+            )))
+        } else {
+            // Take exactly the number of required hosts to ensure consistency.
+            Ok(control_plane_hosts
+                .into_iter()
+                .take(REQUIRED_HOSTS)
+                .collect())
+        }
+    }
+
+    async fn persist_network_bond(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+        hosts: &Vec<PhysicalHost>,
+    ) -> Result<(), InterpretError> {
+        info!("Ensuring persistent bonding");
+
+        let score = HostNetworkConfigurationScore {
+            hosts: hosts.clone(),
+        };
+        score.interpret(inventory, topology).await?;
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl Interpret<HAClusterTopology> for OKDSetupPersistNetworkBondInterpet {
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OKDSetupPersistNetworkBondInterpet")
+    }
+
+    fn get_version(&self) -> Version {
+        self.version.clone()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        self.status.clone()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        vec![]
+    }
+
+    async fn execute(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+    ) -> Result<Outcome, InterpretError> {
+        let nodes = self.get_nodes(inventory, topology).await?;
+
+        let res = self.persist_network_bond(inventory, topology, &nodes).await;
+
+        match res {
+            Ok(_) => Ok(Outcome::success(
+                "Network bond successfully persisted".into(),
+            )),
+            Err(_) => Err(InterpretError::new(
+                "Failed to persist network bond".to_string(),
+            )),
+        }
+    }
+}

harmony/src/modules/okd/crd/mod.rs

@@ -1,41 +1 @@
-use kube::CustomResource;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-
 pub mod nmstate;
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(
-    group = "operators.coreos.com",
-    version = "v1",
-    kind = "OperatorGroup",
-    namespaced
-)]
-#[serde(rename_all = "camelCase")]
-pub struct OperatorGroupSpec {
-    pub target_namespaces: Vec<String>,
-}
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(
-    group = "operators.coreos.com",
-    version = "v1alpha1",
-    kind = "Subscription",
-    namespaced
-)]
-#[serde(rename_all = "camelCase")]
-pub struct SubscriptionSpec {
-    pub name: String,
-    pub source: String,
-    pub source_namespace: String,
-    pub channel: Option<String>,
-    pub install_plan_approval: Option<InstallPlanApproval>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub enum InstallPlanApproval {
-    #[serde(rename = "Automatic")]
-    Automatic,
-    #[serde(rename = "Manual")]
-    Manual,
-}

harmony/src/modules/okd/crd/nmstate.rs

@@ -6,9 +6,16 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)]
+#[kube(
+    group = "nmstate.io",
+    version = "v1",
+    kind = "NMState",
+    plural = "nmstates",
+    namespaced = false
+)]
 #[serde(rename_all = "camelCase")]
 pub struct NMStateSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub probe_configuration: Option<ProbeConfig>,
 }
 
@@ -44,6 +51,7 @@ pub struct ProbeDns {
 )]
 #[serde(rename_all = "camelCase")]
 pub struct NodeNetworkConfigurationPolicySpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub node_selector: Option<BTreeMap<String, String>>,
     pub desired_state: DesiredStateSpec,
 }
@@ -58,37 +66,64 @@ pub struct DesiredStateSpec {
 #[serde(rename_all = "kebab-case")]
 pub struct InterfaceSpec {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
     pub r#type: String,
     pub state: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub copy_mac_from: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mtu: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub controller: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ipv4: Option<IpStackSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ipv6: Option<IpStackSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ethernet: Option<EthernetSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub link_aggregation: Option<BondSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<VlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vxlan: Option<VxlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_vtap: Option<MacVtapSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_vlan: Option<MacVlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub infiniband: Option<InfinibandSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub linux_bridge: Option<LinuxBridgeSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ovs_bridge: Option<OvsBridgeSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ethtool: Option<EthtoolSpec>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct IpStackSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub autoconf: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub address: Option<Vec<IpAddressSpec>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_dns: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_gateway: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_routes: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp_client_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp_duid: Option<String>,
 }
 
@@ -102,8 +137,11 @@ pub struct IpAddressSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct EthernetSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub speed: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub duplex: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_negotiation: Option<bool>,
 }
 
@@ -112,6 +150,7 @@ pub struct EthernetSpec {
 pub struct BondSpec {
     pub mode: String,
     pub ports: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<BTreeMap<String, Value>>,
 }
 
@@ -120,6 +159,7 @@ pub struct BondSpec {
 pub struct VlanSpec {
     pub base_iface: String,
     pub id: u16,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub protocol: Option<String>,
 }
 
@@ -129,8 +169,11 @@ pub struct VxlanSpec {
     pub base_iface: String,
     pub id: u32,
     pub remote: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub local: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub learning: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub destination_port: Option<u16>,
 }
 
@@ -139,6 +182,7 @@ pub struct VxlanSpec {
 pub struct MacVtapSpec {
     pub base_iface: String,
     pub mode: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub promiscuous: Option<bool>,
 }
 
@@ -147,6 +191,7 @@ pub struct MacVtapSpec {
 pub struct MacVlanSpec {
     pub base_iface: String,
     pub mode: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub promiscuous: Option<bool>,
 }
 
@@ -161,25 +206,35 @@ pub struct InfinibandSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgeSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<LinuxBridgeOptions>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ports: Option<Vec<LinuxBridgePort>>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgeOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_ageing_time: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub multicast_snooping: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub stp: Option<StpOptions>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct StpOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub forward_delay: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub hello_time: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub max_age: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub priority: Option<u16>,
 }
 
@@ -187,15 +242,20 @@ pub struct StpOptions {
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgePort {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<LinuxBridgePortVlan>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgePortVlan {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub trunk_tags: Option<Vec<VlanTag>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub tag: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enable_native: Option<bool>,
 }
 
@@ -203,6 +263,7 @@ pub struct LinuxBridgePortVlan {
 #[serde(rename_all = "kebab-case")]
 pub struct VlanTag {
     pub id: u16,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub id_range: Option<VlanIdRange>,
 }
 
@@ -216,15 +277,20 @@ pub struct VlanIdRange {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct OvsBridgeSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<OvsBridgeOptions>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ports: Option<Vec<OvsPortSpec>>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct OvsBridgeOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub stp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub rstp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mcast_snooping_enable: Option<bool>,
 }
 
@@ -232,8 +298,11 @@ pub struct OvsBridgeOptions {
 #[serde(rename_all = "kebab-case")]
 pub struct OvsPortSpec {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub link_aggregation: Option<BondSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<LinuxBridgePortVlan>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub r#type: Option<String>,
 }
 
@@ -246,6 +315,8 @@ pub struct EthtoolSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct EthtoolFecSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<String>,
 }

harmony/src/modules/okd/host_network.rs

@@ -39,30 +39,70 @@ impl HostNetworkConfigurationInterpret {
         &self,
         topology: &T,
         host: &PhysicalHost,
-    ) -> Result<(), InterpretError> {
-        let switch_ports = self.collect_switch_ports_for_host(topology, host).await?;
-        if !switch_ports.is_empty() {
-            topology
-                .configure_host_network(host, HostNetworkConfig { switch_ports })
-                .await
-                .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
+        current_host: &usize,
+        total_hosts: &usize,
+    ) -> Result<HostNetworkConfig, InterpretError> {
+        if host.network.is_empty() {
+            info!("[Host {current_host}/{total_hosts}] No interfaces to configure, skipping");
+            return Ok(HostNetworkConfig {
+                host_id: host.id.clone(),
+                switch_ports: vec![],
+            });
         }
 
-        Ok(())
+        let switch_ports = self
+            .collect_switch_ports_for_host(topology, host, current_host, total_hosts)
+            .await?;
+
+        let config = HostNetworkConfig {
+            host_id: host.id.clone(),
+            switch_ports,
+        };
+
+        if !config.switch_ports.is_empty() {
+            info!(
+                "[Host {current_host}/{total_hosts}] Found {} ports for {} interfaces",
+                config.switch_ports.len(),
+                host.network.len()
+            );
+
+            info!("[Host {current_host}/{total_hosts}] Configuring host network...");
+            topology
+                .configure_host_network(&config)
+                .await
+                .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
+        } else {
+            info!(
+                "[Host {current_host}/{total_hosts}] No ports found for {} interfaces, skipping",
+                host.network.len()
+            );
+        }
+
+        Ok(config)
     }
 
     async fn collect_switch_ports_for_host<T: Topology + Switch>(
         &self,
         topology: &T,
         host: &PhysicalHost,
+        current_host: &usize,
+        total_hosts: &usize,
     ) -> Result<Vec<SwitchPort>, InterpretError> {
         let mut switch_ports = vec![];
 
+        if host.network.is_empty() {
+            return Ok(switch_ports);
+        }
+
+        info!("[Host {current_host}/{total_hosts}] Collecting ports on switch...");
         for network_interface in &host.network {
             let mac_address = network_interface.mac_address;
 
             match topology.get_port_for_mac_address(&mac_address).await {
                 Ok(Some(port)) => {
+                    info!(
+                        "[Host {current_host}/{total_hosts}] Found port '{port}' for '{mac_address}'"
+                    );
                     switch_ports.push(SwitchPort {
                         interface: NetworkInterface {
                             name: network_interface.name.clone(),
@@ -73,7 +113,7 @@ impl HostNetworkConfigurationInterpret {
                         port,
                     });
                 }
-                Ok(None) => debug!("No port found for host '{}', skipping", host.id),
+                Ok(None) => debug!("No port found for '{mac_address}', skipping"),
                 Err(e) => {
                     return Err(InterpretError::new(format!(
                         "Failed to get port for host '{}': {}",
@@ -85,6 +125,47 @@ impl HostNetworkConfigurationInterpret {
 
         Ok(switch_ports)
     }
+
+    fn format_host_configuration(&self, configs: Vec<HostNetworkConfig>) -> Vec<String> {
+        let mut report = vec![
+            "Network Configuration Report".to_string(),
+            "------------------------------------------------------------------".to_string(),
+        ];
+
+        for config in configs {
+            let host = self
+                .score
+                .hosts
+                .iter()
+                .find(|h| h.id == config.host_id)
+                .unwrap();
+
+            println!("[Host] {host}");
+
+            if config.switch_ports.is_empty() {
+                report.push(format!(
+                    "⏭️ Host {}: SKIPPED (No matching switch ports found)",
+                    config.host_id
+                ));
+            } else {
+                let mappings: Vec<String> = config
+                    .switch_ports
+                    .iter()
+                    .map(|p| format!("[{} -> {}]", p.interface.name, p.port))
+                    .collect();
+
+                report.push(format!(
+                    "✅ Host {}: Bonded {} port(s) {}",
+                    config.host_id,
+                    config.switch_ports.len(),
+                    mappings.join(", ")
+                ));
+            }
+        }
+        report
+            .push("------------------------------------------------------------------".to_string());
+        report
+    }
 }
 
 #[async_trait]
@@ -114,27 +195,38 @@ impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
             return Ok(Outcome::noop("No hosts to configure".into()));
         }
 
-        info!(
-            "Started network configuration for {} host(s)...",
-            self.score.hosts.len()
-        );
+        let host_count = self.score.hosts.len();
+        info!("Started network configuration for {host_count} host(s)...",);
 
+        info!("Setting up switch with sane defaults...");
         topology
             .setup_switch()
             .await
             .map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?;
+        info!("Switch ready");
+
+        let mut current_host = 1;
+        let mut host_configurations = vec![];
 
-        let mut configured_host_count = 0;
         for host in &self.score.hosts {
-            self.configure_network_for_host(topology, host).await?;
-            configured_host_count += 1;
-        }
+            let host_configuration = self
+                .configure_network_for_host(topology, host, &current_host, &host_count)
+                .await?;
 
-        if configured_host_count > 0 {
-            Ok(Outcome::success(format!(
-                "Configured {configured_host_count}/{} host(s)",
-                self.score.hosts.len()
-            )))
+            host_configurations.push(host_configuration);
+            current_host += 1;
+        }
+        if current_host > 1 {
+            let details = self.format_host_configuration(host_configurations);
+
+            Ok(Outcome::success_with_details(
+                format!(
+                    "Configured {}/{} host(s)",
+                    current_host - 1,
+                    self.score.hosts.len()
+                ),
+                details,
+            ))
         } else {
             Ok(Outcome::noop("No hosts configured".into()))
         }
@@ -209,6 +301,7 @@ mod tests {
         assert_that!(*configured_host_networks).contains_exactly(vec![(
             HOST_ID.clone(),
             HostNetworkConfig {
+                host_id: HOST_ID.clone(),
                 switch_ports: vec![SwitchPort {
                     interface: EXISTING_INTERFACE.clone(),
                     port: PORT.clone(),
@@ -234,6 +327,7 @@ mod tests {
         assert_that!(*configured_host_networks).contains_exactly(vec![(
             HOST_ID.clone(),
             HostNetworkConfig {
+                host_id: HOST_ID.clone(),
                 switch_ports: vec![
                     SwitchPort {
                         interface: EXISTING_INTERFACE.clone(),
@@ -263,6 +357,7 @@ mod tests {
             (
                 HOST_ID.clone(),
                 HostNetworkConfig {
+                    host_id: HOST_ID.clone(),
                     switch_ports: vec![SwitchPort {
                         interface: EXISTING_INTERFACE.clone(),
                         port: PORT.clone(),
@@ -272,6 +367,7 @@ mod tests {
             (
                 ANOTHER_HOST_ID.clone(),
                 HostNetworkConfig {
+                    host_id: ANOTHER_HOST_ID.clone(),
                     switch_ports: vec![SwitchPort {
                         interface: ANOTHER_EXISTING_INTERFACE.clone(),
                         port: ANOTHER_PORT.clone(),
@@ -382,11 +478,10 @@ mod tests {
 
         async fn configure_host_network(
             &self,
-            host: &PhysicalHost,
-            config: HostNetworkConfig,
+            config: &HostNetworkConfig,
         ) -> Result<(), SwitchError> {
             let mut configured_host_networks = self.configured_host_networks.lock().unwrap();
-            configured_host_networks.push((host.id.clone(), config.clone()));
+            configured_host_networks.push((config.host_id.clone(), config.clone()));
 
             Ok(())
         }

harmony/src/modules/okd/installation.rs

@@ -50,7 +50,7 @@
 use crate::{
     modules::okd::{
         OKDSetup01InventoryScore, OKDSetup02BootstrapScore, OKDSetup03ControlPlaneScore,
-        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore,
+        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, OKDSetupPersistNetworkBondScore,
         bootstrap_06_installation_report::OKDSetup06InstallationReportScore,
     },
     score::Score,
@@ -65,6 +65,7 @@ impl OKDInstallationPipeline {
             Box::new(OKDSetup01InventoryScore::new()),
             Box::new(OKDSetup02BootstrapScore::new()),
             Box::new(OKDSetup03ControlPlaneScore::new()),
+            Box::new(OKDSetupPersistNetworkBondScore::new()),
             Box::new(OKDSetup04WorkersScore::new()),
             Box::new(OKDSetup05SanityCheckScore::new()),
             Box::new(OKDSetup06InstallationReportScore::new()),

harmony/src/modules/okd/mod.rs

@@ -6,6 +6,7 @@ mod bootstrap_05_sanity_check;
 mod bootstrap_06_installation_report;
 pub mod bootstrap_dhcp;
 pub mod bootstrap_load_balancer;
+mod bootstrap_persist_network_bond;
 pub mod dhcp;
 pub mod dns;
 pub mod installation;
@@ -19,5 +20,6 @@ pub use bootstrap_03_control_plane::*;
 pub use bootstrap_04_workers::*;
 pub use bootstrap_05_sanity_check::*;
 pub use bootstrap_06_installation_report::*;
+pub use bootstrap_persist_network_bond::*;
 pub mod crd;
 pub mod host_network;

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -12,7 +12,8 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::C
 use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
 use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
     Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
-    GrafanaDatasourceSpec, GrafanaSpec,
+    GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec,
+    GrafanaValueFrom, GrafanaValueSource,
 };
 use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
     PrometheusRule, PrometheusRuleSpec, RuleGroup,
@@ -39,7 +40,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct K8sPrometheusCRDAlertingScore {
@@ -49,7 +50,7 @@ pub struct K8sPrometheusCRDAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Score<T>
     for K8sPrometheusCRDAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
@@ -75,7 +76,7 @@ pub struct K8sPrometheusCRDAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Interpret<T>
     for K8sPrometheusCRDAlertingInterpret
 {
     async fn execute(
@@ -466,10 +467,13 @@ impl K8sPrometheusCRDAlertingInterpret {
             match_labels: label.clone(),
             match_expressions: vec![],
         };
-        let mut json_data = BTreeMap::new();
-        json_data.insert("timeInterval".to_string(), "5s".to_string());
         let namespace = self.sender.namespace.clone();
-
+        let json_data = GrafanaDatasourceJsonData {
+            time_interval: Some("5s".to_string()),
+            http_header_name1: None,
+            tls_skip_verify: Some(true),
+            oauth_pass_thru: Some(true),
+        };
         let json = build_default_dashboard(&namespace);
 
         let graf_data_source = GrafanaDatasource {
@@ -495,7 +499,11 @@ impl K8sPrometheusCRDAlertingInterpret {
                         "http://prometheus-operated.{}.svc.cluster.local:9090",
                         self.sender.namespace.clone()
                     ),
+                    secure_json_data: None,
+                    is_default: None,
+                    editable: None,
                 },
+                values_from: None,
             },
         };
 
@@ -516,7 +524,9 @@ impl K8sPrometheusCRDAlertingInterpret {
             spec: GrafanaDashboardSpec {
                 resync_period: Some("30s".to_string()),
                 instance_selector: labels.clone(),
-                json,
+                json: Some(json),
+                grafana_com: None,
+                datasources: None,
             },
         };
 

harmony/src/modules/prometheus/prometheus.rs

@@ -9,11 +9,17 @@ use crate::{
 };
 
 #[async_trait]
-pub trait PrometheusApplicationMonitoring<S: AlertSender> {
+pub trait PrometheusMonitoring<S: AlertSender> {
     async fn install_prometheus(
         &self,
         sender: &S,
         inventory: &Inventory,
         receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
     ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &S,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
 }

harmony/src/modules/prometheus/rhob_alerting_score.rs

@@ -38,7 +38,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct RHOBAlertingScore {
@@ -48,8 +48,8 @@ pub struct RHOBAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Score<T> for RHOBAlertingScore
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Score<T>
+    for RHOBAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
         Box::new(RHOBAlertingInterpret {
@@ -74,8 +74,8 @@ pub struct RHOBAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Interpret<T> for RHOBAlertingInterpret
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Interpret<T>
+    for RHOBAlertingInterpret
 {
     async fn execute(
         &self,

harmony_cli/src/cli_reporter.rs

@@ -40,7 +40,7 @@ pub fn init() {
                 HarmonyEvent::HarmonyFinished => {
                     if !details.is_empty() {
                         println!(
-                            "\n{} All done! Here's what's next for you:",
+                            "\n{} All done! Here's a few info for you:",
                             theme::EMOJI_SUMMARY
                         );
                         for detail in details.iter() {

opnsense-config-xml/src/data/opnsense.rs

@@ -216,7 +216,7 @@ pub struct System {
     pub maximumfrags: Option<MaybeString>,
     pub aliasesresolveinterval: Option<MaybeString>,
     pub maximumtableentries: Option<MaybeString>,
-    pub language: String,
+    pub language: Option<MaybeString>,
     pub dnsserver: Option<MaybeString>,
     pub dns1gw: Option<String>,
     pub dns2gw: Option<String>,