Compare commits

..

5 Commits

82 changed files with 1121 additions and 3286 deletions

36
Cargo.lock generated
View File

@@ -1719,24 +1719,6 @@ dependencies = [
"url",
]
[[package]]
name = "example-ha-cluster"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",
"env_logger",
"harmony",
"harmony_macros",
"harmony_secret",
"harmony_tui",
"harmony_types",
"log",
"serde",
"tokio",
"url",
]
[[package]]
name = "example-kube-rs"
version = "0.1.0"
@@ -1856,24 +1838,6 @@ dependencies = [
[[package]]
name = "example-opnsense"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",
"env_logger",
"harmony",
"harmony_cli",
"harmony_macros",
"harmony_secret",
"harmony_types",
"log",
"serde",
"tokio",
"url",
]
[[package]]
name = "example-opnsense-2"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",

View File

@@ -31,7 +31,6 @@ pub struct BrocadeOptions {
pub struct TimeoutConfig {
pub shell_ready: Duration,
pub command_execution: Duration,
pub command_output: Duration,
pub cleanup: Duration,
pub message_wait: Duration,
}
@@ -41,7 +40,6 @@ impl Default for TimeoutConfig {
Self {
shell_ready: Duration::from_secs(10),
command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while
command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output"
cleanup: Duration::from_secs(10),
message_wait: Duration::from_millis(500),
}

View File

@@ -3,7 +3,6 @@ use std::str::FromStr;
use async_trait::async_trait;
use harmony_types::switch::{PortDeclaration, PortLocation};
use log::{debug, info};
use regex::Regex;
use crate::{
BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo,
@@ -104,37 +103,13 @@ impl NetworkOperatingSystemClient {
};
Some(Ok(InterfaceInfo {
name: format!("{interface_type} {port_location}"),
name: format!("{} {}", interface_type, port_location),
port_location,
interface_type,
operating_mode,
status,
}))
}
fn map_configure_interfaces_error(&self, err: Error) -> Error {
debug!("[Brocade] {err}");
if let Error::CommandError(message) = &err {
if message.contains("switchport")
&& message.contains("Cannot configure aggregator member")
{
let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap();
if let Some(caps) = re.captures(message) {
let interface_type = &caps[1];
let port_location = &caps[2];
let interface = format!("{interface_type} {port_location}");
return Error::CommandError(format!(
"Cannot configure interface '{interface}', it is a member of a port-channel (LAG)"
));
}
}
}
err
}
}
#[async_trait]
@@ -222,10 +197,11 @@ impl BrocadeClient for NetworkOperatingSystemClient {
commands.push("exit".into());
}
commands.push("write memory".into());
self.shell
.run_commands(commands, ExecutionMode::Regular)
.await
.map_err(|err| self.map_configure_interfaces_error(err))?;
.await?;
info!("[Brocade] Interfaces configured.");
@@ -237,7 +213,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
let output = self
.shell
.run_command("show port-channel summary", ExecutionMode::Regular)
.run_command("show port-channel", ExecutionMode::Regular)
.await?;
let used_ids: Vec<u8> = output
@@ -272,12 +248,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
ports: &[PortLocation],
) -> Result<(), Error> {
info!(
"[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}",
ports
.iter()
.map(|p| format!("{p}"))
.collect::<Vec<String>>()
.join(", ")
"[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}"
);
let interfaces = self.get_interfaces().await?;
@@ -305,6 +276,8 @@ impl BrocadeClient for NetworkOperatingSystemClient {
commands.push("exit".into());
}
commands.push("write memory".into());
self.shell
.run_commands(commands, ExecutionMode::Regular)
.await?;
@@ -321,6 +294,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
"configure terminal".into(),
format!("no interface port-channel {}", channel_name),
"exit".into(),
"write memory".into(),
];
self.shell

View File

@@ -211,7 +211,7 @@ impl BrocadeSession {
let mut output = Vec::new();
let start = Instant::now();
let read_timeout = Duration::from_millis(500);
let log_interval = Duration::from_secs(5);
let log_interval = Duration::from_secs(3);
let mut last_log = Instant::now();
loop {
@@ -221,9 +221,7 @@ impl BrocadeSession {
));
}
if start.elapsed() > self.options.timeouts.command_output
&& last_log.elapsed() > log_interval
{
if start.elapsed() > Duration::from_secs(5) && last_log.elapsed() > log_interval {
info!("[Brocade] Waiting for command output...");
last_log = Instant::now();
}
@@ -278,7 +276,7 @@ impl BrocadeSession {
let output_lower = output.to_lowercase();
if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) {
return Err(Error::CommandError(format!(
"Command error: {}",
"Command '{command}' failed: {}",
output.trim()
)));
}

View File

@@ -27,6 +27,7 @@ async fn main() {
};
let application = Arc::new(RustWebapp {
name: "example-monitoring".to_string(),
dns: "example-monitoring.harmony.mcd".to_string(),
project_root: PathBuf::from("./examples/rust/webapp"),
framework: Some(RustWebFramework::Leptos),
service_port: 3000,

View File

@@ -1,21 +0,0 @@
[package]
name = "example-ha-cluster"
edition = "2024"
version.workspace = true
readme.workspace = true
license.workspace = true
publish = false
[dependencies]
harmony = { path = "../../harmony" }
harmony_tui = { path = "../../harmony_tui" }
harmony_types = { path = "../../harmony_types" }
cidr = { workspace = true }
tokio = { workspace = true }
harmony_macros = { path = "../../harmony_macros" }
log = { workspace = true }
env_logger = { workspace = true }
url = { workspace = true }
harmony_secret = { path = "../../harmony_secret" }
brocade = { path = "../../brocade" }
serde = { workspace = true }

View File

@@ -1,15 +0,0 @@
## OPNSense demo
Download the virtualbox snapshot from {{TODO URL}}
Start the virtualbox image
This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
Credentials are opnsense default (root/opnsense)
Run the project with the correct ip address on the command line :
```bash
cargo run -p example-opnsense -- 192.168.5.229
```

View File

@@ -1,141 +0,0 @@
use std::{
net::{IpAddr, Ipv4Addr},
sync::Arc,
};
use brocade::BrocadeOptions;
use cidr::Ipv4Cidr;
use harmony::{
hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
inventory::Inventory,
modules::{
dummy::{ErrorScore, PanicScore, SuccessScore},
http::StaticFilesHttpScore,
okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
opnsense::OPNsenseShellCommandScore,
tftp::TftpScore,
},
topology::{LogicalHost, UnmanagedRouter},
};
use harmony_macros::{ip, mac_address};
use harmony_secret::{Secret, SecretManager};
use harmony_types::net::Url;
use serde::{Deserialize, Serialize};
#[tokio::main]
async fn main() {
let firewall = harmony::topology::LogicalHost {
ip: ip!("192.168.5.229"),
name: String::from("opnsense-1"),
};
let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
.await
.expect("Failed to get credentials");
let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
let brocade_options = Some(BrocadeOptions {
dry_run: *harmony::config::DRY_RUN,
..Default::default()
});
let switch_client = BrocadeSwitchClient::init(
&switches,
&switch_auth.username,
&switch_auth.password,
brocade_options,
)
.await
.expect("Failed to connect to switch");
let switch_client = Arc::new(switch_client);
let opnsense = Arc::new(
harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
);
let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
let gateway_ip = IpAddr::V4(gateway_ipv4);
let topology = harmony::topology::HAClusterTopology {
kubeconfig: None,
domain_name: "demo.harmony.mcd".to_string(),
router: Arc::new(UnmanagedRouter::new(
gateway_ip,
Ipv4Cidr::new(lan_subnet, 24).unwrap(),
)),
load_balancer: opnsense.clone(),
firewall: opnsense.clone(),
tftp_server: opnsense.clone(),
http_server: opnsense.clone(),
dhcp_server: opnsense.clone(),
dns_server: opnsense.clone(),
control_plane: vec![LogicalHost {
ip: ip!("10.100.8.20"),
name: "cp0".to_string(),
}],
bootstrap_host: LogicalHost {
ip: ip!("10.100.8.20"),
name: "cp0".to_string(),
},
workers: vec![],
switch_client: switch_client.clone(),
};
let inventory = Inventory {
location: Location::new(
"232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
"wk".to_string(),
),
switch: SwitchGroup::from([]),
firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
storage_host: vec![],
worker_host: vec![],
control_plane_host: vec![
PhysicalHost::empty(HostCategory::Server)
.mac_address(mac_address!("08:00:27:62:EC:C3")),
],
};
// TODO regroup smaller scores in a larger one such as this
// let okd_boostrap_preparation();
let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
let dns_score = OKDDnsScore::new(&topology);
let load_balancer_score = OKDLoadBalancerScore::new(&topology);
let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
let http_score = StaticFilesHttpScore {
folder_to_serve: Some(Url::LocalFolder(
"./data/watchguard/pxe-http-files".to_string(),
)),
files: vec![],
remote_path: None,
};
harmony_tui::run(
inventory,
topology,
vec![
Box::new(dns_score),
Box::new(dhcp_score),
Box::new(load_balancer_score),
Box::new(tftp_score),
Box::new(http_score),
Box::new(OPNsenseShellCommandScore {
opnsense: opnsense.get_opnsense_config(),
command: "touch /tmp/helloharmonytouching".to_string(),
}),
Box::new(SuccessScore {}),
Box::new(ErrorScore {}),
Box::new(PanicScore {}),
],
)
.await
.unwrap();
}
#[derive(Secret, Serialize, Deserialize, Debug)]
pub struct BrocadeSwitchAuth {
pub username: String,
pub password: String,
}

View File

@@ -61,7 +61,6 @@ async fn main() {
let gateway_ipv4 = Ipv4Addr::new(192, 168, 33, 1);
let gateway_ip = IpAddr::V4(gateway_ipv4);
let topology = harmony::topology::HAClusterTopology {
kubeconfig: None,
domain_name: "ncd0.harmony.mcd".to_string(), // TODO this must be set manually correctly
// when setting up the opnsense firewall
router: Arc::new(UnmanagedRouter::new(

View File

@@ -59,7 +59,6 @@ pub async fn get_topology() -> HAClusterTopology {
let gateway_ipv4 = ipv4!("192.168.1.1");
let gateway_ip = IpAddr::V4(gateway_ipv4);
harmony::topology::HAClusterTopology {
kubeconfig: None,
domain_name: "demo.harmony.mcd".to_string(),
router: Arc::new(UnmanagedRouter::new(
gateway_ip,

View File

@@ -54,7 +54,6 @@ pub async fn get_topology() -> HAClusterTopology {
let gateway_ipv4 = ipv4!("192.168.1.1");
let gateway_ip = IpAddr::V4(gateway_ipv4);
harmony::topology::HAClusterTopology {
kubeconfig: None,
domain_name: "demo.harmony.mcd".to_string(),
router: Arc::new(UnmanagedRouter::new(
gateway_ip,

View File

@@ -8,7 +8,7 @@ publish = false
[dependencies]
harmony = { path = "../../harmony" }
harmony_cli = { path = "../../harmony_cli" }
harmony_tui = { path = "../../harmony_tui" }
harmony_types = { path = "../../harmony_types" }
cidr = { workspace = true }
tokio = { workspace = true }

View File

@@ -1,23 +1,15 @@
# OPNSense Demo
## OPNSense demo
This example demonstrate how to manage an Opnsense server with harmony.
Download the virtualbox snapshot from {{TODO URL}}
todo: add more info
Start the virtualbox image
## Demo instructions
This virtualbox image is configured to use a bridge on the host's physical interface, make sure the bridge is up and the virtual machine can reach internet.
todo: add detailed instructions
Credentials are opnsense default (root/opnsense)
- setup the example execution environment
- setup your system configuration
- topology
- scores
- secrets
- build
- execute
- verify/inspect
## Example execution
See [learning tool documentation](./scripts/README.md)
Run the project with the correct ip address on the command line :
```bash
cargo run -p example-opnsense -- 192.168.5.229
```

View File

@@ -1,4 +0,0 @@
export HARMONY_SECRET_NAMESPACE=example-opnsense
export HARMONY_SECRET_STORE=file
export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info
export RUST_LOG=info

View File

@@ -1,38 +0,0 @@
## réseau
- fonctionne uniquement avec connection filaire, pas wifi
- on crée des bridges pour le réseau WAN et LAN pour qu'ils soient facilement accessible sur le poste de travail
```
# paramètres
PRIMARY_PROFIL=enpXsY
NETWORK_LABEL=harmony
WAN_BRIDGE=$NETWORL_LABEL-wan-brd # max 15 char
LAN_NIC=$NETWORK_LABEL-lan-nic
LAN_BRIDGE=$NETWORK_LABEL-lan-brd
# Setup WAN bridge
nmcli c down $PRIMARY
nmcli c add type bridge ifname $WAN_BRIDGE con-name $WAN_BRIDGE
nmcli c add type bridge-slave ifname $PRIMARY_PROFILE master $WAN_BRIDGE ipv4.method auto
# nmcli c up $WAN_BRIDGE
# Setup LAN nic
sudo modprobe dummy
sudo ip link add $LAN_NIC type dummy
ip tuntap add dev $LAN_NIC mode tap user root # todo: why user root?
# Setup LAN bridge
nmcli c add type bridge ifname $LAN_BRIDGE con-name $LAN_BRIDGE
nmcli c add type bridge-slave ifname $LAN_NIC master $LAN_BRIDGE ipv4.method auto
nmcli c up $LAN_BRIDGE
```
LAN bridge do not have an IP address and appear down. But it is successfully used by opnsense and can be accessed from the host network.
## config opnsense minimale
- ssh enabled

View File

@@ -1,96 +0,0 @@
# Virtualized Execution Environment for Harmony
Scripts included in this directory have 3 purposes:
- automate initial setup of localhost or VM (nested virtualization) for this example
- prototype a solution for an 'OpensenseLocalhostTopology'
- prototype
This exprimentation aim to find an approach suitable for using harmony on virtualised execution environment such that:
- it straights forward for a user with minimal knowledge to start testing harmony
- installation and execution have **minimal impact on the user desktop**
## Usage
### Installation
1. download this directory
2. add this directory in your PATH (example `. setup`)
```
# show help page
harmony-vee
# show active configurations
harmony-vee config
# show what will be modified at installation
harmony-vee install --dry-run
# install
harmony-vee install
# show what will be modified at unistallation
harmony-vee uninstall --dry-run
```
### Create and start a new Virtual Execution Environment
```
# Create a HVEE to test opnsense_score
harmony-vee init opnsense_score
# List existing HVEE
harmony-vee list
# Show HVEE information including devices ip and vault type/location
harmony-vee show opnsense_score
# Start/Stop a HVEE
harmony-vee stop opnsense_score
# Destroy a HVEE instance
harmony-vee destroy opnsense_score
```
### Variable d'environnement
```
## directory containing harmony-ve data
# HVE_ROOT=~/.harmony-ve
## OPNSENSE SRC
# main mirror
# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
# first alternative mirror
# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
## Network
# HVE_NETWORK_LABEL=harmony
```
## Remarks
- A nested VM setup could be safer
## Architecture
### Learning environment directly on host
![localhost case](./doc/automate-opnsense-example-localhost.drawio.png)
### Learning environment nested in a "workspace vm"
![localhost case](./doc/automate-opnsense-example-nested-virtualization.drawio.png)

View File

@@ -1,17 +0,0 @@
#! /bin/bash
_warn(){ >&2 echo "WARNING: $*" ; }
_fatal(){
>&2 echo "FATAL ERROR: $*"
>&2 echo stopping...
exit 1
}
pushd () {
command pushd "$@" > /dev/null
}
popd () {
command popd "$@" > /dev/null
}

View File

@@ -1,31 +0,0 @@
# Conventions:
# - Namespaced with HVE, short for Harmony Virtualised Execution Environment
# - Prefixed values used internally
# - Not prefixed may be supercharged by the user
# Root of harmony data
_HVE_ROOT=${HVE_ROOT:-$HOME/harmony-ve}
[ -d "$_HVE_ROOT" ] || mkdir -p "${_HVE_ROOT}"
_HVE_SRC_IMG=${_HVE_ROOT}/src/images
[ -d "$_HVE_SRC_IMG" ] || mkdir -p "${_HVE_SRC_IMG}"
_HVE_IMG=${_HVE_ROOT}/images
[ -d "$_HVE_IMG" ] || mkdir -p "$_HVE_IMG"
# Opnsense
_HVE_OPNSENSE_URL=${HVE_OPNSENSE_URL:-https://pkg.opnsense.org/releases}
# first alternative mirror
_HVE_OPNSENSE_URL_ALT1=${HVE_OPNSENSE_URL_ALT1:-https://mirror.vraphim.com/opnsense/releases}
_HVE_OPNSENSE_URL_ALT2=${HVE_OPNSENSE_URL_ALT2:-https://mirror.winsub.kr/opnsense/releases}
_HVE_OPNSENSE_SRC_IMG=${_HVE_SRC_IMG}/opnsense
[ -d "$_HVE_OPNSENSE_SRC_IMG" ] || mkdir -p "${_HVE_OPNSENSE_SRC_IMG}"
_HVE_OPNSENSE_IMG=${_HVE_IMG}/opnsense
[ -d "$_HVE_OPNSENSE_IMG" ] || mkdir -p "${_HVE_OPNSENSE_IMG}"
# Network
_HVE_NETWORK=${HVE_NETWORK:-harmony}
_HVE_WAN_BRIDGE=${HVE_WAN_BRIDGE:-${_HVE_NETWORK}-wan-brd}
_HVE_LAN_BRIDGE=${HVE_LAN_BRIDGE:-${_HVE_NETWORK}-lann-brd}

View File

@@ -1,48 +0,0 @@
#! /bin/bash
is_string_empty(){
if [ "${*:-}" != "" ]; then
return 0
else
return 1
fi
}
is_debian_family()(
is_string_empty "$(apt --version 2> /dev/null )"
)
has_ip(){
is_string_empty "$(ip -V 2> /dev/null)"
}
has_virsh(){
is_string_empty "$(virsh --version 2> /dev/null)"
}
has_virt_customize(){
is_string_empty "$(virt-customize --version 2> /dev/null)"
}
has_curl(){
is_string_empty "$(curl --version 2> /dev/null)"
}
has_wget(){
is_string_empty "$(wget --version 2> /dev/null)"
}
install_kvm(){
sudo apt install -y --no-install-recommends qemu-system libvirt-clients libvirt-daemon-system
sudo usermod -aG libvirt "$USER"
# todo: finf how to fix image access out of /var/lib/libvirt/images
sudo setfacl -Rm u:libvirt-qemu:rx $_HVE_IMG
sudo systemctl restart libvirtd
}
install_virt_customize(){
sudo apt install -y libguestfs-tools
}
install_wget(){
sudo apt install -y wget
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 106 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

View File

@@ -1,321 +0,0 @@
<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" version="28.2.9" pages="2">
<diagram name="localhost" id="lK0WmoXmZXwFmV5PC-RW">
<mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="1Ax8jaXdU0J25Zkiwu96-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="300" y="230" width="700" height="550" as="geometry" />
</mxCell>
<mxCell id="VVCo9gNhF-9Hs0fZa6i8-1" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="380" y="240" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="350" y="283" width="230" height="67" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-2" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="590" y="393" width="310" height="270" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-3" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="620" y="403" width="90" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="620" y="250" width="280" height="60" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-5" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="610" y="250" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-6" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="370" y="294" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-7" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
<mxGeometry x="760" y="290" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-8" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-7" target="X0RF5wBsf5JYURxROWwA-9" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-9" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="771" y="342" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-13" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="810" y="290" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-14" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-13" target="X0RF5wBsf5JYURxROWwA-15" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-15" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="820" y="342" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-16" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="760" y="383" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-17" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-16" target="X0RF5wBsf5JYURxROWwA-18" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-18" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="772.5" y="353" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-19" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="809" y="383" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-20" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="X0RF5wBsf5JYURxROWwA-19" target="X0RF5wBsf5JYURxROWwA-21" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-21" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="821.5" y="353" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-22" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="350" y="440" width="230" height="130" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-23" value="&lt;b&gt;Example dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="360" y="450" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-24" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="360" y="510" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-25" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="350" y="590" width="230" height="130" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-26" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="370" y="600" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-27" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="361" y="650" width="208" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-28" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="350" y="353" width="230" height="77" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-29" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="370" y="361.5" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-30" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="620" y="450" width="250" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-31" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="370" y="323" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-33" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs and&lt;/i&gt;&lt;/font&gt;&lt;div&gt;&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;runtime info.&lt;/i&gt;&lt;/font&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="370" y="395" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="360" y="470" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-35" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="626.5" y="264" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-36" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="369" y="620" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="X0RF5wBsf5JYURxROWwA-37" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="626.5" y="420" width="200" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="oOuscOXp9aWETXQepMaW" name="nested-virtualization">
<mxGraphModel dx="1111" dy="487" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="vj3pG7w7rTpS1tnBMssI-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="190" y="60" width="1240" height="660" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-37" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="540" y="210" width="830" height="480" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-2" value="&lt;b&gt;localhost&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="84" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-3" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="223" width="230" height="67" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-4" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="855" y="403.75" width="310" height="266.25" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-5" value="&lt;b&gt;opnsense vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="885" y="413.75" width="90" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-6" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="810" y="230" width="530" height="120" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-7" value="&lt;b&gt;workspace VM network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="825" y="234" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-8" value="&lt;b&gt;Src repo or&lt;/b&gt;&lt;div style=&quot;&quot;&gt;&lt;b&gt;Harmony learning tool&lt;/b&gt;&lt;/div&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="234" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-10" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-9" target="vj3pG7w7rTpS1tnBMssI-11" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-12" target="vj3pG7w7rTpS1tnBMssI-14" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-15" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1025" y="393.75" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-16" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-15" target="vj3pG7w7rTpS1tnBMssI-17" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-17" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="1037.5" y="363.75" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-18" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1074" y="393.75" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-19" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-18" target="vj3pG7w7rTpS1tnBMssI-20" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-20" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="1086.5" y="363.75" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-21" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="380" width="230" height="120" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-22" value="&lt;b&gt;Learning dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="250" y="391" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-23" value="&lt;ul&gt;&lt;li&gt;harmony&lt;/li&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="250" y="450" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-24" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="540" width="230" height="130" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-25" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="550" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-26" value="&lt;ul&gt;&lt;li&gt;Can be mounted locally&lt;br&gt;for persistence&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="251" y="600" width="189" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-27" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="293" width="230" height="67" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-28" value="&lt;b&gt;Local workspace&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="301.5" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-29" value="&lt;ul&gt;&lt;li&gt;provisioned using the learning tool&lt;/li&gt;&lt;li&gt;managed using harmony&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="885" y="460.75" width="250" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-30" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Minimun required to learn Harmony&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="263" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-31" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;A place to store configs&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="260" y="321" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-32" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Modifications of localhost&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="250" y="410" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-34" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="259" y="570" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-35" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;Execution environment&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="891.5" y="430.75" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-38" value="&lt;b&gt;workspace VM&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="580" y="234" width="130" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-39" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="570" y="305" width="230" height="215" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-40" value="&lt;b&gt;workspace VM dependencies&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="580" y="316" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-41" value="&lt;ul&gt;&lt;li&gt;kvm&lt;/li&gt;&lt;li&gt;virt-customize&lt;/li&gt;&lt;li&gt;...&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="580" y="375" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-44" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="570" y="540" width="230" height="130" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-48" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="830" y="277" width="235" height="45" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-45" value="&lt;b&gt;Example resources&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="590" y="550" width="152" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-46" value="&lt;ul&gt;&lt;li&gt;src opnsense images&lt;/li&gt;&lt;li&gt;modified opnsense images&lt;/li&gt;&lt;/ul&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="581" y="600" width="208" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-47" value="&lt;i&gt;&lt;font style=&quot;color: rgb(51, 51, 51);&quot;&gt;Data store (image registry, etc.)&lt;/font&gt;&lt;/i&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="589" y="570" width="200" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-9" value="" style="html=1;rounded=0;direction=south;rotation=90;" parent="1" vertex="1">
<mxGeometry x="1025" y="300.75" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-11" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="1036" y="352.75" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-49" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="1069" y="275" width="251" height="45" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1075" y="300.75" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-14" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="1085" y="352.75" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-50" value="&lt;b&gt;wan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="821.5" y="277" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-51" value="&lt;b&gt;lan&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1055" y="275" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-52" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="1181" y="403.75" width="159" height="116.25" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-53" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-54" target="vj3pG7w7rTpS1tnBMssI-55" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-54" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1211" y="305" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-55" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="1221" y="357" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-58" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1210" y="393.75" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-59" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="vj3pG7w7rTpS1tnBMssI-58" target="vj3pG7w7rTpS1tnBMssI-60" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-60" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="1222.5" y="363.75" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-61" value="&lt;b&gt;other vm&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="1181" y="430" width="90" height="30" as="geometry" />
</mxCell>
<mxCell id="vj3pG7w7rTpS1tnBMssI-63" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="vj3pG7w7rTpS1tnBMssI-24" target="vj3pG7w7rTpS1tnBMssI-44" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="600" y="770" as="sourcePoint" />
<mxPoint x="360" y="885" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-1" value="" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="799" y="84" width="280" height="60" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-2" value="&lt;b&gt;localhost network&lt;/b&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="789" y="84" width="150" height="30" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-6" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="989" y="124" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-7" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-6" target="Za91R7Nqk7Jj5VTRes6Y-8" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-8" value="" style="ellipse;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;" parent="1" vertex="1">
<mxGeometry x="999" y="176" width="8" height="8" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-12" value="" style="html=1;rounded=0;" parent="1" vertex="1">
<mxGeometry x="988" y="217" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-13" value="" style="endArrow=none;html=1;rounded=0;align=center;verticalAlign=top;endFill=0;labelBackgroundColor=none;endSize=2;" parent="1" source="Za91R7Nqk7Jj5VTRes6Y-12" target="Za91R7Nqk7Jj5VTRes6Y-14" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-14" value="" style="shape=requiredInterface;html=1;fontSize=11;align=center;fillColor=none;points=[];aspect=fixed;resizable=0;verticalAlign=bottom;labelPosition=center;verticalLabelPosition=top;flipH=1;rotation=-90;" parent="1" vertex="1">
<mxGeometry x="1000.5" y="187" width="5" height="10" as="geometry" />
</mxCell>
<mxCell id="Za91R7Nqk7Jj5VTRes6Y-15" value="&lt;font color=&quot;#333333&quot;&gt;&lt;i&gt;No modification required&lt;/i&gt;&lt;/font&gt;" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;" parent="1" vertex="1">
<mxGeometry x="805.5" y="98" width="200" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>

View File

@@ -1,13 +0,0 @@
## directory containing harmony-ve data
# HVE_ROOT=~/.harmony-ve
## OPNSENSE SRC
# main mirror
# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
# first alternative mirror
# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
## Network
# HVE_NETWORK_LABEL=harmony

View File

@@ -1,105 +0,0 @@
#! /bin/bash
harmony-ve()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
_short_help(){
cat <<-EOM
NAME
harmony-ve
DESCRIPTION
CLI management toolkit for Harmony Virtualized Execution Environment
SYNOPSYS
harmony-ve [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-ve dependencies # manage localhost depend
harmony-ve network # manage localhost netork
harmony-ve opnsense-img-src # manage opnsense OS source images
harmony-ve opnsense-img # manage opnsense OS images
harmony-ve vm # manage vm
EOM
}
_extra_help(){
cat <<-EOM
DESCRIPTION
Automation CLI to easily provision and manage a Virtualized Execution Environment for testing and learning Harmony.
This tool allows:
- new harmony users to start testing within 15 minutes on their development desktop
- automate virtualized test in pipeline
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
EOM
}
# Implement functions
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
# Commands entrypoints
deps|dependencies)
harmony-ve-dependencies "${@:2}"
;;
net|network)
harmony-ve-network"${@:2}"
;;
img-src|opnsense-img-src)
harmony-ve-opnsense-img-src "${@:2}"
;;
img|opnsense-img)
harmony-ve-opnsense-img "${@:2}"
;;
vm)
harmony-ve-vm "${@:2}"
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve "${@}"

View File

@@ -1,125 +0,0 @@
#! /bin/bash
#
# virt-install <= virtinst
# quemu-img
harmony-ve-dependencies()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
_short_help(){
cat <<-EOM
NAME
harmony-ve-dependencies
DESCRIPTION
Manage localhost dependencies needed for Harmony Virtual Execution Environment
SYNOPSYS
devops-dependencies [GLOBAL_OPTIONS] COMMAND [OPTIONS]
devops check # Check that dependencies are installed
devops install # Install missing dependencies
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
EOM
}
_check_dependencies(){
. "${SCRIPTS_DIR}/dependencies-management"
missing=0
NEED_IP=false
NEED_KVM=false
NEED_VIRT_CUSTOMIZE=false
NEED_WGET=false
is_debian_family || _fatal only debian based version is supported
has_ip || {
missing=$(( missing + 1));
_warn "ip command is missing";
NEED_IP=true
}
has_virsh ||{
missing=$(( missing + 1));
_warn "virsh command is missing";
NEED_KVM=true
}
has_virt_customize || {
missing=$(( missing + 1));
_warn "virt-customize command is missing";
NEED_VIRT_CUSTOMIZE=true
}
has_wget || has_curl || {
missing=$(( missing + 1));
_warn "wget and curl commands are missing"; NEED_WGET=true
}
}
_install_dependencies(){
[ "$NEED_KVM" != "true" ] || install_kvm
[ "$NEED_VIRT_CUSTOMIZE" != "true" ] || install_virt_customize
[ "$NEED_WGET" != "true" ] || install_wget
}
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
cdeps|check-dependencies)
_check_dependencies
if [ "$missing" -gt 0 ]; then
exit 1
fi
_warn No missing dependencies
;;
ideps|install-dependencies)
_check_dependencies
_install_dependencies
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-dependencies "${@}"

View File

@@ -1,232 +0,0 @@
#! /bin/bash
# todo: allow wan to switch from ethernet to wifi
harmony-ve-network()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
_short_help(){
cat <<-EOM
NAME
harmony-ve-network
DESCRIPTION
Modify localhost network for Harmony Virtual Execution Environment
SYNOPSYS
harmony-ve-network [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-ve-network check
harmony-ve-network setup
harmony-ve-network cleanup
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
IMPLEMENTATION NOTES
- use the network manager present to add 2 bridges (wan + lan)
EOM
}
# dependency management
_is_service_used(){
service=$1
sudo systemctl list-unit-files $service || return 1
sudo systemctl status --no-pager $service || return 1
}
# Implement functions
_list_bridges(){
ip -o link show type bridge | awk '{print $2}' | sed 's/://g'
}
_is_a_bridge(){
bridge=$1
matched=$(_list_bridges | grep "$bridge")
[ "$matched" = "$bridge" ] || return 1
}
_bridge_is_up(){
_fatal Not implemented
}
_rename_nmcli_profile(){
device=$1
profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$device:" | cut -d':' -f2)
[ "$profile" != "" ] || _fatal Failed to find nmcli profile
sudo nmcli con mod "$profile" con-name "$device"
}
_create_a_bridge_using_networkmanager(){
bridge=$1
profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$PRIMARY_INTERFACE:" | cut -d':' -f2)
nmcli conn delete "$profile"
nmcli conn add type bridge ifname $bridge con-name $bridge || _fatal Fail to create a bridge using nmcli
nmcli con add type bridge-slave ifname $PRIMARY_INTERFACE master $bridge || _fatal Fail to create a slave-interface using nmcli
nmcli con up $bridge || _fatal Fail to set interface up using nmcli
sudo systemctl restart NetworkManager.service
# todo: use a check loop until connection with a timeout
#sleep 10
#ping nationtech.io | _fatal Internet connection lost
}
_delete_a_bridge_using_networkmanager(){
device=$1
nmcli conn delete bridge-slave-$PRIMARY_INTERFACE
nmcli conn delete $device
nmcli con add type ethernet ifname $PRIMARY_INTERFACE con-name $PRIMARY_INTERFACE autoconnect yes ipv4.method auto ipv6.method ignore
nmcli conn up "$PRIMARY_INTERFACE"
sudo systemctl restart NetworkManager.service
# todo: use a check loop until connection with a timeout
#sleep 10
#ping nationtech.io | _fatal Internet connection lost
}
_create_a_bridge(){
bridge=$1
[ $USE_NETWORKMANAGER = 0 ] | _fatal "Only NetworkManager is implemented"
_create_a_bridge_using_networkmanager $bridge
}
_setup_a_bridge(){
$bridge
bridge_exist=1
bridge_is_up=1
bridge_has_ip=1
bridge_has_route=1
bridge_is_working=1
_is_a_bridge $bridge && bridge_exists=0 || _create_a_bridge $bridge || _fatal Fail to create a bridge
}
_get_networkmanager_profile_from_device(){
device=$1
profile=$(nmcli -t -f DEVICE,NAME c show --active | grep "^$device:" | cut -d':' -f2)
[ "$profile" != "" ] || _fatal Fail to retreive nmcli profile
echo "$profile"
}
_find_primary_interface(){
PRIMARY_INTERFACE=$(ip route | grep '^default' | sed 's/ dev /!/g' | cut -d'!' -f 2 | awk '{ print $1 }' )
[ "$PRIMARY_INTERFACE" != "" ] || _fatal Fail to find the primary interface
}
_find_used_network_manager(){
_is_service_used NetworkManager.service && USE_NETWORKMANAGER=0 || USE_NETWORKMANAGER=1
_is_service_used systemd-networkd.service && USE_SYSTEMD_NETWORKD=0 || USE_SYSTEMD_NETWORKD=1
_is_service_used dhcpd.service && USE_DHCPD=0 || USE_DHCPD=1
USE_MANUAL=0 && [ $USE_NETWORKMANAGER = 0 ] || [ $USE_SYSTEMD_NETWORKD = 0 ] || [ $USE_DHCPD = 0 ] || USE_MANUAL=0
}
_connect(){
_find_used_network_manager
_find_primary_interface
_setup_a_bridge $_HVE_WAN_BRIDGE
_setup_a_bridge $_HVE_LAN_BRIDGE
}
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
connect)
_connect "${@:2}"
;;
disconnect)
_disconnect "${@:2}"
;;
dev)
"${@:2}"
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-network "${@}"

View File

@@ -1,150 +0,0 @@
#! /bin/bash
harmony-ve-opnsense-img()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
export PATH=$SCRIPTS_DIR:$PATH
_short_help(){
cat <<-EOM
NAME
harmony-ve-opnsense-img
DESCRIPTION
Manage opnsense images needed by Harmony Virtual Execution Environment
SYNOPSYS
harmony-vee-opnsense-img [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-ve-opnsense-img list
harmony-ve-opnsense-img init NAME VERSION
harmony-ve-opnsense-img start NAME
harmony-ve-opnsense-img update NAME
harmony-ve-opnsense-img delete [NAME]
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
EOM
}
# assertions
_assert_image_do_not_exists(){
name=$1
[ ! -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "An image '$name' already exists"
}
_assert_image_exists(){
name=$1
[ -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "Image '$name' do not exists"
}
# Implement functions
_init(){
name=$1
version=${2}
_assert_image_do_not_exists $name
mkdir -p "${_HVE_OPNSENSE_IMG}/$name"
harmony-ve opnsense-img-src download $version
sudo qemu-img convert -f raw -O qcow2 "$_HVE_OPNSENSE_SRC_IMG/OPNsense-${version}-nano-amd64.img" "/var/lib/libvirt/images/opnsense-$name.qcow2"
cat <<-EOM > "$_HVE_OPNSENSE_IMG/$name/$name.sh"
virt-install \
--name $name \
--os-variant freebsd14.0 \
--vcpus=2,sockets=1,cores=2,threads=1 \
--memory 4096 \
--disk path="/var/lib/libvirt/images/opnsense-$name.qcow2" \
--network bridge=${_HVE_WAN_BRIDGE},model=virtio \
--network bridge=${_HVE_LAN_BRIDGE},model=virtio \
--graphics none \
--console pty,target_type=serial \
--import \
--autostart
EOM
chmod +x "$_HVE_OPNSENSE_IMG/$name/$name.sh"
}
_start(){
name=$1
_assert_image_exists $name
"$_HVE_OPNSENSE_IMG/$name/$name.sh"
}
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
# Commands entrypoints
init)
_init "${@:2}"
;;
start)
_start "${@:2}"
;;
delete)
rm -r ${_HVE_OPNSENSE_IMG}/"$2"
;;
ls|list)
ls ${_HVE_OPNSENSE_IMG} | cat
;;
show)
ls ${_HVE_OPNSENSE_IMG}/"$2" | cat
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img "${@}"

View File

@@ -1,310 +0,0 @@
#! /bin/bash
harmony-ve-opnsense-img-src()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
_short_help(){
cat <<-EOM
NAME
harmony-ve-opnsense-img-src
DESCRIPTION
Manage opnsense source images needed by Harmony Virtual Execution Environment
SYNOPSYS
harmony-vee-opnsense-img-src [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-vee-opnsense-img-src list [--remote]
harmony-vee-opnsense-img-src download [VERSION]
harmony-vee-opnsense-img-src check [VERSION]
harmony-vee-opnsense-img-src delete [VERSION]
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
DETAILS
- for 'list', show local images available
- for 'list --remote', show available upstream images
- for 'download', when no VERSION is specified, use the latest
- for 'delete', when no VERSION is specified, delete all image
- use the 'nano' flavor
EOM
}
# Implement functions
_parse_version_from_image_file_string(){
#https://pkg.opnsense.org/releases/25.7/OPNsense-25.7-nano-amd64.img.bz2
echo "$1" | cut -d '/' -f 5
}
_list_local_images(){
ls "${_HVE_OPNSENSE_SRC_IMG}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
}
_list_remote_images(){
curl -L -s "${_HVE_OPNSENSE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2 | cut -d '/' -f 1 | sort -r
}
_latest_version(){
_list_remote_images | head -n 1
}
_is_downloaded(){
version=$1
name="${_HVE_OPNSENSE_SRC_IMG}/OPNsense-${version}-nano-amd64.img"
[ -f "$name" ] && return 0 || return 1
}
_is_valid_version(){
version=${1}
matched_version=$(_list_remote_images | grep "$version")
[ "$matched_version" != "" ] && return 0 || return 1
}
_download_img(){
version=$1
_download_crypto_files $version
name="OPNsense-${version}-nano-amd64.img"
compressed_name=$name.bz2
_is_downloaded $version && {
_warn "Image '$name' is already downloaded"
} || {
url=$_HVE_OPNSENSE_URL/$version/$compressed_name
>&2 echo DOWNLOAD $url
wget -q -c "${url}"
_verify_image_checksum $version
>&2 echo DECOMPRESS $url
bzip2 -d $compressed_name
}
}
_compare_files_checksum(){
file1=$1
file2=$2
sha256_1=$(openssl sha256 $file1 | cut -d" " -f2)
sha256_2=$(openssl sha256 $file2 | cut -d" " -f2)
[ "$sha256_1" = "$sha256_2" ] || return 1
}
_download_crypto_files(){
# see: https://docs.opnsense.org/manual/install.html#download-and-verification
version=$1
# download multiple pubkeys from different server
pubkey="OPNsense-${version}.pub"
rm -f $pubkey $pubkey.sig $pubkey.alt1 $pubkey.alt2
url=$_HVE_OPNSENSE_URL/$version/$pubkey
wget -q -c "${url}"
# failing:
wget -q -c "${url}.sig"
rm -f /tmp/file.sig
openssl base64 -d -in $pubkey.sig -out /tmp/file.sig
openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $pubkey || _fatal "Can't verify the signature of the public key"
url_alt1=$_HVE_OPNSENSE_URL_ALT1/$version/$pubkey
wget -q -c -O "$pubkey.alt1" "${url_alt1}"
url_alt2=$_HVE_OPNSENSE_URL_ALT2/$version/$pubkey
wget -q -c -O "$pubkey.alt2" "${url_alt2}"
_compare_files_checksum $pubkey $pubkey.alt1 || _fatal "Fail to compare pubkeys" ;
_compare_files_checksum $pubkey $pubkey.alt2 || _fatal "Fail to compare pubkeys" ;
img_sig="OPNsense-${version}-nano-amd64.img.sig"
sha256_name="OPNsense-${version}-checksums-amd64.sha256"
sha256_sig=$sha256_name.sig
[ ! -f "$img_sig" ] || rm $img_sig
[ ! -f "$sha256_name" ] || rm $sha256_name
[ ! -f "$sha256_sig" ] || rm $sha256_sig
for file in $img_sig $sha256_name $sha256_sig;
do
url=$_HVE_OPNSENSE_URL/$version/$file
wget -q -c "${url}"
done
rm -f /tmp/file.sig
openssl base64 -d -in $sha256_sig -out /tmp/file.sig
openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $sha256_name || _fatal "Can't verify the signature of the checksum file"
}
_download(){
version=${1:-}
[ "${version:-}" != "" ] || _fatal "Must pass a VERSION for downloading"
_is_valid_version $version || _fatal "'$version' is not a valid version number"
_download_img ${version}
}
_verify_image_checksum(){
version=$1
name="OPNsense-${version}-nano-amd64.img.bz2"
sha256_file="OPNsense-${version}-checksums-amd64.sha256"
sha256=$(cat $sha256_file | grep "$name" | cut -d'=' -f 2 | tr -s [:space:])
echo "$sha256 $name" | sha256sum -c || _fatal "Checksum failed for '$name'"
}
_verify_image_signature(){
version=$1
# download multiple pubkeys from different server
pubkey="OPNsense-${version}.pub"
img_name="OPNsense-${version}-nano-amd64.img"
img_sig="${img_name}.sig"
rm -f /tmp/file.sig
openssl base64 -d -in $img_sig -out /tmp/file.sig
openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $img_name || _fatal "Can't verify image signature"
}
_check(){
version=${1:-}
if [ "${version:-}" = "" ] ; then
for version in $(_list_local_images);
do
>&2 echo check $version
_download_crypto_files $version
_verify_image_signature $version
done
else
_download_crypto_files $version
_verify_image_signature $version
fi
}
_delete(){
version=${1:-}
if [ -z "${version:-1}" ]; then
_clear
rm -f *.img
else
rm -f *$version*.img
fi
}
_clear(){
rm -f *.pub *.sig *.bz2 *.alt1 *.alt2 *.sha256
}
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
ls|list)
if [ "${2:-}" == "" ]; then
_list_local_images
elif [ "${2:-}" == "--remote" ]; then
_list_remote_images
else
_warn "Unknown option '$2'"
fi
;;
download)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_download "${2:-"$(_latest_version)"}"
popd
;;
delete)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_delete "${@:2}"
popd
;;
check)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_check "${@:2}"
popd
;;
show)
ls $_HVE_OPNSENSE_SRC_IMG | cat
;;
clear)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_clear "${@:2}"
popd
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img-src "${@}"

View File

@@ -1,77 +0,0 @@
#! /bin/bash
harmony-ve-vm()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
_short_help(){
cat <<-EOM
NAME
harmony-ve-mv
DESCRIPTION
Manage virtalized hosts (VM) dependencies by Harmony Virtual Execution Environment
SYNOPSYS
harmony-ve-vm [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-ve-vm list
harmony-ve-vm create
harmony-ve-vm start
harmony-ve-vm stop
harmony-ve-vm login
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
EOM
}
# Implement functions
case "${1:-}" in
"")
_short_help
;;
-h|--help)
_short_help
_extra_help
;;
# Commands entrypoints
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-vm "${@}"

View File

@@ -1,75 +0,0 @@
#! /bin/bash
learn-harmony()(
set -eu
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
export PATH=$SCRIPTS_DIR:$PATH
_short_help(){
cat <<-EOM
NAME
learn-harmony -- Harmony Learning Tool prototype
($(basename ${BASE_DIR}) example)
SYNOPSYS
learn-harmony [GLOBAL_OPTIONS] COMMAND [OPTIONS]
learn-harmony list # List learning steps
learn-harmony show STEP # Show instruction of step STEP
learn-harmony check [STEP] # Verify that your ready to begin step STEP+1
EOM
}
_extra_help(){
cat <<-EOM
GLOBAL_OPTIONS
-d Debug mode.
WARNINGS
This script is experimetal. Use with caution.
EOM
}
case "${1:-}" in
-h|--help|"")
_short_help
_extra_help
;;
ls|list)
echo "not implemented"
;;
show)
echo "not implemented"
;;
verify)
echo "not implemented"
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;
esac
)
[ "$0" != "${BASH_SOURCE}" ] || learn-harmony "${@}"

View File

@@ -1,4 +0,0 @@
#! /bin/bash
export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")"):"${PATH}"

View File

@@ -1,70 +1,133 @@
use harmony::{
config::secret::OPNSenseFirewallCredentials,
infra::opnsense::OPNSenseFirewall,
inventory::Inventory,
modules::{dhcp::DhcpScore, opnsense::OPNsenseShellCommandScore},
topology::LogicalHost,
use std::{
net::{IpAddr, Ipv4Addr},
sync::Arc,
};
use harmony_macros::{ip, ipv4};
use brocade::BrocadeOptions;
use cidr::Ipv4Cidr;
use harmony::{
hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
inventory::Inventory,
modules::{
dummy::{ErrorScore, PanicScore, SuccessScore},
http::StaticFilesHttpScore,
okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
opnsense::OPNsenseShellCommandScore,
tftp::TftpScore,
},
topology::{LogicalHost, UnmanagedRouter},
};
use harmony_macros::{ip, mac_address};
use harmony_secret::{Secret, SecretManager};
use harmony_types::net::Url;
use serde::{Deserialize, Serialize};
#[tokio::main]
async fn main() {
let firewall = LogicalHost {
ip: ip!("192.168.1.1"),
let firewall = harmony::topology::LogicalHost {
ip: ip!("192.168.5.229"),
name: String::from("opnsense-1"),
};
let opnsense_auth = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>()
let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
.await
.expect("Failed to get credentials");
let opnsense = OPNSenseFirewall::new(
firewall,
None,
&opnsense_auth.username,
&opnsense_auth.password,
let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
let brocade_options = Some(BrocadeOptions {
dry_run: *harmony::config::DRY_RUN,
..Default::default()
});
let switch_client = BrocadeSwitchClient::init(
&switches,
&switch_auth.username,
&switch_auth.password,
brocade_options,
)
.await;
.await
.expect("Failed to connect to switch");
let dhcp_score = DhcpScore {
dhcp_range: (
ipv4!("192.168.1.100").into(),
ipv4!("192.168.1.150").into(),
),
host_binding: vec![],
next_server: None,
boot_filename: None,
filename: None,
filename64: None,
filenameipxe: Some("filename.ipxe".to_string()),
domain: None,
let switch_client = Arc::new(switch_client);
let opnsense = Arc::new(
harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
);
let lan_subnet = Ipv4Addr::new(10, 100, 8, 0);
let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
let gateway_ip = IpAddr::V4(gateway_ipv4);
let topology = harmony::topology::HAClusterTopology {
domain_name: "demo.harmony.mcd".to_string(),
router: Arc::new(UnmanagedRouter::new(
gateway_ip,
Ipv4Cidr::new(lan_subnet, 24).unwrap(),
)),
load_balancer: opnsense.clone(),
firewall: opnsense.clone(),
tftp_server: opnsense.clone(),
http_server: opnsense.clone(),
dhcp_server: opnsense.clone(),
dns_server: opnsense.clone(),
control_plane: vec![LogicalHost {
ip: ip!("10.100.8.20"),
name: "cp0".to_string(),
}],
bootstrap_host: LogicalHost {
ip: ip!("10.100.8.20"),
name: "cp0".to_string(),
},
workers: vec![],
switch_client: switch_client.clone(),
};
// let dns_score = OKDDnsScore::new(&topology);
// let load_balancer_score = OKDLoadBalancerScore::new(&topology);
//
// let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
// let http_score = StaticFilesHttpScore {
// folder_to_serve: Some(Url::LocalFolder(
// "./data/watchguard/pxe-http-files".to_string(),
// )),
// files: vec![],
// remote_path: None,
// };
let opnsense_config = opnsense.get_opnsense_config();
harmony_cli::run(
Inventory::autoload(),
opnsense,
vec![
Box::new(dhcp_score),
Box::new(OPNsenseShellCommandScore {
opnsense: opnsense_config,
command: "touch /tmp/helloharmonytouching_2".to_string(),
}),
let inventory = Inventory {
location: Location::new(
"232 des Éperviers, Wendake, Qc, G0A 4V0".to_string(),
"wk".to_string(),
),
switch: SwitchGroup::from([]),
firewall_mgmt: Box::new(OPNSenseManagementInterface::new()),
storage_host: vec![],
worker_host: vec![],
control_plane_host: vec![
PhysicalHost::empty(HostCategory::Server)
.mac_address(mac_address!("08:00:27:62:EC:C3")),
],
};
// TODO regroup smaller scores in a larger one such as this
// let okd_boostrap_preparation();
let dhcp_score = OKDDhcpScore::new(&topology, &inventory);
let dns_score = OKDDnsScore::new(&topology);
let load_balancer_score = OKDLoadBalancerScore::new(&topology);
let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
let http_score = StaticFilesHttpScore {
folder_to_serve: Some(Url::LocalFolder(
"./data/watchguard/pxe-http-files".to_string(),
)),
files: vec![],
remote_path: None,
};
harmony_tui::run(
inventory,
topology,
vec![
Box::new(dns_score),
Box::new(dhcp_score),
Box::new(load_balancer_score),
Box::new(tftp_score),
Box::new(http_score),
Box::new(OPNsenseShellCommandScore {
opnsense: opnsense.get_opnsense_config(),
command: "touch /tmp/helloharmonytouching".to_string(),
}),
Box::new(SuccessScore {}),
Box::new(ErrorScore {}),
Box::new(PanicScore {}),
],
None,
)
.await
.unwrap();

View File

@@ -16,6 +16,7 @@ use harmony_types::net::Url;
async fn main() {
let application = Arc::new(RustWebapp {
name: "test-rhob-monitoring".to_string(),
dns: "test-rhob-monitoring.harmony.mcd".to_string(),
project_root: PathBuf::from("./webapp"), // Relative from 'harmony-path' param
framework: Some(RustWebFramework::Leptos),
service_port: 3000,

View File

@@ -19,6 +19,7 @@ use harmony_macros::hurl;
async fn main() {
let application = Arc::new(RustWebapp {
name: "harmony-example-rust-webapp".to_string(),
dns: "harmony-example-rust-webapp.harmony.mcd".to_string(),
project_root: PathBuf::from("./webapp"),
framework: Some(RustWebFramework::Leptos),
service_port: 3000,

View File

@@ -2,12 +2,11 @@ use harmony::{
inventory::Inventory,
modules::{
application::{
ApplicationScore, RustWebFramework, RustWebapp,
features::{PackagingDeployment, rhob_monitoring::Monitoring},
features::{rhob_monitoring::Monitoring, PackagingDeployment}, ApplicationScore, RustWebFramework, RustWebapp
},
monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
},
topology::K8sAnywhereTopology,
topology::{K8sAnywhereTopology, LocalhostTopology},
};
use harmony_macros::hurl;
use std::{path::PathBuf, sync::Arc};
@@ -22,8 +21,8 @@ async fn main() {
});
let discord_webhook = DiscordWebhook {
name: "harmony_demo".to_string(),
url: hurl!("http://not_a_url.com"),
name: "harmony-demo".to_string(),
url: hurl!("https://discord.com/api/webhooks/1415391405681021050/V6KzV41vQ7yvbn7BchejRu9C8OANxy0i2ESZOz2nvCxG8xAY3-2i3s5MS38k568JKTzH"),
};
let app = ApplicationScore {

View File

@@ -3,7 +3,7 @@ use harmony::{
modules::{
application::{
ApplicationScore, RustWebFramework, RustWebapp,
features::{Monitoring, PackagingDeployment},
features::{PackagingDeployment, rhob_monitoring::Monitoring},
},
monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
},
@@ -16,6 +16,7 @@ use std::{path::PathBuf, sync::Arc};
async fn main() {
let application = Arc::new(RustWebapp {
name: "harmony-example-tryrust".to_string(),
dns: "tryrust.example.harmony.mcd".to_string(),
project_root: PathBuf::from("./tryrust.org"), // <== Project root, in this case it is a
// submodule
framework: Some(RustWebFramework::Leptos),

View File

@@ -67,16 +67,16 @@ impl<T: Topology> Maestro<T> {
}
}
fn is_topology_initialized(&self) -> bool {
self.topology_state.status == TopologyStatus::Success
|| self.topology_state.status == TopologyStatus::Noop
}
pub fn register_all(&mut self, mut scores: ScoreVec<T>) {
let mut score_mut = self.scores.write().expect("Should acquire lock");
score_mut.append(&mut scores);
}
fn is_topology_initialized(&self) -> bool {
self.topology_state.status == TopologyStatus::Success
|| self.topology_state.status == TopologyStatus::Noop
}
pub async fn interpret(&self, score: Box<dyn Score<T>>) -> Result<Outcome, InterpretError> {
if !self.is_topology_initialized() {
warn!(

View File

@@ -4,16 +4,19 @@ use harmony_types::{
net::{MacAddress, Url},
switch::PortLocation,
};
use k8s_openapi::api::core::v1::Namespace;
use kube::api::ObjectMeta;
use log::debug;
use log::info;
use crate::modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy};
use crate::topology::PxeOptions;
use crate::{data::FileContent, modules::okd::crd::nmstate::NMState};
use crate::{
executors::ExecutorError, modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec,
use crate::data::FileContent;
use crate::executors::ExecutorError;
use crate::hardware::PhysicalHost;
use crate::modules::okd::crd::{
InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
};
use crate::topology::PxeOptions;
use super::{
DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
@@ -39,7 +42,6 @@ pub struct HAClusterTopology {
pub bootstrap_host: LogicalHost,
pub control_plane: Vec<LogicalHost>,
pub workers: Vec<LogicalHost>,
pub kubeconfig: Option<String>,
}
#[async_trait]
@@ -58,17 +60,9 @@ impl Topology for HAClusterTopology {
#[async_trait]
impl K8sclient for HAClusterTopology {
async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
match &self.kubeconfig {
None => Ok(Arc::new(
K8sClient::try_default().await.map_err(|e| e.to_string())?,
)),
Some(kubeconfig) => {
let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else {
return Err("Failed to create k8s client".to_string());
};
Ok(Arc::new(client))
}
}
Ok(Arc::new(
K8sClient::try_default().await.map_err(|e| e.to_string())?,
))
}
}
@@ -94,47 +88,59 @@ impl HAClusterTopology {
}
async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
// FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate)
debug!("Installing NMState operator...");
let k8s_client = self.k8s_client().await?;
debug!("Installing NMState controller...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
debug!("Creating NMState namespace...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
debug!("Creating NMState service account...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
debug!("Creating NMState role...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
debug!("Creating NMState role binding...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
debug!("Creating NMState operator...");
k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml
").unwrap(), Some("nmstate"))
.await
.map_err(|e| e.to_string())?;
let nmstate_namespace = Namespace {
metadata: ObjectMeta {
name: Some("openshift-nmstate".to_string()),
finalizers: Some(vec!["kubernetes".to_string()]),
..Default::default()
},
..Default::default()
};
debug!("Creating NMState namespace: {nmstate_namespace:#?}");
k8s_client
.wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None)
.await?;
.apply(&nmstate_namespace, None)
.await
.map_err(|e| e.to_string())?;
let nmstate_operator_group = OperatorGroup {
metadata: ObjectMeta {
name: Some("openshift-nmstate".to_string()),
namespace: Some("openshift-nmstate".to_string()),
..Default::default()
},
spec: OperatorGroupSpec {
target_namespaces: vec!["openshift-nmstate".to_string()],
},
};
debug!("Creating NMState operator group: {nmstate_operator_group:#?}");
k8s_client
.apply(&nmstate_operator_group, None)
.await
.map_err(|e| e.to_string())?;
let nmstate_subscription = Subscription {
metadata: ObjectMeta {
name: Some("kubernetes-nmstate-operator".to_string()),
namespace: Some("openshift-nmstate".to_string()),
..Default::default()
},
spec: SubscriptionSpec {
channel: Some("stable".to_string()),
install_plan_approval: Some(InstallPlanApproval::Automatic),
name: "kubernetes-nmstate-operator".to_string(),
source: "redhat-operators".to_string(),
source_namespace: "openshift-marketplace".to_string(),
},
};
debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}");
k8s_client
.apply(&nmstate_subscription, None)
.await
.map_err(|e| e.to_string())?;
let nmstate = NMState {
metadata: ObjectMeta {
@@ -156,7 +162,11 @@ impl HAClusterTopology {
42 // FIXME: Find a better way to declare the bond id
}
async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
async fn configure_bond(
&self,
host: &PhysicalHost,
config: &HostNetworkConfig,
) -> Result<(), SwitchError> {
self.ensure_nmstate_operator_installed()
.await
.map_err(|e| {
@@ -165,33 +175,29 @@ impl HAClusterTopology {
))
})?;
let bond_config = self.create_bond_configuration(config);
debug!(
"Applying NMState bond config for host {}: {bond_config:#?}",
config.host_id
);
let bond_config = self.create_bond_configuration(host, config);
debug!("Configuring bond for host {host:?}: {bond_config:#?}");
self.k8s_client()
.await
.unwrap()
.apply(&bond_config, None)
.await
.map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?;
.unwrap();
Ok(())
todo!()
}
fn create_bond_configuration(
&self,
host: &PhysicalHost,
config: &HostNetworkConfig,
) -> NodeNetworkConfigurationPolicy {
let host_name = &config.host_id;
let host_name = host.id.clone();
let bond_id = self.get_next_bond_id();
let bond_name = format!("bond{bond_id}");
info!("Configuring bond '{bond_name}' for host '{host_name}'...");
let mut bond_mtu: Option<u32> = None;
let mut copy_mac_from: Option<String> = None;
let mut bond_mac_address: Option<String> = None;
let mut bond_ports = Vec::new();
let mut interfaces: Vec<nmstate::InterfaceSpec> = Vec::new();
@@ -217,14 +223,14 @@ impl HAClusterTopology {
..Default::default()
});
bond_ports.push(interface_name.clone());
bond_ports.push(interface_name);
// Use the first port's details for the bond mtu and mac address
if bond_mtu.is_none() {
bond_mtu = Some(switch_port.interface.mtu);
}
if copy_mac_from.is_none() {
copy_mac_from = Some(interface_name);
if bond_mac_address.is_none() {
bond_mac_address = Some(switch_port.interface.mac_address.to_string());
}
}
@@ -233,7 +239,8 @@ impl HAClusterTopology {
description: Some(format!("Network bond for host {host_name}")),
r#type: "bond".to_string(),
state: "up".to_string(),
copy_mac_from,
mtu: bond_mtu,
mac_address: bond_mac_address,
ipv4: Some(nmstate::IpStackSpec {
dhcp: Some(true),
enabled: Some(true),
@@ -268,12 +275,16 @@ impl HAClusterTopology {
}
}
async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
async fn configure_port_channel(
&self,
host: &PhysicalHost,
config: &HostNetworkConfig,
) -> Result<(), SwitchError> {
debug!("Configuring port channel: {config:#?}");
let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
self.switch_client
.configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports)
.configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
.await
.map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
@@ -288,7 +299,6 @@ impl HAClusterTopology {
};
Self {
kubeconfig: None,
domain_name: "DummyTopology".to_string(),
router: dummy_infra.clone(),
load_balancer: dummy_infra.clone(),
@@ -470,9 +480,13 @@ impl Switch for HAClusterTopology {
Ok(port)
}
async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
self.configure_bond(config).await?;
self.configure_port_channel(config).await
async fn configure_host_network(
&self,
host: &PhysicalHost,
config: HostNetworkConfig,
) -> Result<(), SwitchError> {
self.configure_bond(host, &config).await?;
self.configure_port_channel(host, &config).await
}
}

View File

@@ -1,12 +1,10 @@
use std::time::Duration;
use std::{collections::HashMap, time::Duration};
use derive_new::new;
use k8s_openapi::{
ClusterResourceScope, NamespaceResourceScope,
api::{
apps::v1::Deployment,
core::v1::{Pod, ServiceAccount},
},
api::{apps::v1::Deployment, core::v1::Pod},
apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition,
apimachinery::pkg::version::Info,
};
use kube::{
@@ -14,7 +12,6 @@ use kube::{
api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
config::{KubeConfigOptions, Kubeconfig},
core::ErrorResponse,
discovery::{ApiCapabilities, Scope},
error::DiscoveryError,
runtime::reflector::Lookup,
};
@@ -23,12 +20,11 @@ use kube::{
api::{ApiResource, GroupVersionKind},
runtime::wait::await_condition,
};
use log::{debug, error, info, trace, warn};
use log::{debug, error, info, trace};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::json;
use serde_json::{json, Value};
use similar::TextDiff;
use tokio::{io::AsyncReadExt, time::sleep};
use url::Url;
#[derive(new, Clone)]
pub struct K8sClient {
@@ -62,9 +58,146 @@ impl K8sClient {
})
}
pub async fn service_account_api(&self, namespace: &str) -> Api<ServiceAccount> {
let api: Api<ServiceAccount> = Api::namespaced(self.client.clone(), namespace);
api
// Returns true if any deployment in the given namespace matching the label selector
// has status.availableReplicas > 0 (or condition Available=True).
pub async fn has_healthy_deployment_with_label(
&self,
namespace: &str,
label_selector: &str,
) -> Result<bool, Error> {
let api: Api<Deployment> = Api::namespaced(self.client.clone(), namespace);
let lp = ListParams::default().labels(label_selector);
let list = api.list(&lp).await?;
for d in list.items {
// Check AvailableReplicas > 0 or Available condition
let available = d
.status
.as_ref()
.and_then(|s| s.available_replicas)
.unwrap_or(0);
if available > 0 {
return Ok(true);
}
// Fallback: scan conditions
if let Some(conds) = d.status.as_ref().and_then(|s| s.conditions.as_ref()) {
if conds.iter().any(|c| {
c.type_ == "Available"
&& c.status == "True"
}) {
return Ok(true);
}
}
}
Ok(false)
}
// Cluster-wide: returns namespaces that have at least one healthy deployment
// matching the label selector (equivalent to kubectl -A -l ...).
pub async fn list_namespaces_with_healthy_deployments(
&self,
label_selector: &str,
) -> Result<Vec<String>, Error> {
let api: Api<Deployment> = Api::all(self.client.clone());
let lp = ListParams::default().labels(label_selector);
let list = api.list(&lp).await?;
let mut healthy_ns: HashMap<String, bool> = HashMap::new();
for d in list.items {
let ns = match d.metadata.namespace.clone() {
Some(n) => n,
None => continue,
};
let available = d
.status
.as_ref()
.and_then(|s| s.available_replicas)
.unwrap_or(0);
let is_healthy = if available > 0 {
true
} else {
d.status
.as_ref()
.and_then(|s| s.conditions.as_ref())
.map(|conds| {
conds.iter().any(|c| {
c.type_ == "Available"
&& c.status == "True"
})
})
.unwrap_or(false)
};
if is_healthy {
healthy_ns.insert(ns, true);
}
}
Ok(healthy_ns.into_keys().collect())
}
// Get the application-controller ServiceAccount name (fallback to default)
pub async fn get_argocd_controller_sa_name(&self, ns: &str) -> Result<String, Error> {
let api: Api<Deployment> = Api::namespaced(self.client.clone(), ns);
let lp = ListParams::default().labels("app.kubernetes.io/component=controller");
let list = api.list(&lp).await?;
if let Some(dep) = list.items.get(0) {
if let Some(sa) = dep
.spec
.as_ref()
.and_then(|ds| ds.template.spec.as_ref())
.and_then(|ps| ps.service_account_name.clone())
{
return Ok(sa);
}
}
Ok("argocd-application-controller".to_string())
}
// List ClusterRoleBindings dynamically and return as JSON values
pub async fn list_clusterrolebindings_json(&self) -> Result<Vec<Value>, Error> {
let gvk = kube::api::GroupVersionKind::gvk(
"rbac.authorization.k8s.io",
"v1",
"ClusterRoleBinding",
);
let ar = kube::api::ApiResource::from_gvk(&gvk);
let api: Api<kube::api::DynamicObject> = Api::all_with(self.client.clone(), &ar);
let crbs = api.list(&ListParams::default()).await?;
let mut out = Vec::new();
for o in crbs {
let v = serde_json::to_value(&o).unwrap_or(Value::Null);
out.push(v);
}
Ok(out)
}
// Determine if Argo controller in ns has cluster-wide permissions via CRBs
// TODO This does not belong in the generic k8s client, should be refactored at some point
pub async fn is_argocd_cluster_wide(&self, ns: &str) -> Result<bool, Error> {
let sa = self.get_argocd_controller_sa_name(ns).await?;
let crbs = self.list_clusterrolebindings_json().await?;
let sa_user = format!("system:serviceaccount:{}:{}", ns, sa);
for crb in crbs {
if let Some(subjects) = crb.get("subjects").and_then(|s| s.as_array()) {
for subj in subjects {
let kind = subj.get("kind").and_then(|v| v.as_str()).unwrap_or("");
let name = subj.get("name").and_then(|v| v.as_str()).unwrap_or("");
let subj_ns = subj.get("namespace").and_then(|v| v.as_str()).unwrap_or("");
if (kind == "ServiceAccount" && name == sa && subj_ns == ns)
|| (kind == "User" && name == sa_user)
{
return Ok(true);
}
}
}
}
Ok(false)
}
pub async fn has_crd(&self, name: &str) -> Result<bool, Error> {
let api: Api<CustomResourceDefinition> = Api::all(self.client.clone());
let lp = ListParams::default().fields(&format!("metadata.name={}", name));
let crds = api.list(&lp).await?;
Ok(!crds.items.is_empty())
}
pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
@@ -90,8 +223,7 @@ impl K8sClient {
} else {
Api::default_namespaced_with(self.client.clone(), &gvk)
};
resource.get(name).await
Ok(resource.get(name).await?)
}
pub async fn get_deployment(
@@ -106,9 +238,8 @@ impl K8sClient {
debug!("getting default namespace deployment");
Api::default_namespaced(self.client.clone())
};
debug!("getting deployment {} in ns {}", name, namespace.unwrap());
deps.get_opt(name).await
Ok(deps.get_opt(name).await?)
}
pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result<Option<Pod>, Error> {
@@ -117,8 +248,7 @@ impl K8sClient {
} else {
Api::default_namespaced(self.client.clone())
};
pods.get_opt(name).await
Ok(pods.get_opt(name).await?)
}
pub async fn scale_deployment(
@@ -161,9 +291,9 @@ impl K8sClient {
pub async fn wait_until_deployment_ready(
&self,
name: &str,
name: String,
namespace: Option<&str>,
timeout: Option<Duration>,
timeout: Option<u64>,
) -> Result<(), String> {
let api: Api<Deployment>;
@@ -173,9 +303,9 @@ impl K8sClient {
api = Api::default_namespaced(self.client.clone());
}
let establish = await_condition(api, name, conditions::is_deployment_completed());
let timeout = timeout.unwrap_or(Duration::from_secs(120));
let res = tokio::time::timeout(timeout, establish).await;
let establish = await_condition(api, name.as_str(), conditions::is_deployment_completed());
let t = timeout.unwrap_or(300);
let res = tokio::time::timeout(std::time::Duration::from_secs(t), establish).await;
if res.is_ok() {
Ok(())
@@ -265,7 +395,7 @@ impl K8sClient {
if let Some(s) = status.status {
let mut stdout_buf = String::new();
if let Some(mut stdout) = process.stdout() {
if let Some(mut stdout) = process.stdout().take() {
stdout
.read_to_string(&mut stdout_buf)
.await
@@ -371,14 +501,14 @@ impl K8sClient {
Ok(current) => {
trace!("Received current value {current:#?}");
// The resource exists, so we calculate and display a diff.
println!("\nPerforming dry-run for resource: '{name}'");
println!("\nPerforming dry-run for resource: '{}'", name);
let mut current_yaml = serde_yaml::to_value(&current).unwrap_or_else(|_| {
panic!("Could not serialize current value : {current:#?}")
});
if current_yaml.is_mapping() && current_yaml.get("status").is_some() {
let map = current_yaml.as_mapping_mut().unwrap();
let removed = map.remove_entry("status");
trace!("Removed status {removed:?}");
trace!("Removed status {:?}", removed);
} else {
trace!(
"Did not find status entry for current object {}/{}",
@@ -407,14 +537,14 @@ impl K8sClient {
similar::ChangeTag::Insert => "+",
similar::ChangeTag::Equal => " ",
};
print!("{sign}{change}");
print!("{}{}", sign, change);
}
// In a dry run, we return the new resource state that would have been applied.
Ok(resource.clone())
}
Err(Error::Api(ErrorResponse { code: 404, .. })) => {
// The resource does not exist, so the "diff" is the entire new resource.
println!("\nPerforming dry-run for new resource: '{name}'");
println!("\nPerforming dry-run for new resource: '{}'", name);
println!(
"Resource does not exist. It would be created with the following content:"
);
@@ -423,14 +553,14 @@ impl K8sClient {
// Print each line of the new resource with a '+' prefix.
for line in new_yaml.lines() {
println!("+{line}");
println!("+{}", line);
}
// In a dry run, we return the new resource state that would have been created.
Ok(resource.clone())
}
Err(e) => {
// Another API error occurred.
error!("Failed to get resource '{name}': {e}");
error!("Failed to get resource '{}': {}", name, e);
Err(e)
}
}
@@ -445,7 +575,7 @@ impl K8sClient {
where
K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize,
<K as Resource>::Scope: ApplyStrategy<K>,
<K as Resource>::DynamicType: Default,
<K as kube::Resource>::DynamicType: Default,
{
let mut result = Vec::new();
for r in resource.iter() {
@@ -510,7 +640,10 @@ impl K8sClient {
// 6. Apply the object to the cluster using Server-Side Apply.
// This will create the resource if it doesn't exist, or update it if it does.
println!("Applying '{name}' in namespace '{namespace}'...",);
println!(
"Applying Argo Application '{}' in namespace '{}'...",
name, namespace
);
let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name
let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?;
@@ -519,51 +652,6 @@ impl K8sClient {
Ok(())
}
/// Apply a resource from a URL
///
/// It is the equivalent of `kubectl apply -f <url>`
pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> {
let patch_params = PatchParams::apply("harmony");
let discovery = kube::Discovery::new(self.client.clone()).run().await?;
let yaml = reqwest::get(url)
.await
.expect("Could not get URL")
.text()
.await
.expect("Could not get content from URL");
for doc in multidoc_deserialize(&yaml).expect("failed to parse YAML from file") {
let obj: DynamicObject =
serde_yaml::from_value(doc).expect("cannot apply without valid YAML");
let namespace = obj.metadata.namespace.as_deref().or(ns);
let type_meta = obj
.types
.as_ref()
.expect("cannot apply object without valid TypeMeta");
let gvk = GroupVersionKind::try_from(type_meta)
.expect("cannot apply object without valid GroupVersionKind");
let name = obj.name_any();
if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) {
let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false);
trace!(
"Applying {}: \n{}",
gvk.kind,
serde_yaml::to_string(&obj).expect("Failed to serialize YAML")
);
let data: serde_json::Value =
serde_json::to_value(&obj).expect("Failed to serialize JSON");
let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?;
debug!("applied {} {}", gvk.kind, name);
} else {
warn!("Cannot apply document for unknown {gvk:?}");
}
}
Ok(())
}
pub(crate) async fn from_kubeconfig(path: &str) -> Option<K8sClient> {
let k = match Kubeconfig::read_from(path) {
Ok(k) => k,
@@ -583,31 +671,6 @@ impl K8sClient {
}
}
fn get_dynamic_api(
resource: ApiResource,
capabilities: ApiCapabilities,
client: Client,
ns: Option<&str>,
all: bool,
) -> Api<DynamicObject> {
if capabilities.scope == Scope::Cluster || all {
Api::all_with(client, &resource)
} else if let Some(namespace) = ns {
Api::namespaced_with(client, namespace, &resource)
} else {
Api::default_namespaced_with(client, &resource)
}
}
fn multidoc_deserialize(data: &str) -> Result<Vec<serde_yaml::Value>, serde_yaml::Error> {
use serde::Deserialize;
let mut docs = vec![];
for de in serde_yaml::Deserializer::from_str(data) {
docs.push(serde_yaml::Value::deserialize(de)?);
}
Ok(docs)
}
pub trait ApplyStrategy<K: Resource> {
fn get_api(client: &Client, ns: Option<&str>) -> Api<K>;
}

View File

@@ -1,13 +1,8 @@
use std::{collections::BTreeMap, process::Command, sync::Arc, time::Duration};
use std::{process::Command, sync::Arc};
use async_trait::async_trait;
use base64::{Engine, engine::general_purpose};
use k8s_openapi::api::{
core::v1::Secret,
rbac::v1::{ClusterRoleBinding, RoleRef, Subject},
};
use kube::api::{DynamicObject, GroupVersionKind, ObjectMeta};
use log::{debug, info, warn};
use kube::api::GroupVersionKind;
use log::{debug, info, trace, warn};
use serde::Serialize;
use tokio::sync::OnceCell;
@@ -17,26 +12,14 @@ use crate::{
inventory::Inventory,
modules::{
k3d::K3DInstallationScore,
k8s::ingress::{K8sIngressScore, PathType},
monitoring::{
grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score},
kube_prometheus::crd::{
crd_alertmanager_config::CRDPrometheus,
crd_grafana::{
Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard,
GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource,
GrafanaDatasourceConfig, GrafanaDatasourceJsonData,
GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec,
},
crd_prometheuses::LabelSelector,
prometheus_operator::prometheus_operator_helm_chart_score,
rhob_alertmanager_config::RHOBObservability,
service_monitor::ServiceMonitor,
},
monitoring::kube_prometheus::crd::{
crd_alertmanager_config::CRDPrometheus,
prometheus_operator::prometheus_operator_helm_chart_score,
rhob_alertmanager_config::RHOBObservability,
},
prometheus::{
k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
prometheus::PrometheusMonitoring, rhob_alerting_score::RHOBAlertingScore,
prometheus::PrometheusApplicationMonitoring, rhob_alerting_score::RHOBAlertingScore,
},
},
score::Score,
@@ -88,6 +71,7 @@ pub struct K8sAnywhereTopology {
#[async_trait]
impl K8sclient for K8sAnywhereTopology {
async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
trace!("getting k8s client");
let state = match self.k8s_state.get() {
Some(state) => state,
None => return Err("K8s state not initialized yet".to_string()),
@@ -103,172 +87,41 @@ impl K8sclient for K8sAnywhereTopology {
}
#[async_trait]
impl Grafana for K8sAnywhereTopology {
async fn ensure_grafana_operator(
&self,
inventory: &Inventory,
) -> Result<PreparationOutcome, PreparationError> {
debug!("ensure grafana operator");
let client = self.k8s_client().await.unwrap();
let grafana_gvk = GroupVersionKind {
group: "grafana.integreatly.org".to_string(),
version: "v1beta1".to_string(),
kind: "Grafana".to_string(),
};
let name = "grafanas.grafana.integreatly.org";
let ns = "grafana";
let grafana_crd = client
.get_resource_json_value(name, Some(ns), &grafana_gvk)
.await;
match grafana_crd {
Ok(_) => {
return Ok(PreparationOutcome::Success {
details: "Found grafana CRDs in cluster".to_string(),
});
}
Err(_) => {
return self
.install_grafana_operator(inventory, Some("grafana"))
.await;
}
};
}
async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError> {
let ns = "grafana";
let mut label = BTreeMap::new();
label.insert("dashboards".to_string(), "grafana".to_string());
let label_selector = LabelSelector {
match_labels: label.clone(),
match_expressions: vec![],
};
let client = self.k8s_client().await?;
let grafana = self.build_grafana(ns, &label);
client.apply(&grafana, Some(ns)).await?;
//TODO change this to a ensure ready or something better than just a timeout
client
.wait_until_deployment_ready(
"grafana-grafana-deployment",
Some("grafana"),
Some(Duration::from_secs(30)),
)
.await?;
let sa_name = "grafana-grafana-sa";
let token_secret_name = "grafana-sa-token-secret";
let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns);
client.apply(&sa_token_secret, Some(ns)).await?;
let secret_gvk = GroupVersionKind {
group: "".to_string(),
version: "v1".to_string(),
kind: "Secret".to_string(),
};
let secret = client
.get_resource_json_value(token_secret_name, Some(ns), &secret_gvk)
.await?;
let token = format!(
"Bearer {}",
self.extract_and_normalize_token(&secret).unwrap()
);
debug!("creating grafana clusterrole binding");
let clusterrolebinding =
self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns);
client.apply(&clusterrolebinding, Some(ns)).await?;
debug!("creating grafana datasource crd");
let thanos_url = format!(
"https://{}",
self.get_domain("thanos-querier-openshift-monitoring")
.await
.unwrap()
);
let thanos_openshift_datasource = self.build_grafana_datasource(
"thanos-openshift-monitoring",
ns,
&label_selector,
&thanos_url,
&token,
);
client.apply(&thanos_openshift_datasource, Some(ns)).await?;
debug!("creating grafana dashboard crd");
let dashboard = self.build_grafana_dashboard(ns, &label_selector);
client.apply(&dashboard, Some(ns)).await?;
debug!("creating grafana ingress");
let grafana_ingress = self.build_grafana_ingress(ns).await;
grafana_ingress
.interpret(&Inventory::empty(), self)
.await
.map_err(|e| PreparationError::new(e.to_string()))?;
Ok(PreparationOutcome::Success {
details: "Installed grafana composants".to_string(),
})
}
}
#[async_trait]
impl PrometheusMonitoring<CRDPrometheus> for K8sAnywhereTopology {
impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
async fn install_prometheus(
&self,
sender: &CRDPrometheus,
_inventory: &Inventory,
_receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
) -> Result<PreparationOutcome, PreparationError> {
let client = self.k8s_client().await?;
for monitor in sender.service_monitor.iter() {
client
.apply(monitor, Some(&sender.namespace))
.await
.map_err(|e| PreparationError::new(e.to_string()))?;
}
Ok(PreparationOutcome::Success {
details: "successfuly installed prometheus components".to_string(),
})
}
async fn ensure_prometheus_operator(
&self,
sender: &CRDPrometheus,
_inventory: &Inventory,
inventory: &Inventory,
receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
) -> Result<PreparationOutcome, PreparationError> {
let po_result = self.ensure_prometheus_operator(sender).await?;
match po_result {
PreparationOutcome::Success { details: _ } => {
debug!("Detected prometheus crds operator present in cluster.");
return Ok(po_result);
}
PreparationOutcome::Noop => {
debug!("Skipping Prometheus CR installation due to missing operator.");
return Ok(po_result);
}
if po_result == PreparationOutcome::Noop {
debug!("Skipping Prometheus CR installation due to missing operator.");
return Ok(po_result);
}
let result = self
.get_k8s_prometheus_application_score(sender.clone(), receivers)
.await
.interpret(inventory, self)
.await;
match result {
Ok(outcome) => match outcome.status {
InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success {
details: outcome.message,
}),
InterpretStatus::NOOP => Ok(PreparationOutcome::Noop),
_ => Err(PreparationError::new(outcome.message)),
},
Err(err) => Err(PreparationError::new(err.to_string())),
}
}
}
#[async_trait]
impl PrometheusMonitoring<RHOBObservability> for K8sAnywhereTopology {
impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology {
async fn install_prometheus(
&self,
sender: &RHOBObservability,
@@ -302,14 +155,6 @@ impl PrometheusMonitoring<RHOBObservability> for K8sAnywhereTopology {
Err(err) => Err(PreparationError::new(err.to_string())),
}
}
async fn ensure_prometheus_operator(
&self,
sender: &RHOBObservability,
inventory: &Inventory,
) -> Result<PreparationOutcome, PreparationError> {
todo!()
}
}
impl Serialize for K8sAnywhereTopology {
@@ -371,180 +216,6 @@ impl K8sAnywhereTopology {
.await
}
fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option<String> {
let token_b64 = secret
.data
.get("token")
.or_else(|| secret.data.get("data").and_then(|d| d.get("token")))
.and_then(|v| v.as_str())?;
let bytes = general_purpose::STANDARD.decode(token_b64).ok()?;
let s = String::from_utf8(bytes).ok()?;
let cleaned = s
.trim_matches(|c: char| c.is_whitespace() || c == '\0')
.to_string();
Some(cleaned)
}
pub fn build_cluster_rolebinding(
&self,
service_account_name: &str,
clusterrole_name: &str,
ns: &str,
) -> ClusterRoleBinding {
ClusterRoleBinding {
metadata: ObjectMeta {
name: Some(format!("{}-view-binding", service_account_name)),
..Default::default()
},
role_ref: RoleRef {
api_group: "rbac.authorization.k8s.io".into(),
kind: "ClusterRole".into(),
name: clusterrole_name.into(),
},
subjects: Some(vec![Subject {
kind: "ServiceAccount".into(),
name: service_account_name.into(),
namespace: Some(ns.into()),
..Default::default()
}]),
}
}
pub fn build_sa_token_secret(
&self,
secret_name: &str,
service_account_name: &str,
ns: &str,
) -> Secret {
let mut annotations = BTreeMap::new();
annotations.insert(
"kubernetes.io/service-account.name".to_string(),
service_account_name.to_string(),
);
Secret {
metadata: ObjectMeta {
name: Some(secret_name.into()),
namespace: Some(ns.into()),
annotations: Some(annotations),
..Default::default()
},
type_: Some("kubernetes.io/service-account-token".to_string()),
..Default::default()
}
}
fn build_grafana_datasource(
&self,
name: &str,
ns: &str,
label_selector: &LabelSelector,
url: &str,
token: &str,
) -> GrafanaDatasource {
let mut json_data = BTreeMap::new();
json_data.insert("timeInterval".to_string(), "5s".to_string());
GrafanaDatasource {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(ns.to_string()),
..Default::default()
},
spec: GrafanaDatasourceSpec {
instance_selector: label_selector.clone(),
allow_cross_namespace_import: Some(true),
values_from: None,
datasource: GrafanaDatasourceConfig {
access: "proxy".to_string(),
name: name.to_string(),
r#type: "prometheus".to_string(),
url: url.to_string(),
database: None,
json_data: Some(GrafanaDatasourceJsonData {
time_interval: Some("60s".to_string()),
http_header_name1: Some("Authorization".to_string()),
tls_skip_verify: Some(true),
oauth_pass_thru: Some(true),
}),
secure_json_data: Some(GrafanaDatasourceSecureJsonData {
http_header_value1: Some(format!("Bearer {token}")),
}),
is_default: Some(false),
editable: Some(true),
},
},
}
}
fn build_grafana_dashboard(
&self,
ns: &str,
label_selector: &LabelSelector,
) -> GrafanaDashboard {
let graf_dashboard = GrafanaDashboard {
metadata: ObjectMeta {
name: Some(format!("grafana-dashboard-{}", ns)),
namespace: Some(ns.to_string()),
..Default::default()
},
spec: GrafanaDashboardSpec {
resync_period: Some("30s".to_string()),
instance_selector: label_selector.clone(),
datasources: Some(vec![GrafanaDashboardDatasource {
input_name: "DS_PROMETHEUS".to_string(),
datasource_name: "thanos-openshift-monitoring".to_string(),
}]),
json: None,
grafana_com: Some(GrafanaCom {
id: 17406,
revision: None,
}),
},
};
graf_dashboard
}
fn build_grafana(&self, ns: &str, labels: &BTreeMap<String, String>) -> GrafanaCRD {
let grafana = GrafanaCRD {
metadata: ObjectMeta {
name: Some(format!("grafana-{}", ns)),
namespace: Some(ns.to_string()),
labels: Some(labels.clone()),
..Default::default()
},
spec: GrafanaSpec {
config: None,
admin_user: None,
admin_password: None,
ingress: None,
persistence: None,
resources: None,
},
};
grafana
}
async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore {
let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap();
let name = format!("{}-grafana", ns);
let backend_service = format!("grafana-{}-service", ns);
K8sIngressScore {
name: fqdn::fqdn!(&name),
host: fqdn::fqdn!(&domain),
backend_service: fqdn::fqdn!(&backend_service),
port: 3000,
path: Some("/".to_string()),
path_type: Some(PathType::Prefix),
namespace: Some(fqdn::fqdn!(&ns)),
ingress_class_name: Some("openshift-default".to_string()),
}
}
async fn get_cluster_observability_operator_prometheus_application_score(
&self,
sender: RHOBObservability,
@@ -562,14 +233,13 @@ impl K8sAnywhereTopology {
&self,
sender: CRDPrometheus,
receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
service_monitors: Option<Vec<ServiceMonitor>>,
) -> K8sPrometheusCRDAlertingScore {
return K8sPrometheusCRDAlertingScore {
K8sPrometheusCRDAlertingScore {
sender,
receivers: receivers.unwrap_or_default(),
service_monitors: service_monitors.unwrap_or_default(),
service_monitors: vec![],
prometheus_rules: vec![],
};
}
}
async fn openshift_ingress_operator_available(&self) -> Result<(), PreparationError> {
@@ -837,30 +507,6 @@ impl K8sAnywhereTopology {
details: "prometheus operator present in cluster".into(),
})
}
async fn install_grafana_operator(
&self,
inventory: &Inventory,
ns: Option<&str>,
) -> Result<PreparationOutcome, PreparationError> {
let namespace = ns.unwrap_or("grafana");
info!("installing grafana operator in ns {namespace}");
let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await;
let mut namespace_scope = false;
if tenant.is_some() {
namespace_scope = true;
}
let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope)
.interpret(inventory, self)
.await
.map_err(|e| PreparationError::new(e.to_string()));
Ok(PreparationOutcome::Success {
details: format!(
"Successfully installed grafana operator in ns {}",
ns.unwrap()
),
})
}
}
#[derive(Clone, Debug)]
@@ -975,36 +621,56 @@ impl TenantManager for K8sAnywhereTopology {
#[async_trait]
impl Ingress for K8sAnywhereTopology {
//TODO this is specifically for openshift/okd which violates the k8sanywhere idea
async fn get_domain(&self, service: &str) -> Result<String, PreparationError> {
use log::{trace, debug, warn};
let client = self.k8s_client().await?;
if let Some(Some(k8s_state)) = self.k8s_state.get() {
match k8s_state.source {
K8sSource::LocalK3d => Ok(format!("{service}.local.k3d")),
K8sSource::LocalK3d => {
// Local developer UX
return Ok(format!("{service}.local.k3d"));
}
K8sSource::Kubeconfig => {
self.openshift_ingress_operator_available().await?;
trace!("K8sSource is kubeconfig; attempting to detect domain");
let gvk = GroupVersionKind {
group: "operator.openshift.io".into(),
version: "v1".into(),
kind: "IngressController".into(),
};
let ic = client
.get_resource_json_value(
"default",
Some("openshift-ingress-operator"),
&gvk,
)
.await
.map_err(|_| {
PreparationError::new("Failed to fetch IngressController".to_string())
})?;
// 1) Try OpenShift IngressController domain (backward compatible)
if self.openshift_ingress_operator_available().await.is_ok() {
trace!("OpenShift ingress operator detected; using IngressController");
let gvk = GroupVersionKind {
group: "operator.openshift.io".into(),
version: "v1".into(),
kind: "IngressController".into(),
};
let ic = client
.get_resource_json_value("default", Some("openshift-ingress-operator"), &gvk)
.await
.map_err(|_| PreparationError::new("Failed to fetch IngressController".to_string()))?;
match ic.data["status"]["domain"].as_str() {
Some(domain) => Ok(format!("{service}.{domain}")),
None => Err(PreparationError::new("Could not find domain".to_string())),
if let Some(domain) = ic.data["status"]["domain"].as_str() {
return Ok(format!("{service}.{domain}"));
} else {
warn!("OpenShift IngressController present but no status.domain set");
}
} else {
trace!("OpenShift ingress operator not detected; trying generic Kubernetes");
}
// 2) Try NGINX Ingress Controller common setups
// 2.a) Well-known namespace/name for the controller Service
// - upstream default: namespace "ingress-nginx", service "ingress-nginx-controller"
// - some distros: "ingress-nginx-controller" svc in "ingress-nginx" ns
// If found with LoadBalancer ingress hostname, use its base domain.
if let Some(domain) = try_nginx_lb_domain(&client).await? {
return Ok(format!("{service}.{domain}"));
}
// 3) Fallback: internal cluster DNS suffix (service.namespace.svc.cluster.local)
// We don't have tenant namespace here, so we fallback to 'default' with a warning.
warn!("Could not determine external ingress domain; falling back to internal-only DNS");
let internal = format!("{service}.default.svc.cluster.local");
Ok(internal)
}
}
} else {
@@ -1014,3 +680,57 @@ impl Ingress for K8sAnywhereTopology {
}
}
}
async fn try_nginx_lb_domain(client: &K8sClient) -> Result<Option<String>, PreparationError> {
use log::{trace, debug};
// Try common service path: svc/ingress-nginx-controller in ns/ingress-nginx
let svc_gvk = GroupVersionKind {
group: "".into(), // core
version: "v1".into(),
kind: "Service".into(),
};
let candidates = [
("ingress-nginx", "ingress-nginx-controller"),
("ingress-nginx", "ingress-nginx-controller-internal"),
("ingress-nginx", "ingress-nginx"), // some charts name the svc like this
("kube-system", "ingress-nginx-controller"), // less common but seen
];
for (ns, name) in candidates {
trace!("Checking NGINX Service {ns}/{name} for LoadBalancer hostname");
if let Ok(svc) = client.get_resource_json_value(ns, Some(name), &svc_gvk).await {
let lb_hosts = svc.data["status"]["loadBalancer"]["ingress"].as_array().cloned().unwrap_or_default();
for entry in lb_hosts {
if let Some(host) = entry.get("hostname").and_then(|v| v.as_str()) {
debug!("Found NGINX LB hostname: {host}");
if let Some(domain) = extract_base_domain(host) {
return Ok(Some(domain.to_string()));
} else {
return Ok(Some(host.to_string())); // already a domain
}
}
if let Some(ip) = entry.get("ip").and_then(|v| v.as_str()) {
// If only an IP is exposed, we can't create a hostname; return None to keep searching
debug!("NGINX LB exposes IP {ip} (no hostname); skipping");
}
}
}
}
Ok(None)
}
fn extract_base_domain(host: &str) -> Option<String> {
// For a host like a1b2c3d4e5f6abcdef.elb.amazonaws.com -> base domain elb.amazonaws.com
// For a managed DNS like xyz.example.com -> base domain example.com (keep 2+ labels)
// Heuristic: keep last 2 labels by default; special-case known multi-label TLDs if needed.
let parts: Vec<&str> = host.split('.').collect();
if parts.len() >= 2 {
// Very conservative: last 2 labels
Some(parts[parts.len() - 2..].join("."))
} else {
None
}
}

View File

@@ -1,6 +1,5 @@
mod ha_cluster;
pub mod ingress;
pub mod opnsense;
use harmony_types::net::IpAddress;
mod host_binding;
mod http;
@@ -187,7 +186,7 @@ impl TopologyState {
}
}
#[derive(Debug)]
#[derive(Debug, PartialEq)]
pub enum DeploymentTarget {
LocalDev,
Staging,

View File

@@ -9,7 +9,6 @@ use std::{
use async_trait::async_trait;
use derive_new::new;
use harmony_types::{
id::Id,
net::{IpAddress, MacAddress},
switch::PortLocation,
};
@@ -192,12 +191,15 @@ pub trait Switch: Send + Sync {
mac_address: &MacAddress,
) -> Result<Option<PortLocation>, SwitchError>;
async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>;
async fn configure_host_network(
&self,
host: &PhysicalHost,
config: HostNetworkConfig,
) -> Result<(), SwitchError>;
}
#[derive(Clone, Debug, PartialEq)]
pub struct HostNetworkConfig {
pub host_id: Id,
pub switch_ports: Vec<SwitchPort>,
}

View File

@@ -31,7 +31,6 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
debug!("hit sender configure for AlertingInterpret");
self.sender.configure(inventory, topology).await?;
for receiver in self.receivers.iter() {
receiver.install(&self.sender).await?;
@@ -87,5 +86,4 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
#[async_trait]
pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
fn clone_box(&self) -> Box<dyn ScrapeTarget<S>>;
}

View File

@@ -1,23 +0,0 @@
use async_trait::async_trait;
use log::info;
use crate::{
infra::opnsense::OPNSenseFirewall,
topology::{PreparationError, PreparationOutcome, Topology},
};
#[async_trait]
impl Topology for OPNSenseFirewall {
async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
// FIXME we should be initializing the opnsense config here instead of
// OPNSenseFirewall::new as this causes the config to be loaded too early in
// harmony initialization process
let details = "OPNSenseFirewall topology is ready".to_string();
info!("{}", details);
Ok(PreparationOutcome::Success { details })
}
fn name(&self) -> &str {
"OPNSenseFirewall"
}
}

View File

@@ -11,7 +11,7 @@ pub struct InventoryRepositoryFactory;
impl InventoryRepositoryFactory {
pub async fn build() -> Result<Box<dyn InventoryRepository>, RepoError> {
Ok(Box::new(
SqliteInventoryRepository::new(&DATABASE_URL).await?,
SqliteInventoryRepository::new(&(*DATABASE_URL)).await?,
))
}
}

View File

@@ -10,7 +10,7 @@ use super::OPNSenseFirewall;
#[async_trait]
impl DnsServer for OPNSenseFirewall {
async fn register_hosts(&self, hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
async fn register_hosts(&self, _hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
todo!("Refactor this to use dnsmasq")
// let mut writable_opnsense = self.opnsense_config.write().await;
// let mut dns = writable_opnsense.dns();
@@ -68,7 +68,7 @@ impl DnsServer for OPNSenseFirewall {
self.host.clone()
}
async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> {
async fn register_dhcp_leases(&self, _register: bool) -> Result<(), ExecutorError> {
todo!("Refactor this to use dnsmasq")
// let mut writable_opnsense = self.opnsense_config.write().await;
// let mut dns = writable_opnsense.dns();

View File

@@ -25,8 +25,6 @@ impl OPNSenseFirewall {
self.host.ip
}
/// panics : if the opnsense config file cannot be loaded by the underlying opnsense_config
/// crate
pub async fn new(host: LogicalHost, port: Option<u16>, username: &str, password: &str) -> Self {
Self {
opnsense_config: Arc::new(RwLock::new(

View File

@@ -21,7 +21,7 @@ pub struct Helm {
pub skip_schema_validation: Option<bool>,
pub version: Option<String>,
pub kube_version: Option<String>,
pub api_versions: Vec<String>,
// pub api_versions: Vec<String>,
pub namespace: Option<String>,
}
@@ -105,7 +105,7 @@ impl Default for ArgoApplication {
skip_schema_validation: None,
version: None,
kube_version: None,
api_versions: vec![],
// api_versions: vec![],
namespace: None,
},
path: "".to_string(),
@@ -155,7 +155,7 @@ impl From<CDApplicationConfig> for ArgoApplication {
skip_schema_validation: None,
version: None,
kube_version: None,
api_versions: vec![],
// api_versions: vec![],
namespace: None,
},
},
@@ -181,13 +181,11 @@ impl From<CDApplicationConfig> for ArgoApplication {
}
impl ArgoApplication {
pub fn to_yaml(&self) -> serde_yaml::Value {
pub fn to_yaml(&self, target_namespace: Option<&str>) -> serde_yaml::Value {
let name = &self.name;
let namespace = if let Some(ns) = self.namespace.as_ref() {
ns
} else {
"argocd"
};
let default_ns = "argocd".to_string();
let namespace: &str =
target_namespace.unwrap_or(self.namespace.as_ref().unwrap_or(&default_ns));
let project = &self.project;
let yaml_str = format!(
@@ -285,7 +283,7 @@ mod tests {
skip_schema_validation: None,
version: None,
kube_version: None,
api_versions: vec![],
// api_versions: vec![],
namespace: None,
},
path: "".to_string(),
@@ -345,7 +343,7 @@ spec:
assert_eq!(
expected_yaml_output.trim(),
serde_yaml::to_string(&app.clone().to_yaml())
serde_yaml::to_string(&app.clone().to_yaml(None))
.unwrap()
.trim()
);

View File

@@ -1,22 +1,21 @@
use async_trait::async_trait;
use harmony_macros::hurl;
use kube::{Api, api::GroupVersionKind};
use log::{debug, warn};
use log::{debug, info, trace, warn};
use non_blank_string_rs::NonBlankString;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{process::Command, str::FromStr, sync::Arc};
use std::{str::FromStr, sync::Arc};
use crate::{
data::Version,
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::Inventory,
modules::helm::chart::{HelmChartScore, HelmRepository},
score::Score,
topology::{
HelmCommand, K8sclient, PreparationError, PreparationOutcome, Topology, ingress::Ingress,
k8s::K8sClient,
modules::{
argocd::{ArgoDeploymentType, detect_argo_deployment_type},
helm::chart::{HelmChartScore, HelmRepository},
},
score::Score,
topology::{HelmCommand, K8sclient, Topology, ingress::Ingress, k8s::K8sClient},
};
use harmony_types::id::Id;
@@ -25,6 +24,7 @@ use super::ArgoApplication;
#[derive(Debug, Serialize, Clone)]
pub struct ArgoHelmScore {
pub namespace: String,
// TODO: remove and rely on topology (it now knows the flavor)
pub openshift: bool,
pub argo_apps: Vec<ArgoApplication>,
}
@@ -55,29 +55,101 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let k8s_client = topology.k8s_client().await?;
let svc = format!("argo-{}", self.score.namespace.clone());
trace!("Starting ArgoInterpret execution {self:?}");
let k8s_client: Arc<K8sClient> = topology.k8s_client().await?;
trace!("Got k8s client");
let desired_ns = self.score.namespace.clone();
debug!("ArgoInterpret detecting cluster configuration");
let svc = format!("argo-{}", desired_ns);
let domain = topology.get_domain(&svc).await?;
let helm_score =
argo_helm_chart_score(&self.score.namespace, self.score.openshift, &domain);
debug!("Resolved Argo service domain for '{}': {}", svc, domain);
helm_score.interpret(inventory, topology).await?;
// Detect current Argo deployment type
let current = detect_argo_deployment_type(&k8s_client, &desired_ns).await?;
info!("Detected Argo deployment type: {:?}", current);
// Decide control namespace and whether we must install
let (control_ns, must_install) = match current.clone() {
ArgoDeploymentType::NotInstalled => {
info!(
"Argo CD not installed. Will install via Helm into namespace '{}'.",
desired_ns
);
(desired_ns.clone(), true)
}
ArgoDeploymentType::AvailableInDesiredNamespace(ns) => {
info!(
"Argo CD already installed by Harmony in '{}'. Skipping install.",
ns
);
(ns, false)
}
ArgoDeploymentType::InstalledClusterWide(ns) => {
info!(
"Argo CD installed cluster-wide in namespace '{}'.",
ns
);
(ns, false)
}
ArgoDeploymentType::InstalledNamespaceScoped(ns) => {
// TODO we could support this use case by installing a new argo instance. But that
// means handling a few cases that are out of scope for now :
// - Wether argo operator is installed
// - Managing CRD versions compatibility
// - Potentially handling the various k8s flavors and setups we might encounter
//
// There is a possibility that the helm chart already handles most or even all of these use cases but they are out of scope for now.
let msg = format!(
"Argo CD found in '{}' but it is namespace-scoped and not supported for attachment yet.",
ns
);
warn!("{}", msg);
return Err(InterpretError::new(msg));
}
};
info!("ArgoCD will be installed : {must_install} . Current argocd status : {current:?} ");
if must_install {
let helm_score = argo_helm_chart_score(&desired_ns, self.score.openshift, &domain);
info!(
"Installing Argo CD via Helm into namespace '{}' ...",
desired_ns
);
helm_score.interpret(inventory, topology).await?;
info!("Argo CD install complete in '{}'.", desired_ns);
}
let yamls: Vec<serde_yaml::Value> = self
.argo_apps
.iter()
.map(|a| a.to_yaml(Some(&control_ns)))
.collect();
info!(
"Applying {} Argo application object(s) into control namespace '{}'.",
yamls.len(),
control_ns
);
k8s_client
.apply_yaml_many(&self.argo_apps.iter().map(|a| a.to_yaml()).collect(), None)
.apply_yaml_many(&yamls, Some(control_ns.as_str()))
.await
.unwrap();
.map_err(|e| InterpretError::new(format!("Failed applying Argo CRs: {e}")))?;
Ok(Outcome::success_with_details(
format!(
"ArgoCD {} {}",
self.argo_apps.len(),
match self.argo_apps.len() {
1 => "application",
_ => "applications",
if self.argo_apps.len() == 1 {
"application"
} else {
"applications"
}
),
vec![format!("argo application: http://{}", domain)],
vec![
format!("control_namespace={}", control_ns),
format!("argo ui: http://{}", domain),
],
))
}
@@ -86,7 +158,7 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
}
fn get_version(&self) -> Version {
todo!()
Version::from("0.1.0").unwrap()
}
fn get_status(&self) -> InterpretStatus {
@@ -94,39 +166,7 @@ impl<T: Topology + K8sclient + HelmCommand + Ingress> Interpret<T> for ArgoInter
}
fn get_children(&self) -> Vec<Id> {
todo!()
}
}
impl ArgoInterpret {
pub async fn get_host_domain(
&self,
client: Arc<K8sClient>,
openshift: bool,
) -> Result<String, InterpretError> {
//This should be the job of the topology to determine if we are in
//openshift, potentially we need on openshift topology the same way we create a
//localhosttopology
match openshift {
true => {
let gvk = GroupVersionKind {
group: "operator.openshift.io".into(),
version: "v1".into(),
kind: "IngressController".into(),
};
let ic = client
.get_resource_json_value("default", Some("openshift-ingress-operator"), &gvk)
.await?;
match ic.data["status"]["domain"].as_str() {
Some(domain) => return Ok(domain.to_string()),
None => return Err(InterpretError::new("Could not find domain".to_string())),
}
}
false => {
todo!()
}
};
vec![]
}
}

View File

@@ -2,11 +2,7 @@ use crate::modules::application::{
Application, ApplicationFeature, InstallationError, InstallationOutcome,
};
use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
use crate::modules::monitoring::grafana::grafana::Grafana;
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
ServiceMonitor, ServiceMonitorSpec,
};
use crate::topology::MultiTargetTopology;
use crate::topology::ingress::Ingress;
use crate::{
@@ -18,7 +14,7 @@ use crate::{
topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
};
use crate::{
modules::prometheus::prometheus::PrometheusMonitoring,
modules::prometheus::prometheus::PrometheusApplicationMonitoring,
topology::oberservability::monitoring::AlertReceiver,
};
use async_trait::async_trait;
@@ -26,7 +22,6 @@ use base64::{Engine as _, engine::general_purpose};
use harmony_secret::SecretManager;
use harmony_secret_derive::Secret;
use harmony_types::net::Url;
use kube::api::ObjectMeta;
use log::{debug, info};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -45,8 +40,7 @@ impl<
+ TenantManager
+ K8sclient
+ MultiTargetTopology
+ PrometheusMonitoring<CRDPrometheus>
+ Grafana
+ PrometheusApplicationMonitoring<CRDPrometheus>
+ Ingress
+ std::fmt::Debug,
> ApplicationFeature<T> for Monitoring
@@ -63,20 +57,10 @@ impl<
.unwrap_or_else(|| self.application.name());
let domain = topology.get_domain("ntfy").await.unwrap();
let app_service_monitor = ServiceMonitor {
metadata: ObjectMeta {
name: Some(self.application.name()),
namespace: Some(namespace.clone()),
..Default::default()
},
spec: ServiceMonitorSpec::default(),
};
let mut alerting_score = ApplicationMonitoringScore {
sender: CRDPrometheus {
namespace: namespace.clone(),
client: topology.k8s_client().await.unwrap(),
service_monitor: vec![app_service_monitor],
},
application: self.application.clone(),
receivers: self.alert_receiver.clone(),

View File

@@ -10,12 +10,11 @@ use crate::{
data::Version,
inventory::Inventory,
modules::application::{
ApplicationFeature, HelmPackage, InstallationError, InstallationOutcome, OCICompliant,
features::{ArgoApplication, ArgoHelmScore},
features::{ArgoApplication, ArgoHelmScore}, webapp::Webapp, ApplicationFeature, HelmPackage, InstallationError, InstallationOutcome, OCICompliant
},
score::Score,
topology::{
DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology, ingress::Ingress,
ingress::Ingress, DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology
},
};
@@ -47,11 +46,11 @@ use crate::{
/// - ArgoCD to install/upgrade/rollback/inspect k8s resources
/// - Kubernetes for runtime orchestration
#[derive(Debug, Default, Clone)]
pub struct PackagingDeployment<A: OCICompliant + HelmPackage> {
pub struct PackagingDeployment<A: OCICompliant + HelmPackage + Webapp> {
pub application: Arc<A>,
}
impl<A: OCICompliant + HelmPackage> PackagingDeployment<A> {
impl<A: OCICompliant + HelmPackage + Webapp> PackagingDeployment<A> {
async fn deploy_to_local_k3d(
&self,
app_name: String,
@@ -137,7 +136,7 @@ impl<A: OCICompliant + HelmPackage> PackagingDeployment<A> {
#[async_trait]
impl<
A: OCICompliant + HelmPackage + Clone + 'static,
A: OCICompliant + HelmPackage + Webapp + Clone + 'static,
T: Topology + HelmCommand + MultiTargetTopology + K8sclient + Ingress + 'static,
> ApplicationFeature<T> for PackagingDeployment<A>
{
@@ -146,10 +145,15 @@ impl<
topology: &T,
) -> Result<InstallationOutcome, InstallationError> {
let image = self.application.image_name();
let domain = topology
let domain = if topology.current_target() == DeploymentTarget::Production {
self.application.dns()
} else {
topology
.get_domain(&self.application.name())
.await
.map_err(|e| e.to_string())?;
.map_err(|e| e.to_string())?
};
// TODO Write CI/CD workflow files
// we can autotedect the CI type using the remote url (default to github action for github
@@ -194,7 +198,7 @@ impl<
openshift: true,
argo_apps: vec![ArgoApplication::from(CDApplicationConfig {
// helm pull oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart --version 0.1.0
version: Version::from("0.1.0").unwrap(),
version: Version::from("0.2.1").unwrap(),
helm_chart_repo_url: "hub.nationtech.io/harmony".to_string(),
helm_chart_name: format!("{}-chart", self.application.name()),
values_overrides: None,

View File

@@ -3,7 +3,6 @@ use std::sync::Arc;
use crate::modules::application::{
Application, ApplicationFeature, InstallationError, InstallationOutcome,
};
use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
use crate::modules::monitoring::application_monitoring::rhobs_application_monitoring_score::ApplicationRHOBMonitoringScore;
use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability;
@@ -18,7 +17,7 @@ use crate::{
topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
};
use crate::{
modules::prometheus::prometheus::PrometheusMonitoring,
modules::prometheus::prometheus::PrometheusApplicationMonitoring,
topology::oberservability::monitoring::AlertReceiver,
};
use async_trait::async_trait;
@@ -42,7 +41,7 @@ impl<
+ MultiTargetTopology
+ Ingress
+ std::fmt::Debug
+ PrometheusMonitoring<RHOBObservability>,
+ PrometheusApplicationMonitoring<RHOBObservability>,
> ApplicationFeature<T> for Monitoring
{
async fn ensure_installed(

View File

@@ -2,6 +2,7 @@ mod feature;
pub mod features;
pub mod oci;
mod rust;
mod webapp;
use std::sync::Arc;
pub use feature::*;

View File

@@ -16,6 +16,7 @@ use tar::{Builder, Header};
use walkdir::WalkDir;
use crate::config::{REGISTRY_PROJECT, REGISTRY_URL};
use crate::modules::application::webapp::Webapp;
use crate::{score::Score, topology::Topology};
use super::{Application, ApplicationFeature, ApplicationInterpret, HelmPackage, OCICompliant};
@@ -60,6 +61,10 @@ pub struct RustWebapp {
pub project_root: PathBuf,
pub service_port: u32,
pub framework: Option<RustWebFramework>,
/// Host name that will be used in production environment.
///
/// This is the place to put the public host name if this is a public facing webapp.
pub dns: String,
}
impl Application for RustWebapp {
@@ -68,6 +73,12 @@ impl Application for RustWebapp {
}
}
impl Webapp for RustWebapp {
fn dns(&self) -> String {
self.dns.clone()
}
}
#[async_trait]
impl HelmPackage for RustWebapp {
async fn build_push_helm_package(
@@ -194,10 +205,10 @@ impl RustWebapp {
Some(body_full(tar_data.into())),
);
while let Some(mut msg) = image_build_stream.next().await {
while let Some(msg) = image_build_stream.next().await {
trace!("Got bollard msg {msg:?}");
match msg {
Ok(mut msg) => {
Ok(msg) => {
if let Some(progress) = msg.progress_detail {
info!(
"Build progress {}/{}",
@@ -257,7 +268,6 @@ impl RustWebapp {
".harmony_generated",
"harmony",
"node_modules",
"Dockerfile.harmony",
];
let mut entries: Vec<_> = WalkDir::new(project_root)
.into_iter()
@@ -461,52 +471,53 @@ impl RustWebapp {
let (image_repo, image_tag) = image_url.rsplit_once(':').unwrap_or((image_url, "latest"));
let app_name = &self.name;
let service_port = self.service_port;
// Create Chart.yaml
let chart_yaml = format!(
r#"
apiVersion: v2
name: {}
description: A Helm chart for the {} web application.
name: {chart_name}
description: A Helm chart for the {app_name} web application.
type: application
version: 0.1.0
appVersion: "{}"
version: 0.2.1
appVersion: "{image_tag}"
"#,
chart_name, self.name, image_tag
);
fs::write(chart_dir.join("Chart.yaml"), chart_yaml)?;
// Create values.yaml
let values_yaml = format!(
r#"
# Default values for {}.
# Default values for {chart_name}.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: {}
repository: {image_repo}
pullPolicy: IfNotPresent
# Overridden by the chart's appVersion
tag: "{}"
tag: "{image_tag}"
service:
type: ClusterIP
port: {}
port: {service_port}
ingress:
enabled: true
tls: true
# Annotations for cert-manager to handle SSL.
annotations:
# Add other annotations like nginx ingress class if needed
# kubernetes.io/ingress.class: nginx
hosts:
- host: {}
- host: {domain}
paths:
- path: /
pathType: ImplementationSpecific
"#,
chart_name, image_repo, image_tag, self.service_port, domain,
);
fs::write(chart_dir.join("values.yaml"), values_yaml)?;
@@ -583,7 +594,11 @@ spec:
);
fs::write(templates_dir.join("deployment.yaml"), deployment_yaml)?;
let service_port = self.service_port;
// Create templates/ingress.yaml
// TODO get issuer name and tls config from topology as it may be different from one
// cluster to another, also from one version to another
let ingress_yaml = format!(
r#"
{{{{- if $.Values.ingress.enabled -}}}}
@@ -596,13 +611,11 @@ metadata:
spec:
{{{{- if $.Values.ingress.tls }}}}
tls:
{{{{- range $.Values.ingress.tls }}}}
- hosts:
{{{{- range .hosts }}}}
- {{{{ . | quote }}}}
- secretName: {{{{ include "chart.fullname" . }}}}-tls
hosts:
{{{{- range $.Values.ingress.hosts }}}}
- {{{{ .host | quote }}}}
{{{{- end }}}}
secretName: {{{{ .secretName }}}}
{{{{- end }}}}
{{{{- end }}}}
rules:
{{{{- range $.Values.ingress.hosts }}}}
@@ -616,12 +629,11 @@ spec:
service:
name: {{{{ include "chart.fullname" $ }}}}
port:
number: {{{{ $.Values.service.port | default {} }}}}
number: {{{{ $.Values.service.port | default {service_port} }}}}
{{{{- end }}}}
{{{{- end }}}}
{{{{- end }}}}
"#,
self.service_port
);
fs::write(templates_dir.join("ingress.yaml"), ingress_yaml)?;

View File

@@ -0,0 +1,7 @@
use super::Application;
use async_trait::async_trait;
#[async_trait]
pub trait Webapp: Application {
fn dns(&self) -> String;
}

View File

@@ -0,0 +1,203 @@
use std::sync::Arc;
use log::{debug, info};
use crate::{interpret::InterpretError, topology::k8s::K8sClient};
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ArgoScope {
ClusterWide(String),
NamespaceScoped(String),
}
#[derive(Clone, Debug)]
pub struct DiscoveredArgo {
pub control_namespace: String,
pub scope: ArgoScope,
pub has_crds: bool,
pub has_applicationset: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ArgoDeploymentType {
NotInstalled,
AvailableInDesiredNamespace(String),
InstalledClusterWide(String),
InstalledNamespaceScoped(String),
}
pub async fn discover_argo_all(
k8s: &Arc<K8sClient>,
) -> Result<Vec<DiscoveredArgo>, InterpretError> {
use log::{debug, info, trace, warn};
trace!("Starting Argo discovery");
// CRDs
let mut has_crds = true;
let required_crds = vec!["applications.argoproj.io", "appprojects.argoproj.io"];
trace!("Checking required Argo CRDs: {:?}", required_crds);
for crd in required_crds {
trace!("Verifying CRD presence: {crd}");
let crd_exists = k8s.has_crd(crd).await.map_err(|e| {
InterpretError::new(format!("Failed to verify existence of CRD {crd}: {e}"))
})?;
debug!("CRD {crd} exists: {crd_exists}");
if !crd_exists {
info!(
"Missing Argo CRD {crd}, looks like Argo CD is not installed (or partially installed)"
);
has_crds = false;
break;
}
}
trace!(
"Listing namespaces with healthy Argo CD deployments using selector app.kubernetes.io/part-of=argocd"
);
let mut candidate_namespaces = k8s
.list_namespaces_with_healthy_deployments("app.kubernetes.io/part-of=argocd")
.await
.map_err(|e| InterpretError::new(format!("List healthy argocd deployments: {e}")))?;
trace!(
"Listing namespaces with healthy Argo CD deployments using selector app.kubernetes.io/name=argo-cd"
);
candidate_namespaces.append(
&mut k8s
.list_namespaces_with_healthy_deployments("app.kubernetes.io/name=argo-cd")
.await
.map_err(|e| InterpretError::new(format!("List healthy argocd deployments: {e}")))?,
);
debug!(
"Discovered {} candidate namespace(s) for Argo CD: {:?}",
candidate_namespaces.len(),
candidate_namespaces
);
let mut found = Vec::new();
for ns in candidate_namespaces {
trace!("Evaluating namespace '{ns}' for Argo CD instance");
// Require the application-controller to be healthy (sanity check)
trace!(
"Checking healthy deployment with label app.kubernetes.io/name=argocd-application-controller in namespace '{ns}'"
);
let controller_ok = k8s
.has_healthy_deployment_with_label(
&ns,
"app.kubernetes.io/name=argocd-application-controller",
)
.await
.unwrap_or_else(|e| {
warn!(
"Error while checking application-controller health in namespace '{ns}': {e}"
);
false
}) || k8s
.has_healthy_deployment_with_label(
&ns,
"app.kubernetes.io/component=controller",
)
.await
.unwrap_or_else(|e| {
warn!(
"Error while checking application-controller health in namespace '{ns}': {e}"
);
false
});
debug!("Namespace '{ns}': application-controller healthy = {controller_ok}");
if !controller_ok {
trace!("Skipping namespace '{ns}' because application-controller is not healthy");
continue;
}
trace!("Determining Argo CD scope for namespace '{ns}' (cluster-wide vs namespace-scoped)");
let scope = match k8s.is_argocd_cluster_wide(&ns).await {
Ok(true) => {
debug!("Namespace '{ns}' identified as cluster-wide Argo CD control plane");
ArgoScope::ClusterWide(ns.to_string())
}
Ok(false) => {
debug!("Namespace '{ns}' identified as namespace-scoped Argo CD control plane");
ArgoScope::NamespaceScoped(ns.to_string())
}
Err(e) => {
warn!(
"Failed to determine Argo CD scope for namespace '{ns}': {e}. Assuming namespace-scoped."
);
ArgoScope::NamespaceScoped(ns.to_string())
}
};
trace!("Checking optional ApplicationSet CRD (applicationsets.argoproj.io)");
let has_applicationset = match k8s.has_crd("applicationsets.argoproj.io").await {
Ok(v) => {
debug!("applicationsets.argoproj.io present: {v}");
v
}
Err(e) => {
warn!("Failed to check applicationsets.argoproj.io CRD: {e}. Assuming absent.");
false
}
};
let argo = DiscoveredArgo {
control_namespace: ns.clone(),
scope,
has_crds,
has_applicationset,
};
debug!("Discovered Argo instance in '{ns}': {argo:?}");
found.push(argo);
}
if found.is_empty() {
info!("No Argo CD installations discovered");
} else {
info!(
"Argo CD discovery complete: {} instance(s) found",
found.len()
);
}
Ok(found)
}
pub async fn detect_argo_deployment_type(
k8s: &Arc<K8sClient>,
desired_namespace: &str,
) -> Result<ArgoDeploymentType, InterpretError> {
let discovered = discover_argo_all(k8s).await?;
debug!("Discovered argo instances {discovered:?}");
if discovered.is_empty() {
return Ok(ArgoDeploymentType::NotInstalled);
}
if let Some(d) = discovered
.iter()
.find(|d| d.control_namespace == desired_namespace)
{
return Ok(ArgoDeploymentType::AvailableInDesiredNamespace(
d.control_namespace.clone(),
));
}
if let Some(d) = discovered
.iter()
.find(|d| matches!(d.scope, ArgoScope::ClusterWide(_)))
{
return Ok(ArgoDeploymentType::InstalledClusterWide(
d.control_namespace.clone(),
));
}
Ok(ArgoDeploymentType::InstalledNamespaceScoped(
discovered[0].control_namespace.clone(),
))
}

View File

@@ -90,12 +90,12 @@ impl<T: Topology> Interpret<T> for DiscoverInventoryAgentInterpret {
// refactoring to do it now
let harmony_inventory_agent::hwinfo::PhysicalHost {
storage_drives,
storage_controller,
storage_controller: _,
memory_modules,
cpus,
chipset,
chipset: _,
network_interfaces,
management_interface,
management_interface: _,
host_uuid,
} = host;

View File

@@ -38,15 +38,13 @@ impl<
+ 'static
+ Send
+ Clone,
T: Topology + K8sclient,
T: Topology,
> Score<T> for K8sResourceScore<K>
where
<K as kube::Resource>::DynamicType: Default,
{
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(K8sResourceInterpret {
score: self.clone(),
})
todo!()
}
fn name(&self) -> String {

View File

@@ -17,3 +17,4 @@ pub mod prometheus;
pub mod storage;
pub mod tenant;
pub mod tftp;
pub mod argocd;

View File

@@ -1,23 +1,21 @@
use std::sync::Arc;
use log::debug;
use async_trait::async_trait;
use serde::Serialize;
use crate::{
interpret::Interpret,
data::Version,
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::Inventory,
modules::{
application::Application,
monitoring::{
grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
},
prometheus::prometheus::PrometheusMonitoring,
monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
prometheus::prometheus::PrometheusApplicationMonitoring,
},
score::Score,
topology::{
K8sclient, Topology,
oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget},
},
topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
};
use harmony_types::id::Id;
#[derive(Debug, Clone, Serialize)]
pub struct ApplicationMonitoringScore {
@@ -26,16 +24,12 @@ pub struct ApplicationMonitoringScore {
pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
}
impl<T: Topology + PrometheusMonitoring<CRDPrometheus> + K8sclient + Grafana> Score<T>
impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
for ApplicationMonitoringScore
{
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
debug!("creating alerting interpret");
Box::new(AlertingInterpret {
sender: self.sender.clone(),
receivers: self.receivers.clone(),
rules: vec![],
scrape_targets: None,
Box::new(ApplicationMonitoringInterpret {
score: self.clone(),
})
}
@@ -46,3 +40,55 @@ impl<T: Topology + PrometheusMonitoring<CRDPrometheus> + K8sclient + Grafana> Sc
)
}
}
#[derive(Debug)]
pub struct ApplicationMonitoringInterpret {
score: ApplicationMonitoringScore,
}
#[async_trait]
impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
for ApplicationMonitoringInterpret
{
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let result = topology
.install_prometheus(
&self.score.sender,
inventory,
Some(self.score.receivers.clone()),
)
.await;
match result {
Ok(outcome) => match outcome {
PreparationOutcome::Success { details: _ } => {
Ok(Outcome::success("Prometheus installed".into()))
}
PreparationOutcome::Noop => {
Ok(Outcome::noop("Prometheus installation skipped".into()))
}
},
Err(err) => Err(InterpretError::from(err)),
}
}
fn get_name(&self) -> InterpretName {
InterpretName::ApplicationMonitoring
}
fn get_version(&self) -> Version {
todo!()
}
fn get_status(&self) -> InterpretStatus {
todo!()
}
fn get_children(&self) -> Vec<Id> {
todo!()
}
}

View File

@@ -12,7 +12,7 @@ use crate::{
monitoring::kube_prometheus::crd::{
crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability,
},
prometheus::prometheus::PrometheusMonitoring,
prometheus::prometheus::PrometheusApplicationMonitoring,
},
score::Score,
topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
@@ -26,7 +26,7 @@ pub struct ApplicationRHOBMonitoringScore {
pub receivers: Vec<Box<dyn AlertReceiver<RHOBObservability>>>,
}
impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Score<T>
impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Score<T>
for ApplicationRHOBMonitoringScore
{
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
@@ -49,7 +49,7 @@ pub struct ApplicationRHOBMonitoringInterpret {
}
#[async_trait]
impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Interpret<T>
impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Interpret<T>
for ApplicationRHOBMonitoringInterpret
{
async fn execute(

View File

@@ -1,17 +0,0 @@
use async_trait::async_trait;
use k8s_openapi::Resource;
use crate::{
inventory::Inventory,
topology::{PreparationError, PreparationOutcome},
};
#[async_trait]
pub trait Grafana {
async fn ensure_grafana_operator(
&self,
inventory: &Inventory,
) -> Result<PreparationOutcome, PreparationError>;
async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError>;
}

View File

@@ -1,28 +1,27 @@
use harmony_macros::hurl;
use non_blank_string_rs::NonBlankString;
use std::{collections::HashMap, str::FromStr};
use std::str::FromStr;
use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
use crate::modules::helm::chart::HelmChartScore;
pub fn grafana_helm_chart_score(ns: &str) -> HelmChartScore {
let values = r#"
rbac:
namespaced: true
sidecar:
dashboards:
enabled: true
"#
.to_string();
pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore {
let mut values_overrides = HashMap::new();
values_overrides.insert(
NonBlankString::from_str("namespaceScope").unwrap(),
namespace_scope.to_string(),
);
HelmChartScore {
namespace: Some(NonBlankString::from_str(ns).unwrap()),
release_name: NonBlankString::from_str("grafana-operator").unwrap(),
chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(),
release_name: NonBlankString::from_str("grafana").unwrap(),
chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana").unwrap(),
chart_version: None,
values_overrides: Some(values_overrides),
values_yaml: None,
values_overrides: None,
values_yaml: Some(values.to_string()),
create_namespace: true,
install_only: true,
repository: Some(HelmRepository::new(
"grafana".to_string(),
hurl!("https://grafana.github.io/helm-charts"),
true,
)),
repository: None,
}
}

View File

@@ -1,2 +1 @@
pub mod grafana;
pub mod helm;

View File

@@ -1,25 +1,12 @@
use std::sync::Arc;
use async_trait::async_trait;
use kube::CustomResource;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::{
interpret::{InterpretError, Outcome},
inventory::Inventory,
modules::{
monitoring::{
grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor,
},
prometheus::prometheus::PrometheusMonitoring,
},
topology::{
K8sclient, Topology,
installable::Installable,
k8s::K8sClient,
oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget},
},
use crate::topology::{
k8s::K8sClient,
oberservability::monitoring::{AlertReceiver, AlertSender},
};
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
@@ -39,7 +26,6 @@ pub struct AlertmanagerConfigSpec {
pub struct CRDPrometheus {
pub namespace: String,
pub client: Arc<K8sClient>,
pub service_monitor: Vec<ServiceMonitor>,
}
impl AlertSender for CRDPrometheus {
@@ -54,12 +40,6 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
}
}
impl Clone for Box<dyn ScrapeTarget<CRDPrometheus>> {
fn clone(&self) -> Self {
self.clone_box()
}
}
impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where
@@ -68,24 +48,3 @@ impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
todo!()
}
}
#[async_trait]
impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus> + Grafana> Installable<T>
for CRDPrometheus
{
async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> {
topology.ensure_grafana_operator(inventory).await?;
topology.ensure_prometheus_operator(self, inventory).await?;
Ok(())
}
async fn ensure_installed(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<(), InterpretError> {
topology.install_grafana().await?;
topology.install_prometheus(&self, inventory, None).await?;
Ok(())
}
}

View File

@@ -103,34 +103,9 @@ pub struct GrafanaDashboardSpec {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resync_period: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub datasources: Option<Vec<GrafanaDashboardDatasource>>,
pub instance_selector: LabelSelector,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub json: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grafana_com: Option<GrafanaCom>,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaDashboardDatasource {
pub input_name: String,
pub datasource_name: String,
}
// ------------------------------------------------------------------------------------------------
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaCom {
pub id: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub revision: Option<u32>,
pub json: String,
}
// ------------------------------------------------------------------------------------------------
@@ -151,79 +126,20 @@ pub struct GrafanaDatasourceSpec {
pub allow_cross_namespace_import: Option<bool>,
pub datasource: GrafanaDatasourceConfig,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub values_from: Option<Vec<GrafanaValueFrom>>,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaValueFrom {
pub target_path: String,
pub value_from: GrafanaValueSource,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaValueSource {
pub secret_key_ref: GrafanaSecretKeyRef,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaSecretKeyRef {
pub name: String,
pub key: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaDatasourceConfig {
pub access: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub database: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub json_data: Option<BTreeMap<String, String>>,
pub name: String,
pub r#type: String,
pub url: String,
/// Represents jsonData in the GrafanaDatasource spec
#[serde(default, skip_serializing_if = "Option::is_none")]
pub json_data: Option<GrafanaDatasourceJsonData>,
/// Represents secureJsonData (secrets)
#[serde(default, skip_serializing_if = "Option::is_none")]
pub secure_json_data: Option<GrafanaDatasourceSecureJsonData>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub is_default: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub editable: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaDatasourceJsonData {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub time_interval: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub http_header_name1: Option<String>,
/// Disable TLS skip verification (false = verify)
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tls_skip_verify: Option<bool>,
/// Auth type - set to "forward" for OpenShift OAuth identity
#[serde(default, skip_serializing_if = "Option::is_none")]
pub oauth_pass_thru: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct GrafanaDatasourceSecureJsonData {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub http_header_value1: Option<String>,
}
// ------------------------------------------------------------------------------------------------
#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]

View File

@@ -1,12 +1,8 @@
use std::collections::BTreeMap;
use kube::CustomResource;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::{
LabelSelector, PrometheusSpec,
};
use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector;
/// MonitoringStack CRD for monitoring.rhobs/v1alpha1
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]

View File

@@ -100,7 +100,11 @@ impl<T: Topology + HelmCommand + K8sclient + MultiTargetTopology> Interpret<T> f
info!("deploying ntfy...");
client
.wait_until_deployment_ready("ntfy", Some(self.score.namespace.as_str()), None)
.wait_until_deployment_ready(
"ntfy".to_string(),
Some(self.score.namespace.as_str()),
None,
)
.await?;
info!("ntfy deployed");

View File

@@ -114,7 +114,7 @@ impl Prometheus {
};
if let Some(ns) = namespace.as_deref() {
grafana_helm_chart_score(ns, false)
grafana_helm_chart_score(ns)
.interpret(inventory, topology)
.await
} else {

View File

@@ -73,8 +73,4 @@ impl ScrapeTarget<CRDPrometheus> for Server {
self.name.clone()
)))
}
fn clone_box(&self) -> Box<dyn ScrapeTarget<CRDPrometheus>> {
Box::new(self.clone())
}
}

View File

@@ -5,8 +5,10 @@ use crate::{
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::{HostRole, Inventory},
modules::{
dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
dhcp::DhcpHostBindingScore,
http::IPxeMacBootFileScore,
inventory::DiscoverHostForRoleScore,
okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl},
},
score::Score,
topology::{HAClusterTopology, HostBinding},
@@ -203,6 +205,28 @@ impl OKDSetup03ControlPlaneInterpret {
Ok(())
}
/// Placeholder for automating network bonding configuration.
async fn persist_network_bond(
&self,
inventory: &Inventory,
topology: &HAClusterTopology,
hosts: &Vec<PhysicalHost>,
) -> Result<(), InterpretError> {
info!("[ControlPlane] Ensuring persistent bonding");
let score = HostNetworkConfigurationScore {
hosts: hosts.clone(),
};
score.interpret(inventory, topology).await?;
inquire::Confirm::new(
"Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
)
.prompt()
.map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
Ok(())
}
}
#[async_trait]
@@ -241,6 +265,10 @@ impl Interpret<HAClusterTopology> for OKDSetup03ControlPlaneInterpret {
// 4. Reboot the nodes to start the OS installation.
self.reboot_targets(&nodes).await?;
// 5. Placeholder for post-boot network configuration (e.g., bonding).
self.persist_network_bond(inventory, topology, &nodes)
.await?;
// TODO: Implement a step to wait for the control plane nodes to join the cluster
// and for the cluster operators to become available. This would be similar to
// the `wait-for bootstrap-complete` command.

View File

@@ -1,130 +0,0 @@
use crate::{
data::Version,
hardware::PhysicalHost,
infra::inventory::InventoryRepositoryFactory,
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::{HostRole, Inventory},
modules::okd::host_network::HostNetworkConfigurationScore,
score::Score,
topology::HAClusterTopology,
};
use async_trait::async_trait;
use derive_new::new;
use harmony_types::id::Id;
use log::info;
use serde::Serialize;
// -------------------------------------------------------------------------------------------------
// Persist Network Bond
// - Persist bonding via NMState
// - Persist port channels on the Switch
// -------------------------------------------------------------------------------------------------
#[derive(Debug, Clone, Serialize, new)]
pub struct OKDSetupPersistNetworkBondScore {}
impl Score<HAClusterTopology> for OKDSetupPersistNetworkBondScore {
fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
Box::new(OKDSetupPersistNetworkBondInterpet::new())
}
fn name(&self) -> String {
"OKDSetupPersistNetworkBondScore".to_string()
}
}
#[derive(Debug, Clone)]
pub struct OKDSetupPersistNetworkBondInterpet {
version: Version,
status: InterpretStatus,
}
impl OKDSetupPersistNetworkBondInterpet {
pub fn new() -> Self {
let version = Version::from("1.0.0").unwrap();
Self {
version,
status: InterpretStatus::QUEUED,
}
}
/// Ensures that three physical hosts are discovered and available for the ControlPlane role.
/// It will trigger discovery if not enough hosts are found.
async fn get_nodes(
&self,
_inventory: &Inventory,
_topology: &HAClusterTopology,
) -> Result<Vec<PhysicalHost>, InterpretError> {
const REQUIRED_HOSTS: usize = 3;
let repo = InventoryRepositoryFactory::build().await?;
let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
if control_plane_hosts.len() < REQUIRED_HOSTS {
Err(InterpretError::new(format!(
"OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.",
REQUIRED_HOSTS,
control_plane_hosts.len()
)))
} else {
// Take exactly the number of required hosts to ensure consistency.
Ok(control_plane_hosts
.into_iter()
.take(REQUIRED_HOSTS)
.collect())
}
}
async fn persist_network_bond(
&self,
inventory: &Inventory,
topology: &HAClusterTopology,
hosts: &Vec<PhysicalHost>,
) -> Result<(), InterpretError> {
info!("Ensuring persistent bonding");
let score = HostNetworkConfigurationScore {
hosts: hosts.clone(),
};
score.interpret(inventory, topology).await?;
Ok(())
}
}
#[async_trait]
impl Interpret<HAClusterTopology> for OKDSetupPersistNetworkBondInterpet {
fn get_name(&self) -> InterpretName {
InterpretName::Custom("OKDSetupPersistNetworkBondInterpet")
}
fn get_version(&self) -> Version {
self.version.clone()
}
fn get_status(&self) -> InterpretStatus {
self.status.clone()
}
fn get_children(&self) -> Vec<Id> {
vec![]
}
async fn execute(
&self,
inventory: &Inventory,
topology: &HAClusterTopology,
) -> Result<Outcome, InterpretError> {
let nodes = self.get_nodes(inventory, topology).await?;
let res = self.persist_network_bond(inventory, topology, &nodes).await;
match res {
Ok(_) => Ok(Outcome::success(
"Network bond successfully persisted".into(),
)),
Err(_) => Err(InterpretError::new(
"Failed to persist network bond".to_string(),
)),
}
}
}

View File

@@ -1 +1,41 @@
use kube::CustomResource;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
pub mod nmstate;
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
#[kube(
group = "operators.coreos.com",
version = "v1",
kind = "OperatorGroup",
namespaced
)]
#[serde(rename_all = "camelCase")]
pub struct OperatorGroupSpec {
pub target_namespaces: Vec<String>,
}
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
#[kube(
group = "operators.coreos.com",
version = "v1alpha1",
kind = "Subscription",
namespaced
)]
#[serde(rename_all = "camelCase")]
pub struct SubscriptionSpec {
pub name: String,
pub source: String,
pub source_namespace: String,
pub channel: Option<String>,
pub install_plan_approval: Option<InstallPlanApproval>,
}
#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
pub enum InstallPlanApproval {
#[serde(rename = "Automatic")]
Automatic,
#[serde(rename = "Manual")]
Manual,
}

View File

@@ -6,16 +6,9 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
#[kube(
group = "nmstate.io",
version = "v1",
kind = "NMState",
plural = "nmstates",
namespaced = false
)]
#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)]
#[serde(rename_all = "camelCase")]
pub struct NMStateSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub probe_configuration: Option<ProbeConfig>,
}
@@ -51,7 +44,6 @@ pub struct ProbeDns {
)]
#[serde(rename_all = "camelCase")]
pub struct NodeNetworkConfigurationPolicySpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub node_selector: Option<BTreeMap<String, String>>,
pub desired_state: DesiredStateSpec,
}
@@ -66,64 +58,37 @@ pub struct DesiredStateSpec {
#[serde(rename_all = "kebab-case")]
pub struct InterfaceSpec {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub r#type: String,
pub state: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub mac_address: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub copy_mac_from: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mtu: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub controller: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ipv4: Option<IpStackSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ipv6: Option<IpStackSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ethernet: Option<EthernetSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub link_aggregation: Option<BondSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vlan: Option<VlanSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vxlan: Option<VxlanSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mac_vtap: Option<MacVtapSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mac_vlan: Option<MacVlanSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub infiniband: Option<InfinibandSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub linux_bridge: Option<LinuxBridgeSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ovs_bridge: Option<OvsBridgeSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ethtool: Option<EthtoolSpec>,
}
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct IpStackSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dhcp: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub autoconf: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub address: Option<Vec<IpAddressSpec>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_dns: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_gateway: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_routes: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dhcp_client_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dhcp_duid: Option<String>,
}
@@ -137,11 +102,8 @@ pub struct IpAddressSpec {
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct EthernetSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub speed: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub duplex: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_negotiation: Option<bool>,
}
@@ -150,7 +112,6 @@ pub struct EthernetSpec {
pub struct BondSpec {
pub mode: String,
pub ports: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub options: Option<BTreeMap<String, Value>>,
}
@@ -159,7 +120,6 @@ pub struct BondSpec {
pub struct VlanSpec {
pub base_iface: String,
pub id: u16,
#[serde(skip_serializing_if = "Option::is_none")]
pub protocol: Option<String>,
}
@@ -169,11 +129,8 @@ pub struct VxlanSpec {
pub base_iface: String,
pub id: u32,
pub remote: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub local: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub learning: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub destination_port: Option<u16>,
}
@@ -182,7 +139,6 @@ pub struct VxlanSpec {
pub struct MacVtapSpec {
pub base_iface: String,
pub mode: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub promiscuous: Option<bool>,
}
@@ -191,7 +147,6 @@ pub struct MacVtapSpec {
pub struct MacVlanSpec {
pub base_iface: String,
pub mode: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub promiscuous: Option<bool>,
}
@@ -206,35 +161,25 @@ pub struct InfinibandSpec {
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct LinuxBridgeSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub options: Option<LinuxBridgeOptions>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ports: Option<Vec<LinuxBridgePort>>,
}
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct LinuxBridgeOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub mac_ageing_time: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub multicast_snooping: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stp: Option<StpOptions>,
}
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct StpOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub forward_delay: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hello_time: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_age: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub priority: Option<u16>,
}
@@ -242,20 +187,15 @@ pub struct StpOptions {
#[serde(rename_all = "kebab-case")]
pub struct LinuxBridgePort {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub vlan: Option<LinuxBridgePortVlan>,
}
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct LinuxBridgePortVlan {
#[serde(skip_serializing_if = "Option::is_none")]
pub mode: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub trunk_tags: Option<Vec<VlanTag>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tag: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub enable_native: Option<bool>,
}
@@ -263,7 +203,6 @@ pub struct LinuxBridgePortVlan {
#[serde(rename_all = "kebab-case")]
pub struct VlanTag {
pub id: u16,
#[serde(skip_serializing_if = "Option::is_none")]
pub id_range: Option<VlanIdRange>,
}
@@ -277,20 +216,15 @@ pub struct VlanIdRange {
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct OvsBridgeSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub options: Option<OvsBridgeOptions>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ports: Option<Vec<OvsPortSpec>>,
}
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct OvsBridgeOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub stp: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rstp: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mcast_snooping_enable: Option<bool>,
}
@@ -298,11 +232,8 @@ pub struct OvsBridgeOptions {
#[serde(rename_all = "kebab-case")]
pub struct OvsPortSpec {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub link_aggregation: Option<BondSpec>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vlan: Option<LinuxBridgePortVlan>,
#[serde(skip_serializing_if = "Option::is_none")]
pub r#type: Option<String>,
}
@@ -315,8 +246,6 @@ pub struct EthtoolSpec {
#[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "kebab-case")]
pub struct EthtoolFecSpec {
#[serde(skip_serializing_if = "Option::is_none")]
pub auto: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mode: Option<String>,
}

View File

@@ -39,70 +39,30 @@ impl HostNetworkConfigurationInterpret {
&self,
topology: &T,
host: &PhysicalHost,
current_host: &usize,
total_hosts: &usize,
) -> Result<HostNetworkConfig, InterpretError> {
if host.network.is_empty() {
info!("[Host {current_host}/{total_hosts}] No interfaces to configure, skipping");
return Ok(HostNetworkConfig {
host_id: host.id.clone(),
switch_ports: vec![],
});
}
let switch_ports = self
.collect_switch_ports_for_host(topology, host, current_host, total_hosts)
.await?;
let config = HostNetworkConfig {
host_id: host.id.clone(),
switch_ports,
};
if !config.switch_ports.is_empty() {
info!(
"[Host {current_host}/{total_hosts}] Found {} ports for {} interfaces",
config.switch_ports.len(),
host.network.len()
);
info!("[Host {current_host}/{total_hosts}] Configuring host network...");
) -> Result<(), InterpretError> {
let switch_ports = self.collect_switch_ports_for_host(topology, host).await?;
if !switch_ports.is_empty() {
topology
.configure_host_network(&config)
.configure_host_network(host, HostNetworkConfig { switch_ports })
.await
.map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
} else {
info!(
"[Host {current_host}/{total_hosts}] No ports found for {} interfaces, skipping",
host.network.len()
);
}
Ok(config)
Ok(())
}
async fn collect_switch_ports_for_host<T: Topology + Switch>(
&self,
topology: &T,
host: &PhysicalHost,
current_host: &usize,
total_hosts: &usize,
) -> Result<Vec<SwitchPort>, InterpretError> {
let mut switch_ports = vec![];
if host.network.is_empty() {
return Ok(switch_ports);
}
info!("[Host {current_host}/{total_hosts}] Collecting ports on switch...");
for network_interface in &host.network {
let mac_address = network_interface.mac_address;
match topology.get_port_for_mac_address(&mac_address).await {
Ok(Some(port)) => {
info!(
"[Host {current_host}/{total_hosts}] Found port '{port}' for '{mac_address}'"
);
switch_ports.push(SwitchPort {
interface: NetworkInterface {
name: network_interface.name.clone(),
@@ -113,7 +73,7 @@ impl HostNetworkConfigurationInterpret {
port,
});
}
Ok(None) => debug!("No port found for '{mac_address}', skipping"),
Ok(None) => debug!("No port found for host '{}', skipping", host.id),
Err(e) => {
return Err(InterpretError::new(format!(
"Failed to get port for host '{}': {}",
@@ -125,47 +85,6 @@ impl HostNetworkConfigurationInterpret {
Ok(switch_ports)
}
fn format_host_configuration(&self, configs: Vec<HostNetworkConfig>) -> Vec<String> {
let mut report = vec![
"Network Configuration Report".to_string(),
"------------------------------------------------------------------".to_string(),
];
for config in configs {
let host = self
.score
.hosts
.iter()
.find(|h| h.id == config.host_id)
.unwrap();
println!("[Host] {host}");
if config.switch_ports.is_empty() {
report.push(format!(
"⏭️ Host {}: SKIPPED (No matching switch ports found)",
config.host_id
));
} else {
let mappings: Vec<String> = config
.switch_ports
.iter()
.map(|p| format!("[{} -> {}]", p.interface.name, p.port))
.collect();
report.push(format!(
"✅ Host {}: Bonded {} port(s) {}",
config.host_id,
config.switch_ports.len(),
mappings.join(", ")
));
}
}
report
.push("------------------------------------------------------------------".to_string());
report
}
}
#[async_trait]
@@ -195,38 +114,27 @@ impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
return Ok(Outcome::noop("No hosts to configure".into()));
}
let host_count = self.score.hosts.len();
info!("Started network configuration for {host_count} host(s)...",);
info!(
"Started network configuration for {} host(s)...",
self.score.hosts.len()
);
info!("Setting up switch with sane defaults...");
topology
.setup_switch()
.await
.map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?;
info!("Switch ready");
let mut current_host = 1;
let mut host_configurations = vec![];
let mut configured_host_count = 0;
for host in &self.score.hosts {
let host_configuration = self
.configure_network_for_host(topology, host, &current_host, &host_count)
.await?;
host_configurations.push(host_configuration);
current_host += 1;
self.configure_network_for_host(topology, host).await?;
configured_host_count += 1;
}
if current_host > 1 {
let details = self.format_host_configuration(host_configurations);
Ok(Outcome::success_with_details(
format!(
"Configured {}/{} host(s)",
current_host - 1,
self.score.hosts.len()
),
details,
))
if configured_host_count > 0 {
Ok(Outcome::success(format!(
"Configured {configured_host_count}/{} host(s)",
self.score.hosts.len()
)))
} else {
Ok(Outcome::noop("No hosts configured".into()))
}
@@ -301,7 +209,6 @@ mod tests {
assert_that!(*configured_host_networks).contains_exactly(vec![(
HOST_ID.clone(),
HostNetworkConfig {
host_id: HOST_ID.clone(),
switch_ports: vec![SwitchPort {
interface: EXISTING_INTERFACE.clone(),
port: PORT.clone(),
@@ -327,7 +234,6 @@ mod tests {
assert_that!(*configured_host_networks).contains_exactly(vec![(
HOST_ID.clone(),
HostNetworkConfig {
host_id: HOST_ID.clone(),
switch_ports: vec![
SwitchPort {
interface: EXISTING_INTERFACE.clone(),
@@ -357,7 +263,6 @@ mod tests {
(
HOST_ID.clone(),
HostNetworkConfig {
host_id: HOST_ID.clone(),
switch_ports: vec![SwitchPort {
interface: EXISTING_INTERFACE.clone(),
port: PORT.clone(),
@@ -367,7 +272,6 @@ mod tests {
(
ANOTHER_HOST_ID.clone(),
HostNetworkConfig {
host_id: ANOTHER_HOST_ID.clone(),
switch_ports: vec![SwitchPort {
interface: ANOTHER_EXISTING_INTERFACE.clone(),
port: ANOTHER_PORT.clone(),
@@ -478,10 +382,11 @@ mod tests {
async fn configure_host_network(
&self,
config: &HostNetworkConfig,
host: &PhysicalHost,
config: HostNetworkConfig,
) -> Result<(), SwitchError> {
let mut configured_host_networks = self.configured_host_networks.lock().unwrap();
configured_host_networks.push((config.host_id.clone(), config.clone()));
configured_host_networks.push((host.id.clone(), config.clone()));
Ok(())
}

View File

@@ -50,7 +50,7 @@
use crate::{
modules::okd::{
OKDSetup01InventoryScore, OKDSetup02BootstrapScore, OKDSetup03ControlPlaneScore,
OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, OKDSetupPersistNetworkBondScore,
OKDSetup04WorkersScore, OKDSetup05SanityCheckScore,
bootstrap_06_installation_report::OKDSetup06InstallationReportScore,
},
score::Score,
@@ -65,7 +65,6 @@ impl OKDInstallationPipeline {
Box::new(OKDSetup01InventoryScore::new()),
Box::new(OKDSetup02BootstrapScore::new()),
Box::new(OKDSetup03ControlPlaneScore::new()),
Box::new(OKDSetupPersistNetworkBondScore::new()),
Box::new(OKDSetup04WorkersScore::new()),
Box::new(OKDSetup05SanityCheckScore::new()),
Box::new(OKDSetup06InstallationReportScore::new()),

View File

@@ -6,7 +6,6 @@ mod bootstrap_05_sanity_check;
mod bootstrap_06_installation_report;
pub mod bootstrap_dhcp;
pub mod bootstrap_load_balancer;
mod bootstrap_persist_network_bond;
pub mod dhcp;
pub mod dns;
pub mod installation;
@@ -20,6 +19,5 @@ pub use bootstrap_03_control_plane::*;
pub use bootstrap_04_workers::*;
pub use bootstrap_05_sanity_check::*;
pub use bootstrap_06_installation_report::*;
pub use bootstrap_persist_network_bond::*;
pub mod crd;
pub mod host_network;

View File

@@ -12,8 +12,7 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::C
use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec,
GrafanaValueFrom, GrafanaValueSource,
GrafanaDatasourceSpec, GrafanaSpec,
};
use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
PrometheusRule, PrometheusRuleSpec, RuleGroup,
@@ -40,7 +39,7 @@ use crate::{
};
use harmony_types::id::Id;
use super::prometheus::PrometheusMonitoring;
use super::prometheus::PrometheusApplicationMonitoring;
#[derive(Clone, Debug, Serialize)]
pub struct K8sPrometheusCRDAlertingScore {
@@ -50,7 +49,7 @@ pub struct K8sPrometheusCRDAlertingScore {
pub prometheus_rules: Vec<RuleGroup>,
}
impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Score<T>
impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
for K8sPrometheusCRDAlertingScore
{
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
@@ -76,7 +75,7 @@ pub struct K8sPrometheusCRDAlertingInterpret {
}
#[async_trait]
impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Interpret<T>
impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
for K8sPrometheusCRDAlertingInterpret
{
async fn execute(
@@ -467,13 +466,10 @@ impl K8sPrometheusCRDAlertingInterpret {
match_labels: label.clone(),
match_expressions: vec![],
};
let mut json_data = BTreeMap::new();
json_data.insert("timeInterval".to_string(), "5s".to_string());
let namespace = self.sender.namespace.clone();
let json_data = GrafanaDatasourceJsonData {
time_interval: Some("5s".to_string()),
http_header_name1: None,
tls_skip_verify: Some(true),
oauth_pass_thru: Some(true),
};
let json = build_default_dashboard(&namespace);
let graf_data_source = GrafanaDatasource {
@@ -499,11 +495,7 @@ impl K8sPrometheusCRDAlertingInterpret {
"http://prometheus-operated.{}.svc.cluster.local:9090",
self.sender.namespace.clone()
),
secure_json_data: None,
is_default: None,
editable: None,
},
values_from: None,
},
};
@@ -524,9 +516,7 @@ impl K8sPrometheusCRDAlertingInterpret {
spec: GrafanaDashboardSpec {
resync_period: Some("30s".to_string()),
instance_selector: labels.clone(),
json: Some(json),
grafana_com: None,
datasources: None,
json,
},
};

View File

@@ -9,17 +9,11 @@ use crate::{
};
#[async_trait]
pub trait PrometheusMonitoring<S: AlertSender> {
pub trait PrometheusApplicationMonitoring<S: AlertSender> {
async fn install_prometheus(
&self,
sender: &S,
inventory: &Inventory,
receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
) -> Result<PreparationOutcome, PreparationError>;
async fn ensure_prometheus_operator(
&self,
sender: &S,
inventory: &Inventory,
) -> Result<PreparationOutcome, PreparationError>;
}

View File

@@ -38,7 +38,7 @@ use crate::{
};
use harmony_types::id::Id;
use super::prometheus::PrometheusMonitoring;
use super::prometheus::PrometheusApplicationMonitoring;
#[derive(Clone, Debug, Serialize)]
pub struct RHOBAlertingScore {
@@ -48,8 +48,8 @@ pub struct RHOBAlertingScore {
pub prometheus_rules: Vec<RuleGroup>,
}
impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Score<T>
for RHOBAlertingScore
impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
Score<T> for RHOBAlertingScore
{
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
Box::new(RHOBAlertingInterpret {
@@ -74,8 +74,8 @@ pub struct RHOBAlertingInterpret {
}
#[async_trait]
impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Interpret<T>
for RHOBAlertingInterpret
impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
Interpret<T> for RHOBAlertingInterpret
{
async fn execute(
&self,

View File

@@ -40,7 +40,7 @@ pub fn init() {
HarmonyEvent::HarmonyFinished => {
if !details.is_empty() {
println!(
"\n{} All done! Here's a few info for you:",
"\n{} All done! Here's what's next for you:",
theme::EMOJI_SUMMARY
);
for detail in details.iter() {

View File

@@ -216,7 +216,7 @@ pub struct System {
pub maximumfrags: Option<MaybeString>,
pub aliasesresolveinterval: Option<MaybeString>,
pub maximumtableentries: Option<MaybeString>,
pub language: Option<MaybeString>,
pub language: String,
pub dnsserver: Option<MaybeString>,
pub dns1gw: Option<String>,
pub dns2gw: Option<String>,