enhance hvee environment variables

This commit is contained in:
2025-11-10 13:22:25 -05:00
parent 06c9d78049
commit 28476fbac4
13 changed files with 435 additions and 71 deletions

36
Cargo.lock generated
View File

@@ -1719,6 +1719,24 @@ dependencies = [
"url",
]
[[package]]
name = "example-ha-cluster"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",
"env_logger",
"harmony",
"harmony_macros",
"harmony_secret",
"harmony_tui",
"harmony_types",
"log",
"serde",
"tokio",
"url",
]
[[package]]
name = "example-kube-rs"
version = "0.1.0"
@@ -1838,6 +1856,24 @@ dependencies = [
[[package]]
name = "example-opnsense"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",
"env_logger",
"harmony",
"harmony_cli",
"harmony_macros",
"harmony_secret",
"harmony_types",
"log",
"serde",
"tokio",
"url",
]
[[package]]
name = "example-opnsense-2"
version = "0.1.0"
dependencies = [
"brocade",
"cidr",

View File

@@ -1,3 +1,4 @@
export HARMONY_SECRET_NAMESPACE=example-opnsense
export HARMONY_SECRET_STORE=file
export HARMONY_DATABASE_URL=sqlite://harmony_vms.sqlite RUST_LOG=info
export RUST_LOG=info

View File

@@ -1,4 +1,4 @@
# Example scripts
# Virtualized Execution Environment for Harmony
Scripts included in this directory have 3 purposes:
@@ -6,17 +6,91 @@ Scripts included in this directory have 3 purposes:
- prototype a solution for an 'OpensenseLocalhostTopology'
- prototype
This exprimentation aim to find an approach suitable for using harmony on virtualised execution environment such that:
- it straights forward for a user with minimal knowledge to start testing harmony
- installation and execution have **minimal impact on the user desktop**
## Usage
### Installation
1. download this directory
2. add this directory in your PATH (example `. setup`)
```
# show help page
harmony-vee
# show active configurations
harmony-vee config
# show what will be modified at installation
harmony-vee install --dry-run
# install
harmony-vee install
# show what will be modified at unistallation
harmony-vee uninstall --dry-run
```
### Create and start a new Virtual Execution Environment
```
# Create a HVEE to test opnsense_score
harmony-vee init opnsense_score
# List existing HVEE
harmony-vee list
# Show HVEE information including devices ip and vault type/location
harmony-vee show opnsense_score
# Start/Stop a HVEE
harmony-vee stop opnsense_score
# Destroy a HVEE instance
harmony-vee destroy opnsense_score
```
### Variable d'environnement
```
## directory containing harmony-ve data
# HVE_ROOT=~/.harmony-ve
## OPNSENSE SRC
# main mirror
# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
# first alternative mirror
# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
## Network
# HVE_NETWORK_LABEL=harmony
```
## Remarks
- A nested VM setup should be safer
- A nested VM setup could be safer
## Automation
### Localhost learning environment
## Architecture
### Learning environment directly on host
![localhost case](./doc/automate-opnsense-example-localhost.drawio.png)
### Virtualized learning environment
### Learning environment nested in a "workspace vm"
![localhost case](./doc/automate-opnsense-example-nested-virtualization.drawio.png)

View File

@@ -0,0 +1,31 @@
# Conventions:
# - Namespaced with HVE, short for Harmony Virtualised Execution Environment
# - Prefixed values used internally
# - Not prefixed may be supercharged by the user
# Root of harmony data
_HVE_ROOT=${HVE_ROOT:-$HOME/harmony-ve}
[ -d "$_HVE_ROOT" ] || mkdir -p "${_HVE_ROOT}"
_HVE_SRC_IMG=${_HVE_ROOT}/src/images
[ -d "$_HVE_SRC_IMG" ] || mkdir -p "${_HVE_SRC_IMG}"
_HVE_IMG=${_HVE_ROOT}/images
[ -d "$_HVE_IMG" ] || mkdir -p "$_HVE_IMG"
# Opnsense
_HVE_OPNSENSE_URL=${HVE_OPNSENSE_URL:-https://pkg.opnsense.org/releases}
# first alternative mirror
_HVE_OPNSENSE_URL_ALT1=${HVE_OPNSENSE_URL_ALT1:-https://mirror.vraphim.com/opnsense/releases}
_HVE_OPNSENSE_URL_ALT2=${HVE_OPNSENSE_URL_ALT2:-https://mirror.winsub.kr/opnsense/releases}
_HVE_OPNSENSE_SRC_IMG=${_HVE_SRC_IMG}/opnsense
[ -d "$_HVE_OPNSENSE_SRC_IMG" ] || mkdir -p "${_HVE_OPNSENSE_SRC_IMG}"
_HVE_OPNSENSE_IMG=${_HVE_IMG}/opnsense
[ -d "$_HVE_OPNSENSE_IMG" ] || mkdir -p "${_HVE_OPNSENSE_IMG}"
# Network
_HVE_NETWORK=${HVE_NETWORK:-harmony}
_HVE_WAN_BRIDGE=${HVE_WAN_BRIDGE:-${_HVE_NETWORK}-wan-brd}
_HVE_LAN_BRIDGE=${HVE_LAN_BRIDGE:-${_HVE_NETWORK}-lann-brd}

View File

@@ -32,7 +32,10 @@ has_wget(){
install_kvm(){
sudo apt install -y --no-install-recommends qemu-system libvirt-clients libvirt-daemon-system
sudo adduser "$USER" libvirt
sudo usermod -aG libvirt "$USER"
# todo: finf how to fix image access out of /var/lib/libvirt/images
sudo setfacl -Rm u:libvirt-qemu:rx $_HVE_IMG
sudo systemctl restart libvirtd
}
install_virt_customize(){

View File

@@ -0,0 +1,13 @@
## directory containing harmony-ve data
# HVE_ROOT=~/.harmony-ve
## OPNSENSE SRC
# main mirror
# HVE_OPNSENSE_URL=https://pkg.opnsense.org/releases
# first alternative mirror
# HVE_OPENSENSE_URL_ALT1=https://mirror.vraphim.com/opnsense/releases
# HVE_OPNSENSE_URL_ALT2=https://mirror.winsub.kr/opnsense/releases
## Network
# HVE_NETWORK_LABEL=harmony

View File

@@ -1,4 +1,9 @@
#! /bin/bash
#
# virt-install <= virtinst
# quemu-img
harmony-ve-dependencies()(

View File

@@ -1,5 +1,7 @@
#! /bin/bash
# todo: allow wan to switch from ethernet to wifi
harmony-ve-network()(
@@ -8,7 +10,9 @@ harmony-ve-network()(
[ "${1:-}" != "-d" ] || { set -x ; shift ; }
trap '[ "$?" = "0" ] || >&2 echo ABNORMAL TERMINATION' EXIT
BASE_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")/..")
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
_short_help(){
@@ -44,13 +48,157 @@ EOM
WARNINGS
This script is experimetal. Use with caution.
IMPLEMENTATION NOTES
- use the network manager present to add 2 bridges (wan + lan)
EOM
}
# dependency management
_is_service_used(){
service=$1
sudo systemctl list-unit-files $service || return 1
sudo systemctl status --no-pager $service || return 1
}
# Implement functions
_list_bridges(){
ip -o link show type bridge | awk '{print $2}' | sed 's/://g'
}
_is_a_bridge(){
bridge=$1
matched=$(_list_bridges | grep "$bridge")
[ "$matched" = "$bridge" ] || return 1
}
_bridge_is_up(){
_fatal Not implemented
}
_rename_nmcli_profile(){
device=$1
profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$device:" | cut -d':' -f2)
[ "$profile" != "" ] || _fatal Failed to find nmcli profile
sudo nmcli con mod "$profile" con-name "$device"
}
_create_a_bridge_using_networkmanager(){
bridge=$1
profile=$(nmcli -t -f DEVICE,UUID c show --active | grep "^$PRIMARY_INTERFACE:" | cut -d':' -f2)
nmcli conn delete "$profile"
nmcli conn add type bridge ifname $bridge con-name $bridge || _fatal Fail to create a bridge using nmcli
nmcli con add type bridge-slave ifname $PRIMARY_INTERFACE master $bridge || _fatal Fail to create a slave-interface using nmcli
nmcli con up $bridge || _fatal Fail to set interface up using nmcli
sudo systemctl restart NetworkManager.service
# todo: use a check loop until connection with a timeout
#sleep 10
#ping nationtech.io | _fatal Internet connection lost
}
_delete_a_bridge_using_networkmanager(){
device=$1
nmcli conn delete bridge-slave-$PRIMARY_INTERFACE
nmcli conn delete $device
nmcli con add type ethernet ifname $PRIMARY_INTERFACE con-name $PRIMARY_INTERFACE autoconnect yes ipv4.method auto ipv6.method ignore
nmcli conn up "$PRIMARY_INTERFACE"
sudo systemctl restart NetworkManager.service
# todo: use a check loop until connection with a timeout
#sleep 10
#ping nationtech.io | _fatal Internet connection lost
}
_create_a_bridge(){
bridge=$1
[ $USE_NETWORKMANAGER = 0 ] | _fatal "Only NetworkManager is implemented"
_create_a_bridge_using_networkmanager $bridge
}
_setup_a_bridge(){
$bridge
bridge_exist=1
bridge_is_up=1
bridge_has_ip=1
bridge_has_route=1
bridge_is_working=1
_is_a_bridge $bridge && bridge_exists=0 || _create_a_bridge $bridge || _fatal Fail to create a bridge
}
_get_networkmanager_profile_from_device(){
device=$1
profile=$(nmcli -t -f DEVICE,NAME c show --active | grep "^$device:" | cut -d':' -f2)
[ "$profile" != "" ] || _fatal Fail to retreive nmcli profile
echo "$profile"
}
_find_primary_interface(){
PRIMARY_INTERFACE=$(ip route | grep '^default' | sed 's/ dev /!/g' | cut -d'!' -f 2 | awk '{ print $1 }' )
[ "$PRIMARY_INTERFACE" != "" ] || _fatal Fail to find the primary interface
}
_find_used_network_manager(){
_is_service_used NetworkManager.service && USE_NETWORKMANAGER=0 || USE_NETWORKMANAGER=1
_is_service_used systemd-networkd.service && USE_SYSTEMD_NETWORKD=0 || USE_SYSTEMD_NETWORKD=1
_is_service_used dhcpd.service && USE_DHCPD=0 || USE_DHCPD=1
USE_MANUAL=0 && [ $USE_NETWORKMANAGER = 0 ] || [ $USE_SYSTEMD_NETWORKD = 0 ] || [ $USE_DHCPD = 0 ] || USE_MANUAL=0
}
_connect(){
_find_used_network_manager
_find_primary_interface
_setup_a_bridge $_HVE_WAN_BRIDGE
_setup_a_bridge $_HVE_LAN_BRIDGE
}
case "${1:-}" in
"")
@@ -60,7 +208,17 @@ case "${1:-}" in
_short_help
_extra_help
;;
# Commands entrypoints
connect)
_connect "${@:2}"
;;
disconnect)
_disconnect "${@:2}"
;;
dev)
"${@:2}"
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1

View File

@@ -10,11 +10,9 @@ harmony-ve-opnsense-img()(
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
DEFAULT_KVM_IMG_DIR=/var/lib/libvirt/images
KVM_IMG_DIR=${KVM_IMG_DIR:-$DEFAULT_KVM_IMG_DIR}
[ -d "$KVM_IMG_DIR" ] || mkdir -p "${KVM_IMG_DIR}"
export PATH=$SCRIPTS_DIR:$PATH
_short_help(){
@@ -31,10 +29,12 @@ harmony-ve-opnsense-img()(
harmony-vee-opnsense-img [GLOBAL_OPTIONS] COMMAND [OPTIONS]
harmony-vee-opnsense-img list [--remote]
harmony-vee-opnsense-img new NAME
harmony-vee-opnsense-img update NAME
harmony-vee-opnsense-img delete [NAME]
harmony-ve-opnsense-img list
harmony-ve-opnsense-img init NAME VERSION
harmony-ve-opnsense-img start NAME
harmony-ve-opnsense-img update NAME
harmony-ve-opnsense-img delete [NAME]
EOM
@@ -56,7 +56,61 @@ EOM
}
# assertions
_assert_image_do_not_exists(){
name=$1
[ ! -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "An image '$name' already exists"
}
_assert_image_exists(){
name=$1
[ -d "$_HVE_OPNSENSE_IMG/$name" ] || _fatal "Image '$name' do not exists"
}
# Implement functions
_init(){
name=$1
version=${2}
_assert_image_do_not_exists $name
mkdir -p "${_HVE_OPNSENSE_IMG}/$name"
harmony-ve opnsense-img-src download $version
sudo qemu-img convert -f raw -O qcow2 "$_HVE_OPNSENSE_SRC_IMG/OPNsense-${version}-nano-amd64.img" "/var/lib/libvirt/images/opnsense-$name.qcow2"
cat <<-EOM > "$_HVE_OPNSENSE_IMG/$name/$name.sh"
virt-install \
--name $name \
--os-variant freebsd14.0 \
--vcpus=2,sockets=1,cores=2,threads=1 \
--memory 4096 \
--disk path="/var/lib/libvirt/images/opnsense-$name.qcow2" \
--network bridge=${_HVE_WAN_BRIDGE},model=virtio \
--network bridge=${_HVE_LAN_BRIDGE},model=virtio \
--graphics none \
--console pty,target_type=serial \
--import \
--autostart
EOM
chmod +x "$_HVE_OPNSENSE_IMG/$name/$name.sh"
}
_start(){
name=$1
_assert_image_exists $name
"$_HVE_OPNSENSE_IMG/$name/$name.sh"
}
case "${1:-}" in
@@ -68,7 +122,22 @@ case "${1:-}" in
_extra_help
;;
# Commands entrypoints
*)
init)
_init "${@:2}"
;;
start)
_start "${@:2}"
;;
delete)
rm -r ${_HVE_OPNSENSE_IMG}/"$2"
;;
ls|list)
ls ${_HVE_OPNSENSE_IMG} | cat
;;
show)
ls ${_HVE_OPNSENSE_IMG}/"$2" | cat
;;
*)
_warn "Unknown COMMAND '$1'"
exit 1
;;

View File

@@ -10,13 +10,8 @@ harmony-ve-opnsense-img-src()(
SCRIPTS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE}")")
. "${SCRIPTS_DIR}/common"
. "${SCRIPTS_DIR}/default-env-var"
OPNSENSE_SRC_BASE_URL=https://pkg.opnsense.org/releases
ALT1_SRC_BASE_URL=https://mirror.vraphim.com/opnsense/releases
ALT2_SRC_BASE_URL=https://mirror.winsub.kr/opnsense/releases
DEFAULT_KVM_IMG_DIR=$HOME/.harmony-ve/img-src
KVM_IMG_DIR=${KVM_IMG_DIR:-$DEFAULT_KVM_IMG_DIR}
[ -d "$KVM_IMG_DIR" ] || mkdir -p "${KVM_IMG_DIR}"
_short_help(){
@@ -75,12 +70,12 @@ EOM
_list_local_images(){
ls "${KVM_IMG_DIR}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
ls "${_HVE_OPNSENSE_SRC_IMG}" | grep "OPNsense-" | grep "\-nano\-amd64\.img" | cut -d'-' -f 2 | sort -u -r
}
_list_remote_images(){
curl -L -s "${OPNSENSE_SRC_BASE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2 | cut -d '/' -f 1 | sort -r
curl -L -s "${_HVE_OPNSENSE_URL}" | sed 's/</\n</g' | grep href | grep 2 | cut -d'>' -f 2 | cut -d '/' -f 1 | sort -r
}
_latest_version(){
@@ -89,7 +84,7 @@ EOM
_is_downloaded(){
version=$1
name="${KVM_IMG_DIR}/OPNsense-${version}-nano-amd64.img"
name="${_HVE_OPNSENSE_SRC_IMG}/OPNsense-${version}-nano-amd64.img"
[ -f "$name" ] && return 0 || return 1
}
@@ -104,7 +99,6 @@ EOM
_download_crypto_files $version
pushd "${KVM_IMG_DIR}"
name="OPNsense-${version}-nano-amd64.img"
compressed_name=$name.bz2
@@ -112,7 +106,7 @@ EOM
_is_downloaded $version && {
_warn "Image '$name' is already downloaded"
} || {
url=$OPNSENSE_SRC_BASE_URL/$version/$compressed_name
url=$_HVE_OPNSENSE_URL/$version/$compressed_name
>&2 echo DOWNLOAD $url
wget -q -c "${url}"
@@ -121,8 +115,6 @@ EOM
>&2 echo DECOMPRESS $url
bzip2 -d $compressed_name
}
popd
}
@@ -142,14 +134,13 @@ EOM
# see: https://docs.opnsense.org/manual/install.html#download-and-verification
version=$1
pushd "${KVM_IMG_DIR}"
# download multiple pubkeys from different server
pubkey="OPNsense-${version}.pub"
rm -f $pubkey $pubkey.sig $pubkey.alt1 $pubkey.alt2
url=$OPNSENSE_SRC_BASE_URL/$version/$pubkey
url=$_HVE_OPNSENSE_URL/$version/$pubkey
wget -q -c "${url}"
# failing:
@@ -158,17 +149,15 @@ EOM
openssl base64 -d -in $pubkey.sig -out /tmp/file.sig
openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $pubkey || _fatal "Can't verify the signature of the public key"
url_alt1=$ALT1_SRC_BASE_URL/$version/$pubkey
url_alt1=$_HVE_OPNSENSE_URL_ALT1/$version/$pubkey
wget -q -c -O "$pubkey.alt1" "${url_alt1}"
url_alt2=$ALT2_SRC_BASE_URL/$version/$pubkey
url_alt2=$_HVE_OPNSENSE_URL_ALT2/$version/$pubkey
wget -q -c -O "$pubkey.alt2" "${url_alt2}"
_compare_files_checksum $pubkey $pubkey.alt1 || { popd && _fatal "Fail to compare pubkeys" ; }
_compare_files_checksum $pubkey $pubkey.alt1 || _fatal "Fail to compare pubkeys" ;
_compare_files_checksum $pubkey $pubkey.alt2 || { popd && _fatal "Fail to compare pubkeys" ; }
_compare_files_checksum $pubkey $pubkey.alt2 || _fatal "Fail to compare pubkeys" ;
@@ -184,7 +173,7 @@ EOM
for file in $img_sig $sha256_name $sha256_sig;
do
url=$OPNSENSE_SRC_BASE_URL/$version/$file
url=$_HVE_OPNSENSE_URL/$version/$file
wget -q -c "${url}"
done
@@ -193,8 +182,6 @@ EOM
openssl base64 -d -in $sha256_sig -out /tmp/file.sig
openssl dgst -sha256 -verify $pubkey -signature /tmp/file.sig $sha256_name || _fatal "Can't verify the signature of the checksum file"
popd
}
_download(){
@@ -211,27 +198,18 @@ EOM
_verify_image_checksum(){
failed=1
version=$1
name="OPNsense-${version}-nano-amd64.img.bz2"
sha256_file="OPNsense-${version}-checksums-amd64.sha256"
pushd "${KVM_IMG_DIR}"
sha256=$(cat $sha256_file | grep "$name" | cut -d'=' -f 2 | tr -s [:space:])
echo "$sha256 $name" | sha256sum -c || failed=0
popd
[ "$failed" = "1" ] || _fatal "Checksum failed for '$name'"
echo "$sha256 $name" | sha256sum -c || _fatal "Checksum failed for '$name'"
}
_verify_image_signature(){
version=$1
pushd "${KVM_IMG_DIR}"
# download multiple pubkeys from different server
pubkey="OPNsense-${version}.pub"
@@ -265,25 +243,16 @@ EOM
version=${1:-}
if [ -z "${version:-1}" ]; then
_clear
pushd "${KVM_IMG_DIR}"
rm -f *.img
popd
else
pushd "${KVM_IMG_DIR}"
rm -f *$version*.img
popd
fi
}
_clear(){
pushd "${KVM_IMG_DIR}"
rm -f *.pub *.sig *.bz2 *.alt1 *.alt2 *.sha256
popd
}
case "${1:-}" in
@@ -306,19 +275,27 @@ case "${1:-}" in
fi
;;
download)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_download "${2:-"$(_latest_version)"}"
popd
;;
delete)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_delete "${@:2}"
popd
;;
check)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_check "${@:2}"
popd
;;
show)
ls $KVM_IMG_DIR | cat
ls $_HVE_OPNSENSE_SRC_IMG | cat
;;
clear)
pushd "${_HVE_OPNSENSE_SRC_IMG}"
_clear "${@:2}"
popd
;;
*)
_warn "Unknown COMMAND '$1'"
@@ -331,6 +308,3 @@ esac
[ "$0" != "${BASH_SOURCE}" ] || harmony-ve-opnsense-img-src "${@}"
# todo: refactor
# todo: enhance output management

View File

@@ -0,0 +1,4 @@
#! /bin/bash
export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")"):"${PATH}"

View File

@@ -1,4 +0,0 @@
#! /bin/bash
export PATH=$(readlink -f "$(dirname "${BASH_SOURCE}")/scripts"):"${PATH}"

View File

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
#[tokio::main]
async fn main() {
let firewall = LogicalHost {
ip: ip!("192.168.55.1"),
ip: ip!("192.168.1.1"),
name: String::from("opnsense-1"),
};
@@ -30,8 +30,8 @@ async fn main() {
let dhcp_score = DhcpScore {
dhcp_range: (
ipv4!("192.168.55.100").into(),
ipv4!("192.168.55.150").into(),
ipv4!("192.168.1.100").into(),
ipv4!("192.168.1.150").into(),
),
host_binding: vec![],
next_server: None,