Files

691 lines
24 KiB
Rust

//! OPNsense firewall pair integration example.
//!
//! Boots two OPNsense VMs, bootstraps both (login, SSH, webgui port),
//! then applies `FirewallPairTopology` + `CarpVipScore` for CARP HA testing.
//!
//! Both VMs share a LAN bridge but boot with the same default IP (.1).
//! The bootstrap sequence disables one VM's LAN NIC while bootstrapping
//! the other, then changes IPs via the API to avoid conflicts.
//!
//! # Usage
//!
//! ```bash
//! cargo run -p opnsense-pair-integration -- --check # verify prerequisites
//! cargo run -p opnsense-pair-integration -- --boot # boot + bootstrap both VMs
//! cargo run -p opnsense-pair-integration # run pair integration test
//! cargo run -p opnsense-pair-integration -- --full # boot + bootstrap + test (CI mode)
//! cargo run -p opnsense-pair-integration -- --status # check both VMs
//! cargo run -p opnsense-pair-integration -- --clean # tear down everything
//! ```
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use harmony::config::secret::{OPNSenseApiCredentials, OPNSenseFirewallCredentials};
use harmony::infra::opnsense::OPNSenseFirewall;
use harmony::inventory::Inventory;
use harmony::modules::kvm::config::init_executor;
use harmony::modules::kvm::{
BootDevice, ForwardMode, KvmExecutor, NetworkConfig, NetworkRef, VmConfig,
};
use harmony::modules::opnsense::bootstrap::OPNsenseBootstrap;
use harmony::modules::opnsense::firewall::{FilterRuleDef, FirewallRuleScore};
use harmony::modules::opnsense::vip::VipDef;
use harmony::modules::opnsense::vlan::{VlanDef, VlanScore};
use harmony::score::Score;
use harmony::topology::{CarpVipScore, FirewallPairTopology, LogicalHost};
use harmony_types::firewall::{Direction, FirewallAction, IpProtocol, NetworkProtocol, VipMode};
use log::info;
const OPNSENSE_IMG_URL: &str =
"https://mirror.ams1.nl.leaseweb.net/opnsense/releases/26.1/OPNsense-26.1-nano-amd64.img.bz2";
const OPNSENSE_IMG_NAME: &str = "OPNsense-26.1-nano-amd64.img";
const VM_PRIMARY: &str = "opn-pair-primary";
const VM_BACKUP: &str = "opn-pair-backup";
const NET_LAN: &str = "opn-pair-lan";
/// Both VMs boot on this IP (OPNsense default, ignores injected config.xml).
/// We bootstrap one at a time by toggling LAN NICs, then change IPs via the API.
const BOOT_IP: &str = "192.168.1.1";
const HOST_IP: &str = "192.168.1.10";
/// After bootstrap, primary gets .2, backup gets .3, CARP VIP stays at .1
const PRIMARY_IP: &str = "192.168.1.2";
const BACKUP_IP: &str = "192.168.1.3";
const CARP_VIP: &str = "192.168.1.1";
const API_PORT: u16 = 9443;
const CARP_PASSWORD: &str = "pair-test-carp";
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
harmony_cli::cli_logger::init();
let args: Vec<String> = std::env::args().collect();
if args.iter().any(|a| a == "--check") {
return check_prerequisites();
}
if args.iter().any(|a| a == "--download") {
download_image().await?;
return Ok(());
}
let executor = init_executor()?;
if args.iter().any(|a| a == "--clean") {
return clean(&executor).await;
}
if args.iter().any(|a| a == "--status") {
return status(&executor).await;
}
if args.iter().any(|a| a == "--boot") {
let img_path = download_image().await?;
return boot_pair(&executor, &img_path).await;
}
if args.iter().any(|a| a == "--full") {
let img_path = download_image().await?;
boot_pair(&executor, &img_path).await?;
return run_pair_test().await;
}
// Default: run pair test (assumes VMs are bootstrapped)
check_prerequisites()?;
run_pair_test().await
}
// ── Phase 1: Boot and bootstrap both VMs ───────────────────────────
async fn boot_pair(
executor: &KvmExecutor,
img_path: &Path,
) -> Result<(), Box<dyn std::error::Error>> {
info!("Creating shared LAN network and two OPNsense VMs...");
// Create the shared LAN network
let network = NetworkConfig::builder(NET_LAN)
.bridge("virbr-pair")
.subnet(HOST_IP, 24)
.forward(ForwardMode::Nat)
.build();
executor.ensure_network(network).await?;
// Prepare disk images for both VMs
for vm_name in [VM_PRIMARY, VM_BACKUP] {
prepare_vm_disk(vm_name, img_path)?;
}
// Define and start both VMs (2 NICs each: LAN + WAN)
for vm_name in [VM_PRIMARY, VM_BACKUP] {
let disk = image_dir().join(format!("{vm_name}.qcow2"));
let vm = VmConfig::builder(vm_name)
.vcpus(1)
.memory_mib(1024)
.disk_from_path(disk.to_string_lossy().to_string())
.network(NetworkRef::named(NET_LAN)) // vtnet0 = LAN
.network(NetworkRef::named("default")) // vtnet1 = WAN
.boot_order([BootDevice::Disk])
.build();
executor.ensure_vm(vm).await?;
executor.start_vm(vm_name).await?;
}
// Get MAC addresses for LAN NICs (first interface on each VM)
let primary_interfaces = executor.list_interfaces(VM_PRIMARY).await?;
let backup_interfaces = executor.list_interfaces(VM_BACKUP).await?;
let primary_lan_mac = &primary_interfaces[0].mac;
let backup_lan_mac = &backup_interfaces[0].mac;
info!("Primary LAN MAC: {primary_lan_mac}, Backup LAN MAC: {backup_lan_mac}");
// ── Sequential bootstrap with NIC juggling ─────────────────────
//
// Both VMs boot on .1 (OPNsense default). We disable backup's LAN
// NIC so primary gets exclusive access to .1, bootstrap it, change
// its IP, then do the same for backup.
// Step 1: Disable backup's LAN NIC
info!("Disabling backup LAN NIC for primary bootstrap...");
executor
.set_interface_link(VM_BACKUP, backup_lan_mac, false)
.await?;
// Step 2: Wait for primary web UI and bootstrap
info!("Waiting for primary web UI at https://{BOOT_IP}...");
wait_for_https(BOOT_IP, 443).await?;
bootstrap_vm("primary", BOOT_IP).await?;
// Step 3: Change primary's LAN IP from .1 to .2 via API
info!("Changing primary LAN IP to {PRIMARY_IP}...");
change_lan_ip_via_ssh(BOOT_IP, PRIMARY_IP, 24).await?;
// Step 4: Wait for primary to come back on new IP
info!("Waiting for primary on new IP {PRIMARY_IP}:{API_PORT}...");
OPNsenseBootstrap::wait_for_ready(
&format!("https://{PRIMARY_IP}:{API_PORT}"),
std::time::Duration::from_secs(60),
)
.await?;
// Step 5: Disable primary's LAN NIC, enable backup's
info!("Swapping NICs: disabling primary, enabling backup...");
executor
.set_interface_link(VM_PRIMARY, primary_lan_mac, false)
.await?;
executor
.set_interface_link(VM_BACKUP, backup_lan_mac, true)
.await?;
// Step 6: Wait for backup web UI and bootstrap
info!("Waiting for backup web UI at https://{BOOT_IP}...");
wait_for_https(BOOT_IP, 443).await?;
bootstrap_vm("backup", BOOT_IP).await?;
// Step 7: Change backup's LAN IP from .1 to .3 via API
info!("Changing backup LAN IP to {BACKUP_IP}...");
change_lan_ip_via_ssh(BOOT_IP, BACKUP_IP, 24).await?;
// Step 8: Re-enable primary's LAN NIC
info!("Re-enabling primary LAN NIC...");
executor
.set_interface_link(VM_PRIMARY, primary_lan_mac, true)
.await?;
// Step 9: Wait for both to be reachable on their final IPs
info!("Waiting for both VMs on final IPs...");
OPNsenseBootstrap::wait_for_ready(
&format!("https://{PRIMARY_IP}:{API_PORT}"),
std::time::Duration::from_secs(60),
)
.await?;
OPNsenseBootstrap::wait_for_ready(
&format!("https://{BACKUP_IP}:{API_PORT}"),
std::time::Duration::from_secs(60),
)
.await?;
println!();
println!("OPNsense firewall pair is running and bootstrapped:");
println!(" Primary: https://{PRIMARY_IP}:{API_PORT} (root/opnsense)");
println!(" Backup: https://{BACKUP_IP}:{API_PORT} (root/opnsense)");
println!(" CARP VIP: {CARP_VIP} (will be configured by pair scores)");
println!();
println!("Run the pair integration test:");
println!(" cargo run -p opnsense-pair-integration");
Ok(())
}
async fn bootstrap_vm(role: &str, ip: &str) -> Result<(), Box<dyn std::error::Error>> {
info!("Bootstrapping {role} firewall at {ip}...");
let bootstrap = OPNsenseBootstrap::new(&format!("https://{ip}"));
bootstrap.login("root", "opnsense").await?;
bootstrap.abort_wizard().await?;
bootstrap.enable_ssh(true, true).await?;
bootstrap.set_webgui_port(API_PORT, ip, false).await?;
// Wait for webgui on new port
OPNsenseBootstrap::wait_for_ready(
&format!("https://{ip}:{API_PORT}"),
std::time::Duration::from_secs(120),
)
.await?;
// Verify SSH
for _ in 0..15 {
if check_tcp_port(ip, 22).await {
break;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if !check_tcp_port(ip, 22).await {
return Err(format!("SSH not reachable on {role} after bootstrap").into());
}
info!("{role} bootstrap complete");
Ok(())
}
/// Change the LAN interface IP via SSH (using OPNsense's ifconfig + config edit).
async fn change_lan_ip_via_ssh(
current_ip: &str,
new_ip: &str,
subnet: u8,
) -> Result<(), Box<dyn std::error::Error>> {
use opnsense_config::config::{OPNsenseShell, SshCredentials, SshOPNSenseShell};
let ssh_config = Arc::new(russh::client::Config {
inactivity_timeout: None,
..<_>::default()
});
let credentials = SshCredentials::Password {
username: "root".to_string(),
password: "opnsense".to_string(),
};
let ip: IpAddr = current_ip.parse()?;
let shell = SshOPNSenseShell::new((ip, 22), credentials, ssh_config);
// Use a PHP script to update config.xml and apply
let php_script = format!(
r#"<?php
require_once '/usr/local/etc/inc/config.inc';
$config = OPNsense\Core\Config::getInstance();
$config->object()->interfaces->lan->ipaddr = '{new_ip}';
$config->object()->interfaces->lan->subnet = '{subnet}';
$config->save();
echo "OK\n";
"#
);
shell
.write_content_to_file(&php_script, "/tmp/change_ip.php")
.await?;
let output = shell
.exec("php /tmp/change_ip.php && rm /tmp/change_ip.php && configctl interface reconfigure lan")
.await?;
info!("IP change result: {}", output.trim());
Ok(())
}
// ── Phase 2: Pair integration test ─────────────────────────────────
async fn run_pair_test() -> Result<(), Box<dyn std::error::Error>> {
// Verify both VMs are reachable
info!("Checking primary at {PRIMARY_IP}:{API_PORT}...");
if !check_tcp_port(PRIMARY_IP, API_PORT).await {
return Err(format!("Primary not reachable at {PRIMARY_IP}:{API_PORT}").into());
}
info!("Checking backup at {BACKUP_IP}:{API_PORT}...");
if !check_tcp_port(BACKUP_IP, API_PORT).await {
return Err(format!("Backup not reachable at {BACKUP_IP}:{API_PORT}").into());
}
// Create API keys on both
info!("Creating API keys...");
let primary_ip: IpAddr = PRIMARY_IP.parse()?;
let backup_ip: IpAddr = BACKUP_IP.parse()?;
let (primary_key, primary_secret) = create_api_key_ssh(&primary_ip).await?;
let (backup_key, backup_secret) = create_api_key_ssh(&backup_ip).await?;
info!("API keys created for both firewalls");
// Build FirewallPairTopology
let primary_host = LogicalHost {
ip: primary_ip.into(),
name: VM_PRIMARY.to_string(),
};
let backup_host = LogicalHost {
ip: backup_ip.into(),
name: VM_BACKUP.to_string(),
};
let primary_api_creds = OPNSenseApiCredentials {
key: primary_key.clone(),
secret: primary_secret.clone(),
};
let backup_api_creds = OPNSenseApiCredentials {
key: backup_key.clone(),
secret: backup_secret.clone(),
};
let ssh_creds = OPNSenseFirewallCredentials {
username: "root".to_string(),
password: "opnsense".to_string(),
};
let primary_fw = OPNSenseFirewall::with_api_port(
primary_host,
None,
API_PORT,
&primary_api_creds,
&ssh_creds,
)
.await;
let backup_fw =
OPNSenseFirewall::with_api_port(backup_host, None, API_PORT, &backup_api_creds, &ssh_creds)
.await;
let pair = FirewallPairTopology {
primary: primary_fw,
backup: backup_fw,
};
// Build pair scores
let carp_score = CarpVipScore {
vips: vec![VipDef {
mode: VipMode::Carp,
interface: "lan".to_string(),
subnet: CARP_VIP.to_string(),
subnet_bits: 24,
vhid: Some(1),
advbase: Some(1),
advskew: None, // handled by CarpVipScore (primary=0, backup=100)
password: Some(CARP_PASSWORD.to_string()),
peer: None,
}],
backup_advskew: Some(100),
};
let vlan_score = VlanScore {
vlans: vec![VlanDef {
parent_interface: "vtnet0".to_string(),
tag: 100,
description: "pair-test-vlan-100".to_string(),
}],
};
let fw_rule_score = FirewallRuleScore {
rules: vec![FilterRuleDef {
action: FirewallAction::Pass,
direction: Direction::In,
interface: "lan".to_string(),
ip_protocol: IpProtocol::Inet,
protocol: NetworkProtocol::Icmp,
source_net: "any".to_string(),
destination_net: "any".to_string(),
destination_port: None,
gateway: None,
description: "pair-test-allow-icmp".to_string(),
log: false,
}],
};
// Run pair scores
info!("Running pair scores...");
let scores: Vec<Box<dyn Score<FirewallPairTopology>>> = vec![
Box::new(carp_score),
Box::new(vlan_score),
Box::new(fw_rule_score),
];
let args = harmony_cli::Args {
yes: true,
filter: None,
interactive: false,
all: true,
number: 0,
list: false,
};
harmony_cli::run_cli(Inventory::autoload(), pair, scores, args).await?;
// Verify CARP VIPs via API
info!("Verifying CARP VIPs...");
let primary_client = opnsense_api::OpnsenseClient::builder()
.base_url(format!("https://{PRIMARY_IP}:{API_PORT}/api"))
.auth_from_key_secret(&primary_key, &primary_secret)
.skip_tls_verify()
.timeout_secs(60)
.build()?;
let backup_client = opnsense_api::OpnsenseClient::builder()
.base_url(format!("https://{BACKUP_IP}:{API_PORT}/api"))
.auth_from_key_secret(&backup_key, &backup_secret)
.skip_tls_verify()
.timeout_secs(60)
.build()?;
let primary_vips: serde_json::Value = primary_client
.get_typed("interfaces", "vip_settings", "searchItem")
.await?;
let backup_vips: serde_json::Value = backup_client
.get_typed("interfaces", "vip_settings", "searchItem")
.await?;
let primary_vip_count = primary_vips["rowCount"].as_i64().unwrap_or(0);
let backup_vip_count = backup_vips["rowCount"].as_i64().unwrap_or(0);
info!(" Primary VIPs: {primary_vip_count}");
info!(" Backup VIPs: {backup_vip_count}");
assert!(primary_vip_count >= 1, "Primary should have at least 1 VIP");
assert!(backup_vip_count >= 1, "Backup should have at least 1 VIP");
// Verify VLANs on both
let primary_vlans: serde_json::Value = primary_client
.get_typed("interfaces", "vlan_settings", "get")
.await?;
let backup_vlans: serde_json::Value = backup_client
.get_typed("interfaces", "vlan_settings", "get")
.await?;
let p_vlan_count = primary_vlans["vlan"]["vlan"]
.as_object()
.map(|m| m.len())
.unwrap_or(0);
let b_vlan_count = backup_vlans["vlan"]["vlan"]
.as_object()
.map(|m| m.len())
.unwrap_or(0);
info!(" Primary VLANs: {p_vlan_count}");
info!(" Backup VLANs: {b_vlan_count}");
assert!(p_vlan_count >= 1, "Primary should have at least 1 VLAN");
assert!(b_vlan_count >= 1, "Backup should have at least 1 VLAN");
println!();
println!("PASSED - OPNsense firewall pair integration test:");
println!(
" - CarpVipScore: CARP VIP {CARP_VIP} on both (primary advskew=0, backup advskew=100)"
);
println!(" - VlanScore: VLAN 100 on both");
println!(" - FirewallRuleScore: ICMP allow on both");
println!();
println!("VMs are running. Use --clean to tear down.");
Ok(())
}
// ── Helpers ────────────────────────────────────────────────────────
fn prepare_vm_disk(vm_name: &str, img_path: &Path) -> Result<(), Box<dyn std::error::Error>> {
let vm_raw = image_dir().join(format!("{vm_name}.img"));
if !vm_raw.exists() {
info!("Copying nano image for {vm_name}...");
std::fs::copy(img_path, &vm_raw)?;
info!("Injecting config.xml for {vm_name}...");
let config =
harmony::modules::opnsense::image::minimal_config_xml("vtnet1", "vtnet0", BOOT_IP, 24);
harmony::modules::opnsense::image::replace_config_xml(&vm_raw, &config)?;
}
let vm_disk = image_dir().join(format!("{vm_name}.qcow2"));
if !vm_disk.exists() {
info!("Converting {vm_name} to qcow2...");
run_cmd(
"qemu-img",
&[
"convert",
"-f",
"raw",
"-O",
"qcow2",
&vm_raw.to_string_lossy(),
&vm_disk.to_string_lossy(),
],
)?;
run_cmd("qemu-img", &["resize", &vm_disk.to_string_lossy(), "4G"])?;
}
Ok(())
}
fn check_prerequisites() -> Result<(), Box<dyn std::error::Error>> {
let mut ok = true;
for (cmd, test_args) in [
("virsh", vec!["-c", "qemu:///system", "version"]),
("qemu-img", vec!["--version"]),
("bunzip2", vec!["--help"]),
] {
match std::process::Command::new(cmd).args(&test_args).output() {
Ok(out) if out.status.success() => println!("[ok] {cmd}"),
_ => {
println!("[FAIL] {cmd}");
ok = false;
}
}
}
if !ok {
return Err("Prerequisites not met".into());
}
println!("All prerequisites met.");
Ok(())
}
fn run_cmd(cmd: &str, args: &[&str]) -> Result<(), Box<dyn std::error::Error>> {
let status = std::process::Command::new(cmd).args(args).status()?;
if !status.success() {
return Err(format!("{cmd} failed").into());
}
Ok(())
}
fn image_dir() -> PathBuf {
let dir = std::env::var("HARMONY_KVM_IMAGE_DIR").unwrap_or_else(|_| {
dirs::data_dir()
.unwrap_or_else(|| PathBuf::from("/tmp"))
.join("harmony")
.join("kvm")
.join("images")
.to_string_lossy()
.to_string()
});
PathBuf::from(dir)
}
async fn download_image() -> Result<PathBuf, Box<dyn std::error::Error>> {
let dir = image_dir();
std::fs::create_dir_all(&dir)?;
let img_path = dir.join(OPNSENSE_IMG_NAME);
if img_path.exists() {
info!("Image cached: {}", img_path.display());
return Ok(img_path);
}
let bz2_path = dir.join(format!("{OPNSENSE_IMG_NAME}.bz2"));
if !bz2_path.exists() {
info!("Downloading OPNsense nano image (~350MB)...");
let response = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?
.get(OPNSENSE_IMG_URL)
.send()
.await?;
if !response.status().is_success() {
return Err(format!("Download failed: HTTP {}", response.status()).into());
}
let bytes = response.bytes().await?;
std::fs::write(&bz2_path, &bytes)?;
}
info!("Decompressing...");
run_cmd("bunzip2", &["--keep", &bz2_path.to_string_lossy()])?;
Ok(img_path)
}
async fn clean(executor: &KvmExecutor) -> Result<(), Box<dyn std::error::Error>> {
info!("Cleaning up pair integration...");
for vm_name in [VM_PRIMARY, VM_BACKUP] {
let _ = executor.destroy_vm(vm_name).await;
let _ = executor.undefine_vm(vm_name).await;
for ext in ["img", "qcow2"] {
let path = image_dir().join(format!("{vm_name}.{ext}"));
if path.exists() {
std::fs::remove_file(&path)?;
info!("Removed: {}", path.display());
}
}
}
let _ = executor.delete_network(NET_LAN).await;
info!("Done.");
Ok(())
}
async fn status(executor: &KvmExecutor) -> Result<(), Box<dyn std::error::Error>> {
for (vm_name, ip) in [(VM_PRIMARY, PRIMARY_IP), (VM_BACKUP, BACKUP_IP)] {
match executor.vm_status(vm_name).await {
Ok(s) => {
let api = check_tcp_port(ip, API_PORT).await;
let ssh = check_tcp_port(ip, 22).await;
println!("{vm_name}: {s:?}");
println!(" LAN IP: {ip}");
println!(
" API: {}",
if api { "responding" } else { "not responding" }
);
println!(
" SSH: {}",
if ssh { "responding" } else { "not responding" }
);
}
Err(_) => println!("{vm_name}: not found"),
}
}
Ok(())
}
async fn wait_for_https(ip: &str, port: u16) -> Result<(), Box<dyn std::error::Error>> {
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.timeout(std::time::Duration::from_secs(5))
.build()?;
let url = format!("https://{ip}:{port}");
for i in 0..60 {
if client.get(&url).send().await.is_ok() {
info!("Web UI responding at {url} (attempt {i})");
return Ok(());
}
if i % 10 == 0 {
info!("Waiting for {url}... (attempt {i})");
}
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
Err(format!("{url} did not respond within 5 minutes").into())
}
async fn check_tcp_port(ip: &str, port: u16) -> bool {
tokio::time::timeout(
std::time::Duration::from_secs(3),
tokio::net::TcpStream::connect(format!("{ip}:{port}")),
)
.await
.map(|r| r.is_ok())
.unwrap_or(false)
}
async fn create_api_key_ssh(ip: &IpAddr) -> Result<(String, String), Box<dyn std::error::Error>> {
use opnsense_config::config::{OPNsenseShell, SshCredentials, SshOPNSenseShell};
let ssh_config = Arc::new(russh::client::Config {
inactivity_timeout: None,
..<_>::default()
});
let credentials = SshCredentials::Password {
username: "root".to_string(),
password: "opnsense".to_string(),
};
let shell = SshOPNSenseShell::new((*ip, 22), credentials, ssh_config);
let php_script = r#"<?php
require_once '/usr/local/etc/inc/config.inc';
$key = bin2hex(random_bytes(20));
$secret = bin2hex(random_bytes(40));
$config = OPNsense\Core\Config::getInstance();
foreach ($config->object()->system->user as $user) {
if ((string)$user->name === 'root') {
if (!isset($user->apikeys)) { $user->addChild('apikeys'); }
$item = $user->apikeys->addChild('item');
$item->addChild('key', $key);
$item->addChild('secret', crypt($secret, '$6$' . bin2hex(random_bytes(8)) . '$'));
$config->save();
echo $key . "\n" . $secret . "\n";
exit(0);
}
}
echo "ERROR: root user not found\n";
exit(1);
"#;
shell
.write_content_to_file(php_script, "/tmp/create_api_key.php")
.await?;
let output = shell
.exec("php /tmp/create_api_key.php && rm /tmp/create_api_key.php")
.await?;
let lines: Vec<&str> = output.trim().lines().collect();
if lines.len() >= 2 && !lines[0].starts_with("ERROR") {
Ok((lines[0].to_string(), lines[1].to_string()))
} else {
Err(format!("API key creation failed: {output}").into())
}
}