harmony/examples/opnsense_vm_integration/src/main.rs

//! OPNsense VM integration example.
//!
//! Fully unattended workflow — no manual browser interaction required:
//!
//! 1. `--boot` — creates a KVM VM, waits for web UI, bootstraps SSH + webgui port
//! 2. (default run) — creates API key via SSH, installs packages, runs Scores
//! 3. `--full` — does both in a single invocation (CI-friendly)
//!
//! # Usage
//!
//! ```bash
//! cargo run -p opnsense-vm-integration -- --check     # verify prerequisites
//! cargo run -p opnsense-vm-integration -- --download   # download OPNsense image
//! cargo run -p opnsense-vm-integration -- --boot       # create VM + automated bootstrap
//! cargo run -p opnsense-vm-integration                 # run integration test
//! cargo run -p opnsense-vm-integration -- --full       # boot + bootstrap + test (CI mode)
//! cargo run -p opnsense-vm-integration -- --status     # check VM state
//! cargo run -p opnsense-vm-integration -- --clean      # tear down everything
//! ```

use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use harmony::config::secret::{OPNSenseApiCredentials, OPNSenseFirewallCredentials};
use harmony::hardware::{HostCategory, PhysicalHost};
use harmony::infra::opnsense::OPNSenseFirewall;
use harmony::inventory::Inventory;
use harmony::modules::dhcp::DhcpScore;
use harmony::modules::kvm::config::init_executor;
use harmony::modules::kvm::{
    BootDevice, ForwardMode, KvmExecutor, NetworkConfig, NetworkRef, VmConfig,
};
use harmony::modules::load_balancer::LoadBalancerScore;
use harmony::modules::opnsense::bootstrap::OPNsenseBootstrap;
use harmony::modules::opnsense::dnat::{DnatRuleDef, DnatScore};
use harmony::modules::opnsense::firewall::{
    BinatRuleDef, BinatScore, FilterRuleDef, FirewallRuleScore, OutboundNatScore, SnatRuleDef,
};
use harmony::modules::opnsense::lagg::{LaggDef, LaggScore};
use harmony::modules::opnsense::node_exporter::NodeExporterScore;
use harmony::modules::opnsense::vip::{VipDef, VipScore};
use harmony::modules::opnsense::vlan::{VlanDef, VlanScore};
use harmony::modules::tftp::TftpScore;
use harmony::score::Score;
use harmony::topology::{
    BackendServer, HealthCheck, HostBinding, HostConfig, LoadBalancerService, LogicalHost,
};
use harmony_inventory_agent::hwinfo::NetworkInterface;
use harmony_macros::ip;
use harmony_types::firewall::{
    Direction, FirewallAction, IpProtocol, LaggProtocol, NetworkProtocol, VipMode,
};
use harmony_types::id::Id;
use harmony_types::net::{MacAddress, Url};
use log::{info, warn};

const OPNSENSE_IMG_URL: &str =
    "https://mirror.ams1.nl.leaseweb.net/opnsense/releases/26.1/OPNsense-26.1-nano-amd64.img.bz2";
const OPNSENSE_IMG_NAME: &str = "OPNsense-26.1-nano-amd64.img";

const VM_NAME: &str = "opn-integration";
const NET_NAME: &str = "opn-test";
// OPNsense nano defaults LAN to 192.168.1.1/24.
// The libvirt network uses the same subnet so the host can reach the VM.
const HOST_IP: &str = "192.168.1.10";
const OPN_LAN_IP: &str = "192.168.1.1";
/// Web GUI/API port — moved from 443 to avoid HAProxy conflicts.
/// Set in the manual step: System > Settings > Administration > TCP Port.
const OPN_API_PORT: u16 = 9443;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    harmony_cli::cli_logger::init();

    let args: Vec<String> = std::env::args().collect();

    if args.iter().any(|a| a == "--setup") {
        print_setup();
        return Ok(());
    }
    if args.iter().any(|a| a == "--check") {
        return check_prerequisites();
    }
    if args.iter().any(|a| a == "--download") {
        download_image().await?;
        return Ok(());
    }

    let executor = init_executor()?;

    if args.iter().any(|a| a == "--clean") {
        return clean(&executor).await;
    }
    if args.iter().any(|a| a == "--status") {
        return status(&executor).await;
    }
    if args.iter().any(|a| a == "--boot") {
        let img_path = download_image().await?;
        return boot_vm(&executor, &img_path).await;
    }

    if args.iter().any(|a| a == "--full") {
        // CI mode: boot + bootstrap + integration test in one shot
        let img_path = download_image().await?;
        boot_vm(&executor, &img_path).await?;
        return run_integration().await;
    }

    // Default: run the integration test (assumes VM is booted + bootstrapped)
    check_prerequisites()?;
    run_integration().await
}

// ── Phase 1: Boot VM ────────────────────────────────────────────────────

async fn boot_vm(
    executor: &KvmExecutor,
    img_path: &Path,
) -> Result<(), Box<dyn std::error::Error>> {
    info!("Creating network and OPNsense VM...");

    let network = NetworkConfig::builder(NET_NAME)
        .bridge("virbr-opn")
        .subnet(HOST_IP, 24)
        .forward(ForwardMode::Nat)
        .build();
    executor.ensure_network(network).await?;

    // Copy and convert the nano image
    let vm_raw = image_dir().join(format!("{VM_NAME}-boot.img"));
    if !vm_raw.exists() {
        info!("Copying nano image...");
        std::fs::copy(img_path, &vm_raw)?;

        // Inject config.xml with virtio interface names
        info!("Injecting config.xml for virtio NICs...");
        let config = harmony::modules::opnsense::image::minimal_config_xml(
            "vtnet1", "vtnet0", OPN_LAN_IP, 24,
        );
        harmony::modules::opnsense::image::replace_config_xml(&vm_raw, &config)?;
    }

    let vm_disk = image_dir().join(format!("{VM_NAME}-boot.qcow2"));
    if !vm_disk.exists() {
        info!("Converting to qcow2...");
        run_cmd(
            "qemu-img",
            &[
                "convert",
                "-f",
                "raw",
                "-O",
                "qcow2",
                &vm_raw.to_string_lossy(),
                &vm_disk.to_string_lossy(),
            ],
        )?;
        run_cmd("qemu-img", &["resize", &vm_disk.to_string_lossy(), "4G"])?;
    }

    let vm = VmConfig::builder(VM_NAME)
        .vcpus(3)
        .memory_mib(2048)
        .disk_from_path(vm_disk.to_string_lossy().to_string())
        .network(NetworkRef::named(NET_NAME)) // vtnet0 = LAN
        .network(NetworkRef::named("default")) // vtnet1 = WAN
        .network(NetworkRef::named(NET_NAME)) // vtnet2 = LAGG member 1
        .network(NetworkRef::named(NET_NAME)) // vtnet3 = LAGG member 2
        .boot_order([BootDevice::Disk])
        .build();

    executor.ensure_vm(vm).await?;
    executor.start_vm(VM_NAME).await?;
    info!("VM started. Waiting for web UI at https://{OPN_LAN_IP} ...");

    wait_for_https(OPN_LAN_IP, 443).await?;

    // ── Automated bootstrap (replaces manual browser interaction) ───
    info!("Bootstrapping OPNsense: login, abort wizard, enable SSH, set webgui port...");
    let bootstrap = OPNsenseBootstrap::new(&format!("https://{OPN_LAN_IP}"));
    bootstrap.login("root", "opnsense").await?;
    bootstrap.abort_wizard().await?;
    bootstrap.enable_ssh(true, true).await?;
    bootstrap
        .set_webgui_port(OPN_API_PORT, OPN_LAN_IP, false)
        .await?;

    // Wait for the web UI to come back on the new port
    info!("Waiting for web UI on new port {OPN_API_PORT}...");
    if let Err(e) = OPNsenseBootstrap::wait_for_ready(
        &format!("https://{OPN_LAN_IP}:{OPN_API_PORT}"),
        std::time::Duration::from_secs(120),
    )
    .await
    {
        warn!("Web UI did not come up on port {OPN_API_PORT}: {e}");
        info!("Running diagnostics via SSH...");
        match OPNsenseBootstrap::diagnose_via_ssh(OPN_LAN_IP).await {
            Ok(report) => {
                info!("Diagnostic report:\n{}", report);
            }
            Err(diag_err) => warn!("Diagnostics failed: {diag_err}"),
        }
        return Err(e.into());
    }

    // Verify SSH is reachable
    info!("Verifying SSH is reachable...");
    for _ in 0..30 {
        if check_tcp_port(OPN_LAN_IP, 22).await {
            break;
        }
        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
    }
    if !check_tcp_port(OPN_LAN_IP, 22).await {
        return Err("SSH did not become reachable after bootstrap".into());
    }

    println!();
    println!("OPNsense VM is running and fully bootstrapped:");
    println!("  Web UI: https://{OPN_LAN_IP}:{OPN_API_PORT}");
    println!("  SSH:    root@{OPN_LAN_IP} (password: opnsense)");
    println!("  Login:  root / opnsense");
    println!();
    println!("Run the integration test:");
    println!("  cargo run -p opnsense-vm-integration");
    println!();
    println!("Or use --full to boot + test in one shot (CI mode):");
    println!("  cargo run -p opnsense-vm-integration -- --full");

    Ok(())
}

// ── Phase 2: Integration test ───────────────────────────────────────────

async fn run_integration() -> Result<(), Box<dyn std::error::Error>> {
    let vm_ip: IpAddr = OPN_LAN_IP.parse().unwrap();

    // Verify SSH is reachable (bootstrap should have enabled it)
    info!("Checking SSH at {OPN_LAN_IP}:22...");
    if !check_tcp_port(OPN_LAN_IP, 22).await {
        eprintln!("SSH is not reachable at {OPN_LAN_IP}:22");
        eprintln!("Run '--boot' first (it will automatically enable SSH).");
        return Err("SSH not available".into());
    }
    info!("SSH is reachable");

    // Create API key
    info!("Creating API key via SSH...");
    let (api_key, api_secret) = create_api_key_ssh(&vm_ip).await?;
    info!("API key created: {}...", &api_key[..api_key.len().min(12)]);

    // Build topology
    let firewall_host = LogicalHost {
        ip: vm_ip,
        name: VM_NAME.to_string(),
    };
    let api_creds = OPNSenseApiCredentials {
        key: api_key.clone(),
        secret: api_secret.clone(),
    };
    let ssh_creds = OPNSenseFirewallCredentials {
        username: "root".to_string(),
        password: "opnsense".to_string(),
    };
    let opnsense =
        OPNSenseFirewall::with_api_port(firewall_host, None, OPN_API_PORT, &api_creds, &ssh_creds)
            .await;

    // Install packages
    let config = opnsense.get_opnsense_config();
    if !config.is_package_installed("os-haproxy").await {
        info!("Installing os-haproxy (may need firmware update first)...");
        match config.install_package("os-haproxy").await {
            Ok(()) => info!("os-haproxy installed"),
            Err(e) => {
                warn!("os-haproxy install failed: {e}");
                info!("Attempting firmware update...");
                // Trigger firmware update then retry
                let _: serde_json::Value = config
                    .client()
                    .post_typed("core", "firmware", "update", None::<&()>)
                    .await
                    .map_err(|e| format!("firmware update failed: {e}"))?;
                // Poll for completion
                for _ in 0..120 {
                    tokio::time::sleep(std::time::Duration::from_secs(5)).await;
                    let status: serde_json::Value = match config
                        .client()
                        .get_typed("core", "firmware", "upgradestatus")
                        .await
                    {
                        Ok(s) => s,
                        Err(_) => continue, // VM may be rebooting
                    };
                    if status["status"].as_str() == Some("done")
                        || status["status"].as_str() == Some("reboot")
                    {
                        break;
                    }
                }
                info!("Firmware updated, retrying package install...");
                // Wait for API to come back — try configured port first
                // (config.xml persists across reboots, so port stays at 9443)
                wait_for_https(OPN_LAN_IP, OPN_API_PORT).await?;
                // Extra settle time — web UI responds before API backend is ready
                tokio::time::sleep(std::time::Duration::from_secs(10)).await;
                config.install_package("os-haproxy").await?;
            }
        }
    } else {
        info!("os-haproxy already installed");
    }

    // ── Build and run all Scores ──────────────────────────────────────
    info!("Running all Scores (run 1)...");
    let scores = build_all_scores()?;
    let args = harmony_cli::Args {
        yes: true,
        filter: None,
        interactive: false,
        all: true,
        number: 0,
        list: false,
    };
    harmony_cli::run_cli(Inventory::autoload(), opnsense, scores, args).await?;

    // ── Verify via typed API ──────────────────────────────────────────
    info!("Verifying all Scores via typed API...");
    let client = opnsense_api::OpnsenseClient::builder()
        .base_url(format!("https://{OPN_LAN_IP}:{OPN_API_PORT}/api"))
        .auth_from_key_secret(&api_key, &api_secret)
        .skip_tls_verify()
        .timeout_secs(60)
        .build()?;

    let state1 = verify_state(&client).await?;
    state1.print("Run 1");
    state1.assert_minimum_counts();

    // ── Idempotency test: run ALL Scores a second time ─────────────
    info!("=== IDEMPOTENCY TEST: Running all Scores a SECOND time ===");
    let scores_round2 = build_all_scores()?;
    let firewall_host2 = LogicalHost {
        ip: vm_ip,
        name: VM_NAME.to_string(),
    };
    let opnsense2 =
        OPNSenseFirewall::with_api_port(firewall_host2, None, OPN_API_PORT, &api_creds, &ssh_creds)
            .await;
    let args2 = harmony_cli::Args {
        yes: true,
        filter: None,
        interactive: false,
        all: true,
        number: 0,
        list: false,
    };
    harmony_cli::run_cli(Inventory::autoload(), opnsense2, scores_round2, args2).await?;

    let state2 = verify_state(&client).await?;
    state2.print("Run 2");

    // ── Assert counts are UNCHANGED ────────────────────────────────
    info!("Verifying idempotency — counts must be unchanged...");
    assert_eq!(
        state1.haproxy_frontends, state2.haproxy_frontends,
        "HAProxy frontends changed after 2nd run! {} -> {}",
        state1.haproxy_frontends, state2.haproxy_frontends
    );
    assert_eq!(
        state1.dnsmasq_hosts, state2.dnsmasq_hosts,
        "Dnsmasq hosts changed after 2nd run! {} -> {}",
        state1.dnsmasq_hosts, state2.dnsmasq_hosts
    );
    assert_eq!(
        state1.dnsmasq_ranges, state2.dnsmasq_ranges,
        "DHCP ranges changed after 2nd run! {} -> {}",
        state1.dnsmasq_ranges, state2.dnsmasq_ranges
    );
    assert_eq!(
        state1.vlan_count, state2.vlan_count,
        "VLANs changed after 2nd run! {} -> {}",
        state1.vlan_count, state2.vlan_count
    );
    assert_eq!(
        state1.firewall_rules, state2.firewall_rules,
        "Firewall rules changed after 2nd run! {} -> {}",
        state1.firewall_rules, state2.firewall_rules
    );
    assert_eq!(
        state1.vip_count, state2.vip_count,
        "VIPs changed after 2nd run! {} -> {}",
        state1.vip_count, state2.vip_count
    );
    assert_eq!(
        state1.dnat_rules, state2.dnat_rules,
        "DNAT rules changed after 2nd run! {} -> {}",
        state1.dnat_rules, state2.dnat_rules
    );
    assert_eq!(
        state1.lagg_count, state2.lagg_count,
        "LAGGs changed after 2nd run! {} -> {}",
        state1.lagg_count, state2.lagg_count
    );

    // Clean up temp files
    let _ = std::fs::remove_dir_all(std::env::temp_dir().join("harmony-tftp-test"));

    println!();
    println!("PASSED — All OPNsense integration tests successful:");
    println!("  Run 1: all entities created correctly");
    println!("  Run 2: idempotency verified — zero duplicates");
    println!();
    println!("VM is running at {OPN_LAN_IP}. Use --clean to tear down.");
    Ok(())
}

// ── Verification ───────────────────────────────────────────────────────

/// Snapshot of OPNsense entity counts, used to verify idempotency.
#[derive(Debug, PartialEq)]
struct StateSnapshot {
    haproxy_frontends: usize,
    dnsmasq_hosts: usize,
    dnsmasq_ranges: usize,
    vlan_count: usize,
    firewall_rules: usize,
    snat_rules: usize,
    vip_count: usize,
    dnat_rules: usize,
    lagg_count: usize,
}

impl StateSnapshot {
    fn print(&self, label: &str) {
        info!("State after {label}:");
        info!("  HAProxy frontends: {}", self.haproxy_frontends);
        info!("  Dnsmasq hosts: {}", self.dnsmasq_hosts);
        info!("  Dnsmasq DHCP ranges: {}", self.dnsmasq_ranges);
        info!("  VLANs: {}", self.vlan_count);
        info!("  Firewall rules: {}", self.firewall_rules);
        info!("  SNAT rules: {}", self.snat_rules);
        info!("  VIPs: {}", self.vip_count);
        info!("  DNat rules: {}", self.dnat_rules);
        info!("  LAGGs: {}", self.lagg_count);
    }

    fn assert_minimum_counts(&self) {
        assert!(
            self.haproxy_frontends >= 2,
            "Expected >= 2 HAProxy frontends, got {}",
            self.haproxy_frontends
        );
        assert!(
            self.dnsmasq_hosts >= 2,
            "Expected >= 2 dnsmasq hosts, got {}",
            self.dnsmasq_hosts
        );
        assert!(
            self.dnsmasq_ranges >= 1,
            "Expected >= 1 DHCP range, got {}",
            self.dnsmasq_ranges
        );
        assert!(
            self.vlan_count >= 2,
            "Expected >= 2 VLANs, got {}",
            self.vlan_count
        );
        assert!(
            self.firewall_rules >= 1,
            "Expected >= 1 firewall rule, got {}",
            self.firewall_rules
        );
        assert!(
            self.vip_count >= 1,
            "Expected >= 1 VIP, got {}",
            self.vip_count
        );
        assert!(
            self.dnat_rules >= 1,
            "Expected >= 1 DNat rule, got {}",
            self.dnat_rules
        );
        assert!(
            self.lagg_count >= 1,
            "Expected >= 1 LAGG, got {}",
            self.lagg_count
        );
    }
}

/// Query OPNsense to get current entity counts.
///
/// Uses a mix of GET endpoints (for settings that return full objects)
/// and search endpoints (for paginated rule lists).
async fn verify_state(
    client: &opnsense_api::OpnsenseClient,
) -> Result<StateSnapshot, Box<dyn std::error::Error>> {
    // Search endpoints — use raw JSON because OPNsense returns rowCount in the response
    let fw_rules: serde_json::Value = client.get_typed("firewall", "filter", "searchRule").await?;
    let firewall_rules = fw_rules["rowCount"].as_i64().unwrap_or(0) as usize;

    let snat_rules_resp: serde_json::Value = client
        .get_typed("firewall", "source_nat", "searchRule")
        .await?;
    let snat_rules = snat_rules_resp["rowCount"].as_i64().unwrap_or(0) as usize;

    let dnat_rules_resp: serde_json::Value =
        client.get_typed("firewall", "d_nat", "searchRule").await?;
    let dnat_rules = dnat_rules_resp["rowCount"].as_i64().unwrap_or(0) as usize;

    let vips_resp: serde_json::Value = client
        .get_typed("interfaces", "vip_settings", "searchItem")
        .await?;
    let vip_count = vips_resp["rowCount"].as_i64().unwrap_or(0) as usize;

    // GET endpoints — return full settings objects
    let haproxy: serde_json::Value = client.get_typed("haproxy", "settings", "get").await?;
    let haproxy_frontends = haproxy["haproxy"]["frontends"]["frontend"]
        .as_object()
        .map(|m| m.len())
        .unwrap_or(0);

    let dnsmasq: serde_json::Value = client.get_typed("dnsmasq", "settings", "get").await?;
    let dnsmasq_hosts = dnsmasq["dnsmasq"]["hosts"]
        .as_object()
        .map(|m| m.len())
        .unwrap_or(0);
    let dnsmasq_ranges = dnsmasq["dnsmasq"]["dhcp_ranges"]
        .as_object()
        .map(|m| m.len())
        .unwrap_or(0);

    let vlans: serde_json::Value = client
        .get_typed("interfaces", "vlan_settings", "get")
        .await?;
    let vlan_count = vlans["vlan"]["vlan"]
        .as_object()
        .map(|m| m.len())
        .unwrap_or(0);

    let laggs: serde_json::Value = client
        .get_typed("interfaces", "lagg_settings", "get")
        .await?;
    let lagg_count = laggs["lagg"]["lagg"]
        .as_object()
        .map(|m| m.len())
        .unwrap_or(0);

    Ok(StateSnapshot {
        haproxy_frontends,
        dnsmasq_hosts,
        dnsmasq_ranges,
        vlan_count,
        firewall_rules,
        snat_rules,
        vip_count,
        dnat_rules,
        lagg_count,
    })
}

type FirewallScore = Box<dyn Score<OPNSenseFirewall>>;
type BuildScoresResult = Result<Vec<FirewallScore>, Box<dyn std::error::Error>>;

/// Build all test Scores — extracted so we can call it for both run 1 and run 2.
fn build_all_scores() -> BuildScoresResult {
    let lb_score = LoadBalancerScore {
        public_services: vec![
            LoadBalancerService {
                listening_port: format!("{OPN_LAN_IP}:16443").parse()?,
                backend_servers: vec![
                    BackendServer {
                        address: "10.50.0.10".into(),
                        port: 6443,
                    },
                    BackendServer {
                        address: "10.50.0.11".into(),
                        port: 6443,
                    },
                    BackendServer {
                        address: "10.50.0.12".into(),
                        port: 6443,
                    },
                ],
                health_check: Some(HealthCheck::TCP(None)),
            },
            LoadBalancerService {
                listening_port: format!("{OPN_LAN_IP}:18443").parse()?,
                backend_servers: vec![
                    BackendServer {
                        address: "10.50.0.10".into(),
                        port: 443,
                    },
                    BackendServer {
                        address: "10.50.0.11".into(),
                        port: 443,
                    },
                ],
                health_check: Some(HealthCheck::TCP(None)),
            },
        ],
        private_services: vec![],
    };

    let dhcp_score = DhcpScore::new(
        vec![
            make_host_binding(
                "node1",
                ip!("192.168.1.50"),
                [0x52, 0x54, 0x00, 0xAA, 0xBB, 0x01],
            ),
            make_host_binding(
                "node2",
                ip!("192.168.1.51"),
                [0x52, 0x54, 0x00, 0xAA, 0xBB, 0x02],
            ),
        ],
        None,
        None,
        None,
        None,
        None,
        (ip!("192.168.1.100"), ip!("192.168.1.200")),
        Some("test.local".to_string()),
    );

    let tftp_dir = std::env::temp_dir().join("harmony-tftp-test");
    std::fs::create_dir_all(&tftp_dir)?;
    std::fs::write(tftp_dir.join("test.txt"), "harmony integration test\n")?;
    let tftp_score = TftpScore::new(Url::LocalFolder(tftp_dir.to_string_lossy().to_string()));

    let vlan_score = VlanScore {
        vlans: vec![
            VlanDef {
                parent_interface: "vtnet0".to_string(),
                tag: 100,
                description: "test-vlan-100".to_string(),
            },
            VlanDef {
                parent_interface: "vtnet0".to_string(),
                tag: 200,
                description: "test-vlan-200".to_string(),
            },
        ],
    };

    let fw_rule_score = FirewallRuleScore {
        rules: vec![FilterRuleDef {
            action: FirewallAction::Pass,
            direction: Direction::In,
            interface: "lan".to_string(),
            ip_protocol: IpProtocol::Inet,
            protocol: NetworkProtocol::Tcp,
            source_net: "any".to_string(),
            destination_net: "any".to_string(),
            destination_port: Some("8080".to_string()),
            gateway: None,
            description: "harmony-test-allow-8080".to_string(),
            log: true,
        }],
    };

    let snat_score = OutboundNatScore {
        rules: vec![SnatRuleDef {
            interface: "wan".to_string(),
            ip_protocol: IpProtocol::Inet,
            protocol: NetworkProtocol::Any,
            source_net: "192.168.1.0/24".to_string(),
            destination_net: "any".to_string(),
            target: "wanip".to_string(),
            description: "harmony-test-snat-lan".to_string(),
            log: false,
            nonat: false,
        }],
    };

    let binat_score = BinatScore {
        rules: vec![BinatRuleDef {
            interface: "wan".to_string(),
            source_net: "192.168.1.50".to_string(),
            external: "10.0.0.50".to_string(),
            description: "harmony-test-binat".to_string(),
            log: false,
        }],
    };

    let vip_score = VipScore {
        vips: vec![VipDef {
            mode: VipMode::IpAlias,
            interface: "lan".to_string(),
            subnet: "192.168.1.250".to_string(),
            subnet_bits: 32,
            vhid: None,
            advbase: None,
            advskew: None,
            password: None,
            peer: None,
        }],
    };

    let dnat_score = DnatScore {
        rules: vec![DnatRuleDef {
            interface: "wan".to_string(),
            ip_protocol: IpProtocol::Inet,
            protocol: NetworkProtocol::Tcp,
            destination: "wanip".to_string(),
            destination_port: "8443".to_string(),
            target: "192.168.1.50".to_string(),
            local_port: Some("443".to_string()),
            description: "harmony-test-dnat-8443".to_string(),
            log: false,
            register_rule: true,
        }],
    };

    let lagg_score = LaggScore {
        laggs: vec![LaggDef {
            members: vec!["vtnet2".to_string(), "vtnet3".to_string()],
            protocol: LaggProtocol::Failover,
            description: "harmony-test-lagg".to_string(),
            mtu: None,
            lacp_fast_timeout: false,
        }],
    };

    // WebGuiConfigScore runs first: moves webgui to 9443 so HAProxy can bind 443.
    // This is an explicit Score (not hidden in bootstrap) — see docs/architecture-challenges.md
    // for discussion of Score ordering/dependency.
    let webgui_score = harmony::modules::opnsense::webgui::WebGuiConfigScore {
        port: OPN_API_PORT,
        disable_http_redirect: false,
    };

    Ok(vec![
        Box::new(webgui_score),
        Box::new(lb_score),
        Box::new(dhcp_score),
        Box::new(tftp_score),
        Box::new(NodeExporterScore {}),
        Box::new(vlan_score),
        Box::new(fw_rule_score),
        Box::new(snat_score),
        Box::new(binat_score),
        Box::new(vip_score),
        Box::new(dnat_score),
        Box::new(lagg_score),
    ])
}

// ── Helpers ─────────────────────────────────────────────────────────────

fn print_setup() {
    println!("Run the setup script for sudo-less libvirt access:");
    println!("  ./examples/opnsense_vm_integration/setup-libvirt.sh");
    println!();
    println!("Verify with:");
    println!("  cargo run -p opnsense-vm-integration -- --check");
}

fn check_prerequisites() -> Result<(), Box<dyn std::error::Error>> {
    let mut ok = true;

    let libvirtd = std::process::Command::new("systemctl")
        .args(["is-active", "libvirtd"])
        .output();
    match libvirtd {
        Ok(out) if out.status.success() => println!("[ok] libvirtd is running"),
        _ => {
            println!("[FAIL] libvirtd is not running");
            ok = false;
        }
    }

    let virsh = std::process::Command::new("virsh")
        .args(["-c", "qemu:///system", "version"])
        .output();
    match virsh {
        Ok(out) if out.status.success() => {
            let v = String::from_utf8_lossy(&out.stdout);
            println!("[ok] virsh connects: {}", v.lines().next().unwrap_or("?"));
        }
        _ => {
            println!("[FAIL] Cannot connect to qemu:///system");
            ok = false;
        }
    }

    let pool = std::process::Command::new("virsh")
        .args(["-c", "qemu:///system", "pool-info", "default"])
        .output();
    match pool {
        Ok(out) if out.status.success() => println!("[ok] Default storage pool exists"),
        _ => {
            println!("[FAIL] Default storage pool not found");
            ok = false;
        }
    }

    if which("bunzip2") {
        println!("[ok] bunzip2 available");
    } else {
        println!("[FAIL] bunzip2 not found");
        ok = false;
    }

    if which("qemu-img") {
        println!("[ok] qemu-img available");
    } else {
        println!("[FAIL] qemu-img not found");
        ok = false;
    }

    // Check Docker + libvirt FORWARD conflict
    if which("docker") {
        let fw_backend = std::fs::read_to_string("/etc/libvirt/network.conf").unwrap_or_default();
        if fw_backend
            .lines()
            .any(|l| l.trim().starts_with("firewall_backend") && l.contains("iptables"))
        {
            println!("[ok] libvirt uses iptables backend (Docker compatible)");
        } else {
            println!("[WARN] Docker detected but libvirt uses nftables backend");
            println!("       VM NAT may not work. Run setup-libvirt.sh to fix.");
        }
    }

    if !ok {
        println!("\nRun --setup for setup instructions.");
        return Err("Prerequisites not met".into());
    }
    println!("\nAll prerequisites met.");
    Ok(())
}

fn which(cmd: &str) -> bool {
    std::process::Command::new("which")
        .arg(cmd)
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn run_cmd(cmd: &str, args: &[&str]) -> Result<(), Box<dyn std::error::Error>> {
    let status = std::process::Command::new(cmd).args(args).status()?;
    if !status.success() {
        return Err(format!("{cmd} failed").into());
    }
    Ok(())
}

fn image_dir() -> PathBuf {
    let dir = std::env::var("HARMONY_KVM_IMAGE_DIR").unwrap_or_else(|_| {
        dirs::data_dir()
            .unwrap_or_else(|| PathBuf::from("/tmp"))
            .join("harmony")
            .join("kvm")
            .join("images")
            .to_string_lossy()
            .to_string()
    });
    PathBuf::from(dir)
}

/// FIXME this should be using the harmony-asset crate
async fn download_image() -> Result<PathBuf, Box<dyn std::error::Error>> {
    let dir = image_dir();
    std::fs::create_dir_all(&dir)?;
    let img_path = dir.join(OPNSENSE_IMG_NAME);

    if img_path.exists() {
        info!("Image cached: {}", img_path.display());
        return Ok(img_path);
    }

    let bz2_path = dir.join(format!("{OPNSENSE_IMG_NAME}.bz2"));
    if !bz2_path.exists() {
        info!("Downloading OPNsense nano image (~350MB)...");
        let response = reqwest::Client::builder()
            .timeout(std::time::Duration::from_secs(600))
            .build()?
            .get(OPNSENSE_IMG_URL)
            .send()
            .await?;
        if !response.status().is_success() {
            return Err(format!("Download failed: HTTP {}", response.status()).into());
        }
        let bytes = response.bytes().await?;
        std::fs::write(&bz2_path, &bytes)?;
        info!("Downloaded {} bytes", bytes.len());
    }

    info!("Decompressing...");
    run_cmd("bunzip2", &["--keep", &bz2_path.to_string_lossy()])?;
    info!("Image ready: {}", img_path.display());
    Ok(img_path)
}

async fn clean(executor: &KvmExecutor) -> Result<(), Box<dyn std::error::Error>> {
    info!("Cleaning up...");
    let _ = executor.destroy_vm(VM_NAME).await;
    let _ = executor.undefine_vm(VM_NAME).await;
    let _ = executor.delete_network(NET_NAME).await;
    for ext in ["img", "qcow2"] {
        let path = image_dir().join(format!("{VM_NAME}-boot.{ext}"));
        if path.exists() {
            std::fs::remove_file(&path)?;
            info!("Removed: {}", path.display());
        }
    }
    info!("Done. (Original image cached at {})", image_dir().display());
    Ok(())
}

async fn status(executor: &KvmExecutor) -> Result<(), Box<dyn std::error::Error>> {
    match executor.vm_status(VM_NAME).await {
        Ok(s) => {
            println!("{VM_NAME}: {s:?}");
            if let Ok(Some(ip)) = executor.vm_ip(VM_NAME).await {
                println!("  WAN IP: {ip}");
            }
            println!("  LAN IP: {OPN_LAN_IP} (static)");
            let https_default = check_tcp_port(OPN_LAN_IP, 443).await;
            let https_custom = check_tcp_port(OPN_LAN_IP, OPN_API_PORT).await;
            let ssh = check_tcp_port(OPN_LAN_IP, 22).await;
            if https_custom {
                println!("  API:   responding on port {OPN_API_PORT}");
            } else if https_default {
                println!("  API:   responding on port 443 (change to {OPN_API_PORT} in web UI)");
            } else {
                println!("  API:   not responding");
            }
            println!(
                "  SSH:   {}",
                if ssh { "responding" } else { "not responding" }
            );
        }
        Err(_) => println!("{VM_NAME}: not found (run --boot first)"),
    }
    Ok(())
}

/// Wait for the OPNsense web UI to respond.
///
/// Tries the target port first, then falls back to port 443 (the OPNsense
/// default) on each attempt. This handles the case where the VM boots fresh
/// with port 443, then gets reconfigured to a custom port.
async fn wait_for_https(ip: &str, port: u16) -> Result<(), Box<dyn std::error::Error>> {
    let client = reqwest::Client::builder()
        .danger_accept_invalid_certs(true)
        .timeout(std::time::Duration::from_secs(5))
        .build()?;

    let urls: Vec<String> = if port == 443 {
        vec![format!("https://{ip}")]
    } else {
        vec![format!("https://{ip}:{port}"), format!("https://{ip}")]
    };

    for i in 0..120 {
        for url in &urls {
            if client.get(url).send().await.is_ok() {
                info!("Web UI responding at {url} (attempt {i})");
                return Ok(());
            }
        }
        if i % 10 == 0 {
            info!("Waiting for OPNsense... (attempt {i})");
        }
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
    }
    Err("OPNsense web UI did not respond within 10 minutes".into())
}

async fn check_tcp_port(ip: &str, port: u16) -> bool {
    tokio::time::timeout(
        std::time::Duration::from_secs(3),
        tokio::net::TcpStream::connect(format!("{ip}:{port}")),
    )
    .await
    .map(|r| r.is_ok())
    .unwrap_or(false)
}

/// Build a HostBinding from a name, IP, and MAC bytes for use with DhcpScore.
fn make_host_binding(name: &str, ip: IpAddr, mac: [u8; 6]) -> HostBinding {
    let logical = LogicalHost {
        ip,
        name: name.to_string(),
    };
    let physical = PhysicalHost {
        id: Id::from(name.to_string()),
        category: HostCategory::Server,
        network: vec![NetworkInterface {
            name: "eth0".to_string(),
            mac_address: MacAddress(mac),
            speed_mbps: None,
            is_up: true,
            mtu: 1500,
            ipv4_addresses: vec![ip.to_string()],
            ipv6_addresses: vec![],
            driver: String::new(),
            firmware_version: None,
        }],
        storage: vec![],
        labels: vec![],
        memory_modules: vec![],
        cpus: vec![],
    };
    HostBinding::new(logical, physical, HostConfig::new(None))
}

async fn create_api_key_ssh(ip: &IpAddr) -> Result<(String, String), Box<dyn std::error::Error>> {
    use opnsense_config::config::{OPNsenseShell, SshCredentials, SshOPNSenseShell};

    let ssh_config = Arc::new(russh::client::Config {
        inactivity_timeout: None,
        ..<_>::default()
    });
    let credentials = SshCredentials::Password {
        username: "root".to_string(),
        password: "opnsense".to_string(),
    };
    let shell = SshOPNSenseShell::new((*ip, 22), credentials, ssh_config);

    let php_script = r#"<?php
require_once '/usr/local/etc/inc/config.inc';
$key = bin2hex(random_bytes(20));
$secret = bin2hex(random_bytes(40));
$config = OPNsense\Core\Config::getInstance();
foreach ($config->object()->system->user as $user) {
    if ((string)$user->name === 'root') {
        if (!isset($user->apikeys)) { $user->addChild('apikeys'); }
        $item = $user->apikeys->addChild('item');
        $item->addChild('key', $key);
        $item->addChild('secret', crypt($secret, '$6$' . bin2hex(random_bytes(8)) . '$'));
        $config->save();
        echo $key . "\n" . $secret . "\n";
        exit(0);
    }
}
echo "ERROR: root user not found\n";
exit(1);
"#;

    info!("Writing API key script...");
    shell
        .write_content_to_file(php_script, "/tmp/create_api_key.php")
        .await?;

    info!("Executing API key generation...");
    let output = shell
        .exec("php /tmp/create_api_key.php && rm /tmp/create_api_key.php")
        .await?;

    let lines: Vec<&str> = output.trim().lines().collect();
    if lines.len() >= 2 && !lines[0].starts_with("ERROR") {
        Ok((lines[0].to_string(), lines[1].to_string()))
    } else {
        Err(format!("API key creation failed: {output}").into())
    }
}