commit

.dockerignore

@@ -1,6 +1,2 @@
 target/
 Dockerfile
-.git
-data
-target
-demos

.gitea/workflows/check.yml

@@ -15,4 +15,4 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run check script
-        run: bash build/check.sh
+        run: bash check.sh

.gitignore

@@ -24,11 +24,3 @@ Cargo.lock
 
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
-
-.harmony_generated
-
-# Useful to create ignore folders for temp files and notes
-ignore
-
-# Generated book
-book

.sqlx/query-24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b.json

@@ -1,26 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "SELECT host_id, installation_device FROM host_role_mapping WHERE role = ?",
-  "describe": {
-    "columns": [
-      {
-        "name": "host_id",
-        "ordinal": 0,
-        "type_info": "Text"
-      },
-      {
-        "name": "installation_device",
-        "ordinal": 1,
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": [
-      false,
-      true
-    ]
-  },
-  "hash": "24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b"
-}

.sqlx/query-2ea29df2326f7c84bd4100ad510a3fd4878dc2e217dc83f9bf45a402dfd62a91.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT host_id FROM host_role_mapping WHERE role = ?",
+  "describe": {
+    "columns": [
+      {
+        "name": "host_id",
+        "ordinal": 0,
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "2ea29df2326f7c84bd4100ad510a3fd4878dc2e217dc83f9bf45a402dfd62a91"
+}

.sqlx/query-df7a7c9cfdd0972e2e0ce7ea444ba8bc9d708a4fb89d5593a0be2bbebde62aff.json

@@ -1,12 +1,12 @@
 {
   "db_name": "SQLite",
-  "query": "\n        INSERT INTO host_role_mapping (host_id, role, installation_device)\n        VALUES (?, ?, ?)\n        ",
+  "query": "\n        INSERT INTO host_role_mapping (host_id, role)\n        VALUES (?, ?)\n        ",
   "describe": {
     "columns": [],
     "parameters": {
-      "Right": 3
+      "Right": 2
     },
     "nullable": []
   },
-  "hash": "6fcc29cfdbdf3b2cee94a4844e227f09b245dd8f079832a9a7b774151cb03af6"
+  "hash": "df7a7c9cfdd0972e2e0ce7ea444ba8bc9d708a4fb89d5593a0be2bbebde62aff"
 }

Cargo.toml

@@ -1,13 +1,12 @@
 [workspace]
 resolver = "2"
 members = [
-  "examples/*",
   "private_repos/*",
+  "examples/*",
   "harmony",
   "harmony_types",
   "harmony_macros",
   "harmony_tui",
-  "harmony_execution",
   "opnsense-config",
   "opnsense-config-xml",
   "harmony_cli",
@@ -15,17 +14,7 @@ members = [
   "harmony_composer",
   "harmony_inventory_agent",
   "harmony_secret_derive",
-  "harmony_secret",
+  "harmony_secret", "adr/agent_discovery/mdns",
-  "examples/kvm_okd_ha_cluster",
-  "examples/example_linux_vm",
-  "harmony_config_derive",
-  "harmony_config",
-  "brocade",
-  "harmony_agent",
-  "harmony_agent/deploy",
-  "harmony_node_readiness",
-  "harmony-k8s",
-  "harmony_assets",
 ]
 
 [workspace.package]
@@ -40,13 +29,10 @@ derive-new = "0.7"
 async-trait = "0.1"
 tokio = { version = "1.40", features = [
   "io-std",
-  "io-util",
   "fs",
   "macros",
   "rt-multi-thread",
 ] }
-tokio-retry = "0.3.0"
-tokio-util = "0.7.15"
 cidr = { features = ["serde"], version = "0.2" }
 russh = "0.45"
 russh-keys = "0.45"
@@ -61,7 +47,6 @@ kube = { version = "1.1.0", features = [
   "jsonpatch",
 ] }
 k8s-openapi = { version = "0.25", features = ["v1_30"] }
-# TODO replace with https://github.com/bourumir-wyngs/serde-saphyr as serde_yaml is deprecated https://github.com/sebastienrousseau/serde_yml
 serde_yaml = "0.9"
 serde-value = "0.7"
 http = "1.2"
@@ -77,18 +62,9 @@ base64 = "0.22.1"
 tar = "0.4.44"
 lazy_static = "1.5.0"
 directories = "6.0.0"
-futures-util = "0.3"
 thiserror = "2.0.14"
 serde = { version = "1.0.209", features = ["derive", "rc"] }
 serde_json = "1.0.127"
 askama = "0.14"
 sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite" ] }
-reqwest = { version = "0.12", features = [
+reqwest = { version = "0.12", features = ["blocking", "stream", "rustls-tls", "http2", "json"], default-features = false }
-  "blocking",
-  "stream",
-  "rustls-tls",
-  "http2",
-  "json",
-], default-features = false }
-assertor = "0.0.4"
-tokio-test = "0.4"

README.md

@@ -1,121 +1,90 @@
-# Harmony
+# Harmony : Open-source infrastructure orchestration that treats your platform like first-class code
-
-**Infrastructure orchestration that treats your platform like first-class code.**
-
-Harmony is an open-source framework that brings the rigor of software engineering to infrastructure management. Write Rust code to define what you want, and Harmony handles the rest — from local development to production clusters.
 
 _By [NationTech](https://nationtech.io)_
 
-[![Build](https://git.nationtech.io/NationTech/harmony/actions/workflows/check.yml/badge.svg)](https://git.nationtech.io/NationTech/harmony)
+[![Build](https://git.nationtech.io/NationTech/harmony/actions/workflows/check.yml/badge.svg)](https://git.nationtech.io/nationtech/harmony)
 [![License](https://img.shields.io/badge/license-AGPLv3-blue?style=flat-square)](LICENSE)
 
----
+### Unify
 
-## The Problem Harmony Solves
+- **Project Scaffolding**
+- **Infrastructure Provisioning**
+- **Application Deployment**
+- **Day-2 operations**
 
-Modern infrastructure is messy. Your Kubernetes cluster needs monitoring. Your bare-metal servers need provisioning. Your applications need deployments. Each comes with its own tooling, its own configuration format, and its own failure modes.
+All in **one strongly-typed Rust codebase**.
 
-**What if you could describe your entire platform in one consistent language?**
+### Deploy anywhere
 
-That's Harmony. It unifies project scaffolding, infrastructure provisioning, application deployment, and day-2 operations into a single strongly-typed Rust codebase.
+From a **developer laptop** to a **global production cluster**, a single **source of truth** drives the **full software lifecycle.**
 
 ---
 
-## Three Principles That Make the Difference
+## 1 · The Harmony Philosophy
 
-| Principle | What It Means |
+Infrastructure is essential, but it shouldn’t be your core business. Harmony is built on three guiding principles that make modern platforms reliable, repeatable, and easy to reason about.
-|-----------|---------------|
+
-| **Infrastructure as Resilient Code** | Stop fighting with YAML and bash. Write type-safe Rust that you can test, version, and refactor like any other code. |
+| Principle                              | What it means for you                                                                                                                                                                |
-| **Prove It Works Before You Deploy** | Harmony verifies at _compile time_ that your application can actually run on your target infrastructure. No more "the config looks right but it doesn't work" surprises. |
+| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| **One Unified Model** | Software and infrastructure are one system. Deploy from laptop to production cluster without switching contexts or tools. |
+| **Infrastructure as Resilient Code**   | Replace sprawling YAML and bash scripts with type-safe Rust. Test, refactor, and version your platform just like application code.                                                   |
+| **Prove It Works — Before You Deploy** | Harmony uses the compiler to verify that your application’s needs match the target environment’s capabilities at **compile-time**, eliminating an entire class of runtime outages.   |
+| **One Unified Model**                  | Software and infrastructure are a single system. Harmony models them together, enabling deep automation—from bare-metal servers to Kubernetes workloads—with zero context switching. |
+
+These principles surface as simple, ergonomic Rust APIs that let teams focus on their product while trusting the platform underneath.
 
 ---
 
-## How It Works: The Core Concepts
+## 2 · Quick Start
 
-Harmony is built around three concepts that work together:
+The snippet below spins up a complete **production-grade Rust + Leptos Webapp** with monitoring. Swap it for your own scores to deploy anything from microservices to machine-learning pipelines.
-
-### Score — "What You Want"
-
-A `Score` is a declarative description of desired state. Think of it as a "recipe" that says _what_ you want without specifying _how_ to get there.
-
-```rust
-// "I want a PostgreSQL cluster running with default settings"
-let postgres = PostgreSQLScore {
-    config: PostgreSQLConfig {
-        cluster_name: "harmony-postgres-example".to_string(),
-        namespace: "harmony-postgres-example".to_string(),
-        ..Default::default()
-    },
-};
-```
-
-### Topology — "Where It Goes"
-
-A `Topology` represents your infrastructure environment and its capabilities. It answers the question: "What can this environment actually do?"
-
-```rust
-// Deploy to a local K3D cluster, or any Kubernetes cluster via environment variables
-K8sAnywhereTopology::from_env()
-```
-
-### Interpret — "How It Happens"
-
-An `Interpret` is the execution logic that connects your `Score` to your `Topology`. It translates "what you want" into "what the infrastructure does."
-
-**The Compile-Time Check:** Before your code ever runs, Harmony verifies that your `Score` is compatible with your `Topology`. If your application needs a feature your infrastructure doesn't provide, you get a compile error — not a runtime failure.
-
----
-
-## What You Can Deploy
-
-Harmony ships with ready-made Scores for:
-
-**Data Services**
-- PostgreSQL clusters (via CloudNativePG operator)
-- Multi-site PostgreSQL with failover
-
-**Kubernetes**
-- Namespaces, Deployments, Ingress
-- Helm charts
-- cert-manager for TLS
-- Monitoring (Prometheus, alerting, ntfy)
-
-**Bare Metal / Infrastructure**
-- OKD clusters from scratch
-- OPNsense firewalls
-- Network services (DNS, DHCP, TFTP)
-- Brocade switch configuration
-
-**And more:** Application deployment, tenant management, load balancing, and more.
-
----
-
-## Quick Start: Deploy a PostgreSQL Cluster
-
-This example provisions a local Kubernetes cluster (K3D) and deploys a PostgreSQL cluster on it — no external infrastructure required.
 
 ```rust
 use harmony::{
     inventory::Inventory,
-    modules::postgresql::{PostgreSQLScore, capability::PostgreSQLConfig},
+    modules::{
+        application::{
+            ApplicationScore, RustWebFramework, RustWebapp,
+            features::{PackagingDeployment, rhob_monitoring::Monitoring},
+        },
+        monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
+    },
     topology::K8sAnywhereTopology,
 };
+use harmony_macros::hurl;
+use std::{path::PathBuf, sync::Arc};
 
 #[tokio::main]
 async fn main() {
-    let postgres = PostgreSQLScore {
+    let application = Arc::new(RustWebapp {
-        config: PostgreSQLConfig {
+        name: "harmony-example-leptos".to_string(),
-            cluster_name: "harmony-postgres-example".to_string(),
+        project_root: PathBuf::from(".."), // <== Your project root, usually .. if you use the standard `/harmony` folder
-            namespace: "harmony-postgres-example".to_string(),
+        framework: Some(RustWebFramework::Leptos),
-            ..Default::default()
+        service_port: 8080,
-        },
+    });
+
+    // Define your Application deployment and the features you want
+    let app = ApplicationScore {
+        features: vec![
+            Box::new(PackagingDeployment {
+                application: application.clone(),
+            }),
+            Box::new(Monitoring {
+                application: application.clone(),
+                alert_receiver: vec![
+                    Box::new(DiscordWebhook {
+                        name: "test-discord".to_string(),
+                        url: hurl!("https://discord.doesnt.exist.com"), // <== Get your discord webhook url
+                    }),
+                ],
+            }),
+        ],
+        application,
     };
 
     harmony_cli::run(
         Inventory::autoload(),
-        K8sAnywhereTopology::from_env(),
+        K8sAnywhereTopology::from_env(), // <== Deploy to local automatically provisioned local k3d by default or connect to any kubernetes cluster
-        vec![Box::new(postgres)],
+        vec![Box::new(app)],
         None,
     )
     .await
@@ -123,128 +92,70 @@ async fn main() {
 }
 ```
 
-### What this actually does
+Run it:
-
+
-When you compile and run this program:
+```bash
-
+cargo run
-1. **Compiles** the Harmony Score into an executable
+```
-2. **Connects** to `K8sAnywhereTopology` — which auto-provisions a local K3D cluster if none exists
+
-3. **Installs** the CloudNativePG operator into the cluster (one-time setup)
+Harmony analyses the code, shows an execution plan in a TUI, and applies it once you confirm. Same code, same binary—every environment.
-4. **Creates** a PostgreSQL cluster with 1 instance and 1 GiB of storage
+
-5. **Exposes** the PostgreSQL instance as a Kubernetes Service
+---
-
+
-### Prerequisites
+## 3 · Core Concepts
-
+
-- [Rust](https://rust-lang.org/tools/install) (edition 2024)
+| Term             | One-liner                                                                                            |
-- [Docker](https://docs.docker.com/get-docker/) (for the local K3D cluster)
+| ---------------- | ---------------------------------------------------------------------------------------------------- |
-- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (optional, for inspecting the cluster)
+| **Score<T>**     | Declarative description of the desired state (e.g., `LAMPScore`).                                    |
-
+| **Interpret<T>** | Imperative logic that realises a `Score` on a specific environment.                                  |
-### Run it
+| **Topology**     | An environment (local k3d, AWS, bare-metal) exposing verified _Capabilities_ (Kubernetes, DNS, …).   |
+| **Maestro**      | Orchestrator that compiles Scores + Topology, ensuring all capabilities line up **at compile-time**. |
+| **Inventory**    | Optional catalogue of physical assets for bare-metal and edge deployments.                           |
+
+A visual overview is in the diagram below.
+
+[Harmony Core Architecture](docs/diagrams/Harmony_Core_Architecture.drawio.svg)
+
+---
+
+## 4 · Install
+
+Prerequisites:
+
+- Rust
+- Docker (if you deploy locally)
+- `kubectl` / `helm` for Kubernetes-based topologies
 
 ```bash
-# Clone the repository
 git clone https://git.nationtech.io/nationtech/harmony
 cd harmony
-
+cargo build --release          # builds the CLI, TUI and libraries
-# Build the project
-cargo build --release
-
-# Run the example
-cargo run -p example-postgresql
-```
-
-Harmony will print its progress as it sets up the cluster and deploys PostgreSQL. When complete, you can inspect the deployment:
-
-```bash
-kubectl get pods -n harmony-postgres-example
-kubectl get secret -n harmony-postgres-example harmony-postgres-example-db-user -o jsonpath='{.data.password}' | base64 -d
-```
-
-To connect to the database, forward the port:
-```bash
-kubectl port-forward -n harmony-postgres-example svc/harmony-postgres-example-rw 5432:5432
-psql -h localhost -p 5432 -U postgres
-```
-
-To clean up, delete the K3D cluster:
-```bash
-k3d cluster delete harmony-postgres-example
 ```
 
 ---
 
-## Environment Variables
+## 5 · Learning More
 
-`K8sAnywhereTopology::from_env()` reads the following environment variables to determine where and how to connect:
+- **Architectural Decision Records** – dive into the rationale
+  - [ADR-001 · Why Rust](adr/001-rust.md)
+  - [ADR-003 · Infrastructure Abstractions](adr/003-infrastructure-abstractions.md)
+  - [ADR-006 · Secret Management](adr/006-secret-management.md)
+  - [ADR-011 · Multi-Tenant Cluster](adr/011-multi-tenant-cluster.md)
 
-| Variable | Default | Description |
+- **Extending Harmony** – write new Scores / Interprets, add hardware like OPNsense firewalls, or embed Harmony in your own tooling (`/docs`).
-|----------|---------|-------------|
-| `KUBECONFIG` | `~/.kube/config` | Path to your kubeconfig file |
-| `HARMONY_AUTOINSTALL` | `true` | Auto-provision a local K3D cluster if none found |
-| `HARMONY_USE_LOCAL_K3D` | `true` | Always prefer local K3D over remote clusters |
-| `HARMONY_PROFILE` | `dev` | Deployment profile: `dev`, `staging`, or `prod` |
-| `HARMONY_K8S_CONTEXT` | _none_ | Use a specific kubeconfig context |
-| `HARMONY_PUBLIC_DOMAIN` | _none_ | Public domain for ingress endpoints |
 
-To connect to an existing Kubernetes cluster instead of provisioning K3D:
+- **Community** – discussions and roadmap live in [GitLab issues](https://git.nationtech.io/nationtech/harmony/-/issues). PRs, ideas, and feedback are welcome!
-
-```bash
-# Point to your kubeconfig
-export KUBECONFIG=/path/to/your/kubeconfig
-export HARMONY_USE_LOCAL_K3D=false
-export HARMONY_AUTOINSTALL=false
-
-# Then run
-cargo run -p example-postgresql
-```
 
 ---
 
-## Documentation
+## 6 · License
-
-| I want to... | Start here |
-|--------------|------------|
-| Understand the core concepts | [Core Concepts](./docs/concepts.md) |
-| Deploy my first application | [Getting Started Guide](./docs/guides/getting-started.md) |
-| Explore available components | [Scores Catalog](./docs/catalogs/scores.md) · [Topologies Catalog](./docs/catalogs/topologies.md) |
-| See a complete bare-metal deployment | [OKD on Bare Metal](./docs/use-cases/okd-on-bare-metal.md) |
-| Build my own Score or Topology | [Developer Guide](./docs/guides/developer-guide.md) |
-
----
-
-## Why Rust?
-
-We chose Rust for the same reason you might: **reliability through type safety**.
-
-Infrastructure code runs in production. It needs to be correct. Rust's ownership model and type system let us build a framework where:
-
-- Invalid configurations fail at compile time, not at 3 AM
-- Refactoring infrastructure is as safe as refactoring application code
-- The compiler verifies that your platform can actually fulfill your requirements
-
-See [ADR-001 · Why Rust](./adr/001-rust.md) for our full rationale.
-
----
-
-## Architecture Decisions
-
-Harmony's design is documented through Architecture Decision Records (ADRs):
-
-- [ADR-001 · Why Rust](./adr/001-rust.md)
-- [ADR-003 · Infrastructure Abstractions](./adr/003-infrastructure-abstractions.md)
-- [ADR-006 · Secret Management](./adr/006-secret-management.md)
-- [ADR-011 · Multi-Tenant Cluster](./adr/011-multi-tenant-cluster.md)
-
----
-
-## License
 
 Harmony is released under the **GNU AGPL v3**.
 
-> We choose a strong copyleft license to ensure the project—and every improvement to it—remains open and benefits the entire community.
+> We choose a strong copyleft license to ensure the project—and every improvement to it—remains open and benefits the entire community. Fork it, enhance it, even out-innovate us; just keep it open.
 
 See [LICENSE](LICENSE) for the full text.
 
 ---
 
-_Made with ❤️ & 🦀 by NationTech and the Harmony community_
+_Made with ❤️ & 🦀 by the NationTech and the Harmony community_

ROADMAP.md

@@ -1,29 +0,0 @@
-# Harmony Roadmap
-
-Six phases to take Harmony from working prototype to production-ready open-source project.
-
-| # | Phase | Status | Depends On | Detail |
-|---|-------|--------|------------|--------|
-| 1 | [Harden `harmony_config`](ROADMAP/01-config-crate.md) | Not started | — | Test every source, add SQLite backend, wire Zitadel + OpenBao, validate zero-setup UX |
-| 2 | [Migrate to `harmony_config`](ROADMAP/02-refactor-harmony-config.md) | Not started | 1 | Replace all 19 `SecretManager` call sites, deprecate direct `harmony_secret` usage |
-| 3 | [Complete `harmony_assets`](ROADMAP/03-assets-crate.md) | Not started | 1, 2 | Test, refactor k3d and OKD to use it, implement `Url::Url`, remove LFS |
-| 4 | [Publish to GitHub](ROADMAP/04-publish-github.md) | Not started | 3 | Clean history, set up GitHub as community hub, CI on self-hosted runners |
-| 5 | [E2E tests: PostgreSQL & RustFS](ROADMAP/05-e2e-tests-simple.md) | Not started | 1 | k3d-based test harness, two passing E2E tests, CI job |
-| 6 | [E2E tests: OKD HA on KVM](ROADMAP/06-e2e-tests-kvm.md) | Not started | 5 | KVM test infrastructure, full OKD installation test, nightly CI |
-
-## Current State (as of branch `feature/kvm-module`)
-
-- `harmony_config` crate exists with `EnvSource`, `LocalFileSource`, `PromptSource`, `StoreSource`. 12 unit tests. **Zero consumers** in workspace — everything still uses `harmony_secret::SecretManager` directly (19 call sites).
-- `harmony_assets` crate exists with `Asset`, `LocalCache`, `LocalStore`, `S3Store`. **No tests. Zero consumers.** The `k3d` crate has its own `DownloadableAsset` with identical functionality and full test coverage.
-- `harmony_secret` has `LocalFileSecretStore`, `OpenbaoSecretStore` (token/userpass only), `InfisicalSecretStore`. Works but no Zitadel OIDC integration.
-- KVM module exists on this branch with `KvmExecutor`, VM lifecycle, ISO download, two examples (`example_linux_vm`, `kvm_okd_ha_cluster`).
-- RustFS module exists on `feat/rustfs` branch (2 commits ahead of master).
-- 39 example crates, **zero E2E tests**. Unit tests pass across workspace (~240 tests).
-- CI runs `cargo check`, `fmt`, `clippy`, `test` on Gitea. No E2E job.
-
-## Guiding Principles
-
-- **Zero-setup first**: A new user clones, runs `cargo run`, gets prompted for config, values persist to local SQLite. No env vars, no external services required.
-- **Progressive disclosure**: Local SQLite → OpenBao → Zitadel SSO. Each layer is opt-in.
-- **Test what ships**: Every example that works should have an E2E test proving it works.
-- **Community over infrastructure**: GitHub for engagement, self-hosted runners for CI.

ROADMAP/01-config-crate.md

@@ -1,177 +0,0 @@
-# Phase 1: Harden `harmony_config`, Validate UX, Zero-Setup Starting Point
-
-## Goal
-
-Make `harmony_config` production-ready with a seamless first-run experience: clone, run, get prompted, values persist locally. Then progressively add team-scale backends (OpenBao, Zitadel SSO) without changing any calling code.
-
-## Current State
-
-`harmony_config` exists with:
-
-- `Config` trait + `#[derive(Config)]` macro
-- `ConfigManager` with ordered source chain
-- Four `ConfigSource` implementations:
-  - `EnvSource` — reads `HARMONY_CONFIG_{KEY}` env vars
-  - `LocalFileSource` — reads/writes `{key}.json` files from a directory
-  - `PromptSource` — **stub** (returns `None` / no-ops on set)
-  - `StoreSource<S: SecretStore>` — wraps any `harmony_secret::SecretStore` backend
-- 12 unit tests (mock source, env, local file)
-- Global `CONFIG_MANAGER` static with `init()`, `get()`, `get_or_prompt()`, `set()`
-- **Zero workspace consumers** — nothing calls `harmony_config` yet
-
-## Tasks
-
-### 1.1 Add `SqliteSource` as the default zero-setup backend
-
-Replace `LocalFileSource` (JSON files scattered in a directory) with a single SQLite database as the default local backend. `sqlx` with SQLite is already a workspace dependency.
-
-```rust
-// harmony_config/src/source/sqlite.rs
-pub struct SqliteSource {
-    pool: SqlitePool,
-}
-
-impl SqliteSource {
-    /// Opens or creates the database at the given path.
-    /// Creates the `config` table if it doesn't exist.
-    pub async fn open(path: PathBuf) -> Result<Self, ConfigError>
-
-    /// Uses the default Harmony data directory:
-    /// ~/.local/share/harmony/config.db (Linux)
-    pub async fn default() -> Result<Self, ConfigError>
-}
-
-#[async_trait]
-impl ConfigSource for SqliteSource {
-    async fn get(&self, key: &str) -> Result<Option<serde_json::Value>, ConfigError>
-    async fn set(&self, key: &str, value: &serde_json::Value) -> Result<(), ConfigError>
-}
-```
-
-Schema:
-
-```sql
-CREATE TABLE IF NOT EXISTS config (
-    key   TEXT PRIMARY KEY,
-    value TEXT NOT NULL,
-    updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-);
-```
-
-**Tests**:
-- `test_sqlite_set_and_get` — round-trip a `TestConfig` struct
-- `test_sqlite_get_returns_none_when_missing` — key not in DB
-- `test_sqlite_overwrites_on_set` — set twice, get returns latest
-- `test_sqlite_concurrent_access` — two tasks writing different keys simultaneously
-- All tests use `tempfile::NamedTempFile` for the DB path
-
-### 1.1.1 Add Config example to show exact DX and confirm functionality
-
-Create `harmony_config/examples` that show how to use config crate with various backends.
-
-Show how to use the derive macros, how to store secrets in a local backend or a zitadel + openbao backend, how to fetch them from environment variables, etc. Explicitely outline the dependencies for examples with dependencies in a comment at the top. Explain how to configure zitadel + openbao for this backend. The local backend should have zero dependency, zero setup, storing its config/secrets with sane defaults.
-
-Also show that a Config with default values will not prompt for values with defaults.
-
-### 1.2 Make `PromptSource` functional
-
-Currently `PromptSource::get()` returns `None` and `set()` is a no-op. Wire it to `interactive_parse::InteractiveParseObj`:
-
-```rust
-#[async_trait]
-impl ConfigSource for PromptSource {
-    async fn get(&self, _key: &str) -> Result<Option<serde_json::Value>, ConfigError> {
-        // PromptSource never "has" a value — it's always a fallback.
-        // The actual prompting happens in ConfigManager::get_or_prompt().
-        Ok(None)
-    }
-
-    async fn set(&self, _key: &str, _value: &serde_json::Value) -> Result<(), ConfigError> {
-        // Prompt source doesn't persist. Other sources in the chain do.
-        Ok(())
-    }
-}
-```
-
-The prompting logic is already in `ConfigManager::get_or_prompt()` via `T::parse_to_obj()`. The `PromptSource` struct exists mainly to hold the `PROMPT_MUTEX` and potentially a custom writer for TUI integration later.
-
-**Key fix**: Ensure `get_or_prompt()` persists the prompted value to the **first writable source** (SQLite), not to all sources. Current code tries all sources — this is wrong for prompt-then-persist because you don't want to write prompted values to env vars.
-
-```rust
-pub async fn get_or_prompt<T: Config>(&self) -> Result<T, ConfigError> {
-    match self.get::<T>().await {
-        Ok(config) => Ok(config),
-        Err(ConfigError::NotFound { .. }) => {
-            let config = T::parse_to_obj()
-                .map_err(|e| ConfigError::PromptError(e.to_string()))?;
-            let value = serde_json::to_value(&config)
-                .map_err(|e| ConfigError::Serialization { key: T::KEY.to_string(), source: e })?;
-
-            // Persist to the first source that accepts writes (skip EnvSource)
-            for source in &self.sources {
-                if source.set(T::KEY, &value).await.is_ok() {
-                    break;
-                }
-            }
-            Ok(config)
-        }
-        Err(e) => Err(e),
-    }
-}
-```
-
-**Tests**:
-- `test_get_or_prompt_persists_to_first_writable_source` — mock source chain where first source is read-only, second is writable. Verify prompted value lands in second source.
-
-### 1.3 Integration test: full resolution chain
-
-Test the complete priority chain: env > sqlite > prompt.
-
-```rust
-#[tokio::test]
-async fn test_full_resolution_chain() {
-    // 1. No env var, no SQLite entry → prompting would happen
-    //    (test with mock/pre-seeded source instead of real stdin)
-    // 2. Set in SQLite → get() returns SQLite value
-    // 3. Set env var → get() returns env value (overrides SQLite)
-    // 4. Remove env var → get() falls back to SQLite
-}
-
-#[tokio::test]
-async fn test_branch_switching_scenario() {
-    // Simulate: struct shape changes between branches.
-    // Old value in SQLite doesn't match new struct.
-    // get() should return Deserialization error.
-    // get_or_prompt() should re-prompt and overwrite.
-}
-```
-
-### 1.4 Validate Zitadel + OpenBao integration path
-
-This is not about building the full OIDC flow yet. It's about validating that the architecture supports it by adding `StoreSource<OpenbaoSecretStore>` to the source chain.
-
-**Validate**:
-- `ConfigManager::new(vec![EnvSource, SqliteSource, StoreSource<Openbao>])` compiles and works
-- When OpenBao is unreachable, the chain falls through to SQLite gracefully (no panic)
-- When OpenBao has the value, it's returned and SQLite is not queried
-
-**Document** the target Zitadel OIDC flow as an ADR (RFC 8628 device authorization grant), but don't implement it yet. The `StoreSource` wrapping OpenBao with JWT auth is the integration point — Zitadel provides the JWT, OpenBao validates it.
-
-### 1.5 UX validation checklist
-
-Before this phase is done, manually verify:
-
-- [ ] `cargo run --example postgresql` with no env vars → prompts for nothing (postgresql doesn't use secrets yet, but the config system initializes cleanly)
-- [ ] An example that uses `SecretManager` today (e.g., `brocade_snmp_server`) → when migrated to `harmony_config`, first run prompts, second run reads from SQLite
-- [ ] Setting `HARMONY_CONFIG_BrocadeSwitchAuth='{"host":"...","user":"...","password":"..."}'` → skips prompt, uses env value
-- [ ] Deleting `~/.local/share/harmony/config.db` → re-prompts on next run
-
-## Deliverables
-
-- [ ] `SqliteSource` implementation with tests
-- [ ] Functional `PromptSource` (or validated that current `get_or_prompt` flow is correct)
-- [ ] Fix `get_or_prompt` to persist to first writable source, not all sources
-- [ ] Integration tests for full resolution chain
-- [ ] Branch-switching deserialization failure test
-- [ ] `StoreSource<OpenbaoSecretStore>` integration validated (compiles, graceful fallback)
-- [ ] ADR for Zitadel OIDC target architecture

ROADMAP/02-refactor-harmony-config.md

@@ -1,112 +0,0 @@
-# Phase 2: Migrate Workspace to `harmony_config`
-
-## Goal
-
-Replace every direct `harmony_secret::SecretManager` call with `harmony_config` equivalents. After this phase, modules and examples depend only on `harmony_config`. `harmony_secret` becomes an internal implementation detail behind `StoreSource`.
-
-## Current State
-
-19 call sites use `SecretManager::get_or_prompt::<T>()` across:
-
-| Location | Secret Types | Call Sites |
-|----------|-------------|------------|
-| `harmony/src/modules/brocade/brocade_snmp.rs` | `BrocadeSnmpAuth`, `BrocadeSwitchAuth` | 2 |
-| `harmony/src/modules/nats/score_nats_k8s.rs` | `NatsAdmin` | 1 |
-| `harmony/src/modules/okd/bootstrap_02_bootstrap.rs` | `RedhatSecret`, `SshKeyPair` | 2 |
-| `harmony/src/modules/application/features/monitoring.rs` | `NtfyAuth` | 1 |
-| `brocade/examples/main.rs` | `BrocadeSwitchAuth` | 1 |
-| `examples/okd_installation/src/main.rs` + `topology.rs` | `SshKeyPair`, `BrocadeSwitchAuth`, `OPNSenseFirewallConfig` | 3 |
-| `examples/okd_pxe/src/main.rs` + `topology.rs` | `SshKeyPair`, `BrocadeSwitchAuth`, `OPNSenseFirewallCredentials` | 3 |
-| `examples/opnsense/src/main.rs` | `OPNSenseFirewallCredentials` | 1 |
-| `examples/sttest/src/main.rs` + `topology.rs` | `SshKeyPair`, `OPNSenseFirewallConfig` | 2 |
-| `examples/opnsense_node_exporter/` | (has dep but unclear usage) | ~1 |
-| `examples/okd_cluster_alerts/` | (has dep but unclear usage) | ~1 |
-| `examples/brocade_snmp_server/` | (has dep but unclear usage) | ~1 |
-
-## Tasks
-
-### 2.1 Bootstrap `harmony_config` in CLI and TUI entry points
-
-Add `harmony_config::init()` as the first thing that happens in `harmony_cli::run()` and `harmony_tui::run()`.
-
-```rust
-// harmony_cli/src/lib.rs — inside run()
-pub async fn run<T: Topology + Send + Sync + 'static>(
-    inventory: Inventory,
-    topology: T,
-    scores: Vec<Box<dyn Score<T>>>,
-    args_struct: Option<Args>,
-) -> Result<(), Box<dyn std::error::Error>> {
-    // Initialize config system with default source chain
-    let sqlite = Arc::new(SqliteSource::default().await?);
-    let env = Arc::new(EnvSource);
-    harmony_config::init(vec![env, sqlite]).await;
-
-    // ... rest of run()
-}
-```
-
-This replaces the implicit `SecretManager` lazy initialization that currently happens on first `get_or_prompt` call.
-
-### 2.2 Migrate each secret type from `Secret` to `Config`
-
-For each secret struct, change:
-
-```rust
-// Before
-use harmony_secret::Secret;
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, InteractiveParse, Secret)]
-struct BrocadeSwitchAuth { ... }
-
-// After
-use harmony_config::Config;
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, InteractiveParse, Config)]
-struct BrocadeSwitchAuth { ... }
-```
-
-At each call site, change:
-
-```rust
-// Before
-let config = SecretManager::get_or_prompt::<BrocadeSwitchAuth>().await.unwrap();
-
-// After
-let config = harmony_config::get_or_prompt::<BrocadeSwitchAuth>().await.unwrap();
-```
-
-### 2.3 Migration order (low risk to high risk)
-
-1. **`brocade/examples/main.rs`** — 1 call site, isolated example, easy to test manually
-2. **`examples/opnsense/src/main.rs`** — 1 call site, isolated
-3. **`harmony/src/modules/brocade/brocade_snmp.rs`** — 2 call sites, core module but straightforward
-4. **`harmony/src/modules/nats/score_nats_k8s.rs`** — 1 call site
-5. **`harmony/src/modules/application/features/monitoring.rs`** — 1 call site
-6. **`examples/sttest/`** — 2 call sites, has both main.rs and topology.rs patterns
-7. **`examples/okd_installation/`** — 3 call sites, complex topology setup
-8. **`examples/okd_pxe/`** — 3 call sites, similar to okd_installation
-9. **`harmony/src/modules/okd/bootstrap_02_bootstrap.rs`** — 2 call sites, critical OKD bootstrap path
-
-### 2.4 Remove `harmony_secret` from direct dependencies
-
-After all call sites are migrated:
-
-1. Remove `harmony_secret` from `Cargo.toml` of: `harmony`, `brocade`, and all examples that had it
-2. `harmony_config` keeps `harmony_secret` as a dependency (for `StoreSource`)
-3. The `Secret` trait and `SecretManager` remain in `harmony_secret` but are not used directly anymore
-
-### 2.5 Backward compatibility for existing local secrets
-
-Users who already have secrets stored via `LocalFileSecretStore` (JSON files in `~/.local/share/harmony/secrets/`) need a migration path:
-
-- On first run after upgrade, if SQLite has no entry for a key but the old JSON file exists, read from JSON and write to SQLite
-- Or: add `LocalFileSource` as a fallback source at the end of the chain (read-only) for one release cycle
-- Log a deprecation warning when reading from old JSON files
-
-## Deliverables
-
-- [ ] `harmony_config::init()` called in `harmony_cli::run()` and `harmony_tui::run()`
-- [ ] All 19 call sites migrated from `SecretManager` to `harmony_config`
-- [ ] `harmony_secret` removed from direct dependencies of `harmony`, `brocade`, and all examples
-- [ ] Backward compatibility for existing local JSON secrets
-- [ ] All existing unit tests still pass
-- [ ] Manual verification: one migrated example works end-to-end (prompt → persist → read)

ROADMAP/03-assets-crate.md

@@ -1,141 +0,0 @@
-# Phase 3: Complete `harmony_assets`, Refactor Consumers
-
-## Goal
-
-Make `harmony_assets` the single way to manage downloadable binaries and images across Harmony. Eliminate `k3d::DownloadableAsset` duplication, implement `Url::Url` in OPNsense infra, remove LFS-tracked files from git.
-
-## Current State
-
-- `harmony_assets` exists with `Asset`, `LocalCache`, `LocalStore`, `S3Store` (behind feature flag). CLI with `upload`, `download`, `checksum`, `verify` commands. **No tests. Zero consumers.**
-- `k3d/src/downloadable_asset.rs` has the same functionality with full test coverage (httptest mock server, checksum verification, cache hit, 404 handling, checksum failure).
-- `Url::Url` variant in `harmony_types/src/net.rs` exists but is `todo!()` in OPNsense TFTP and HTTP infra layers.
-- OKD modules hardcode `./data/...` paths (`bootstrap_02_bootstrap.rs:84-88`, `ipxe.rs:73`).
-- `data/` directory contains ~3GB of LFS-tracked files (OKD binaries, PXE images, SCOS images).
-
-## Tasks
-
-### 3.1 Port k3d tests to `harmony_assets`
-
-The k3d crate has 5 well-written tests in `downloadable_asset.rs`. Port them to test `harmony_assets::LocalStore`:
-
-```rust
-// harmony_assets/tests/local_store.rs (or in src/ as unit tests)
-
-#[tokio::test]
-async fn test_fetch_downloads_and_verifies_checksum() {
-    // Start httptest server serving a known file
-    // Create Asset with URL pointing to mock server
-    // Fetch via LocalStore
-    // Assert file exists at expected cache path
-    // Assert checksum matches
-}
-
-#[tokio::test]
-async fn test_fetch_returns_cached_file_when_present() {
-    // Pre-populate cache with correct file
-    // Fetch — assert no HTTP request made (mock server not hit)
-}
-
-#[tokio::test]
-async fn test_fetch_fails_on_404() { ... }
-
-#[tokio::test]
-async fn test_fetch_fails_on_checksum_mismatch() { ... }
-
-#[tokio::test]
-async fn test_fetch_with_progress_callback() {
-    // Assert progress callback is called with (bytes_received, total_size)
-}
-```
-
-Add `httptest` to `[dev-dependencies]` of `harmony_assets`.
-
-### 3.2 Refactor `k3d` to use `harmony_assets`
-
-Replace `k3d/src/downloadable_asset.rs` with calls to `harmony_assets`:
-
-```rust
-// k3d/src/lib.rs — in download_latest_release()
-use harmony_assets::{Asset, LocalCache, LocalStore, ChecksumAlgo};
-
-let asset = Asset::new(
-    binary_url,
-    checksum,
-    ChecksumAlgo::SHA256,
-    K3D_BIN_FILE_NAME.to_string(),
-);
-let cache = LocalCache::new(self.base_dir.clone());
-let store = LocalStore::new();
-let path = store.fetch(&asset, &cache, None).await
-    .map_err(|e| format!("Failed to download k3d: {}", e))?;
-```
-
-Delete `k3d/src/downloadable_asset.rs`. Update k3d's `Cargo.toml` to depend on `harmony_assets`.
-
-### 3.3 Define asset metadata as config structs
-
-Following `plan.md` Phase 2, create typed config for OKD assets using `harmony_config`:
-
-```rust
-// harmony/src/modules/okd/config.rs
-#[derive(Config, Serialize, Deserialize, JsonSchema, InteractiveParse)]
-struct OkdInstallerConfig {
-    pub openshift_install_url: String,
-    pub openshift_install_sha256: String,
-    pub scos_kernel_url: String,
-    pub scos_kernel_sha256: String,
-    pub scos_initramfs_url: String,
-    pub scos_initramfs_sha256: String,
-    pub scos_rootfs_url: String,
-    pub scos_rootfs_sha256: String,
-}
-```
-
-First run prompts for URLs/checksums (or uses compiled-in defaults). Values persist to SQLite. Can be overridden via env vars or OpenBao.
-
-### 3.4 Implement `Url::Url` in OPNsense infra layer
-
-In `harmony/src/infra/opnsense/http.rs` and `tftp.rs`, implement the `Url::Url(url)` match arm:
-
-```rust
-// Instead of SCP-ing files to OPNsense:
-// SSH into OPNsense, run: fetch -o /usr/local/http/{path} {url}
-// (FreeBSD-native HTTP client, no extra deps on OPNsense)
-```
-
-This eliminates the manual `scp` workaround and the `inquire::Confirm` prompts in `ipxe.rs:126` and `bootstrap_02_bootstrap.rs:230`.
-
-### 3.5 Refactor OKD modules to use assets + config
-
-In `bootstrap_02_bootstrap.rs`:
-- `openshift-install`: Resolve `OkdInstallerConfig` from `harmony_config`, download via `harmony_assets`, invoke from cache.
-- SCOS images: Pass `Url::Url(scos_kernel_url)` etc. to `StaticFilesHttpScore`. OPNsense fetches from S3 directly.
-- Remove `oc` and `kubectl` from `data/okd/bin/` (never used by code).
-
-In `ipxe.rs`:
-- Replace the folder-to-serve SCP workaround with individual `Url::Url` entries.
-- Remove the `inquire::Confirm` SCP prompts.
-
-### 3.6 Upload assets to S3
-
-- Upload all current `data/` binaries to Ceph S3 bucket with path scheme: `harmony-assets/okd/v{version}/openshift-install`, `harmony-assets/pxe/centos-stream-9/install.img`, etc.
-- Set public-read ACL or configure presigned URL generation.
-- Record S3 URLs and SHA256 checksums as defaults in the config structs.
-
-### 3.7 Remove LFS, clean git
-
-- Remove all LFS-tracked files from the repo.
-- Update `.gitattributes` to remove LFS filters.
-- Keep `data/` in `.gitignore` (it becomes a local cache directory).
-- Optionally use `git filter-repo` or BFG to strip LFS objects from history (required before Phase 4 GitHub publish).
-
-## Deliverables
-
-- [ ] `harmony_assets` has tests ported from k3d pattern (5+ tests with httptest)
-- [ ] `k3d::DownloadableAsset` replaced by `harmony_assets` usage
-- [ ] `OkdInstallerConfig` struct using `harmony_config`
-- [ ] `Url::Url` implemented in OPNsense HTTP and TFTP infra
-- [ ] OKD bootstrap refactored to use lazy-download pattern
-- [ ] Assets uploaded to S3 with documented URLs/checksums
-- [ ] LFS removed, git history cleaned
-- [ ] Repo size small enough for GitHub (~code + templates only)

ROADMAP/04-publish-github.md

@@ -1,110 +0,0 @@
-# Phase 4: Publish to GitHub
-
-## Goal
-
-Make Harmony publicly available on GitHub as the primary community hub for issues, pull requests, and discussions. CI runs on self-hosted runners.
-
-## Prerequisites
-
-- Phase 3 complete: LFS removed, git history cleaned, repo is small
-- README polished with quick-start, architecture overview, examples
-- All existing tests pass
-
-## Tasks
-
-### 4.1 Clean git history
-
-```bash
-# Option A: git filter-repo (preferred)
-git filter-repo --strip-blobs-bigger-than 10M
-
-# Option B: BFG Repo Cleaner
-bfg --strip-blobs-bigger-than 10M
-git reflog expire --expire=now --all
-git gc --prune=now --aggressive
-```
-
-Verify final repo size is reasonable (target: <50MB including all code, docs, templates).
-
-### 4.2 Create GitHub repository
-
-- Create `NationTech/harmony` (or chosen org/name) on GitHub
-- Push cleaned repo as initial commit
-- Set default branch to `main` (rename from `master` if desired)
-
-### 4.3 Set up CI on self-hosted runners
-
-GitHub is the community hub, but CI runs on your own infrastructure. Options:
-
-**Option A: GitHub Actions with self-hosted runners**
-- Register your Gitea runner machines as GitHub Actions self-hosted runners
-- Port `.gitea/workflows/check.yml` to `.github/workflows/check.yml`
-- Same Docker image (`hub.nationtech.io/harmony/harmony_composer:latest`), same commands
-- Pro: native GitHub PR checks, no external service needed
-- Con: runners need outbound access to GitHub API
-
-**Option B: External CI (Woodpecker, Drone, Jenkins)**
-- Use any CI that supports webhooks from GitHub
-- Report status back to GitHub via commit status API / checks API
-- Pro: fully self-hosted, no GitHub dependency for builds
-- Con: extra integration work
-
-**Option C: Keep Gitea CI, mirror from GitHub**
-- GitHub repo has a webhook that triggers Gitea CI on push
-- Gitea reports back to GitHub via commit status API
-- Pro: no migration of CI config
-- Con: fragile webhook chain
-
-**Recommendation**: Option A. GitHub Actions self-hosted runners are straightforward and give the best contributor UX (native PR checks). The workflow files are nearly identical to Gitea workflows.
-
-```yaml
-# .github/workflows/check.yml
-name: Check
-on: [push, pull_request]
-jobs:
-  check:
-    runs-on: self-hosted
-    container:
-      image: hub.nationtech.io/harmony/harmony_composer:latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: bash build/check.sh
-```
-
-### 4.4 Polish documentation
-
-- **README.md**: Quick-start (clone → run → get prompted → see result), architecture diagram (Score → Interpret → Topology), link to docs and examples
-- **CONTRIBUTING.md**: Already exists. Review for GitHub-specific guidance (fork workflow, PR template)
-- **docs/**: Already comprehensive. Verify links work on GitHub rendering
-- **Examples**: Ensure each example has a one-line description in its `Cargo.toml` and a comment block in `main.rs`
-
-### 4.5 License and legal
-
-- Verify workspace `license` field in root `Cargo.toml` is set correctly
-- Add `LICENSE` file at repo root if not present
-- Scan for any proprietary dependencies or hardcoded internal URLs
-
-### 4.6 GitHub repository configuration
-
-- Branch protection on `main`: require PR review, require CI to pass
-- Issue templates: bug report, feature request
-- PR template: checklist (tests pass, docs updated, etc.)
-- Topics/tags: `rust`, `infrastructure-as-code`, `kubernetes`, `orchestration`, `bare-metal`
-- Repository description: "Infrastructure orchestration framework. Declare what you want (Score), describe your infrastructure (Topology), let Harmony figure out how."
-
-### 4.7 Gitea as internal mirror
-
-- Set up Gitea to mirror from GitHub (pull mirror)
-- Internal CI can continue running on Gitea for private/experimental branches
-- Public contributions flow through GitHub
-
-## Deliverables
-
-- [ ] Git history cleaned, repo size <50MB
-- [ ] Public GitHub repository created
-- [ ] CI running on self-hosted runners with GitHub Actions
-- [ ] Branch protection enabled
-- [ ] README polished with quick-start guide
-- [ ] Issue and PR templates created
-- [ ] LICENSE file present
-- [ ] Gitea configured as mirror

ROADMAP/05-e2e-tests-simple.md

@@ -1,255 +0,0 @@
-# Phase 5: E2E Tests for PostgreSQL & RustFS
-
-## Goal
-
-Establish an automated E2E test pipeline that proves working examples actually work. Start with the two simplest k8s-based examples: PostgreSQL and RustFS.
-
-## Prerequisites
-
-- Phase 1 complete (config crate works, bootstrap is clean)
-- `feat/rustfs` branch merged
-
-## Architecture
-
-### Test harness: `tests/e2e/`
-
-A dedicated workspace member crate at `tests/e2e/` that contains:
-
-1. **Shared k3d utilities** — create/destroy clusters, wait for readiness
-2. **Per-example test modules** — each example gets a `#[tokio::test]` function
-3. **Assertion helpers** — wait for pods, check CRDs exist, verify services
-
-```
-tests/
-  e2e/
-    Cargo.toml
-    src/
-      lib.rs          # Shared test utilities
-      k3d.rs          # k3d cluster lifecycle
-      k8s_assert.rs   # K8s assertion helpers
-    tests/
-      postgresql.rs   # PostgreSQL E2E test
-      rustfs.rs       # RustFS E2E test
-```
-
-### k3d cluster lifecycle
-
-```rust
-// tests/e2e/src/k3d.rs
-use k3d_rs::K3d;
-
-pub struct TestCluster {
-    pub name: String,
-    pub k3d: K3d,
-    pub client: kube::Client,
-    reuse: bool,
-}
-
-impl TestCluster {
-    /// Creates a k3d cluster for testing.
-    /// If HARMONY_E2E_REUSE_CLUSTER=1, reuses existing cluster.
-    pub async fn ensure(name: &str) -> Result<Self, String> {
-        let reuse = std::env::var("HARMONY_E2E_REUSE_CLUSTER")
-            .map(|v| v == "1")
-            .unwrap_or(false);
-
-        let base_dir = PathBuf::from("/tmp/harmony-e2e");
-        let k3d = K3d::new(base_dir, Some(name.to_string()));
-
-        let client = k3d.ensure_installed().await?;
-
-        Ok(Self { name: name.to_string(), k3d, client, reuse })
-    }
-
-    /// Returns the kubeconfig path for this cluster.
-    pub fn kubeconfig_path(&self) -> String { ... }
-}
-
-impl Drop for TestCluster {
-    fn drop(&mut self) {
-        if !self.reuse {
-            // Best-effort cleanup
-            let _ = self.k3d.run_k3d_command(["cluster", "delete", &self.name]);
-        }
-    }
-}
-```
-
-### K8s assertion helpers
-
-```rust
-// tests/e2e/src/k8s_assert.rs
-
-/// Wait until a pod matching the label selector is Running in the namespace.
-/// Times out after `timeout` duration.
-pub async fn wait_for_pod_running(
-    client: &kube::Client,
-    namespace: &str,
-    label_selector: &str,
-    timeout: Duration,
-) -> Result<(), String>
-
-/// Assert a CRD instance exists.
-pub async fn assert_resource_exists<K: kube::Resource>(
-    client: &kube::Client,
-    name: &str,
-    namespace: Option<&str>,
-) -> Result<(), String>
-
-/// Install a Helm chart. Returns when all pods in the release are running.
-pub async fn helm_install(
-    release_name: &str,
-    chart: &str,
-    namespace: &str,
-    repo_url: Option<&str>,
-    timeout: Duration,
-) -> Result<(), String>
-```
-
-## Tasks
-
-### 5.1 Create the `tests/e2e/` crate
-
-Add to workspace `Cargo.toml`:
-
-```toml
-[workspace]
-members = [
-  # ... existing members
-  "tests/e2e",
-]
-```
-
-`tests/e2e/Cargo.toml`:
-
-```toml
-[package]
-name = "harmony-e2e-tests"
-edition = "2024"
-publish = false
-
-[dependencies]
-harmony = { path = "../../harmony" }
-harmony_cli = { path = "../../harmony_cli" }
-harmony_types = { path = "../../harmony_types" }
-k3d_rs = { path = "../../k3d", package = "k3d_rs" }
-kube = { workspace = true }
-k8s-openapi = { workspace = true }
-tokio = { workspace = true }
-log = { workspace = true }
-env_logger = { workspace = true }
-
-[dev-dependencies]
-pretty_assertions = { workspace = true }
-```
-
-### 5.2 PostgreSQL E2E test
-
-```rust
-// tests/e2e/tests/postgresql.rs
-use harmony::modules::postgresql::{PostgreSQLScore, capability::PostgreSQLConfig};
-use harmony::topology::K8sAnywhereTopology;
-use harmony::inventory::Inventory;
-use harmony::maestro::Maestro;
-
-#[tokio::test]
-async fn test_postgresql_deploys_on_k3d() {
-    let cluster = TestCluster::ensure("harmony-e2e-pg").await.unwrap();
-
-    // Install CNPG operator via Helm
-    // (K8sAnywhereTopology::ensure_ready() now handles this since
-    //  commit e1183ef "K8s postgresql score now ensures cnpg is installed")
-    // But we may need the Helm chart for non-OKD:
-    helm_install(
-        "cnpg",
-        "cloudnative-pg",
-        "cnpg-system",
-        Some("https://cloudnative-pg.github.io/charts"),
-        Duration::from_secs(120),
-    ).await.unwrap();
-
-    // Configure topology pointing to test cluster
-    let config = K8sAnywhereConfig {
-        kubeconfig: Some(cluster.kubeconfig_path()),
-        use_local_k3d: false,
-        autoinstall: false,
-        use_system_kubeconfig: false,
-        harmony_profile: "dev".to_string(),
-        k8s_context: None,
-    };
-    let topology = K8sAnywhereTopology::with_config(config);
-
-    // Create and run the score
-    let score = PostgreSQLScore {
-        config: PostgreSQLConfig {
-            cluster_name: "e2e-test-pg".to_string(),
-            namespace: "e2e-pg-test".to_string(),
-            ..Default::default()
-        },
-    };
-
-    let mut maestro = Maestro::initialize(Inventory::autoload(), topology).await.unwrap();
-    maestro.register_all(vec![Box::new(score)]);
-
-    let scores = maestro.scores().read().unwrap().first().unwrap().clone_box();
-    let result = maestro.interpret(scores).await;
-    assert!(result.is_ok(), "PostgreSQL score failed: {:?}", result.err());
-
-    // Assert: CNPG Cluster resource exists
-    // (the Cluster CRD is applied — pod readiness may take longer)
-    let client = cluster.client.clone();
-    // ... assert Cluster CRD exists in e2e-pg-test namespace
-}
-```
-
-### 5.3 RustFS E2E test
-
-Similar structure. Details depend on what the RustFS score deploys (likely a Helm chart or k8s resources for MinIO/RustFS).
-
-```rust
-#[tokio::test]
-async fn test_rustfs_deploys_on_k3d() {
-    let cluster = TestCluster::ensure("harmony-e2e-rustfs").await.unwrap();
-    // ... similar pattern: configure topology, create score, interpret, assert
-}
-```
-
-### 5.4 CI job for E2E tests
-
-New workflow file (Gitea or GitHub Actions):
-
-```yaml
-# .gitea/workflows/e2e.yml (or .github/workflows/e2e.yml)
-name: E2E Tests
-on:
-  push:
-    branches: [master, main]
-  # Don't run on every PR — too slow. Run on label or manual trigger.
-  workflow_dispatch:
-
-jobs:
-  e2e:
-    runs-on: self-hosted  # Must have Docker available for k3d
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install k3d
-        run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-
-      - name: Run E2E tests
-        run: cargo test -p harmony-e2e-tests -- --test-threads=1
-        env:
-          RUST_LOG: info
-```
-
-Note `--test-threads=1`: E2E tests create k3d clusters and should not run in parallel (port conflicts, resource contention).
-
-## Deliverables
-
-- [ ] `tests/e2e/` crate added to workspace
-- [ ] Shared test utilities: `TestCluster`, `wait_for_pod_running`, `helm_install`
-- [ ] PostgreSQL E2E test passing
-- [ ] RustFS E2E test passing (after `feat/rustfs` merge)
-- [ ] CI job running E2E tests on push to main
-- [ ] `HARMONY_E2E_REUSE_CLUSTER=1` for fast local iteration

ROADMAP/06-e2e-tests-kvm.md

@@ -1,214 +0,0 @@
-# Phase 6: E2E Tests for OKD HA Cluster on KVM
-
-## Goal
-
-Prove the full OKD bare-metal installation flow works end-to-end using KVM virtual machines. This is the ultimate validation of Harmony's core value proposition: declare an OKD cluster, point it at infrastructure, watch it materialize.
-
-## Prerequisites
-
-- Phase 5 complete (test harness exists, k3d tests passing)
-- `feature/kvm-module` merged to main
-- A CI runner with libvirt/KVM access and nested virtualization support
-
-## Architecture
-
-The KVM branch already has a `kvm_okd_ha_cluster` example that creates:
-
-```
-                        Host bridge (WAN)
-                              |
-                    +--------------------+
-                    |  OPNsense          |  192.168.100.1
-                    |  gateway + PXE     |
-                    +--------+-----------+
-                             |
-                  harmonylan (192.168.100.0/24)
-                   +---------+---------+---------+---------+
-                   |         |         |         |         |
-              +----+---+ +---+---+ +---+---+ +---+---+ +--+----+
-              | cp0    | | cp1   | | cp2   | |worker0| |worker1|
-              | .10    | | .11   | | .12   | | .20   | | .21   |
-              +--------+ +-------+ +-------+ +-------+ +---+---+
-                                                            |
-                                                      +-----+----+
-                                                      | worker2  |
-                                                      | .22      |
-                                                      +----------+
-```
-
-The test needs to orchestrate this entire setup, wait for OKD to converge, and assert the cluster is healthy.
-
-## Tasks
-
-### 6.1 Start with `example_linux_vm` — the simplest KVM test
-
-Before tackling the full OKD stack, validate the KVM module itself with the simplest possible test:
-
-```rust
-// tests/e2e/tests/kvm_linux_vm.rs
-
-#[tokio::test]
-#[ignore] // Requires libvirt access — run with: cargo test -- --ignored
-async fn test_linux_vm_boots_from_iso() {
-    let executor = KvmExecutor::from_env().unwrap();
-
-    // Create isolated network
-    let network = NetworkConfig {
-        name: "e2e-test-net".to_string(),
-        bridge: "virbr200".to_string(),
-        // ...
-    };
-    executor.ensure_network(&network).await.unwrap();
-
-    // Define and start VM
-    let vm_config = VmConfig::builder("e2e-linux-test")
-        .vcpus(1)
-        .memory_gb(1)
-        .disk(5)
-        .network(NetworkRef::named("e2e-test-net"))
-        .cdrom("https://releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso")
-        .boot_order([BootDevice::Cdrom, BootDevice::Disk])
-        .build();
-
-    executor.ensure_vm(&vm_config).await.unwrap();
-    executor.start_vm("e2e-linux-test").await.unwrap();
-
-    // Assert VM is running
-    let status = executor.vm_status("e2e-linux-test").await.unwrap();
-    assert_eq!(status, VmStatus::Running);
-
-    // Cleanup
-    executor.destroy_vm("e2e-linux-test").await.unwrap();
-    executor.undefine_vm("e2e-linux-test").await.unwrap();
-    executor.delete_network("e2e-test-net").await.unwrap();
-}
-```
-
-This test validates:
-- ISO download works (via `harmony_assets` if refactored, or built-in KVM module download)
-- libvirt XML generation is correct
-- VM lifecycle (define → start → status → destroy → undefine)
-- Network creation/deletion
-
-### 6.2 OKD HA Cluster E2E test
-
-The full integration test. This is long-running (30-60 minutes) and should only run nightly or on-demand.
-
-```rust
-// tests/e2e/tests/kvm_okd_ha.rs
-
-#[tokio::test]
-#[ignore] // Requires KVM + significant resources. Run nightly.
-async fn test_okd_ha_cluster_on_kvm() {
-    // 1. Create virtual infrastructure
-    //    - OPNsense gateway VM
-    //    - 3 control plane VMs
-    //    - 3 worker VMs
-    //    - Virtual network (harmonylan)
-
-    // 2. Run OKD installation scores
-    //    (the kvm_okd_ha_cluster example, but as a test)
-
-    // 3. Wait for OKD API server to become reachable
-    //    - Poll https://api.okd.harmonylan:6443 until it responds
-    //    - Timeout: 30 minutes
-
-    // 4. Assert cluster health
-    //    - All nodes in Ready state
-    //    - ClusterVersion reports Available=True
-    //    - Sample workload (nginx) deploys and pod reaches Running
-
-    // 5. Cleanup
-    //    - Destroy all VMs
-    //    - Delete virtual networks
-    //    - Clean up disk images
-}
-```
-
-### 6.3 CI runner requirements
-
-The KVM E2E test needs a runner with:
-
-- **Hardware**: 32GB+ RAM, 8+ CPU cores, 100GB+ disk
-- **Software**: libvirt, QEMU/KVM, `virsh`, nested virtualization enabled
-- **Network**: Outbound internet access (to download ISOs, OKD images)
-- **Permissions**: User in `libvirt` group, or root access
-
-Options:
-- **Dedicated bare-metal machine** registered as a self-hosted GitHub Actions runner
-- **Cloud VM with nested virt** (e.g., GCP n2-standard-8 with `--enable-nested-virtualization`)
-- **Manual trigger only** — developer runs locally, CI just tracks pass/fail
-
-### 6.4 Nightly CI job
-
-```yaml
-# .github/workflows/e2e-kvm.yml
-name: E2E KVM Tests
-on:
-  schedule:
-    - cron: '0 2 * * *'  # 2 AM daily
-  workflow_dispatch:       # Manual trigger
-
-jobs:
-  kvm-tests:
-    runs-on: [self-hosted, kvm]  # Label for KVM-capable runners
-    timeout-minutes: 90
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run KVM E2E tests
-        run: cargo test -p harmony-e2e-tests -- --ignored --test-threads=1
-        env:
-          RUST_LOG: info
-          HARMONY_KVM_URI: qemu:///system
-
-      - name: Cleanup VMs on failure
-        if: failure()
-        run: |
-          virsh list --all --name | grep e2e | xargs -I {} virsh destroy {} || true
-          virsh list --all --name | grep e2e | xargs -I {} virsh undefine {} --remove-all-storage || true
-```
-
-### 6.5 Test resource management
-
-KVM tests create real resources that must be cleaned up even on failure. Implement a test fixture pattern:
-
-```rust
-struct KvmTestFixture {
-    executor: KvmExecutor,
-    vms: Vec<String>,
-    networks: Vec<String>,
-}
-
-impl KvmTestFixture {
-    fn track_vm(&mut self, name: &str) { self.vms.push(name.to_string()); }
-    fn track_network(&mut self, name: &str) { self.networks.push(name.to_string()); }
-}
-
-impl Drop for KvmTestFixture {
-    fn drop(&mut self) {
-        // Best-effort cleanup of all tracked resources
-        for vm in &self.vms {
-            let _ = std::process::Command::new("virsh")
-                .args(["destroy", vm]).output();
-            let _ = std::process::Command::new("virsh")
-                .args(["undefine", vm, "--remove-all-storage"]).output();
-        }
-        for net in &self.networks {
-            let _ = std::process::Command::new("virsh")
-                .args(["net-destroy", net]).output();
-            let _ = std::process::Command::new("virsh")
-                .args(["net-undefine", net]).output();
-        }
-    }
-}
-```
-
-## Deliverables
-
-- [ ] `test_linux_vm_boots_from_iso` — passing KVM smoke test
-- [ ] `test_okd_ha_cluster_on_kvm` — full OKD installation test
-- [ ] `KvmTestFixture` with resource cleanup on test failure
-- [ ] Nightly CI job on KVM-capable runner
-- [ ] Force-cleanup script for leaked VMs/networks
-- [ ] Documentation: how to set up a KVM runner for E2E tests

adr/006-secret-management.md

@@ -2,7 +2,7 @@
 
 ## Status
 
-Rejected : See ADR 020 ./020-interactive-configuration-crate.md
+Proposed
 
 ### TODO [#3](https://git.nationtech.io/NationTech/harmony/issues/3):
 

book.toml

@@ -1,9 +0,0 @@
-[book]
-title = "Harmony"
-description = "Infrastructure orchestration that treats your platform like first-class code"
-src = "docs"
-build-dir = "book"
-authors = ["NationTech"]
-
-[output.html]
-mathjax-support = false

brocade/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "brocade"
-edition = "2024"
-version.workspace = true
-readme.workspace = true
-license.workspace = true
-
-[dependencies]
-async-trait.workspace = true
-harmony_types = { path = "../harmony_types" }
-russh.workspace = true
-russh-keys.workspace = true
-tokio.workspace = true
-log.workspace = true
-env_logger.workspace = true
-regex = "1.11.3"
-harmony_secret = { path = "../harmony_secret" }
-serde.workspace = true
-schemars = "0.8"

brocade/examples/main.rs

@@ -1,75 +0,0 @@
-use std::net::{IpAddr, Ipv4Addr};
-
-use brocade::{BrocadeOptions, ssh};
-use harmony_secret::{Secret, SecretManager};
-use harmony_types::switch::PortLocation;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-
-#[derive(Secret, Clone, Debug, JsonSchema, Serialize, Deserialize)]
-struct BrocadeSwitchAuth {
-    username: String,
-    password: String,
-}
-
-#[tokio::main]
-async fn main() {
-    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
-
-    // let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 250)); // old brocade @ ianlet
-    let ip = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); // brocade @ sto1
-    // let ip = IpAddr::V4(Ipv4Addr::new(192, 168, 4, 11)); // brocade @ st
-    let switch_addresses = vec![ip];
-
-    let config = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .unwrap();
-
-    let brocade = brocade::init(
-        &switch_addresses,
-        &config.username,
-        &config.password,
-        &BrocadeOptions {
-            dry_run: true,
-            ssh: ssh::SshOptions {
-                port: 2222,
-                ..Default::default()
-            },
-            ..Default::default()
-        },
-    )
-    .await
-    .expect("Brocade client failed to connect");
-
-    let entries = brocade.get_stack_topology().await.unwrap();
-    println!("Stack topology: {entries:#?}");
-
-    let entries = brocade.get_interfaces().await.unwrap();
-    println!("Interfaces: {entries:#?}");
-
-    let version = brocade.version().await.unwrap();
-    println!("Version: {version:?}");
-
-    println!("--------------");
-    let mac_adddresses = brocade.get_mac_address_table().await.unwrap();
-    println!("VLAN\tMAC\t\t\tPORT");
-    for mac in mac_adddresses {
-        println!("{}\t{}\t{}", mac.vlan, mac.mac_address, mac.port);
-    }
-
-    println!("--------------");
-    todo!();
-    let channel_name = "1";
-    brocade.clear_port_channel(channel_name).await.unwrap();
-
-    println!("--------------");
-    let channel_id = brocade.find_available_channel_id().await.unwrap();
-
-    println!("--------------");
-    let channel_name = "HARMONY_LAG";
-    let ports = [PortLocation(2, 0, 35)];
-    brocade
-        .create_port_channel(channel_id, channel_name, &ports)
-        .await
-        .unwrap();
-}

brocade/src/fast_iron.rs

@@ -1,228 +0,0 @@
-use super::BrocadeClient;
-use crate::{
-    BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, MacAddressEntry,
-    PortChannelId, PortOperatingMode, parse_brocade_mac_address, shell::BrocadeShell,
-};
-
-use async_trait::async_trait;
-use harmony_types::switch::{PortDeclaration, PortLocation};
-use log::{debug, info};
-use regex::Regex;
-use std::{collections::HashSet, str::FromStr};
-
-#[derive(Debug)]
-pub struct FastIronClient {
-    shell: BrocadeShell,
-    version: BrocadeInfo,
-}
-
-impl FastIronClient {
-    pub fn init(mut shell: BrocadeShell, version_info: BrocadeInfo) -> Self {
-        shell.before_all(vec!["skip-page-display".into()]);
-        shell.after_all(vec!["page".into()]);
-
-        Self {
-            shell,
-            version: version_info,
-        }
-    }
-
-    fn parse_mac_entry(&self, line: &str) -> Option<Result<MacAddressEntry, Error>> {
-        debug!("[Brocade] Parsing mac address entry: {line}");
-        let parts: Vec<&str> = line.split_whitespace().collect();
-        if parts.len() < 3 {
-            return None;
-        }
-
-        let (vlan, mac_address, port) = match parts.len() {
-            3 => (
-                u16::from_str(parts[0]).ok()?,
-                parse_brocade_mac_address(parts[1]).ok()?,
-                parts[2].to_string(),
-            ),
-            _ => (
-                1,
-                parse_brocade_mac_address(parts[0]).ok()?,
-                parts[1].to_string(),
-            ),
-        };
-
-        let port =
-            PortDeclaration::parse(&port).map_err(|e| Error::UnexpectedError(format!("{e}")));
-
-        match port {
-            Ok(p) => Some(Ok(MacAddressEntry {
-                vlan,
-                mac_address,
-                port: p,
-            })),
-            Err(e) => Some(Err(e)),
-        }
-    }
-
-    fn parse_stack_port_entry(&self, line: &str) -> Option<Result<InterSwitchLink, Error>> {
-        debug!("[Brocade] Parsing stack port entry: {line}");
-        let parts: Vec<&str> = line.split_whitespace().collect();
-        if parts.len() < 10 {
-            return None;
-        }
-
-        let local_port = PortLocation::from_str(parts[0]).ok()?;
-
-        Some(Ok(InterSwitchLink {
-            local_port,
-            remote_port: None,
-        }))
-    }
-
-    fn build_port_channel_commands(
-        &self,
-        channel_id: PortChannelId,
-        channel_name: &str,
-        ports: &[PortLocation],
-    ) -> Vec<String> {
-        let mut commands = vec![
-            "configure terminal".to_string(),
-            format!("lag {channel_name} static id {channel_id}"),
-        ];
-
-        for port in ports {
-            commands.push(format!("ports ethernet {port}"));
-        }
-
-        commands.push(format!("primary-port {}", ports[0]));
-        commands.push("deploy".into());
-        commands.push("exit".into());
-        commands.push("write memory".into());
-        commands.push("exit".into());
-
-        commands
-    }
-}
-
-#[async_trait]
-impl BrocadeClient for FastIronClient {
-    async fn version(&self) -> Result<BrocadeInfo, Error> {
-        Ok(self.version.clone())
-    }
-
-    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error> {
-        info!("[Brocade] Showing MAC address table...");
-
-        let output = self
-            .shell
-            .run_command("show mac-address", ExecutionMode::Regular)
-            .await?;
-
-        output
-            .lines()
-            .skip(2)
-            .filter_map(|line| self.parse_mac_entry(line))
-            .collect()
-    }
-
-    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error> {
-        let output = self
-            .shell
-            .run_command("show interface stack-ports", crate::ExecutionMode::Regular)
-            .await?;
-
-        output
-            .lines()
-            .skip(1)
-            .filter_map(|line| self.parse_stack_port_entry(line))
-            .collect()
-    }
-
-    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error> {
-        todo!()
-    }
-
-    async fn configure_interfaces(
-        &self,
-        _interfaces: &Vec<(String, PortOperatingMode)>,
-    ) -> Result<(), Error> {
-        todo!()
-    }
-
-    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error> {
-        info!("[Brocade] Finding next available channel id...");
-
-        let output = self
-            .shell
-            .run_command("show lag", ExecutionMode::Regular)
-            .await?;
-        let re = Regex::new(r"=== LAG .* ID\s+(\d+)").expect("Invalid regex");
-
-        let used_ids: HashSet<u8> = output
-            .lines()
-            .filter_map(|line| {
-                re.captures(line)
-                    .and_then(|c| c.get(1))
-                    .and_then(|id_match| id_match.as_str().parse().ok())
-            })
-            .collect();
-
-        let mut next_id: u8 = 1;
-        loop {
-            if !used_ids.contains(&next_id) {
-                break;
-            }
-            next_id += 1;
-        }
-
-        info!("[Brocade] Found channel id: {next_id}");
-        Ok(next_id)
-    }
-
-    async fn create_port_channel(
-        &self,
-        channel_id: PortChannelId,
-        channel_name: &str,
-        ports: &[PortLocation],
-    ) -> Result<(), Error> {
-        info!(
-            "[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}"
-        );
-
-        let commands = self.build_port_channel_commands(channel_id, channel_name, ports);
-        self.shell
-            .run_commands(commands, ExecutionMode::Privileged)
-            .await?;
-
-        info!("[Brocade] Port-channel '{channel_name}' configured.");
-        Ok(())
-    }
-
-    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error> {
-        info!("[Brocade] Clearing port-channel: {channel_name}");
-
-        let commands = vec![
-            "configure terminal".to_string(),
-            format!("no lag {channel_name}"),
-            "write memory".to_string(),
-        ];
-        self.shell
-            .run_commands(commands, ExecutionMode::Privileged)
-            .await?;
-
-        info!("[Brocade] Port-channel '{channel_name}' cleared.");
-        Ok(())
-    }
-
-    async fn enable_snmp(&self, user_name: &str, auth: &str, des: &str) -> Result<(), Error> {
-        let commands = vec![
-            "configure terminal".into(),
-            "snmp-server view ALL 1 included".into(),
-            "snmp-server group public v3 priv read ALL".into(),
-            format!(
-                "snmp-server user {user_name} groupname public auth md5 auth-password {auth} priv des priv-password {des}"
-            ),
-            "exit".into(),
-        ];
-        self.shell
-            .run_commands(commands, ExecutionMode::Regular)
-            .await?;
-        Ok(())
-    }
-}

brocade/src/lib.rs

@@ -1,352 +0,0 @@
-use std::net::IpAddr;
-use std::{
-    fmt::{self, Display},
-    time::Duration,
-};
-
-use crate::network_operating_system::NetworkOperatingSystemClient;
-use crate::{
-    fast_iron::FastIronClient,
-    shell::{BrocadeSession, BrocadeShell},
-};
-
-use async_trait::async_trait;
-use harmony_types::net::MacAddress;
-use harmony_types::switch::{PortDeclaration, PortLocation};
-use regex::Regex;
-use serde::Serialize;
-
-mod fast_iron;
-mod network_operating_system;
-mod shell;
-pub mod ssh;
-
-#[derive(Default, Clone, Debug)]
-pub struct BrocadeOptions {
-    pub dry_run: bool,
-    pub ssh: ssh::SshOptions,
-    pub timeouts: TimeoutConfig,
-}
-
-#[derive(Clone, Debug)]
-pub struct TimeoutConfig {
-    pub shell_ready: Duration,
-    pub command_execution: Duration,
-    pub command_output: Duration,
-    pub cleanup: Duration,
-    pub message_wait: Duration,
-}
-
-impl Default for TimeoutConfig {
-    fn default() -> Self {
-        Self {
-            shell_ready: Duration::from_secs(10),
-            command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while
-            command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output"
-            cleanup: Duration::from_secs(10),
-            message_wait: Duration::from_millis(500),
-        }
-    }
-}
-
-enum ExecutionMode {
-    Regular,
-    Privileged,
-}
-
-#[derive(Clone, Debug)]
-pub struct BrocadeInfo {
-    os: BrocadeOs,
-    _version: String,
-}
-
-#[derive(Clone, Debug)]
-pub enum BrocadeOs {
-    NetworkOperatingSystem,
-    FastIron,
-    Unknown,
-}
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
-pub struct MacAddressEntry {
-    pub vlan: u16,
-    pub mac_address: MacAddress,
-    pub port: PortDeclaration,
-}
-
-pub type PortChannelId = u8;
-
-/// Represents a single physical or logical link connecting two switches within a stack or fabric.
-///
-/// This structure provides a standardized view of the topology regardless of the
-/// underlying Brocade OS configuration (stacking vs. fabric).
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct InterSwitchLink {
-    /// The local port on the switch where the topology command was run.
-    pub local_port: PortLocation,
-    /// The port on the directly connected neighboring switch.
-    pub remote_port: Option<PortLocation>,
-}
-
-/// Represents the key running configuration status of a single switch interface.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct InterfaceInfo {
-    /// The full configuration name (e.g., "TenGigabitEthernet 1/0/1", "FortyGigabitEthernet 2/0/2").
-    pub name: String,
-    /// The physical location of the interface.
-    pub port_location: PortLocation,
-    /// The parsed type and name prefix of the interface.
-    pub interface_type: InterfaceType,
-    /// The primary configuration mode defining the interface's behavior (L2, L3, Fabric).
-    pub operating_mode: Option<PortOperatingMode>,
-    /// Indicates the current state of the interface.
-    pub status: InterfaceStatus,
-}
-
-/// Categorizes the functional type of a switch interface.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum InterfaceType {
-    /// Physical or virtual Ethernet interface (e.g., TenGigabitEthernet, FortyGigabitEthernet).
-    Ethernet(String),
-}
-
-impl fmt::Display for InterfaceType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            InterfaceType::Ethernet(name) => write!(f, "{name}"),
-        }
-    }
-}
-
-/// Defines the primary configuration mode of a switch interface, representing mutually exclusive roles.
-#[derive(Debug, PartialEq, Eq, Clone, Serialize)]
-pub enum PortOperatingMode {
-    /// The interface is explicitly configured for Brocade fabric roles (ISL or Trunk enabled).
-    Fabric,
-    /// The interface is configured for standard Layer 2 switching as Trunk port (`switchport mode trunk`).
-    Trunk,
-    /// The interface is configured for standard Layer 2 switching as Access port (`switchport` without trunk mode).
-    Access,
-}
-
-/// Defines the possible status of an interface.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum InterfaceStatus {
-    /// The interface is connected.
-    Connected,
-    /// The interface is not connected and is not expected to be.
-    NotConnected,
-    /// The interface is not connected but is expected to be (configured with `no shutdown`).
-    SfpAbsent,
-}
-
-pub async fn init(
-    ip_addresses: &[IpAddr],
-    username: &str,
-    password: &str,
-    options: &BrocadeOptions,
-) -> Result<Box<dyn BrocadeClient + Send + Sync>, Error> {
-    let shell = BrocadeShell::init(ip_addresses, username, password, options).await?;
-
-    let version_info = shell
-        .with_session(ExecutionMode::Regular, |session| {
-            Box::pin(get_brocade_info(session))
-        })
-        .await?;
-
-    Ok(match version_info.os {
-        BrocadeOs::FastIron => Box::new(FastIronClient::init(shell, version_info)),
-        BrocadeOs::NetworkOperatingSystem => {
-            Box::new(NetworkOperatingSystemClient::init(shell, version_info))
-        }
-        BrocadeOs::Unknown => todo!(),
-    })
-}
-
-#[async_trait]
-pub trait BrocadeClient: std::fmt::Debug {
-    /// Retrieves the operating system and version details from the connected Brocade switch.
-    ///
-    /// This is typically the first call made after establishing a connection to determine
-    /// the switch OS family (e.g., FastIron, NOS) for feature compatibility.
-    ///
-    /// # Returns
-    ///
-    /// A `BrocadeInfo` structure containing parsed OS type and version string.
-    async fn version(&self) -> Result<BrocadeInfo, Error>;
-
-    /// Retrieves the dynamically learned MAC address table from the switch.
-    ///
-    /// This is crucial for discovering where specific network endpoints (MAC addresses)
-    /// are currently located on the physical ports.
-    ///
-    /// # Returns
-    ///
-    /// A vector of `MacAddressEntry`, where each entry typically contains VLAN, MAC address,
-    /// and the associated port name/index.
-    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error>;
-
-    /// Derives the physical connections used to link multiple switches together
-    /// to form a single logical entity (stack, fabric, etc.).
-    ///
-    /// This abstracts the underlying configuration (e.g., stack ports, fabric ports)
-    /// to return a standardized view of the topology.
-    ///
-    /// # Returns
-    ///
-    /// A vector of `InterSwitchLink` structs detailing which ports are used for stacking/fabric.
-    /// If the switch is not stacked, returns an empty vector.
-    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error>;
-
-    /// Retrieves the status for all interfaces
-    ///
-    /// # Returns
-    ///
-    /// A vector of `InterfaceInfo` structures.
-    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error>;
-
-    /// Configures a set of interfaces to be operated with a specified mode (access ports, ISL, etc.).
-    async fn configure_interfaces(
-        &self,
-        interfaces: &Vec<(String, PortOperatingMode)>,
-    ) -> Result<(), Error>;
-
-    /// Scans the existing configuration to find the next available (unused)
-    /// Port-Channel ID (`lag` or `trunk`) for assignment.
-    ///
-    /// # Returns
-    ///
-    /// The smallest, unassigned `PortChannelId` within the supported range.
-    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error>;
-
-    /// Creates and configures a new Port-Channel (Link Aggregation Group or LAG)
-    /// using the specified channel ID and ports.
-    ///
-    /// The resulting configuration must be persistent (saved to startup-config).
-    /// Assumes a static LAG configuration mode unless specified otherwise by the implementation.
-    ///
-    /// # Parameters
-    ///
-    /// * `channel_id`: The ID (e.g., 1-128) for the logical port channel.
-    /// * `channel_name`: A descriptive name for the LAG (used in configuration context).
-    /// * `ports`: A slice of `PortLocation` structs defining the physical member ports.
-    async fn create_port_channel(
-        &self,
-        channel_id: PortChannelId,
-        channel_name: &str,
-        ports: &[PortLocation],
-    ) -> Result<(), Error>;
-
-    /// Enables Simple Network Management Protocol (SNMP) server for switch
-    ///
-    /// # Parameters
-    ///
-    ///  * `user_name`: The user name for the snmp server
-    ///  * `auth`: The password for authentication process for verifying the identity of a device
-    ///  * `des`: The Data Encryption Standard algorithm key
-    async fn enable_snmp(&self, user_name: &str, auth: &str, des: &str) -> Result<(), Error>;
-
-    /// Removes all configuration associated with the specified Port-Channel name.
-    ///
-    /// This operation should be idempotent; attempting to clear a non-existent
-    /// channel should succeed (or return a benign error).
-    ///
-    /// # Parameters
-    ///
-    /// * `channel_name`: The name of the Port-Channel (LAG) to delete.
-    ///
-    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error>;
-}
-
-async fn get_brocade_info(session: &mut BrocadeSession) -> Result<BrocadeInfo, Error> {
-    let output = session.run_command("show version").await?;
-
-    if output.contains("Network Operating System") {
-        let re = Regex::new(r"Network Operating System Version:\s*(?P<version>[a-zA-Z0-9.\-]+)")
-            .expect("Invalid regex");
-        let version = re
-            .captures(&output)
-            .and_then(|cap| cap.name("version"))
-            .map(|m| m.as_str().to_string())
-            .unwrap_or_default();
-
-        return Ok(BrocadeInfo {
-            os: BrocadeOs::NetworkOperatingSystem,
-            _version: version,
-        });
-    } else if output.contains("ICX") {
-        let re = Regex::new(r"(?m)^\s*SW: Version\s*(?P<version>[a-zA-Z0-9.\-]+)")
-            .expect("Invalid regex");
-        let version = re
-            .captures(&output)
-            .and_then(|cap| cap.name("version"))
-            .map(|m| m.as_str().to_string())
-            .unwrap_or_default();
-
-        return Ok(BrocadeInfo {
-            os: BrocadeOs::FastIron,
-            _version: version,
-        });
-    }
-
-    Err(Error::UnexpectedError("Unknown Brocade OS version".into()))
-}
-
-fn parse_brocade_mac_address(value: &str) -> Result<MacAddress, String> {
-    let cleaned_mac = value.replace('.', "");
-
-    if cleaned_mac.len() != 12 {
-        return Err(format!("Invalid MAC address: {value}"));
-    }
-
-    let mut bytes = [0u8; 6];
-    for (i, pair) in cleaned_mac.as_bytes().chunks(2).enumerate() {
-        let byte_str = std::str::from_utf8(pair).map_err(|_| "Invalid UTF-8")?;
-        bytes[i] =
-            u8::from_str_radix(byte_str, 16).map_err(|_| format!("Invalid hex in MAC: {value}"))?;
-    }
-
-    Ok(MacAddress(bytes))
-}
-
-#[derive(Debug)]
-pub enum SecurityLevel {
-    AuthPriv(String),
-}
-
-#[derive(Debug)]
-pub enum Error {
-    NetworkError(String),
-    AuthenticationError(String),
-    ConfigurationError(String),
-    TimeoutError(String),
-    UnexpectedError(String),
-    CommandError(String),
-}
-
-impl Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Error::NetworkError(msg) => write!(f, "Network error: {msg}"),
-            Error::AuthenticationError(msg) => write!(f, "Authentication error: {msg}"),
-            Error::ConfigurationError(msg) => write!(f, "Configuration error: {msg}"),
-            Error::TimeoutError(msg) => write!(f, "Timeout error: {msg}"),
-            Error::UnexpectedError(msg) => write!(f, "Unexpected error: {msg}"),
-            Error::CommandError(msg) => write!(f, "{msg}"),
-        }
-    }
-}
-
-impl From<Error> for String {
-    fn from(val: Error) -> Self {
-        format!("{val}")
-    }
-}
-
-impl std::error::Error for Error {}
-
-impl From<russh::Error> for Error {
-    fn from(value: russh::Error) -> Self {
-        Error::NetworkError(format!("Russh client error: {value}"))
-    }
-}

brocade/src/network_operating_system.rs

@@ -1,352 +0,0 @@
-use std::str::FromStr;
-
-use async_trait::async_trait;
-use harmony_types::switch::{PortDeclaration, PortLocation};
-use log::{debug, info};
-use regex::Regex;
-
-use crate::{
-    BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo,
-    InterfaceStatus, InterfaceType, MacAddressEntry, PortChannelId, PortOperatingMode,
-    parse_brocade_mac_address, shell::BrocadeShell,
-};
-
-#[derive(Debug)]
-pub struct NetworkOperatingSystemClient {
-    shell: BrocadeShell,
-    version: BrocadeInfo,
-}
-
-impl NetworkOperatingSystemClient {
-    pub fn init(mut shell: BrocadeShell, version_info: BrocadeInfo) -> Self {
-        shell.before_all(vec!["terminal length 0".into()]);
-
-        Self {
-            shell,
-            version: version_info,
-        }
-    }
-
-    fn parse_mac_entry(&self, line: &str) -> Option<Result<MacAddressEntry, Error>> {
-        debug!("[Brocade] Parsing mac address entry: {line}");
-        let parts: Vec<&str> = line.split_whitespace().collect();
-        if parts.len() < 5 {
-            return None;
-        }
-
-        let (vlan, mac_address, port) = match parts.len() {
-            5 => (
-                u16::from_str(parts[0]).ok()?,
-                parse_brocade_mac_address(parts[1]).ok()?,
-                parts[4].to_string(),
-            ),
-            _ => (
-                u16::from_str(parts[0]).ok()?,
-                parse_brocade_mac_address(parts[1]).ok()?,
-                parts[5].to_string(),
-            ),
-        };
-
-        let port =
-            PortDeclaration::parse(&port).map_err(|e| Error::UnexpectedError(format!("{e}")));
-
-        match port {
-            Ok(p) => Some(Ok(MacAddressEntry {
-                vlan,
-                mac_address,
-                port: p,
-            })),
-            Err(e) => Some(Err(e)),
-        }
-    }
-
-    fn parse_inter_switch_link_entry(&self, line: &str) -> Option<Result<InterSwitchLink, Error>> {
-        debug!("[Brocade] Parsing inter switch link entry: {line}");
-        let parts: Vec<&str> = line.split_whitespace().collect();
-        if parts.len() < 10 {
-            return None;
-        }
-
-        let local_port = PortLocation::from_str(parts[2]).ok()?;
-        let remote_port = PortLocation::from_str(parts[5]).ok()?;
-
-        Some(Ok(InterSwitchLink {
-            local_port,
-            remote_port: Some(remote_port),
-        }))
-    }
-
-    fn parse_interface_status_entry(&self, line: &str) -> Option<Result<InterfaceInfo, Error>> {
-        debug!("[Brocade] Parsing interface status entry: {line}");
-        let parts: Vec<&str> = line.split_whitespace().collect();
-        if parts.len() < 6 {
-            return None;
-        }
-
-        let interface_type = match parts[0] {
-            "Fo" => InterfaceType::Ethernet("FortyGigabitEthernet".to_string()),
-            "Te" => InterfaceType::Ethernet("TenGigabitEthernet".to_string()),
-            _ => return None,
-        };
-        let port_location = PortLocation::from_str(parts[1]).ok()?;
-        let status = match parts[2] {
-            "connected" => InterfaceStatus::Connected,
-            "notconnected" => InterfaceStatus::NotConnected,
-            "sfpAbsent" => InterfaceStatus::SfpAbsent,
-            _ => return None,
-        };
-        let operating_mode = match parts[3] {
-            "ISL" => Some(PortOperatingMode::Fabric),
-            "Trunk" => Some(PortOperatingMode::Trunk),
-            "Access" => Some(PortOperatingMode::Access),
-            "--" => None,
-            _ => return None,
-        };
-
-        Some(Ok(InterfaceInfo {
-            name: format!("{interface_type} {port_location}"),
-            port_location,
-            interface_type,
-            operating_mode,
-            status,
-        }))
-    }
-
-    fn map_configure_interfaces_error(&self, err: Error) -> Error {
-        debug!("[Brocade] {err}");
-
-        if let Error::CommandError(message) = &err {
-            if message.contains("switchport")
-                && message.contains("Cannot configure aggregator member")
-            {
-                let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap();
-
-                if let Some(caps) = re.captures(message) {
-                    let interface_type = &caps[1];
-                    let port_location = &caps[2];
-                    let interface = format!("{interface_type} {port_location}");
-
-                    return Error::CommandError(format!(
-                        "Cannot configure interface '{interface}', it is a member of a port-channel (LAG)"
-                    ));
-                }
-            }
-        }
-
-        err
-    }
-}
-
-#[async_trait]
-impl BrocadeClient for NetworkOperatingSystemClient {
-    async fn version(&self) -> Result<BrocadeInfo, Error> {
-        Ok(self.version.clone())
-    }
-
-    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error> {
-        let output = self
-            .shell
-            .run_command("show mac-address-table", ExecutionMode::Regular)
-            .await?;
-
-        output
-            .lines()
-            .skip(1)
-            .filter_map(|line| self.parse_mac_entry(line))
-            .collect()
-    }
-
-    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error> {
-        let output = self
-            .shell
-            .run_command("show fabric isl", ExecutionMode::Regular)
-            .await?;
-
-        output
-            .lines()
-            .skip(6)
-            .filter_map(|line| self.parse_inter_switch_link_entry(line))
-            .collect()
-    }
-
-    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error> {
-        let output = self
-            .shell
-            .run_command(
-                "show interface status rbridge-id all",
-                ExecutionMode::Regular,
-            )
-            .await?;
-
-        output
-            .lines()
-            .skip(2)
-            .filter_map(|line| self.parse_interface_status_entry(line))
-            .collect()
-    }
-
-    async fn configure_interfaces(
-        &self,
-        interfaces: &Vec<(String, PortOperatingMode)>,
-    ) -> Result<(), Error> {
-        info!("[Brocade] Configuring {} interface(s)...", interfaces.len());
-
-        let mut commands = vec!["configure terminal".to_string()];
-
-        for interface in interfaces {
-            commands.push(format!("interface {}", interface.0));
-
-            match interface.1 {
-                PortOperatingMode::Fabric => {
-                    commands.push("fabric isl enable".into());
-                    commands.push("fabric trunk enable".into());
-                }
-                PortOperatingMode::Trunk => {
-                    commands.push("switchport".into());
-                    commands.push("switchport mode trunk".into());
-                    commands.push("switchport trunk allowed vlan all".into());
-                    commands.push("no switchport trunk tag native-vlan".into());
-                    commands.push("spanning-tree shutdown".into());
-                    commands.push("no fabric isl enable".into());
-                    commands.push("no fabric trunk enable".into());
-                    commands.push("no shutdown".into());
-                }
-                PortOperatingMode::Access => {
-                    commands.push("switchport".into());
-                    commands.push("switchport mode access".into());
-                    commands.push("switchport access vlan 1".into());
-                    commands.push("no spanning-tree shutdown".into());
-                    commands.push("no fabric isl enable".into());
-                    commands.push("no fabric trunk enable".into());
-                }
-            }
-
-            commands.push("no shutdown".into());
-            commands.push("exit".into());
-        }
-
-        self.shell
-            .run_commands(commands, ExecutionMode::Regular)
-            .await
-            .map_err(|err| self.map_configure_interfaces_error(err))?;
-
-        info!("[Brocade] Interfaces configured.");
-
-        Ok(())
-    }
-
-    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error> {
-        info!("[Brocade] Finding next available channel id...");
-
-        let output = self
-            .shell
-            .run_command("show port-channel summary", ExecutionMode::Regular)
-            .await?;
-
-        let used_ids: Vec<u8> = output
-            .lines()
-            .skip(6)
-            .filter_map(|line| {
-                let parts: Vec<&str> = line.split_whitespace().collect();
-                if parts.len() < 8 {
-                    return None;
-                }
-
-                u8::from_str(parts[0]).ok()
-            })
-            .collect();
-
-        let mut next_id: u8 = 1;
-        loop {
-            if !used_ids.contains(&next_id) {
-                break;
-            }
-            next_id += 1;
-        }
-
-        info!("[Brocade] Found channel id: {next_id}");
-        Ok(next_id)
-    }
-
-    async fn create_port_channel(
-        &self,
-        channel_id: PortChannelId,
-        channel_name: &str,
-        ports: &[PortLocation],
-    ) -> Result<(), Error> {
-        info!(
-            "[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}",
-            ports
-                .iter()
-                .map(|p| format!("{p}"))
-                .collect::<Vec<String>>()
-                .join(", ")
-        );
-
-        let interfaces = self.get_interfaces().await?;
-
-        let mut commands = vec![
-            "configure terminal".into(),
-            format!("interface port-channel {}", channel_id),
-            "no shutdown".into(),
-            "exit".into(),
-        ];
-
-        for port in ports {
-            let interface = interfaces.iter().find(|i| i.port_location == *port);
-            let Some(interface) = interface else {
-                continue;
-            };
-
-            commands.push(format!("interface {}", interface.name));
-            commands.push("no switchport".into());
-            commands.push("no ip address".into());
-            commands.push("no fabric isl enable".into());
-            commands.push("no fabric trunk enable".into());
-            commands.push(format!("channel-group {channel_id} mode active"));
-            commands.push("no shutdown".into());
-            commands.push("exit".into());
-        }
-
-        self.shell
-            .run_commands(commands, ExecutionMode::Regular)
-            .await?;
-
-        info!("[Brocade] Port-channel '{channel_name}' configured.");
-
-        Ok(())
-    }
-
-    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error> {
-        info!("[Brocade] Clearing port-channel: {channel_name}");
-
-        let commands = vec![
-            "configure terminal".into(),
-            format!("no interface port-channel {}", channel_name),
-            "exit".into(),
-        ];
-
-        self.shell
-            .run_commands(commands, ExecutionMode::Regular)
-            .await?;
-
-        info!("[Brocade] Port-channel '{channel_name}' cleared.");
-        Ok(())
-    }
-
-    async fn enable_snmp(&self, user_name: &str, auth: &str, des: &str) -> Result<(), Error> {
-        let commands = vec![
-            "configure terminal".into(),
-            "snmp-server view ALL 1 included".into(),
-            "snmp-server group public v3 priv read ALL".into(),
-            format!(
-                "snmp-server user {user_name} groupname public auth md5 auth-password {auth} priv des priv-password {des}"
-            ),
-            "exit".into(),
-        ];
-        self.shell
-            .run_commands(commands, ExecutionMode::Regular)
-            .await?;
-        Ok(())
-    }
-}

brocade/src/shell.rs

@@ -1,367 +0,0 @@
-use std::net::IpAddr;
-use std::time::Duration;
-use std::time::Instant;
-
-use crate::BrocadeOptions;
-use crate::Error;
-use crate::ExecutionMode;
-use crate::TimeoutConfig;
-use crate::ssh;
-
-use log::debug;
-use log::info;
-use russh::ChannelMsg;
-use tokio::time::timeout;
-
-#[derive(Debug)]
-pub struct BrocadeShell {
-    ip: IpAddr,
-    username: String,
-    password: String,
-    options: BrocadeOptions,
-    before_all_commands: Vec<String>,
-    after_all_commands: Vec<String>,
-}
-
-impl BrocadeShell {
-    pub async fn init(
-        ip_addresses: &[IpAddr],
-        username: &str,
-        password: &str,
-        options: &BrocadeOptions,
-    ) -> Result<Self, Error> {
-        let ip = ip_addresses
-            .first()
-            .ok_or_else(|| Error::ConfigurationError("No IP addresses provided".to_string()))?;
-
-        let brocade_ssh_client_options =
-            ssh::try_init_client(username, password, ip, options).await?;
-
-        Ok(Self {
-            ip: *ip,
-            username: username.to_string(),
-            password: password.to_string(),
-            before_all_commands: vec![],
-            after_all_commands: vec![],
-            options: brocade_ssh_client_options,
-        })
-    }
-
-    pub async fn open_session(&self, mode: ExecutionMode) -> Result<BrocadeSession, Error> {
-        BrocadeSession::open(
-            self.ip,
-            self.options.ssh.port,
-            &self.username,
-            &self.password,
-            self.options.clone(),
-            mode,
-        )
-        .await
-    }
-
-    pub async fn with_session<F, R>(&self, mode: ExecutionMode, callback: F) -> Result<R, Error>
-    where
-        F: FnOnce(
-            &mut BrocadeSession,
-        ) -> std::pin::Pin<
-            Box<dyn std::future::Future<Output = Result<R, Error>> + Send + '_>,
-        >,
-    {
-        let mut session = self.open_session(mode).await?;
-
-        let _ = session.run_commands(self.before_all_commands.clone()).await;
-        let result = callback(&mut session).await;
-        let _ = session.run_commands(self.after_all_commands.clone()).await;
-
-        session.close().await?;
-        result
-    }
-
-    pub async fn run_command(&self, command: &str, mode: ExecutionMode) -> Result<String, Error> {
-        let mut session = self.open_session(mode).await?;
-
-        let _ = session.run_commands(self.before_all_commands.clone()).await;
-        let result = session.run_command(command).await;
-        let _ = session.run_commands(self.after_all_commands.clone()).await;
-
-        session.close().await?;
-        result
-    }
-
-    pub async fn run_commands(
-        &self,
-        commands: Vec<String>,
-        mode: ExecutionMode,
-    ) -> Result<(), Error> {
-        let mut session = self.open_session(mode).await?;
-
-        let _ = session.run_commands(self.before_all_commands.clone()).await;
-        let result = session.run_commands(commands).await;
-        let _ = session.run_commands(self.after_all_commands.clone()).await;
-
-        session.close().await?;
-        result
-    }
-
-    pub fn before_all(&mut self, commands: Vec<String>) {
-        self.before_all_commands = commands;
-    }
-
-    pub fn after_all(&mut self, commands: Vec<String>) {
-        self.after_all_commands = commands;
-    }
-}
-
-pub struct BrocadeSession {
-    pub channel: russh::Channel<russh::client::Msg>,
-    pub mode: ExecutionMode,
-    pub options: BrocadeOptions,
-}
-
-impl BrocadeSession {
-    pub async fn open(
-        ip: IpAddr,
-        port: u16,
-        username: &str,
-        password: &str,
-        options: BrocadeOptions,
-        mode: ExecutionMode,
-    ) -> Result<Self, Error> {
-        let client = ssh::create_client(ip, port, username, password, &options).await?;
-        let mut channel = client.channel_open_session().await?;
-
-        channel
-            .request_pty(false, "vt100", 80, 24, 0, 0, &[])
-            .await?;
-        channel.request_shell(false).await?;
-
-        wait_for_shell_ready(&mut channel, &options.timeouts).await?;
-
-        if let ExecutionMode::Privileged = mode {
-            try_elevate_session(&mut channel, username, password, &options.timeouts).await?;
-        }
-
-        Ok(Self {
-            channel,
-            mode,
-            options,
-        })
-    }
-
-    pub async fn close(&mut self) -> Result<(), Error> {
-        debug!("[Brocade] Closing session...");
-
-        self.channel.data(&b"exit\n"[..]).await?;
-        if let ExecutionMode::Privileged = self.mode {
-            self.channel.data(&b"exit\n"[..]).await?;
-        }
-
-        let start = Instant::now();
-        while start.elapsed() < self.options.timeouts.cleanup {
-            match timeout(self.options.timeouts.message_wait, self.channel.wait()).await {
-                Ok(Some(ChannelMsg::Close)) => break,
-                Ok(Some(_)) => continue,
-                Ok(None) | Err(_) => break,
-            }
-        }
-
-        debug!("[Brocade] Session closed.");
-        Ok(())
-    }
-
-    pub async fn run_command(&mut self, command: &str) -> Result<String, Error> {
-        if self.should_skip_command(command) {
-            return Ok(String::new());
-        }
-
-        debug!("[Brocade] Running command: '{command}'...");
-
-        self.channel
-            .data(format!("{}\n", command).as_bytes())
-            .await?;
-        tokio::time::sleep(Duration::from_millis(100)).await;
-
-        let output = self.collect_command_output().await?;
-        let output = String::from_utf8(output)
-            .map_err(|_| Error::UnexpectedError("Invalid UTF-8 in command output".to_string()))?;
-
-        self.check_for_command_errors(&output, command)?;
-        Ok(output)
-    }
-
-    pub async fn run_commands(&mut self, commands: Vec<String>) -> Result<(), Error> {
-        for command in commands {
-            self.run_command(&command).await?;
-        }
-        Ok(())
-    }
-
-    fn should_skip_command(&self, command: &str) -> bool {
-        if (command.starts_with("write") || command.starts_with("deploy")) && self.options.dry_run {
-            info!("[Brocade] Dry-run mode enabled, skipping command: {command}");
-            return true;
-        }
-        false
-    }
-
-    async fn collect_command_output(&mut self) -> Result<Vec<u8>, Error> {
-        let mut output = Vec::new();
-        let start = Instant::now();
-        let read_timeout = Duration::from_millis(500);
-        let log_interval = Duration::from_secs(5);
-        let mut last_log = Instant::now();
-
-        loop {
-            if start.elapsed() > self.options.timeouts.command_execution {
-                return Err(Error::TimeoutError(
-                    "Timeout waiting for command completion.".into(),
-                ));
-            }
-
-            if start.elapsed() > self.options.timeouts.command_output
-                && last_log.elapsed() > log_interval
-            {
-                info!("[Brocade] Waiting for command output...");
-                last_log = Instant::now();
-            }
-
-            match timeout(read_timeout, self.channel.wait()).await {
-                Ok(Some(ChannelMsg::Data { data } | ChannelMsg::ExtendedData { data, .. })) => {
-                    output.extend_from_slice(&data);
-                    let current_output = String::from_utf8_lossy(&output);
-                    if current_output.contains('>') || current_output.contains('#') {
-                        return Ok(output);
-                    }
-                }
-                Ok(Some(ChannelMsg::Eof | ChannelMsg::Close)) => return Ok(output),
-                Ok(Some(ChannelMsg::ExitStatus { exit_status })) => {
-                    debug!("[Brocade] Command exit status: {exit_status}");
-                }
-                Ok(Some(_)) => continue,
-                Ok(None) | Err(_) => {
-                    if output.is_empty() {
-                        if let Ok(None) = timeout(read_timeout, self.channel.wait()).await {
-                            break;
-                        }
-                        continue;
-                    }
-
-                    tokio::time::sleep(Duration::from_millis(100)).await;
-                    let current_output = String::from_utf8_lossy(&output);
-                    if current_output.contains('>') || current_output.contains('#') {
-                        return Ok(output);
-                    }
-                }
-            }
-        }
-
-        Ok(output)
-    }
-
-    fn check_for_command_errors(&self, output: &str, command: &str) -> Result<(), Error> {
-        const ERROR_PATTERNS: &[&str] = &[
-            "invalid input",
-            "syntax error",
-            "command not found",
-            "unknown command",
-            "permission denied",
-            "access denied",
-            "authentication failed",
-            "configuration error",
-            "failed to",
-            "error:",
-        ];
-
-        let output_lower = output.to_lowercase();
-        if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) {
-            return Err(Error::CommandError(format!(
-                "Command error: {}",
-                output.trim()
-            )));
-        }
-
-        if !command.starts_with("show") && output.trim().is_empty() {
-            return Err(Error::CommandError(format!(
-                "Command '{command}' produced no output"
-            )));
-        }
-
-        Ok(())
-    }
-}
-
-async fn wait_for_shell_ready(
-    channel: &mut russh::Channel<russh::client::Msg>,
-    timeouts: &TimeoutConfig,
-) -> Result<(), Error> {
-    let mut buffer = Vec::new();
-    let start = Instant::now();
-
-    while start.elapsed() < timeouts.shell_ready {
-        match timeout(timeouts.message_wait, channel.wait()).await {
-            Ok(Some(ChannelMsg::Data { data })) => {
-                buffer.extend_from_slice(&data);
-                let output = String::from_utf8_lossy(&buffer);
-                let output = output.trim();
-                if output.ends_with('>') || output.ends_with('#') {
-                    debug!("[Brocade] Shell ready");
-                    return Ok(());
-                }
-            }
-            Ok(Some(_)) => continue,
-            Ok(None) => break,
-            Err(_) => continue,
-        }
-    }
-    Ok(())
-}
-
-async fn try_elevate_session(
-    channel: &mut russh::Channel<russh::client::Msg>,
-    username: &str,
-    password: &str,
-    timeouts: &TimeoutConfig,
-) -> Result<(), Error> {
-    channel.data(&b"enable\n"[..]).await?;
-    let start = Instant::now();
-    let mut buffer = Vec::new();
-
-    while start.elapsed() < timeouts.shell_ready {
-        match timeout(timeouts.message_wait, channel.wait()).await {
-            Ok(Some(ChannelMsg::Data { data })) => {
-                buffer.extend_from_slice(&data);
-                let output = String::from_utf8_lossy(&buffer);
-
-                if output.ends_with('#') {
-                    debug!("[Brocade] Privileged mode established");
-                    return Ok(());
-                }
-
-                if output.contains("User Name:") {
-                    channel.data(format!("{}\n", username).as_bytes()).await?;
-                    buffer.clear();
-                } else if output.contains("Password:") {
-                    channel.data(format!("{}\n", password).as_bytes()).await?;
-                    buffer.clear();
-                } else if output.contains('>') {
-                    return Err(Error::AuthenticationError(
-                        "Enable authentication failed".into(),
-                    ));
-                }
-            }
-            Ok(Some(_)) => continue,
-            Ok(None) => break,
-            Err(_) => continue,
-        }
-    }
-
-    let output = String::from_utf8_lossy(&buffer);
-    if output.ends_with('#') {
-        debug!("[Brocade] Privileged mode established");
-        Ok(())
-    } else {
-        Err(Error::AuthenticationError(format!(
-            "Enable failed. Output:\n{output}"
-        )))
-    }
-}

brocade/src/ssh.rs

@@ -1,131 +0,0 @@
-use std::borrow::Cow;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use log::debug;
-use russh::client::Handler;
-use russh::kex::DH_G1_SHA1;
-use russh::kex::ECDH_SHA2_NISTP256;
-use russh_keys::key::SSH_RSA;
-
-use super::BrocadeOptions;
-use super::Error;
-
-#[derive(Clone, Debug)]
-pub struct SshOptions {
-    pub preferred_algorithms: russh::Preferred,
-    pub port: u16,
-}
-
-impl Default for SshOptions {
-    fn default() -> Self {
-        Self {
-            preferred_algorithms: Default::default(),
-            port: 22,
-        }
-    }
-}
-
-impl SshOptions {
-    fn ecdhsa_sha2_nistp256(port: u16) -> Self {
-        Self {
-            preferred_algorithms: russh::Preferred {
-                kex: Cow::Borrowed(&[ECDH_SHA2_NISTP256]),
-                key: Cow::Borrowed(&[SSH_RSA]),
-                ..Default::default()
-            },
-            port,
-            ..Default::default()
-        }
-    }
-
-    fn legacy(port: u16) -> Self {
-        Self {
-            preferred_algorithms: russh::Preferred {
-                kex: Cow::Borrowed(&[DH_G1_SHA1]),
-                key: Cow::Borrowed(&[SSH_RSA]),
-                ..Default::default()
-            },
-            port,
-            ..Default::default()
-        }
-    }
-}
-
-pub struct Client;
-
-#[async_trait]
-impl Handler for Client {
-    type Error = Error;
-
-    async fn check_server_key(
-        &mut self,
-        _server_public_key: &russh_keys::key::PublicKey,
-    ) -> Result<bool, Self::Error> {
-        Ok(true)
-    }
-}
-
-pub async fn try_init_client(
-    username: &str,
-    password: &str,
-    ip: &std::net::IpAddr,
-    base_options: &BrocadeOptions,
-) -> Result<BrocadeOptions, Error> {
-    let mut default = SshOptions::default();
-    default.port = base_options.ssh.port;
-    let ssh_options = vec![
-        default,
-        SshOptions::ecdhsa_sha2_nistp256(base_options.ssh.port),
-        SshOptions::legacy(base_options.ssh.port),
-    ];
-
-    for ssh in ssh_options {
-        let opts = BrocadeOptions {
-            ssh: ssh.clone(),
-            ..base_options.clone()
-        };
-        debug!("Creating client {ip}:{} {username}", ssh.port);
-        let client = create_client(*ip, ssh.port, username, password, &opts).await;
-
-        match client {
-            Ok(_) => {
-                return Ok(opts);
-            }
-            Err(e) => match e {
-                Error::NetworkError(e) => {
-                    if e.contains("No common key exchange algorithm") {
-                        continue;
-                    } else {
-                        return Err(Error::NetworkError(e));
-                    }
-                }
-                _ => return Err(e),
-            },
-        }
-    }
-
-    Err(Error::NetworkError(
-        "Could not establish ssh connection: wrong key exchange algorithm)".to_string(),
-    ))
-}
-
-pub async fn create_client(
-    ip: std::net::IpAddr,
-    port: u16,
-    username: &str,
-    password: &str,
-    options: &BrocadeOptions,
-) -> Result<russh::client::Handle<Client>, Error> {
-    let config = russh::client::Config {
-        preferred: options.ssh.preferred_algorithms.clone(),
-        ..Default::default()
-    };
-    let mut client = russh::client::connect(Arc::new(config), (ip, port), Client {}).await?;
-    if !client.authenticate_password(username, password).await? {
-        return Err(Error::AuthenticationError(
-            "ssh authentication failed".to_string(),
-        ));
-    }
-    Ok(client)
-}

build/book.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-set -e
-
-cd "$(dirname "$0")/.."
-
-cargo install mdbook --locked
-mdbook build
-
-test -f book/index.html || (echo "ERROR: book/index.html not found" && exit 1)
-test -f book/concepts.html || (echo "ERROR: book/concepts.html not found" && exit 1)
-test -f book/guides/getting-started.html || (echo "ERROR: book/guides/getting-started.html not found" && exit 1)

build/ci.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-set -e
-
-cd "$(dirname "$0")/.."
-
-BRANCH="${1:-main}"
-
-echo "=== Running CI for branch: $BRANCH ==="
-
-echo "--- Checking code ---"
-./build/check.sh
-
-echo "--- Building book ---"
-./build/book.sh
-
-echo "=== CI passed ==="

check.sh

@@ -1,8 +1,6 @@
 #!/bin/sh
 set -e
 
-cd "$(dirname "$0")/.."
-
 rustc --version
 cargo check --all-targets --all-features --keep-going
 cargo fmt --check

docs/README.md

@@ -1,37 +1 @@
-# Harmony Documentation Hub
+Not much here yet, see the `adr` folder for now. More to come in time!
-
-Welcome to the Harmony documentation. This is the main entry point for learning everything from core concepts to building your own Score, Topologies, and Capabilities.
-
-## 1. Getting Started
-
-If you're new to Harmony, start here:
-
-- [**Getting Started Guide**](./guides/getting-started.md): A step-by-step tutorial that takes you from an empty project to deploying your first application.
-- [**Core Concepts**](./concepts.md): A high-level overview of the key concepts in Harmony: `Score`, `Topology`, `Capability`, `Inventory`, `Interpret`, ...
-
-## 2. Use Cases & Examples
-
-See how to use Harmony to solve real-world problems.
-
-- [**PostgreSQL on Local K3D**](./use-cases/postgresql-on-local-k3d.md): Deploy a production-grade PostgreSQL cluster on a local K3D cluster. The fastest way to get started.
-- [**OKD on Bare Metal**](./use-cases/okd-on-bare-metal.md): A detailed walkthrough of bootstrapping a high-availability OKD cluster from physical hardware.
-
-## 3. Component Catalogs
-
-Discover existing, reusable components you can use in your Harmony projects.
-
-- [**Scores Catalog**](./catalogs/scores.md): A categorized list of all available `Scores` (the "what").
-- [**Topologies Catalog**](./catalogs/topologies.md): A list of all available `Topologies` (the "where").
-- [**Capabilities Catalog**](./catalogs/capabilities.md): A list of all available `Capabilities` (the "how").
-
-## 4. Developer Guides
-
-Ready to build your own components? These guides show you how.
-
-- [**Writing a Score**](./guides/writing-a-score.md): Learn how to create your own `Score` and `Interpret` logic to define a new desired state.
-- [**Writing a Topology**](./guides/writing-a-topology.md): Learn how to model a new environment (like AWS, GCP, or custom hardware) as a `Topology`.
-- [**Adding Capabilities**](./guides/adding-capabilities.md): See how to add a `Capability` to your custom `Topology`.
-
-## 5. Architecture Decision Records
-
-Harmony's design is documented through Architecture Decision Records (ADRs). See the [ADR Overview](./adr/README.md) for a complete index of all decisions.

docs/SUMMARY.md

@@ -1,53 +0,0 @@
-# Summary
-
-[Harmony Documentation](./README.md)
-
-- [Core Concepts](./concepts.md)
-- [Getting Started Guide](./guides/getting-started.md)
-
-## Use Cases
-
-- [PostgreSQL on Local K3D](./use-cases/postgresql-on-local-k3d.md)
-- [OKD on Bare Metal](./use-cases/okd-on-bare-metal.md)
-
-## Component Catalogs
-
-- [Scores Catalog](./catalogs/scores.md)
-- [Topologies Catalog](./catalogs/topologies.md)
-- [Capabilities Catalog](./catalogs/capabilities.md)
-
-## Developer Guides
-
-- [Developer Guide](./guides/developer-guide.md)
-- [Writing a Score](./guides/writing-a-score.md)
-- [Writing a Topology](./guides/writing-a-topology.md)
-- [Adding Capabilities](./guides/adding-capabilities.md)
-
-## Configuration
-
-- [Configuration](./concepts/configuration.md)
-
-## Architecture Decision Records
-
-- [ADR Overview](./adr/README.md)
-   - [000 · ADR Template](./adr/000-ADR-Template.md)
-   - [001 · Why Rust](./adr/001-rust.md)
-   - [002 · Hexagonal Architecture](./adr/002-hexagonal-architecture.md)
-   - [003 · Infrastructure Abstractions](./adr/003-infrastructure-abstractions.md)
-   - [004 · iPXE](./adr/004-ipxe.md)
-   - [005 · Interactive Project](./adr/005-interactive-project.md)
-   - [006 · Secret Management](./adr/006-secret-management.md)
-   - [007 · Default Runtime](./adr/007-default-runtime.md)
-   - [008 · Score Display Formatting](./adr/008-score-display-formatting.md)
-   - [009 · Helm and Kustomize Handling](./adr/009-helm-and-kustomize-handling.md)
-   - [010 · Monitoring and Alerting](./adr/010-monitoring-and-alerting.md)
-   - [011 · Multi-Tenant Cluster](./adr/011-multi-tenant-cluster.md)
-   - [012 · Project Delivery Automation](./adr/012-project-delivery-automation.md)
-   - [013 · Monitoring Notifications](./adr/013-monitoring-notifications.md)
-   - [015 · Higher Order Topologies](./adr/015-higher-order-topologies.md)
-   - [016 · Harmony Agent and Global Mesh](./adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md)
-   - [017-1 · NATS Clusters Interconnection](./adr/017-1-Nats-Clusters-Interconnection-Topology.md)
-   - [018 · Template Hydration for Workload Deployment](./adr/018-Template-Hydration-For-Workload-Deployment.md)
-   - [019 · Network Bond Setup](./adr/019-Network-bond-setup.md)
-   - [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md)
-   - [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md)

docs/adr/015-higher-order-topologies.md

@@ -1,114 +0,0 @@
-# Architecture Decision Record: Higher-Order Topologies
-
-**Initial Author:** Jean-Gabriel Gill-Couture  
-**Initial Date:** 2025-12-08  
-**Last Updated Date:** 2025-12-08  
-
-## Status
-
-Implemented
-
-## Context
-
-Harmony models infrastructure as **Topologies** (deployment targets like `K8sAnywhereTopology`, `LinuxHostTopology`) implementing **Capabilities** (tech traits like `PostgreSQL`, `Docker`).
-
-**Higher-Order Topologies** (e.g., `FailoverTopology<T>`) compose/orchestrate capabilities *across* multiple underlying topologies (e.g., primary+replica `T`).
-
-Naive design requires manual `impl Capability for HigherOrderTopology<T>` *per T per capability*, causing:
-- **Impl explosion**: N topologies × M capabilities = N×M boilerplate.
-- **ISP violation**: Topologies forced to impl unrelated capabilities.
-- **Maintenance hell**: New topology needs impls for *all* orchestrated capabilities; new capability needs impls for *all* topologies/higher-order.
-- **Barrier to extension**: Users can't easily add topologies without todos/panics.
-
-This makes scaling Harmony impractical as ecosystem grows.
-
-## Decision
-
-Use **blanket trait impls** on higher-order topologies to *automatically* derive orchestration:
-
-````rust
-/// Higher-Order Topology: Orchestrates capabilities across sub-topologies.
-pub struct FailoverTopology<T> {
-    /// Primary sub-topology.
-    primary: T,
-    /// Replica sub-topology.
-    replica: T,
-}
-
-/// Automatically provides PostgreSQL failover for *any* `T: PostgreSQL`.
-/// Delegates to primary for queries; orchestrates deploy across both.
-#[async_trait]
-impl<T: PostgreSQL> PostgreSQL for FailoverTopology<T> {
-    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String> {
-        // Deploy primary; extract certs/endpoint;
-        // deploy replica with pg_basebackup + TLS passthrough.
-        // (Full impl logged/elaborated.)
-    }
-
-    // Delegate queries to primary.
-    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String> {
-        self.primary.get_replication_certs(cluster_name).await
-    }
-    // ...
-}
-
-/// Similarly for other capabilities.
-#[async_trait]
-impl<T: Docker> Docker for FailoverTopology<T> {
-    // Failover Docker orchestration.
-}
-````
-
-**Key properties:**
-- **Auto-derivation**: `Failover<K8sAnywhere>` gets `PostgreSQL` iff `K8sAnywhere: PostgreSQL`.
-- **No boilerplate**: One blanket impl per capability *per higher-order type*.
-
-## Rationale
-
-- **Composition via generics**: Rust trait solver auto-selects impls; zero runtime cost.
-- **Compile-time safety**: Missing `T: Capability` → compile error (no panics).
-- **Scalable**: O(capabilities) impls per higher-order; new `T` auto-works.
-- **ISP-respecting**: Capabilities only surface if sub-topology provides.
-- **Centralized logic**: Orchestration (e.g., cert propagation) in one place.
-
-**Example usage:**
-````rust
-// ✅ Works: K8sAnywhere: PostgreSQL → Failover provides failover PG
-let pg_failover: FailoverTopology<K8sAnywhereTopology> = ...;
-pg_failover.deploy_pg(config).await;
-
-// ✅ Works: LinuxHost: Docker → Failover provides failover Docker
-let docker_failover: FailoverTopology<LinuxHostTopology> = ...;
-docker_failover.deploy_docker(...).await;
-
-// ❌ Compile fail: K8sAnywhere !: Docker
-let invalid: FailoverTopology<K8sAnywhereTopology>;
-invalid.deploy_docker(...); // `T: Docker` bound unsatisfied
-````
-
-## Consequences
-
-**Pros:**
-- **Extensible**: New topology `AWSTopology: PostgreSQL` → instant `Failover<AWSTopology>: PostgreSQL`.
-- **Lean**: No useless impls (e.g., no `K8sAnywhere: Docker`).
-- **Observable**: Logs trace every step.
-
-**Cons:**
-- **Monomorphization**: Generics generate code per T (mitigated: few Ts).
-- **Delegation opacity**: Relies on rustdoc/logs for internals.
-
-## Alternatives considered
-
-| Approach | Pros | Cons |
-|----------|------|------|
-| **Manual per-T impls**<br>`impl PG for Failover<K8s> {..}`<br>`impl PG for Failover<Linux> {..}` | Explicit control | N×M explosion; violates ISP; hard to extend. |
-| **Dynamic trait objects**<br>`Box<dyn AnyCapability>` | Runtime flex | Perf hit; type erasure; error-prone dispatch. |
-| **Mega-topology trait**<br>All-in-one `OrchestratedTopology` | Simple wiring | Monolithic; poor composition. |
-| **Registry dispatch**<br>Runtime capability lookup | Decoupled | Complex; no compile safety; perf/debug overhead. |
-
-**Selected**: Blanket impls leverage Rust generics for safe, zero-cost composition.
-
-## Additional Notes
-
-- Applies to `MultisiteTopology<T>`, `ShardedTopology<T>`, etc.
-- `FailoverTopology` in `failover.rs` is first implementation.

docs/adr/015-higher-order-topologies/example.rs

@@ -1,153 +0,0 @@
-//! Example of Higher-Order Topologies in Harmony.
-//! Demonstrates how `FailoverTopology<T>` automatically provides failover for *any* capability
-//! supported by a sub-topology `T` via blanket trait impls.
-//! 
-//! Key insight: No manual impls per T or capability -- scales effortlessly.
-//! Users can:
-//! - Write new `Topology` (impl capabilities on a struct).
-//! - Compose with `FailoverTopology` (gets capabilities if T has them).
-//! - Compile fails if capability missing (safety).
-
-use async_trait::async_trait;
-use tokio;
-
-/// Capability trait: Deploy and manage PostgreSQL.
-#[async_trait]
-pub trait PostgreSQL {
-    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String>;
-    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String>;
-}
-
-/// Capability trait: Deploy Docker.
-#[async_trait]
-pub trait Docker {
-    async fn deploy_docker(&self) -> Result<String, String>;
-}
-
-/// Configuration for PostgreSQL deployments.
-#[derive(Clone)]
-pub struct PostgreSQLConfig;
-
-/// Replication certificates.
-#[derive(Clone)]
-pub struct ReplicationCerts;
-
-/// Concrete topology: Kubernetes Anywhere (supports PostgreSQL).
-#[derive(Clone)]
-pub struct K8sAnywhereTopology;
-
-#[async_trait]
-impl PostgreSQL for K8sAnywhereTopology {
-    async fn deploy(&self, _config: &PostgreSQLConfig) -> Result<String, String> {
-        // Real impl: Use k8s helm chart, operator, etc.
-        Ok("K8sAnywhere PostgreSQL deployed".to_string())
-    }
-
-    async fn get_replication_certs(&self, _cluster_name: &str) -> Result<ReplicationCerts, String> {
-        Ok(ReplicationCerts)
-    }
-}
-
-/// Concrete topology: Linux Host (supports Docker).
-#[derive(Clone)]
-pub struct LinuxHostTopology;
-
-#[async_trait]
-impl Docker for LinuxHostTopology {
-    async fn deploy_docker(&self) -> Result<String, String> {
-        // Real impl: Install/configure Docker on host.
-        Ok("LinuxHost Docker deployed".to_string())
-    }
-}
-
-/// Higher-Order Topology: Composes multiple sub-topologies (primary + replica).
-/// Automatically derives *all* capabilities of `T` with failover orchestration.
-/// 
-/// - If `T: PostgreSQL`, then `FailoverTopology<T>: PostgreSQL` (blanket impl).
-/// - Same for `Docker`, etc. No boilerplate!
-/// - Compile-time safe: Missing `T: Capability` → error.
-#[derive(Clone)]
-pub struct FailoverTopology<T> {
-    /// Primary sub-topology.
-    pub primary: T,
-    /// Replica sub-topology.
-    pub replica: T,
-}
-
-/// Blanket impl: Failover PostgreSQL if T provides PostgreSQL.
-/// Delegates reads to primary; deploys to both.
-#[async_trait]
-impl<T: PostgreSQL + Send + Sync + Clone> PostgreSQL for FailoverTopology<T> {
-    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String> {
-        // Orchestrate: Deploy primary first, then replica (e.g., via pg_basebackup).
-        let primary_result = self.primary.deploy(config).await?;
-        let replica_result = self.replica.deploy(config).await?;
-        Ok(format!("Failover PG deployed: {} | {}", primary_result, replica_result))
-    }
-
-    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String> {
-        // Delegate to primary (replica follows).
-        self.primary.get_replication_certs(cluster_name).await
-    }
-}
-
-/// Blanket impl: Failover Docker if T provides Docker.
-#[async_trait]
-impl<T: Docker + Send + Sync + Clone> Docker for FailoverTopology<T> {
-    async fn deploy_docker(&self) -> Result<String, String> {
-        // Orchestrate across primary + replica.
-        let primary_result = self.primary.deploy_docker().await?;
-        let replica_result = self.replica.deploy_docker().await?;
-        Ok(format!("Failover Docker deployed: {} | {}", primary_result, replica_result))
-    }
-}
-
-#[tokio::main]
-async fn main() {
-    let config = PostgreSQLConfig;
-
-    println!("=== ✅ PostgreSQL Failover (K8sAnywhere supports PG) ===");
-    let pg_failover = FailoverTopology {
-        primary: K8sAnywhereTopology,
-        replica: K8sAnywhereTopology,
-    };
-    let result = pg_failover.deploy(&config).await.unwrap();
-    println!("Result: {}", result);
-
-    println!("\n=== ✅ Docker Failover (LinuxHost supports Docker) ===");
-    let docker_failover = FailoverTopology {
-        primary: LinuxHostTopology,
-        replica: LinuxHostTopology,
-    };
-    let result = docker_failover.deploy_docker().await.unwrap();
-    println!("Result: {}", result);
-
-    println!("\n=== ❌ Would fail to compile (K8sAnywhere !: Docker) ===");
-    // let invalid = FailoverTopology {
-    //     primary: K8sAnywhereTopology,
-    //     replica: K8sAnywhereTopology,
-    // };
-    // invalid.deploy_docker().await.unwrap(); // Error: `K8sAnywhereTopology: Docker` not satisfied!
-    // Very clear error message :
-    // error[E0599]: the method `deploy_docker` exists for struct `FailoverTopology<K8sAnywhereTopology>`, but its trait bounds were not satisfied
-    //   --> src/main.rs:90:9
-    //    |
-    //  4 | pub struct FailoverTopology<T> {
-    //    | ------------------------------ method `deploy_docker` not found for this struct because it doesn't satisfy `FailoverTopology<K8sAnywhereTopology>: Docker`
-    // ...
-    // 37 | struct K8sAnywhereTopology;
-    //    | -------------------------- doesn't satisfy `K8sAnywhereTopology: Docker`
-    // ...
-    // 90 | invalid.deploy_docker(); // `T: Docker` bound unsatisfied
-    //    |         ^^^^^^^^^^^^^ method cannot be called on `FailoverTopology<K8sAnywhereTopology>` due to unsatisfied trait bounds
-    //    |
-    // note: trait bound `K8sAnywhereTopology: Docker` was not satisfied
-    //   --> src/main.rs:61:9
-    //    |
-    // 61 | impl<T: Docker + Send + Sync> Docker for FailoverTopology<T> {
-    //    |         ^^^^^^                ------     -------------------
-    //    |         |
-    //    |         unsatisfied trait bound introduced here
-    // note: the trait `Docker` must be implemented
-}
-

docs/adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md

@@ -1,90 +0,0 @@
-# Architecture Decision Record: Global Orchestration Mesh & The Harmony Agent
-
-**Status:** Proposed
-**Date:** 2025-12-19
-
-## Context
-
-Harmony is designed to enable a truly decentralized infrastructure where independent clusters—owned by different organizations or running on diverse hardware—can collaborate reliably. This vision combines the decentralization of Web3 with the performance and capabilities of Web2.
-
-Currently, Harmony operates as a stateless CLI tool, invoked manually or via CI runners. While effective for deployment, this model presents a critical limitation: **a CLI cannot react to real-time events.**
-
-To achieve automated failover and dynamic workload management, we need a system that is "always on." Relying on manual intervention or scheduled CI jobs to recover from a cluster failure creates unacceptable latency and prevents us from scaling to thousands of nodes.
-
-Furthermore, we face a challenge in serving diverse workloads:
-*   **Financial workloads** require absolute consistency (CP - Consistency/Partition Tolerance).
-*   **AI/Inference workloads** require maximum availability (AP - Availability/Partition Tolerance).
-
-There are many more use cases, but those are the two extremes.
-
-We need a unified architecture that automates cluster coordination and supports both consistency models without requiring a complete re-architecture in the future.
-
-## Decision
-
-We propose a fundamental architectural evolution. It has been clear since the start of Harmony that it would be necessary to transition Harmony from a purely ephemeral CLI tool to a system that includes a persistent **Harmony Agent**. This Agent will connect to a **Global Orchestration Mesh** based on a strongly consistent protocol.
-
-The proposal consists of four key pillars:
-
-### 1. The Harmony Agent (New Component)
-We will develop a long-running process (Daemon/Agent) to be deployed alongside workloads.
-*   **Shift from CLI:** Unlike the CLI, which applies configuration and exits, the Agent maintains a persistent connection to the mesh.
-*   **Responsibility:** It actively monitors cluster health, participates in consensus, and executes lifecycle commands (start/stop/fence) instantly when the mesh dictates a state change.
-
-### 2. The Technology: NATS JetStream
-We will utilize **NATS JetStream** as the underlying transport and consensus layer for the Agent and the Mesh.
-*   **Why not raw Raft?** Implementing a raw Raft library requires building and maintaining the transport layer, log compaction, snapshotting, and peer discovery manually. NATS JetStream provides a battle-tested, distributed log and Key-Value store (based on Raft) out of the box, along with a high-performance pub/sub system for event propagation.
-*   **Role:** It will act as the "source of truth" for the cluster state.
-
-### 3. Strong Consistency at the Mesh Layer
-The mesh will operate with **Strong Consistency** by default.
-*   All critical cluster state changes (topology updates, lease acquisitions, leadership elections) will require consensus among the Agents.
-*   This ensures that in the event of a network partition, we have a mathematical guarantee of which side holds the valid state, preventing data corruption.
-
-### 4. Public UX: The `FailoverStrategy` Abstraction
-To keep the user experience stable and simple, we will expose the complexity of the mesh through a high-level configuration API, tentatively called `FailoverStrategy`.
-
-The user defines the *intent* in their config, and the Harmony Agent automates the *execution*:
-
-*   **`FailoverStrategy::AbsoluteConsistency`**:
-    *   *Use Case:* Banking, Transactional DBs.
-    *   *Behavior:* If the mesh detects a partition, the Agent on the minority side immediately halts workloads. No split-brain is ever allowed.
-*   **`FailoverStrategy::SplitBrainAllowed`**:
-    *   *Use Case:* LLM Inference, Stateless Web Servers.
-    *   *Behavior:* If a partition occurs, the Agent keeps workloads running to maximize uptime. State is reconciled when connectivity returns.
-
-## Rationale
-
-**The Necessity of an Agent**
-You cannot automate what you do not monitor. Moving to an Agent-based model is the only way to achieve sub-second reaction times to infrastructure failures. It transforms Harmony from a deployment tool into a self-healing platform.
-
-**Scaling & Decentralization**
-To allow independent clusters to collaborate, they need a shared language. A strongly consistent mesh allows Cluster A (Organization X) and Cluster B (Organization Y) to agree on workload placement without a central authority.
-
-**Why Strong Consistency First?**
-It is technically feasible to relax a strongly consistent system to allow for "Split Brain" behavior (AP) when the user requests it. However, it is nearly impossible to take an eventually consistent system and force it to be strongly consistent (CP) later. By starting with strict constraints, we cover the hardest use cases (Finance) immediately.
-
-**Future Topologies**
-While our immediate need is `FailoverTopology` (Multi-site), this architecture supports any future topology logic:
-*   **`CostTopology`**: Agents negotiate to route workloads to the cluster with the cheapest spot instances.
-*   **`HorizontalTopology`**: Spreading a single workload across 100 clusters for massive scale.
-*   **`GeoTopology`**: Ensuring data stays within specific legal jurisdictions.
-
-The mesh provides the *capability* (consensus and messaging); the topology provides the *logic*.
-
-## Consequences
-
-**Positive**
-*   **Automation:** Eliminates manual failover, enabling massive scale.
-*   **Reliability:** Guarantees data safety for critical workloads by default.
-*   **Flexibility:** A single codebase serves both high-frequency trading and AI inference.
-*   **Stability:** The public API remains abstract, allowing us to optimize the mesh internals without breaking user code.
-
-**Negative**
-*   **Deployment Complexity:** Users must now deploy and maintain a running service (the Agent) rather than just downloading a binary.
-*   **Engineering Complexity:** Integrating NATS JetStream and handling distributed state machines is significantly more complex than the current CLI logic.
-
-## Implementation Plan (Short Term)
-1.  **Agent Bootstrap:** Create the initial scaffold for the Harmony Agent (daemon).
-2.  **Mesh Integration:** Prototype NATS JetStream embedding within the Agent.
-3.  **Strategy Implementation:** Add `FailoverStrategy` to the configuration schema and implement the logic in the Agent to read and act on it.
-4.  **Migration:** Transition the current manual failover scripts into event-driven logic handled by the Agent.

docs/adr/017-1-Nats-Clusters-Interconnection-Topology.md

@@ -1,189 +0,0 @@
-### 1. ADR 017-1: NATS Cluster Interconnection & Trust Topology
-
-# Architecture Decision Record: NATS Cluster Interconnection & Trust Topology
-
-**Status:** Proposed
-**Date:** 2026-01-12
-**Precedes:** [017-Staleness-Detection-for-Failover.md]
-
-## Context
-
-In ADR 017, we defined the failover mechanisms for the Harmony mesh. However, for a Primary (Site A) and a Replica (Site B) to communicate securely—or for the Global Mesh to function across disparate locations—we must establish a robust Transport Layer Security (TLS) strategy.
-
-Our primary deployment platform is OKD (Kubernetes). While OKD provides an internal `service-ca`, it is designed primarily for intra-cluster service-to-service communication. It lacks the flexibility required for:
-1.  **Public/External Gateway Identities:** NATS Gateways need to identify themselves via public DNS names or external IPs, not just internal `.svc` cluster domains.
-2.  **Cross-Cluster Trust:** We need a mechanism to allow Cluster A to trust Cluster B without sharing a single private root key.
-
-## Decision
-
-We will implement an **"Islands of Trust"** topology using **cert-manager** on OKD.
-
-### 1. Per-Cluster Certificate Authorities (CA)
-
-* We explicitly **reject** the use of a single "Supercluster CA" shared across all sites.
-    * Instead, every Harmony Cluster (Site A, Site B, etc.) will generate its own unique Self-Signed Root CA managed by `cert-manager` inside that cluster.
-*   **Lifecycle:** Root CAs will have a long duration (e.g., 10 years) to minimize rotation friction, while Leaf Certificates (NATS servers) will remain short-lived (e.g., 90 days) and rotate automatically.
-
-> Note : The decision to have a single CA for various workloads managed by Harmony on each deployment, or to have multiple CA for each service that requires interconnection is not made yet. This ADR leans towards one CA per service. This allows for maximum flexibility. But the direction might change and no clear decision has been made yet. The alternative of establishing that each cluster/harmony deployment has a single identity could make mTLS very simple between tenants.
-
-### 2. Trust Federation via Bundle Exchange
-
-To enable secure communication (mTLS) between clusters (e.g., for NATS Gateways or Leaf Nodes):
-
-*   **No Private Keys are shared.**
-*   We will aggregate the **Public CA Certificates** of all trusted clusters into a shared `ca-bundle.pem`.
-*   This bundle is distributed to the NATS configuration of every node.
-*   **Verification Logic:** When Site A connects to Site B, Site A verifies Site B's certificate against the bundle. Since Site B's CA public key is in the bundle, the connection is accepted.
-
-### 3. Tooling
-
-* We will use **cert-manager** (deployed via Operator on OKD) rather than OKD's built-in `service-ca`. This provides us with standard CRDs (`Issuer`, `Certificate`) to manage the lifecycle, rotation, and complex SANs (Subject Alternative Names) required for external connectivity.
-* Harmony will manage installation, configuration and bundle creation across all sites
-
-## Rationale
-
-**Security Blast Radius (The "Key Leak" Scenario)**
-If we used a single global CA and the private key for Site A was compromised (e.g., physical theft of a server from a basement), the attacker could impersonate *any* site in the global mesh.
-By using Per-Cluster CAs:
-*   If Site A is compromised, only Site A's identity is stolen.
-*   We can "evict" Site A from the mesh simply by removing Site A's Public CA from the `ca-bundle.pem` on the remaining healthy clusters and reloading. The attacker can no longer authenticate.
-
-**Decentralized Autonomy**
-This aligns with the "Humane Computing" vision. A local cluster owns its identity. It does not depend on a central authority to issue its certificates. It can function in isolation (offline) indefinitely without needing to "phone home" to renew credentials.
-
-## Consequences
-
-**Positive**
-*   **High Security:** Compromise of one node does not compromise the global mesh.
-*   **Flexibility:** Easier to integrate with third-party clusters or partners by simply adding their public CA to the bundle.
-*   **Standardization:** `cert-manager` is the industry standard, making the configuration portable to non-OKD K8s clusters if needed.
-
-**Negative**
-*   **Configuration Complexity:** We must manage a mechanism to distribute the `ca-bundle.pem` containing public keys to all sites. This should be automated (e.g., via a Harmony Agent) to ensure timely updates and revocation.
-*   **Revocation Latency:** Revoking a compromised cluster requires updating and reloading the bundle on all other clusters. This is slower than OCSP/CRL but acceptable for infrastructure-level trust if automation is in place.
-
----
-
-# 2. Concrete overview of the process, how it can be implemented manually across multiple OKD clusters
-
-All of this will be automated via Harmony, but to understand correctly the process it is outlined in details here :
-
-## 1. Deploying and Configuring cert-manager on OKD
-
-While OKD has a built-in `service-ca` controller, it is "opinionated" and primarily signs certs for internal services (like `my-svc.my-namespace.svc`). It is **not suitable** for the Harmony Global Mesh because you cannot easily control the Subject Alternative Names (SANs) for external routes (e.g., `nats.site-a.nationtech.io`), nor can you easily export its CA to other clusters.
-
-**The Solution:** Use the **cert-manager Operator for Red Hat OpenShift**.
-
-### Step 1: Install the Operator
-1.  Log in to the OKD Web Console.
-2.  Navigate to **Operators** -> **OperatorHub**.
-3.  Search for **"cert-manager"**.
-4.  Choose the **"cert-manager Operator for Red Hat OpenShift"** (Red Hat provided) or the community version.
-5.  Click **Install**. Use the default settings (Namespace: `cert-manager-operator`).
-
-### Step 2: Create the "Island" CA (The Issuer)
-Once installed, you define your cluster's unique identity. Apply this YAML to your NATS namespace.
-
-```yaml
-# filepath: k8s/01-issuer.yaml
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: harmony-selfsigned-issuer
-  namespace: harmony-nats
-spec:
-  selfSigned: {}
----
-# This generates the unique Root CA for THIS specific cluster
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: harmony-root-ca
-  namespace: harmony-nats
-spec:
-  isCA: true
-  commonName: "harmony-site-a-ca" # CHANGE THIS per cluster (e.g., site-b-ca)
-  duration: 87600h # 10 years
-  renewBefore: 2160h # 3 months before expiry
-  secretName: harmony-root-ca-secret
-  privateKey:
-    algorithm: ECDSA
-    size: 256
-  issuerRef:
-    name: harmony-selfsigned-issuer
-    kind: Issuer
-    group: cert-manager.io
----
-# This Issuer uses the Root CA generated above to sign NATS certs
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: harmony-ca-issuer
-  namespace: harmony-nats
-spec:
-  ca:
-    secretName: harmony-root-ca-secret
-```
-
-### Step 3: Generate the NATS Server Certificate
-This certificate will be used by the NATS server. It includes both internal DNS names (for local clients) and external DNS names (for the global mesh).
-
-```yaml
-# filepath: k8s/02-nats-cert.yaml
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: nats-server-cert
-  namespace: harmony-nats
-spec:
-  secretName: nats-server-tls
-  duration: 2160h # 90 days
-  renewBefore: 360h # 15 days
-  issuerRef:
-    name: harmony-ca-issuer
-    kind: Issuer
-  # CRITICAL: Define all names this server can be reached by
-  dnsNames:
-  - "nats"
-  - "nats.harmony-nats.svc"
-  - "nats.harmony-nats.svc.cluster.local"
-  - "*.nats.harmony-nats.svc.cluster.local"
-  - "nats-gateway.site-a.nationtech.io" # External Route for Mesh
-```
-
-## 2. Implementing the "Islands of Trust" (Trust Bundle)
-
-To make Site A and Site B talk, you need to exchange **Public Keys**.
-
-1.  **Extract Public CA from Site A:**
-    ```bash
-    oc get secret harmony-root-ca-secret -n harmony-nats -o jsonpath='{.data.ca\.crt}' | base64 -d > site-a.crt
-    ```
-2.  **Extract Public CA from Site B:**
-    ```bash
-    oc get secret harmony-root-ca-secret -n harmony-nats -o jsonpath='{.data.ca\.crt}' | base64 -d > site-b.crt
-    ```
-3.  **Create the Bundle:**
-    Combine them into one file.
-    ```bash
-    cat site-a.crt site-b.crt > ca-bundle.crt
-    ```
-4.  **Upload Bundle to Both Clusters:**
-    Create a ConfigMap or Secret in *both* clusters containing this combined bundle.
-    ```bash
-    oc create configmap nats-trust-bundle --from-file=ca.crt=ca-bundle.crt -n harmony-nats
-    ```
-5.  **Configure NATS:**
-    Mount this ConfigMap and point NATS to it.
-
-    ```conf
-    # nats.conf snippet
-    tls {
-      cert_file: "/etc/nats-certs/tls.crt"
-      key_file:  "/etc/nats-certs/tls.key"
-      # Point to the bundle containing BOTH Site A and Site B public CAs
-      ca_file:   "/etc/nats-trust/ca.crt"
-    }
-    ```
-
-This setup ensures that Site A can verify Site B's certificate (signed by `harmony-site-b-ca`) because Site B's CA is in Site A's trust store, and vice versa, without ever sharing the private keys that generated them.

docs/adr/017-2-reviewed-staleness-detection-algorithm.md

@@ -1,238 +0,0 @@
-Here are some rough notes on the previous design :
-
-- We found an issue where there could be primary flapping when network latency is larger than the primary self fencing timeout.
-    - e.g. network latency to get nats ack is 30 seconds (extreme but can happen), and self-fencing happens after 50 seconds. Then at second 50 self-fencing would occur, and then at second 60 ack comes in. At this point we reject the ack as already failed because of timeout. Self fencing happens. But then network latency comes back down to 5 seconds and lets one successful heartbeat through, this means the primary comes back to healthy, and the same thing repeats, so the primary flaps.
-    - At least this does not cause split brain since the replica never times out and wins the leadership write since we validate strict write ordering and we force consensus on writes.
-
-Also, we were seeing that the implementation became more complex. There is a lot of timers to handle and that becomes hard to reason about for edge cases.
-
-So, we came up with a slightly different approach, inspired by k8s liveness probes.
-
-We now want to use a failure and success threshold counter . However, on the replica side, all we can do is use a timer. The timer we can use is time since last primary heartbeat jetstream metadata timestamp. We could also try and mitigate clock skew by measuring time between internal clock and jetstream metadata timestamp when writing our own heartbeat (not for now, but worth thinking about, though I feel like it is useless).
-
-So the current working design is this :
-
-configure :
-- number of consecutive success to mark the node as UP
-- number of consecutive failures to mark the node as DOWN
-- note that success/failure must be consecutive. One success in a row of failures is enough to keep service up. This allows for various configuration profiles, from very stict availability to very lenient depending on the number of failure tolerated and success required to keep the service up.
-    - failure_threshold at 100 will let a service fail (or timeout) 99/100 and stay up
-    - success_threshold at 100 will not bring back up a service until it has succeeded 100 heartbeat in a row
-    - failure threshold at 1 will fail the service at the slightest network latency spike/packet loss
-    - success threshold at 1 will bring the service up very quickly and may cause flapping in unstable network conditions
-
-
-```
-# heartbeat session log
-# failure threshold  : 3
-# success threshold : 2
-
-STATUS UP :
-t=1 probe : fail f=1 s=0
-t=2 probe : fail : f=2 s=0
-t=3 probe : ok f=0 s=1
-t=4 probe : fail f=1 s=0
-```
-
-Scenario : 
-
-failure threshold = 2
-heartbeat timeout = 1s
-total before fencing = 2 * 1 = 2s
-
-staleness detection timer = 2*total before fencing
-
-can we do this simple multiplication that staleness detection timer (time the replica waits since the last primary heartbeat before promoting itself) is double the  time the replica will take before starting the fencing process.
-
----
-
-### Context
-We are designing a **Staleness-Based Failover Algorithm** for the Harmony Agent. The goal is to manage High Availability (HA) for stateful workloads (like PostgreSQL) across decentralized, variable-quality networks ("Micro Data Centers").
-
-We are moving away from complex, synchronized clocks in favor of a **Counter-Based Liveness** approach (inspired by Kubernetes probes) for the Primary, and a **Time-Based Watchdog** for the Replica.
-
-### 1. The Algorithm
-
-#### The Primary (Self-Health & Fencing)
-The Primary validates its own "License to Operate" via a heartbeat loop.
-*   **Loop:** Every `heartbeat_interval` (e.g., 1s), it attempts to write a heartbeat to NATS and check the local DB.
-*   **Counters:** It maintains `consecutive_failures` and `consecutive_successes`.
-*   **State Transition:**
-    *   **To UNHEALTHY:** If `consecutive_failures >= failure_threshold`, the Primary **Fences Self** (stops DB, releases locks).
-    *   **To HEALTHY:** If `consecutive_successes >= success_threshold`, the Primary **Un-fences** (starts DB, acquires locks).
-*   **Reset Logic:** A single success resets the failure counter to 0, and vice versa.
-
-#### The Replica (Staleness Detection)
-The Replica acts as a passive watchdog observing the NATS stream.
-*   **Calculation:** It calculates a `MaxStaleness` timeout.
-    $$ \text{MaxStaleness} = (\text{failure\_threshold} \times \text{heartbeat\_interval}) \times \text{SafetyMultiplier} $$
-    *(We use a SafetyMultiplier of 2 to ensure the Primary has definitely fenced itself before we take over).*
-*   **Action:** If `Time.now() - LastPrimaryHeartbeat > MaxStaleness`, the Replica assumes the Primary is dead and **Promotes Self**.
-
----
-
-### 2. Configuration Trade-offs
-
-The separation of `success` and `failure` thresholds allows us to tune the "personality" of the cluster.
-
-#### Scenario A: The "Nervous" Cluster (High Sensitivity)
-*   **Config:** `failure_threshold: 1`, `success_threshold: 1`
-*   **Behavior:** Fails over immediately upon a single missed packet or slow disk write.
-*   **Pros:** Maximum availability for perfect networks.
-*   **Cons:** **High Flapping Risk.** In a residential network, a microwave turning on might cause a failover.
-
-#### Scenario B: The "Tank" Cluster (High Stability)
-*   **Config:** `failure_threshold: 10`, `success_threshold: 1`
-*   **Behavior:** The node must be consistently broken for 10 seconds (assuming 1s interval) to give up.
-*   **Pros:** Extremely stable on bad networks (e.g., Starlink, 4G). Ignores transient spikes.
-*   **Cons:** **Slow Failover.** Users experience 10+ seconds of downtime before the Replica even *thinks* about taking over.
-
-#### Scenario C: The "Sticky" Cluster (Hysteresis)
-*   **Config:** `failure_threshold: 5`, `success_threshold: 5`
-*   **Behavior:** Hard to kill, hard to bring back.
-*   **Pros:** Prevents "Yo-Yo" effects. If a node fails, it must prove it is *really* stable (5 clean checks in a row) before re-joining the cluster.
-
----
-
-### 3. Failure Modes & Behavior Analysis
-
-Here is how the algorithm handles specific edge cases:
-
-#### Case 1: Immediate Outage (Power Cut / Kernel Panic)
-*   **Event:** Primary vanishes instantly. No more writes to NATS.
-*   **Primary:** Does nothing (it's dead).
-*   **Replica:** Sees the `LastPrimaryHeartbeat` timestamp age. Once it crosses `MaxStaleness`, it promotes itself.
-*   **Outcome:** Clean failover after the timeout duration.
-
-#### Case 2: Network Instability (Packet Loss / Jitter)
-*   **Event:** The Primary fails to write to NATS for 2 cycles due to Wi-Fi interference, then succeeds on the 3rd.
-*   **Config:** `failure_threshold: 5`.
-*   **Primary:**
-    *   $t=1$: Fail (Counter=1)
-    *   $t=2$: Fail (Counter=2)
-    *   $t=3$: Success (Counter resets to 0). **State remains HEALTHY.**
-*   **Replica:** Sees a gap in heartbeats but the timestamp never exceeds `MaxStaleness`.
-*   **Outcome:** No downtime, no failover. The system correctly identified this as noise, not failure.
-
-#### Case 3: High Latency (The "Slow Death")
-*   **Event:** Primary is under heavy load; heartbeats take 1.5s to complete (interval is 1s).
-*   **Primary:** The `timeout` on the heartbeat logic triggers. `consecutive_failures` rises. Eventually, it hits `failure_threshold` and fences itself to prevent data corruption.
-*   **Replica:** Sees the heartbeats stop (or arrive too late). The timestamp ages out.
-*   **Outcome:** Primary fences self -> Replica waits for safety buffer -> Replica promotes. **Split-brain is avoided** because the Primary killed itself *before* the Replica acted (due to the SafetyMultiplier).
-
-#### Case 4: Replica Network Partition
-*   **Event:** Replica loses internet connection; Primary is fine.
-*   **Replica:** Sees `LastPrimaryHeartbeat` age out (because it can't reach NATS). It *wants* to promote itself.
-*   **Constraint:** To promote, the Replica must write to NATS. Since it is partitioned, the NATS write fails.
-*   **Outcome:** The Replica remains in Standby (or fails to promote). The Primary continues serving traffic. **Cluster integrity is preserved.**
-
-
-----
-
-
-### Context & Use Case
-We are implementing a High Availability (HA) Failover Strategy for decentralized "Micro Data Centers." The core challenge is managing stateful workloads (PostgreSQL) over unreliable networks.
-
-We solve this using a **Local Fencing First** approach, backed by **NATS JetStream Strict Ordering** for the final promotion authority.
-
-In CAP theorem terms, we are developing a CP system, intentionally sacrificing availability. In practical terms, we expect an average of two primary outages per year, with a failover delay of around 2 minutes. This translates to an uptime of over five nines. To be precise, 2 outages * 2 minutes = 4 minutes per year = 99.99924% uptime.
-
-### The Algorithm: Local Fencing & Remote Promotion
-
-The safety (data consistency) of the system relies on the time gap between the **Primary giving up (Fencing)** and the **Replica taking over (Promotion)**.
-
-To avoid clock skew issues between agents and datastore (nats), all timestamps comparisons will be done using jetstream metadata. I.E. a harmony agent will never use `Instant::now()` to get a timestamp, it will use `my_last_heartbeat.metadata.timestamp` (conceptually).
-
-#### 1. Configuration
-*   `heartbeat_timeout` (e.g., 1s): Max time allowed for a NATS write/DB check.
-*   `failure_threshold` (e.g., 2): Consecutive failures before self-fencing.
-*   `failover_timeout` (e.g., 5s): Time since last NATS update of Primary heartbeat before Replica promotes.
-    * This timeout must be carefully configured to allow enough time for the primary to fence itself (after `heartbeat_timeout * failure_threshold`) BEFORE the replica gets promoted to avoid a split brain with two primaries.
-        * Implementing this will rely on the actual deployment configuration. For example, a CNPG based PostgreSQL cluster might require a longer gap (such as 30s) than other technologies.
-    * Expires when `replica_heartbeat.metadata.timestamp - primary_heartbeat.metadata.timestamp > failover_timeout`
-
-#### 2. The Primary (Self-Preservation)
-
-The Primary is aggressive about killing itself.
-
-*   It attempts a heartbeat.
-*   If the network latency > `heartbeat_timeout`, the attempt is **cancelled locally** because the heartbeat did not make it back in time.
-*   This counts as a failure and increments the `consecutive_failures` counter.
-*   If `consecutive_failures` hit the threshold, **FENCING occurs immediately**. The database is stopped.
-
-This means that the Primary will fence itself after `heartbeat_timeout * failure_threshold`.
-
-#### 3. The Replica (The Watchdog)
-
-The Replica is patient.
-
-*   It watches the NATS stream to measure if `replica_heartbeat.metadata.timestamp - primary_heartbeat.metadata.timestamp > failover_timeout`
-*   It only attempts promotion if the `failover_timeout` (5s) has passed.
-*   **Crucial:** Careful configuration of the failover_timeout is required. This is the only way to avoid a split brain in case of a network partition where the Primary cannot write its heartbeats in time anymore.
-    * In short, `failover_timeout` should be tuned to be `heartbeat_timeout * failure_threshold + safety_margin`. This `safety_margin` will vary by use case. For example, a CNPG cluster may need 30 seconds to demote a Primary to Replica when fencing is triggered, so `safety_margin` should be at least 30s in that setup.
-
-Since we forcibly fail timeouts after `heartbeat_timeout`, we are guaranteed that the primary will have **started** the fencing process after `heartbeat_timeout * failure_threshold`.
-
-But, in a network split scenario where the failed primary is still accessible by clients but cannot write its heartbeat successfully, there is no way to know if the demotion has actually **completed**.
-
-For example, in a CNPG cluster, the failed Primary agent will attempt to change the CNPG cluster state to read-only. But if anything fails after that attempt (permission error, k8s api failure, CNPG bug, etc) it is possible that the PostgreSQL instance keeps accepting writes.
-
-While this is not a theoretical failure of the agent's algorithm, this is a practical failure where data corruption occurs.
-
-This can be fixed by detecting the demotion failure and escalating the fencing procedure aggressiveness. Harmony being an infrastructure orchestrator, it can easily exert radical measures if given the proper credentials, such as forcibly powering off a server, disconnecting its network in the switch configuration, forcibly kill a pod/container/process, etc.
-
-However, these details are out of scope of this algorithm, as they simply fall under the "fencing procedure".
-
-The implementation of the fencing procedure itself is not relevant. This algorithm's responsibility stops at calling the fencing procedure in the appropriate situation.
-
-#### 4. The Demotion Handshake (Return to Normalcy)
-
-When the original Primary recovers:
-
-1.  It becomes healthy locally but sees `current_primary = Replica`. It waits.
-2.  The Replica (current leader) detects the Original Primary is back (via NATS heartbeats).
-3.  Replica performs a **Clean Demotion**:
-    *   Stops DB.
-    *   Writes `current_primary = None` to NATS.
-4.  Original Primary sees `current_primary = None` and can launch the promotion procedure.
-
-Depending on the implementation, the promotion procedure may require a transition phase. Typically, for a PostgreSQL use case the promoting primary will make sure it has caught up on WAL replication before starting to accept writes.
-
----
-
-### Failure Modes & Behavior Analysis
-
-#### Case 1: Immediate Outage (Power Cut)
-
-*   **Primary:** Dies instantly. Fencing is implicit (machine is off).
-*   **Replica:** Waits for `failover_timeout` (5s). Sees staleness. Promotes self.
-*   **Outcome:** Clean failover after 5s.
-
-// TODO detail what happens when the primary comes back up. We will likely have to tie PostgreSQL's lifecycle (liveness/readiness probes) with the agent to ensure it does not come back up as primary.
-
-#### Case 2: High Network Latency on the Primary (The "Split Brain" Trap)
-
-*   **Scenario:** Network latency spikes to 5s on the Primary, still below `heartbeat_timeout` on the Replica.
-*   **T=0 to T=2 (Primary):** Tries to write. Latency (5s) > Timeout (1s). Fails twice.
-*   **T=2 (Primary):** `consecutive_failures` = 2. **Primary Fences Self.** (Service is DOWN).
-*   **T=2 to T=5 (Cluster):** **Read-Only Phase.** No Primary exists.
-*   **T=5 (Replica):** `failover_timeout` reached. Replica promotes self.
-*   **Outcome:** Safe failover. The "Read-Only Gap" (T=2 to T=5) ensures no Split Brain occurred.
-
-#### Case 3: Replica Network Lag (False Positive)
-
-*   **Scenario:** Replica has high latency, greater than `failover_timeout`; Primary is fine.
-*   **Replica:** Thinks Primary is dead. Tries to promote by setting `cluster_state.current_primary = replica_id`.
-*   **NATS:** Rejects the write because the Primary is still updating the sequence numbers successfully.
-*   **Outcome:** Promotion denied. Primary stays leader.
-
-#### Case 4: Network Instability (Flapping)
-
-*   **Scenario:** Intermittent packet loss.
-*   **Primary:** Fails 1 heartbeat, succeeds the next. `consecutive_failures` resets.
-*   **Replica:** Sees a slight delay in updates, but never reaches `failover_timeout`.
-*   **Outcome:** No Fencing, No Promotion. System rides out the noise.
-
-## Contextual notes
-
-* Clock skew : Tokio relies on monotonic clocks. This means that `tokio::time::sleep(...)` will not be affected by system clock corrections (such as NTP). But monotonic clocks are known to jump forward in some cases such as VM live migrations. This could mean a false timeout of a single heartbeat. If `failure_threshold = 1`, this can mean a false negative on the nodes' health, and a potentially useless demotion.

docs/adr/017-3-revised-staleness-inspired-by-kubernetes.md

@@ -1,107 +0,0 @@
-### Context & Use Case
-We are implementing a High Availability (HA) Failover Strategy for decentralized "Micro Data Centers." The core challenge is managing stateful workloads (PostgreSQL) over unreliable networks.
-
-We solve this using a **Local Fencing First** approach, backed by **NATS JetStream Strict Ordering** for the final promotion authority.
-
-In CAP theorem terms, we are developing a CP system, intentionally sacrificing availability. In practical terms, we expect an average of two primary outages per year, with a failover delay of around 2 minutes. This translates to an uptime of over five nines. To be precise, 2 outages * 2 minutes = 4 minutes per year = 99.99924% uptime.
-
-### The Algorithm: Local Fencing & Remote Promotion
-
-The safety (data consistency) of the system relies on the time gap between the **Primary giving up (Fencing)** and the **Replica taking over (Promotion)**.
-
-To avoid clock skew issues between agents and datastore (nats), all timestamps comparisons will be done using jetstream metadata. I.E. a harmony agent will never use `Instant::now()` to get a timestamp, it will use `my_last_heartbeat.metadata.timestamp` (conceptually).
-
-#### 1. Configuration
-*   `heartbeat_timeout` (e.g., 1s): Max time allowed for a NATS write/DB check.
-*   `failure_threshold` (e.g., 2): Consecutive failures before self-fencing.
-*   `failover_timeout` (e.g., 5s): Time since last NATS update of Primary heartbeat before Replica promotes.
-    * This timeout must be carefully configured to allow enough time for the primary to fence itself (after `heartbeat_timeout * failure_threshold`) BEFORE the replica gets promoted to avoid a split brain with two primaries.
-        * Implementing this will rely on the actual deployment configuration. For example, a CNPG based PostgreSQL cluster might require a longer gap (such as 30s) than other technologies.
-    * Expires when `replica_heartbeat.metadata.timestamp - primary_heartbeat.metadata.timestamp > failover_timeout`
-
-#### 2. The Primary (Self-Preservation)
-
-The Primary is aggressive about killing itself.
-
-*   It attempts a heartbeat.
-*   If the network latency > `heartbeat_timeout`, the attempt is **cancelled locally** because the heartbeat did not make it back in time.
-*   This counts as a failure and increments the `consecutive_failures` counter.
-*   If `consecutive_failures` hit the threshold, **FENCING occurs immediately**. The database is stopped.
-
-This means that the Primary will fence itself after `heartbeat_timeout * failure_threshold`.
-
-#### 3. The Replica (The Watchdog)
-
-The Replica is patient.
-
-*   It watches the NATS stream to measure if `replica_heartbeat.metadata.timestamp - primary_heartbeat.metadata.timestamp > failover_timeout`
-*   It only attempts promotion if the `failover_timeout` (5s) has passed.
-*   **Crucial:** Careful configuration of the failover_timeout is required. This is the only way to avoid a split brain in case of a network partition where the Primary cannot write its heartbeats in time anymore.
-    * In short, `failover_timeout` should be tuned to be `heartbeat_timeout * failure_threshold + safety_margin`. This `safety_margin` will vary by use case. For example, a CNPG cluster may need 30 seconds to demote a Primary to Replica when fencing is triggered, so `safety_margin` should be at least 30s in that setup.
-
-Since we forcibly fail timeouts after `heartbeat_timeout`, we are guaranteed that the primary will have **started** the fencing process after `heartbeat_timeout * failure_threshold`.
-
-But, in a network split scenario where the failed primary is still accessible by clients but cannot write its heartbeat successfully, there is no way to know if the demotion has actually **completed**.
-
-For example, in a CNPG cluster, the failed Primary agent will attempt to change the CNPG cluster state to read-only. But if anything fails after that attempt (permission error, k8s api failure, CNPG bug, etc) it is possible that the PostgreSQL instance keeps accepting writes.
-
-While this is not a theoretical failure of the agent's algorithm, this is a practical failure where data corruption occurs.
-
-This can be fixed by detecting the demotion failure and escalating the fencing procedure aggressiveness. Harmony being an infrastructure orchestrator, it can easily exert radical measures if given the proper credentials, such as forcibly powering off a server, disconnecting its network in the switch configuration, forcibly kill a pod/container/process, etc.
-
-However, these details are out of scope of this algorithm, as they simply fall under the "fencing procedure".
-
-The implementation of the fencing procedure itself is not relevant. This algorithm's responsibility stops at calling the fencing procedure in the appropriate situation.
-
-#### 4. The Demotion Handshake (Return to Normalcy)
-
-When the original Primary recovers:
-
-1.  It becomes healthy locally but sees `current_primary = Replica`. It waits.
-2.  The Replica (current leader) detects the Original Primary is back (via NATS heartbeats).
-3.  Replica performs a **Clean Demotion**:
-    *   Stops DB.
-    *   Writes `current_primary = None` to NATS.
-4.  Original Primary sees `current_primary = None` and can launch the promotion procedure.
-
-Depending on the implementation, the promotion procedure may require a transition phase. Typically, for a PostgreSQL use case the promoting primary will make sure it has caught up on WAL replication before starting to accept writes.
-
----
-
-### Failure Modes & Behavior Analysis
-
-#### Case 1: Immediate Outage (Power Cut)
-
-*   **Primary:** Dies instantly. Fencing is implicit (machine is off).
-*   **Replica:** Waits for `failover_timeout` (5s). Sees staleness. Promotes self.
-*   **Outcome:** Clean failover after 5s.
-
-// TODO detail what happens when the primary comes back up. We will likely have to tie PostgreSQL's lifecycle (liveness/readiness probes) with the agent to ensure it does not come back up as primary.
-
-#### Case 2: High Network Latency on the Primary (The "Split Brain" Trap)
-
-*   **Scenario:** Network latency spikes to 5s on the Primary, still below `heartbeat_timeout` on the Replica.
-*   **T=0 to T=2 (Primary):** Tries to write. Latency (5s) > Timeout (1s). Fails twice.
-*   **T=2 (Primary):** `consecutive_failures` = 2. **Primary Fences Self.** (Service is DOWN).
-*   **T=2 to T=5 (Cluster):** **Read-Only Phase.** No Primary exists.
-*   **T=5 (Replica):** `failover_timeout` reached. Replica promotes self.
-*   **Outcome:** Safe failover. The "Read-Only Gap" (T=2 to T=5) ensures no Split Brain occurred.
-
-#### Case 3: Replica Network Lag (False Positive)
-
-*   **Scenario:** Replica has high latency, greater than `failover_timeout`; Primary is fine.
-*   **Replica:** Thinks Primary is dead. Tries to promote by setting `cluster_state.current_primary = replica_id`.
-*   **NATS:** Rejects the write because the Primary is still updating the sequence numbers successfully.
-*   **Outcome:** Promotion denied. Primary stays leader.
-
-#### Case 4: Network Instability (Flapping)
-
-*   **Scenario:** Intermittent packet loss.
-*   **Primary:** Fails 1 heartbeat, succeeds the next. `consecutive_failures` resets.
-*   **Replica:** Sees a slight delay in updates, but never reaches `failover_timeout`.
-*   **Outcome:** No Fencing, No Promotion. System rides out the noise.
-
-## Contextual notes
-
-* Clock skew : Tokio relies on monotonic clocks. This means that `tokio::time::sleep(...)` will not be affected by system clock corrections (such as NTP). But monotonic clocks are known to jump forward in some cases such as VM live migrations. This could mean a false timeout of a single heartbeat. If `failure_threshold = 1`, this can mean a false negative on the nodes' health, and a potentially useless demotion.
-* `heartbeat_timeout == heartbeat_interval` : We intentionally do not provide two separate settings for the timeout before considering a heartbeat failed and the interval between heartbeats. It could make sense in some configurations where low network latency is required to have a small `heartbeat_timeout = 50ms` and larger `hartbeat_interval == 2s`, but we do not have a practical use case for it yet. And having timeout larger than interval does not make sense in any situation we can think of at the moment. So we decided to have a single value for both, which makes the algorithm easier to reason about and implement.

docs/adr/017-staleness-detection-for-failover.md

@@ -1,95 +0,0 @@
-# Architecture Decision Record: Staleness-Based Failover Mechanism & Observability
-
-**Status:** Proposed
-**Date:** 2026-01-09
-**Precedes:** [016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md](https://git.nationtech.io/NationTech/harmony/raw/branch/master/adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md)
-
-## Context
-
-In ADR 016, we established the **Harmony Agent** and the **Global Orchestration Mesh** (powered by NATS JetStream) as the foundation for our decentralized infrastructure. We defined the high-level need for a `FailoverStrategy` that can support both financial consistency (CP) and AI availability (AP).
-
-However, a specific implementation challenge remains: **How do we reliably detect node failure without losing the ability to debug the event later?**
-
-Standard distributed systems often use "Key Expiration" (TTL) for heartbeats. If a key disappears, the node is presumed dead. While simple, this approach is catastrophic for post-mortem analysis. When the key expires, the evidence of *when* and *how* the failure occurred evaporates.
-
-For NationTech’s vision of **Humane Computing**—where micro datacenters might be heating a family home or running a local business—reliability and diagnosability are paramount. If a cluster fails over, we owe it to the user to provide a clear, historical log of exactly what happened. We cannot build a "wonderful future for computers" on ephemeral, untraceable errors.
-
-## Decision
-
-We will implement a **Staleness Detection** mechanism rather than a Key Expiration mechanism. We will leverage NATS JetStream Key-Value (KV) stores with **History Enabled** to create an immutable audit trail of cluster health.
-
-### 1. The "Black Box" Flight Recorder (NATS Configuration)
-We will utilize a persistent NATS KV bucket named `harmony_failover`.
-*   **Storage:** File (Persistent).
-*   **History:** Set to `64` (or higher). This allows us to query the last 64 heartbeat entries to visualize the exact degradation of the primary node before failure.
-*   **TTL:** None. Data never disappears; it only becomes "stale."
-
-### 2. Data Structures
-We will define two primary schemas to manage the state.
-
-
-**A. The Rules of Engagement (`cluster_config`)**
-This persistent key defines the behavior of the mesh. It allows us to tune failover sensitivity dynamically without redeploying the Agent binary.
-
-```json
-{
-  "primary_site_id": "site-a-basement",
-  "replica_site_id": "site-b-cloud",
-  "failover_timeout_ms": 5000,   // Time before Replica takes over
-  "heartbeat_interval_ms": 1000  // Frequency of Primary updates
-}
-```
-
-> **Note :** The location for this configuration data structure is TBD. See https://git.nationtech.io/NationTech/harmony/issues/206
-
-**B. The Heartbeat (`primary_heartbeat`)**
-The Primary writes this; the Replica watches it.
-
-```json
-{
-  "site_id": "site-a-basement",
-  "status": "HEALTHY",
-  "counter": 10452,
-  "timestamp": 1704661549000
-}
-```
-
-### 3. The Failover Algorithm
-
-**The Primary (Site A) Logic:**
-The Primary's ability to write to the mesh is its "License to Operate."
-1.  **Write Loop:** Attempts to write `primary_heartbeat` every `heartbeat_interval_ms`.
-2.  **Self-Preservation (Fencing):** If the write fails (NATS Ack timeout or NATS unreachable), the Primary **immediately self-demotes**. It assumes it is network-isolated. This prevents Split Brain scenarios where a partitioned Primary continues to accept writes while the Replica promotes itself.
-
-**The Replica (Site B) Logic:**
-The Replica acts as the watchdog.
-1.  **Watch:** Subscribes to updates on `primary_heartbeat`.
-2.  **Staleness Check:** Maintains a local timer. Every time a heartbeat arrives, the timer resets.
-3.  **Promotion:** If the timer exceeds `failover_timeout_ms`, the Replica declares the Primary dead and promotes itself to Leader.
-4.  **Yielding:** If the Replica is Leader, but suddenly receives a valid, new heartbeat from the configured `primary_site_id` (indicating the Primary has recovered), the Replica will voluntarily **demote** itself to restore the preferred topology.
-
-## Rationale
-
-**Observability as a First-Class Citizen**
-By keeping the last 64 heartbeats, we can run `nats kv history` to see the exact timeline. Did the Primary stop suddenly (crash)? or did the heartbeats become erratic and slow before stopping (network congestion)? This data is critical for optimizing the "Micro Data Centers" described in our vision, where internet connections in residential areas may vary in quality.
-
-**Energy Efficiency & Resource Optimization**
-NationTech aims to "maximize the value of our energy." A "flapping" cluster (constantly failing over and back) wastes immense energy in data re-synchronization and startup costs. By making the `failover_timeout_ms` configurable via `cluster_config`, we can tune a cluster heating a greenhouse to be less sensitive (slower failover is fine) compared to a cluster running a payment gateway.
-
-**Decentralized Trust**
-This architecture relies on NATS as the consensus engine. If the Primary is part of the NATS majority, it lives. If it isn't, it dies. This removes ambiguity and allows us to scale to thousands of independent sites without a central "God mode" controller managing every single failover.
-
-## Consequences
-
-**Positive**
-*   **Auditability:** Every failover event leaves a permanent trace in the KV history.
-*   **Safety:** The "Write Ack" check on the Primary provides a strong guarantee against Split Brain in `AbsoluteConsistency` mode.
-*   **Dynamic Tuning:** We can adjust timeouts for specific environments (e.g., high-latency satellite links) by updating a JSON key, requiring no downtime.
-
-**Negative**
-*   **Storage Overhead:** Keeping history requires marginally more disk space on the NATS servers, though for 64 small JSON payloads, this is negligible.
-*   **Clock Skew:** While we rely on NATS server-side timestamps for ordering, extreme clock skew on the client side could confuse the debug logs (though not the failover logic itself).
-
-## Alignment with Vision
-This architecture supports the NationTech goal of a **"Beautifully Integrated Design."** It takes the complex, high-stakes problem of distributed consensus and wraps it in a mechanism that is robust enough for enterprise banking yet flexible enough to manage a basement server heating a swimming pool. It bridges the gap between the reliability of Web2 clouds and the decentralized nature of Web3 infrastructure.
-

docs/adr/018-Template-Hydration-For-Workload-Deployment.md

@@ -1,141 +0,0 @@
-# Architecture Decision Record: Template Hydration for Kubernetes Manifest Generation
-
-Initial Author: Jean-Gabriel Gill-Couture & Sylvain Tremblay
-
-Initial Date: 2025-01-23
-
-Last Updated Date: 2025-01-23
-
-## Status
-
-Implemented
-
-## Context
-
-Harmony's philosophy is built on three guiding principles: Infrastructure as Resilient Code, Prove It Works — Before You Deploy, and One Unified Model. Our goal is to shift validation and verification as left as possible—ideally to compile time—rather than discovering errors at deploy time.
-
-After investigating a few approaches such as compile-checked Askama templates to generate Kubernetes manifests for Helm charts, we found again that this approach suffered from several fundamental limitations:
-
-*   **Late Validation:** Typos in template syntax or field names are only discovered at deployment time, not during compilation. A mistyped `metadata.name` won't surface until Helm attempts to render the template.
-*   **Brittle Maintenance:** Templates are string-based with limited IDE support. Refactoring requires grep-and-replace across YAML-like template files, risking subtle breakage.
-*   **Hard-to-Test Logic:** Testing template output requires mocking the template engine and comparing serialized strings rather than asserting against typed data structures.
-*   **No Type Safety:** There is no guarantee that the generated YAML will be valid Kubernetes resources without runtime validation.
-
-We also faced a strategic choice around Helm: use it as both *templating engine* and *packaging mechanism*, or decouple these concerns. While Helm's ecosystem integration (Harbor, ArgoCD, OCI registry support) is valuable, the Jinja-like templating is at odds with Harmony's "code-first" ethos.
-
-## Decision
-
-We will adopt the **Template Hydration Pattern**—constructing Kubernetes manifests programmatically using strongly-typed `kube-rs` objects, then serializing them to YAML files for packaging into Helm charts.
-
-Specifically:
-
-*   **Write strongly typed `k8s_openapi` Structs:** All Kubernetes resources (Deployment, Service, ConfigMap, etc.) will be constructed using the typed structs generated by `k8s_openapi`.
-*   **Direct Serialization to YAML:** Rather than rendering templates, we use `serde_yaml::to_string()` to serialize typed objects directly into YAML manifests. This way, YAML is only used as a data-transfer format and not a templating/programming language - which it is not.
-*   **Helm as Packaging-Only:** Helm's role is reduced to packaging pre-rendered templates into a tarball and pushing to OCI registries. No template rendering logic resides within Helm.
-*   **Ecosystem Preservation:** The generated Helm charts remain fully compatible with Harbor, ArgoCD, and any Helm-compatible tool—the only difference is that the `templates/` directory contains static YAML files.
-
-The implementation in `backend_app.rs` demonstrates this pattern:
-
-```rust
-let deployment = Deployment {
-    metadata: ObjectMeta {
-        name: Some(self.name.clone()),
-        labels: Some([("app.kubernetes.io/name".to_string(), self.name.clone())].into()),
-        ..Default::default()
-    },
-    spec: Some(DeploymentSpec { /* ... */ }),
-    ..Default::default()
-};
-
-let deployment_yaml = serde_yaml::to_string(&deployment)?;
-fs::write(templates_dir.join("deployment.yaml"), deployment_yaml)?;
-```
-
-## Rationale
-
-**Aligns with "Infrastructure as Resilient Code"**
-
-Harmony's first principle states that infrastructure should be treated like application code. By expressing Kubernetes manifests as Rust structs, we gain:
-
-*   **Refactorability:** Rename a label and the compiler catches all usages.
-*   **IDE Support:** Autocomplete for all Kubernetes API fields; documentation inline.
-*   **Code Navigation:** Jump to definition shows exactly where a value comes from.
-
-**Achieves "Prove It Works — Before You Deploy"**
-
-The compiler now validates that:
-
-*   All required fields are populated (Rust's `Option` type prevents missing fields).
-*   Field types match expectations (ports are integers, not strings).
-*   Enums contain valid values (e.g., `ServiceType::ClusterIP`).
-
-This moves what was runtime validation into compile-time checks, fulfilling the "shift left" promise.
-
-**Enables True Unit Testing**
-
-Developers can now write unit tests that assert directly against typed objects:
-
-```rust
-let deployment = create_deployment(&app);
-assert_eq!(deployment.spec.unwrap().replicas.unwrap(), 3);
-assert_eq!(deployment.metadata.name.unwrap(), "my-app");
-```
-
-No string parsing, no YAML serialization, no fragile assertions against rendered output.
-
-**Preserves Ecosystem Benefits**
-
-By generating standard Helm chart structures, Harmony retains compatibility with:
-
-*   **OCI Registries (Harbor, GHCR):** `helm push` works exactly as before.
-*   **ArgoCD:** Syncs and manages releases using the generated charts.
-*   **Existing Workflows:** Teams already consuming Helm charts see no change.
-
-The Helm tarball becomes a "dumb pipe" for transport, which is arguably its ideal role.
-
-## Consequences
-
-### Positive
-
-*   **Compile-Time Safety:** A broad class of errors (typos, missing fields, type mismatches) is now caught at build time.
-*   **Better Developer Experience:** IDE autocomplete, inline documentation, and refactor support significantly reduce the learning curve for Kubernetes manifests.
-*   **Testability:** Unit tests can validate manifest structure without integration or runtime checks.
-*   **Auditability:** The source-of-truth for manifests is now pure Rust—easier to review in pull requests than template logic scattered across files.
-*   **Future-Extensibility:** CustomResources (CRDs) can be supported via `kopium`-generated Rust types, maintaining the same strong typing.
-
-### Negative
-
-*   **API Schema Drift:** Kubernetes API changes require regenerating `k8s_openapi` types and updating code. A change in a struct field will cause the build to fail—intentionally, but still requiring the pipeline to be updated.
-*   **Verbosity:** Typed construction is more verbose than the equivalent template. Builder patterns or helper functions will be needed to keep code readable.
-*   **Learning Curve:** Contributors must understand both the Kubernetes resource spec *and* the Rust type system, rather than just YAML.
-*   **Debugging Shift:** When debugging generated YAML, you now trace through Rust code rather than template files—more precise but different mental model.
-
-## Alternatives Considered
-
-### 1. Enhance Askama with Compile-Time Validation
-*Pros:* Stay within familiar templating paradigm; minimal code changes.
-*Cons:* Rust's type system cannot fully express Kubernetes schema validation without significant macro boilerplate. Errors would still surface at template evaluation time, not compilation.
-
-### 2. Use Helm SDK Programmatically (Go)
-*Pros:* Direct access to Helm's template engine; no YAML serialization step.
-*Cons:* Would introduce a second language (Go) into a Rust codebase, increasing cognitive load and compilation complexity. No improvement in compile-time safety.
-
-### 3. Raw YAML String Templating (Manual)
-*Pros:* Maximum control; no external dependencies.
-*Cons:* Even more error-prone than Askama; no structure validation; string concatenation errors abound.
-
-### 4. Use Kustomize for All Manifests
-*Pros:* Declarative overlays; standard tool.
-*Cons:* Kustomize is itself a layer over YAML templates with its own DSL. It does not provide compile-time type safety and would require externalizing manifest management outside Harmony's codebase.
-
-__Note that this template hydration architecture still allows to override templates with tools like kustomize when required__
-
-## Additional Notes
-
-**Scalability to Future Topologies**
-
-The Template Hydration pattern enables future Harmony architectures to generate manifests dynamically based on topology context. For example, a `CostTopology` might adjust resource requests based on cluster pricing, manipulating the typed `Deployment::spec` directly before serialization.
-
-**Implementation Status**
-
-As of this writing, the pattern is implemented for `BackendApp` deployments (`backend_app.rs`). The next phase is to extend this pattern across all application modules (`webapp.rs`, etc.) and to standardize on this approach for any new implementations.

docs/adr/019-Network-bond-setup.md

@@ -1,65 +0,0 @@
-# Architecture Decision Record: Network Bonding Configuration via External Automation
-
-Initial Author: Jean-Gabriel Gill-Couture & Sylvain Tremblay
-
-Initial Date: 2026-02-13
-
-Last Updated Date: 2026-02-13
-
-## Status
-
-Accepted
-
-## Context
-
-We need to configure LACP bonds on 10GbE interfaces across all worker nodes in the OpenShift cluster. A significant challenge is that interface names (e.g., `enp1s0f0` vs `ens1f0`) vary across different hardware nodes.
-
-The standard OpenShift mechanism (MachineConfig) applies identical configurations to all nodes in a MachineConfigPool. Since the interface names differ, a single static MachineConfig cannot target specific physical devices across the entire cluster without complex workarounds.
-
-## Decision
-
-We will use the existing "Harmony" automation tool to generate and apply host-specific NetworkManager configuration files directly to the nodes.
-
-1.  Harmony will generate the specific `.nmconnection` files for the bond and slaves based on its inventory of interface names.
-2.  Files will be pushed to `/etc/NetworkManager/system-connections/` on each node.
-3.  Configuration will be applied via `nmcli` reload or a node reboot.
-
-## Rationale
-
-*   **Inventory Awareness:** Harmony already possesses the specific interface mapping data for each host.
-*   **Persistence:** Fedora CoreOS/SCOS allows writing to `/etc`, and these files persist across reboots and OS upgrades (rpm-ostree updates).
-*   **Avoids Complexity:** This approach avoids the operational overhead of creating unique MachineConfigPools for every single host or hardware variant.
-*   **Safety:** Unlike wildcard matching, this ensures explicit interface selection, preventing accidental bonding of reserved interfaces (e.g., future separation of Ceph storage traffic).
-
-## Consequences
-
-**Pros:**
-*   Precise, per-host configuration without polluting the Kubernetes API with hundreds of MachineConfigs.
-*   Standard Linux networking behavior; easy to debug locally.
-*   Prevents accidental interface capture (unlike wildcards).
-
-**Cons:**
-*   **Loss of Declarative K8s State:** The network config is not managed by the Machine Config Operator (MCO).
-*   **Node Replacement Friction:** Newly provisioned nodes (replacements) will boot with default config. Harmony must be run against new nodes manually or via a hook before they can fully join the cluster workload.
-
-## Alternatives considered
-
-1.  **Wildcard Matching in NetworkManager (e.g., `interface-name=enp*`):**
-    *   *Pros:* Single MachineConfig for the whole cluster.
-    *   *Cons:* Rejected because it is too broad. It risks capturing interfaces intended for other purposes (e.g., splitting storage and cluster networks later).
-
-2.  **"Kitchen Sink" Configuration:**
-    *   *Pros:* Single file listing every possible interface name as a slave.
-    *   *Cons:* "Dirty" configuration; results in many inactive connections on every host; brittle if new naming schemes appear.
-
-3.  **Per-Host MachineConfig:**
-    *   *Pros:* Fully declarative within OpenShift.
-    *   *Cons:* Requires a unique `MachineConfigPool` per host, which is an anti-pattern and unmaintainable at scale.
-
-4.  **On-boot Generation Script:**
-    *   *Pros:* Dynamic detection.
-    *   *Cons:* Increases boot complexity; harder to debug if the script fails during startup.
-
-## Additional Notes
-
-While `/etc` is writable and persistent on CoreOS, this configuration falls outside the "Day 1" Ignition process. Operational runbooks must be updated to ensure Harmony runs on any node replacement events.

docs/adr/020-1-zitadel-openbao-secure-config-store.md

@@ -1,233 +0,0 @@
-# ADR 020-1: Zitadel OIDC and OpenBao Integration for the Config Store
-
-Author: Jean-Gabriel Gill-Couture
-
-Date: 2026-03-18
-
-## Status
-
-Proposed
-
-## Context
-
-ADR 020 defines a unified `harmony_config` crate with a `ConfigStore` trait. The default team-oriented backend is OpenBao, which provides encrypted storage, versioned KV, audit logging, and fine-grained access control.
-
-OpenBao requires authentication. The question is how developers authenticate without introducing new credentials to manage.
-
-The goals are:
-
-- **Zero new credentials.** Developers log in with their existing corporate identity (Google Workspace, GitHub, or Microsoft Entra ID / Azure AD).
-- **Headless compatibility.** The flow must work over SSH, inside containers, and in CI — environments with no browser or localhost listener.
-- **Minimal friction.** After a one-time login, authentication should be invisible for weeks of active use.
-- **Centralized offboarding.** Revoking a user in the identity provider must immediately revoke their access to the config store.
-
-## Decision
-
-Developers authenticate to OpenBao through a two-step process: first, they obtain an OIDC token from Zitadel (`sso.nationtech.io`) using the OAuth 2.0 Device Authorization Grant (RFC 8628); then, they exchange that token for a short-lived OpenBao client token via OpenBao's JWT auth method.
-
-### The authentication flow
-
-#### Step 1: Trigger
-
-The `ConfigManager` attempts to resolve a value via the `StoreSource`. The `StoreSource` checks for a cached OpenBao token in `~/.local/share/harmony/session.json`. If the token is missing or expired, authentication begins.
-
-#### Step 2: Device Authorization Request
-
-Harmony sends a `POST` to Zitadel's device authorization endpoint:
-
-```
-POST https://sso.nationtech.io/oauth/v2/device_authorization
-Content-Type: application/x-www-form-urlencoded
-
-client_id=<harmony_client_id>&scope=openid email profile offline_access
-```
-
-Zitadel responds with:
-
-```json
-{
-  "device_code": "dOcbPeysDhT26ZatRh9n7Q",
-  "user_code": "GQWC-FWFK",
-  "verification_uri": "https://sso.nationtech.io/device",
-  "verification_uri_complete": "https://sso.nationtech.io/device?user_code=GQWC-FWFK",
-  "expires_in": 300,
-  "interval": 5
-}
-```
-
-#### Step 3: User prompt
-
-Harmony prints the code and URL to the terminal:
-
-```
-[Harmony] To authenticate, open your browser to:
-          https://sso.nationtech.io/device
-          and enter code: GQWC-FWFK
-
-          Or visit: https://sso.nationtech.io/device?user_code=GQWC-FWFK
-```
-
-If a desktop environment is detected, Harmony also calls `open` / `xdg-open` to launch the browser automatically. The `verification_uri_complete` URL pre-fills the code, so the user only needs to click "Confirm" after logging in.
-
-There is no localhost HTTP listener. The CLI does not need to bind a port or receive a callback. This is what makes the device flow work over SSH, in containers, and through corporate firewalls — unlike the `oc login` approach which spins up a temporary web server to catch a redirect.
-
-#### Step 4: User login
-
-The developer logs in through Zitadel's web UI using one of the configured identity providers:
-
-- **Google Workspace** — for teams using Google as their corporate identity.
-- **GitHub** — for open-source or GitHub-centric teams.
-- **Microsoft Entra ID (Azure AD)** — for enterprise clients, particularly common in Quebec and the broader Canadian public sector.
-
-Zitadel federates the login to the chosen provider. The developer authenticates with their existing corporate credentials. No new password is created.
-
-#### Step 5: Polling
-
-While the user is authenticating in the browser, Harmony polls Zitadel's token endpoint at the interval specified in the device authorization response (typically 5 seconds):
-
-```
-POST https://sso.nationtech.io/oauth/v2/token
-Content-Type: application/x-www-form-urlencoded
-
-grant_type=urn:ietf:params:oauth:grant-type:device_code
-&device_code=dOcbPeysDhT26ZatRh9n7Q
-&client_id=<harmony_client_id>
-```
-
-Before the user completes login, Zitadel responds with `authorization_pending`. Once the user consents, Zitadel returns:
-
-```json
-{
-  "access_token": "...",
-  "token_type": "Bearer",
-  "expires_in": 3600,
-  "refresh_token": "...",
-  "id_token": "eyJhbGciOiJSUzI1NiIs..."
-}
-```
-
-The `scope=offline_access` in the initial request is what causes Zitadel to issue a `refresh_token`.
-
-#### Step 6: OpenBao JWT exchange
-
-Harmony sends the `id_token` (a JWT signed by Zitadel) to OpenBao's JWT auth method:
-
-```
-POST https://secrets.nationtech.io/v1/auth/jwt/login
-Content-Type: application/json
-
-{
-  "role": "harmony-developer",
-  "jwt": "eyJhbGciOiJSUzI1NiIs..."
-}
-```
-
-OpenBao validates the JWT:
-
-1. It fetches Zitadel's public keys from `https://sso.nationtech.io/oauth/v2/keys` (the JWKS endpoint).
-2. It verifies the JWT signature.
-3. It reads the claims (`email`, `groups`, and any custom claims mapped from the upstream identity provider, such as Azure AD tenant or Google Workspace org).
-4. It evaluates the claims against the `bound_claims` and `bound_audiences` configured on the `harmony-developer` role.
-5. If validation passes, OpenBao returns a client token:
-
-```json
-{
-  "auth": {
-    "client_token": "hvs.CAES...",
-    "policies": ["harmony-dev"],
-    "metadata": { "role": "harmony-developer" },
-    "lease_duration": 14400,
-    "renewable": true
-  }
-}
-```
-
-Harmony caches the OpenBao token, the OIDC refresh token, and the token expiry timestamps to `~/.local/share/harmony/session.json` with `0600` file permissions.
-
-### OpenBao storage structure
-
-All configuration and secret state is stored in an OpenBao Versioned KV v2 engine.
-
-Path taxonomy:
-
-```
-harmony/<organization>/<project>/<environment>/<key>
-```
-
-Examples:
-
-```
-harmony/nationtech/my-app/staging/PostgresConfig
-harmony/nationtech/my-app/production/PostgresConfig
-harmony/nationtech/my-app/local-shared/PostgresConfig
-```
-
-The `ConfigClass` (Standard vs. Secret) can influence OpenBao policy structure — for example, `Secret`-class paths could require stricter ACLs or additional audit backends — but the path taxonomy itself does not change. This is an operational concern configured in OpenBao policies, not a structural one enforced by path naming.
-
-### Token lifecycle and silent refresh
-
-The system manages three tokens with different lifetimes:
-
-| Token | TTL | Max TTL | Purpose |
-|---|---|---|---|
-| OpenBao client token | 4 hours | 24 hours | Read/write config store |
-| OIDC ID token | 1 hour | — | Exchange for OpenBao token |
-| OIDC refresh token | 90 days absolute, 30 days inactivity | — | Obtain new ID tokens silently |
-
-The refresh flow, from the developer's perspective:
-
-1. **Same session (< 4 hours since last use).** The cached OpenBao token is still valid. No network call to Zitadel. Fastest path.
-2. **Next day (OpenBao token expired, refresh token valid).** Harmony uses the OIDC `refresh_token` to request a new `id_token` from Zitadel's token endpoint (`grant_type=refresh_token`). It then exchanges the new `id_token` for a fresh OpenBao token. This happens silently. The developer sees no prompt.
-3. **OpenBao token near max TTL (approaching 24 hours of cumulative renewals).** Instead of renewing, Harmony re-authenticates using the refresh token to get a completely fresh OpenBao token. Transparent to the user.
-4. **After 30 days of inactivity.** The OIDC refresh token expires. Harmony falls back to the device flow (Step 2 above) and prompts the user to re-authenticate in the browser. This is the only scenario where a returning developer sees a login prompt.
-5. **User offboarded.** An administrator revokes the user's account or group membership in Zitadel. The next time the refresh token is used, Zitadel rejects it. The device flow also fails because the user can no longer authenticate. Access is terminated without any action needed on the OpenBao side.
-
-OpenBao token renewal uses the `/auth/token/renew-self` endpoint with the `X-Vault-Token` header. Harmony renews proactively at ~75% of the TTL to avoid race conditions.
-
-### OpenBao role configuration
-
-The OpenBao JWT auth role for Harmony developers:
-
-```bash
-bao write auth/jwt/config \
-    oidc_discovery_url="https://sso.nationtech.io" \
-    bound_issuer="https://sso.nationtech.io"
-
-bao write auth/jwt/role/harmony-developer \
-    role_type="jwt" \
-    bound_audiences="<harmony_client_id>" \
-    user_claim="email" \
-    groups_claim="urn:zitadel:iam:org:project:roles" \
-    policies="harmony-dev" \
-    ttl="4h" \
-    max_ttl="24h" \
-    token_type="service"
-```
-
-The `bound_audiences` claim ties the role to the specific Harmony Zitadel application. The `groups_claim` allows mapping Zitadel project roles to OpenBao policies for per-team or per-project access control.
-
-### Self-hosted deployments
-
-For organizations running their own infrastructure, the same architecture applies. The operator deploys Zitadel and OpenBao using Harmony's existing `ZitadelScore` and `OpenbaoScore`. The only configuration needed is three environment variables (or their equivalents in the bootstrap config):
-
-- `HARMONY_SSO_URL` — the Zitadel instance URL.
-- `HARMONY_SECRETS_URL` — the OpenBao instance URL.
-- `HARMONY_SSO_CLIENT_ID` — the Zitadel application client ID.
-
-None of these are secrets. They can be committed to an infrastructure repository or distributed via any convenient channel.
-
-## Consequences
-
-### Positive
-
-- Developers authenticate with existing corporate credentials. No new passwords, no static tokens to distribute.
-- The device flow works in every environment: local terminal, SSH, containers, CI runners, corporate VPNs.
-- Silent token refresh keeps developers authenticated for weeks without any manual intervention.
-- User offboarding is a single action in Zitadel. No OpenBao token rotation or manual revocation required.
-- Azure AD / Microsoft Entra ID support addresses the enterprise and public sector market.
-
-### Negative
-
-- The OAuth state machine (device code polling, token refresh, error handling) adds implementation complexity compared to a static token approach.
-- Developers must have network access to `sso.nationtech.io` and `secrets.nationtech.io` to pull or push configuration state. True offline work falls back to the local file store, which does not sync with the team.
-- The first login per machine requires a browser interaction. Fully headless first-run scenarios (e.g., a fresh CI runner with no pre-seeded tokens) must use `EnvSource` overrides or a service account JWT.

docs/adr/020-interactive-configuration-crate.md

@@ -1,177 +0,0 @@
-# ADR 020: Unified Configuration and Secret Management
-
-Author: Jean-Gabriel Gill-Couture
-
-Date: 2026-03-18
-
-## Status
-
-Proposed
-
-## Context
-
-Harmony's orchestration logic depends on runtime data that falls into two categories:
-
-1. **Secrets** — credentials, tokens, private keys.
-2. **Operational configuration** — deployment targets, host selections, port assignments, reboot decisions, and similar contextual choices.
-
-Both categories share the same fundamental lifecycle: a value must be acquired before execution can proceed, it may come from several backends (environment variable, remote store, interactive prompt), and it must be shareable across a team without polluting the Git repository.
-
-Treating these categories as separate subsystems forces developers to choose between a "config API" and a "secret API" at every call site. The only meaningful difference between the two is how the storage backend handles the data (plaintext vs. encrypted, audited vs. unaudited) and how the CLI displays it (visible vs. masked). That difference belongs in the backend, not in the application code.
-
-Three concrete problems drive this change:
-
-- **Async terminal corruption.** `inquire` prompts assume exclusive terminal ownership. Background tokio tasks emitting log output during a prompt corrupt the terminal state. This is inherent to Harmony's concurrent orchestration model.
-- **Untestable code paths.** Any function containing an inline `inquire` call requires a real TTY to execute. Unit testing is impossible without ignoring the test entirely.
-- **No backend integration.** Inline prompts cannot be answered from a remote store, an environment variable, or a CI pipeline. Every automated deployment that passes through a prompting code path requires a human operator at a terminal.
-
-## Decision
-
-A single workspace crate, `harmony_config`, provides all configuration and secret acquisition for Harmony. It replaces both `harmony_secret` and all inline `inquire` usage.
-
-### Schema in Git, state in the store
-
-The Rust type system serves as the configuration schema. Developers declare what configuration is needed by defining structs:
-
-```rust
-#[derive(Config, Serialize, Deserialize, JsonSchema, InteractiveParse)]
-struct PostgresConfig {
-    pub host: String,
-    pub port: u16,
-    #[config(secret)]
-    pub password: String,
-}
-```
-
-These structs live in Git and evolve with the code. When a branch introduces a new field, Git tracks that schema change. The actual values live in an external store — OpenBao by default. No `.env` files, no JSON config files, no YAML in the repository.
-
-### Data classification
-
-```rust
-/// Tells the storage backend how to handle the data.
-pub enum ConfigClass {
-    /// Plaintext storage is acceptable.
-    Standard,
-    /// Must be encrypted at rest, masked in UI, subject to audit logging.
-    Secret,
-}
-```
-
-Classification is determined at the struct level. A struct with no `#[config(secret)]` fields has `ConfigClass::Standard`. A struct with one or more `#[config(secret)]` fields is elevated to `ConfigClass::Secret`. The struct is always stored as a single cohesive JSON blob; field-level splitting across backends is not a concern of the trait.
-
-The `#[config(secret)]` attribute also instructs the `PromptSource` to mask terminal input for that field during interactive prompting.
-
-### The Config trait
-
-```rust
-pub trait Config: Serialize + DeserializeOwned + JsonSchema + InteractiveParseObj + Sized {
-    /// Stable lookup key. By default, the struct name.
-    const KEY: &'static str;
-
-    /// How the backend should treat this data.
-    const CLASS: ConfigClass;
-}
-```
-
-A `#[derive(Config)]` proc macro generates the implementation. The macro inspects field attributes to determine `CLASS`.
-
-### The ConfigStore trait
-
-```rust
-#[async_trait]
-pub trait ConfigStore: Send + Sync {
-    async fn get(
-        &self,
-        class: ConfigClass,
-        namespace: &str,
-        key: &str,
-    ) -> Result<Option<serde_json::Value>, ConfigError>;
-
-    async fn set(
-        &self,
-        class: ConfigClass,
-        namespace: &str,
-        key: &str,
-        value: &serde_json::Value,
-    ) -> Result<(), ConfigError>;
-}
-```
-
-The `class` parameter is a hint. The store implementation decides what to do with it. An OpenBao store may route `Secret` data to a different path prefix or apply stricter ACLs. A future store could split fields across backends — that is an implementation concern, not a trait concern.
-
-### Resolution chain
-
-The `ConfigManager` tries sources in priority order:
-
-1. **`EnvSource`** — reads `HARMONY_CONFIG_{KEY}` as a JSON string. Override hatch for CI/CD pipelines and containerized environments.
-2. **`StoreSource`** — wraps a `ConfigStore` implementation. For teams, this is the OpenBao backend authenticated via Zitadel OIDC (see ADR 020-1).
-3. **`PromptSource`** — presents an `interactive-parse` prompt on the terminal. Acquires a process-wide async mutex before rendering to prevent log output corruption.
-
-When `PromptSource` obtains a value, the `ConfigManager` persists it back to the `StoreSource` so that subsequent runs — by the same developer or any teammate — resolve without prompting.
-
-Callers that do not include `PromptSource` in their source list never block on a TTY. Test code passes empty source lists and constructs config structs directly.
-
-### Schema versioning
-
-The Rust struct is the schema. When a developer renames a field, removes a field, or changes a type on a branch, the store may still contain data shaped for a previous version of the struct. If another team member who does not yet have that commit runs the code, `serde_json::from_value` will fail on the stale entry.
-
-In the initial implementation, the resolution chain handles this gracefully: a deserialization failure is treated as a cache miss, and the `PromptSource` fires. The prompted value overwrites the stale entry in the store.
-
-This is sufficient for small teams working on short-lived branches. It is not sufficient at scale, where silent re-prompting could mask real configuration drift.
-
-A future iteration will introduce a compile-time schema migration mechanism, similar to how `sqlx` verifies queries against a live database at compile time. The mechanism will:
-
-- Detect schema drift between the Rust struct and the stored JSON.
-- Apply named, ordered migration functions to transform stored data forward.
-- Reject ambiguous migrations at compile time rather than silently corrupting state.
-
-Until that mechanism exists, teams should treat store entries as soft caches: the struct definition is always authoritative, and the store is best-effort.
-
-## Rationale
-
-**Why merge secrets and config into one crate?** Separate crates with nearly identical trait shapes (`Secret` vs `Config`, `SecretStore` vs `ConfigStore`) force developers to make a classification decision at every call site. A unified crate with a `ConfigClass` discriminator moves that decision to the struct definition, where it belongs.
-
-**Why OpenBao as the default backend?** OpenBao is a fully open-source Vault fork under the Linux Foundation. It runs on-premises with no phone-home requirement — a hard constraint for private cloud and regulated environments. Harmony already deploys OpenBao for clients (`OpenbaoScore`), so no new infrastructure is introduced.
-
-**Why not store values in Git (e.g., encrypted YAML)?** Git-tracked config files create merge conflicts, require re-encryption on team membership changes, and leak metadata (file names, key names) even when values are encrypted. Storing state in OpenBao avoids all of these issues and provides audit logging, access control, and versioned KV out of the box.
-
-**Why keep `PromptSource`?** Removing interactive prompts entirely would break the zero-infrastructure bootstrapping path and eliminate human-confirmation safety gates for destructive operations (interface reconfiguration, node reboot). The problem was never that prompts exist — it is that they were unavoidable and untestable. Making `PromptSource` an explicit, opt-in entry in the source list restores control.
-
-## Consequences
-
-### Positive
-
-- A single API surface for all runtime data acquisition.
-- All currently-ignored tests become runnable without TTY access.
-- Async terminal corruption is eliminated by the process-wide prompt mutex.
-- The bootstrapping path requires no infrastructure for a first run; `PromptSource` alone is sufficient.
-- The team path (OpenBao + Zitadel) reuses infrastructure Harmony already deploys.
-- User offboarding is a single Zitadel action.
-
-### Negative
-
-- Migrating all inline `inquire` and `harmony_secret` call sites is a significant refactoring effort.
-- Until the schema migration mechanism is built, store entries for renamed or removed fields become stale and must be re-prompted.
-- The Zitadel device flow introduces a browser step on first login per machine.
-
-## Implementation Plan
-
-### Phase 1: Trait design and crate restructure
-
-Refactor `harmony_config` to define the final `Config`, `ConfigClass`, and `ConfigStore` traits. Update the derive macro to support `#[config(secret)]` and generate the correct `CLASS` constant. Implement `EnvSource` and `PromptSource` against the new traits. Write comprehensive unit tests using mock stores.
-
-### Phase 2: Absorb `harmony_secret`
-
-Migrate the `OpenbaoSecretStore`, `InfisicalSecretStore`, and `LocalFileSecretStore` implementations from `harmony_secret` into `harmony_config` as `ConfigStore` backends. Update all call sites that use `SecretManager::get`, `SecretManager::get_or_prompt`, or `SecretManager::set` to use `harmony_config` equivalents.
-
-### Phase 3: Migrate inline prompts
-
-Replace all inline `inquire` call sites in the `harmony` crate (`infra/brocade.rs`, `infra/network_manager.rs`, `modules/okd/host_network.rs`, and others) with `harmony_config` structs and `get_or_prompt` calls. Un-ignore the affected tests.
-
-### Phase 4: Zitadel and OpenBao integration
-
-Implement the authentication flow described in ADR 020-1. Wire `StoreSource` to use Zitadel OIDC tokens for OpenBao access. Implement token caching and silent refresh.
-
-### Phase 5: Remove `harmony_secret`
-
-Delete the `harmony_secret` and `harmony_secret_derive` crates from the workspace. All functionality now lives in `harmony_config`.

docs/adr/README.md

@@ -1,63 +0,0 @@
-# Architecture Decision Records
-
-An Architecture Decision Record (ADR) documents a significant architectural decision made during the development of Harmony — along with its context, rationale, and consequences.
-
-## Why We Use ADRs
-
-As a platform engineering framework used by a team, Harmony accumulates technical decisions over time. ADRs help us:
-
-- **Track rationale** — understand _why_ a decision was made, not just _what_ was decided
-- ** onboard new contributors** — the "why" is preserved even when team membership changes
-- **Avoid repeating past mistakes** — previous decisions and their context are searchable
-- **Manage technical debt** — ADRs make it easier to revisit and revise past choices
-
-An ADR captures a decision at a point in time. It is not a specification — it is a record of reasoning.
-
-## ADR Format
-
-Every ADR follows this structure:
-
-| Section | Purpose |
-|---------|---------|
-| **Status** | Proposed / Pending / Accepted / Implemented / Deprecated |
-| **Context** | The problem or background — the "why" behind this decision |
-| **Decision** | The chosen solution or direction |
-| **Rationale** | Reasoning behind the decision |
-| **Consequences** | Both positive and negative outcomes |
-| **Alternatives considered** | Other options that were evaluated |
-| **Additional Notes** | Supplementary context, links, or open questions |
-
-## ADR Index
-
-| Number | Title | Status |
-|--------|-------|--------|
-| [000](./000-ADR-Template.md) | ADR Template | Reference |
-| [001](./001-rust.md) | Why Rust | Accepted |
-| [002](./002-hexagonal-architecture.md) | Hexagonal Architecture | Accepted |
-| [003](./003-infrastructure-abstractions.md) | Infrastructure Abstractions | Accepted |
-| [004](./004-ipxe.md) | iPXE | Accepted |
-| [005](./005-interactive-project.md) | Interactive Project | Proposed |
-| [006](./006-secret-management.md) | Secret Management | Accepted |
-| [007](./007-default-runtime.md) | Default Runtime | Accepted |
-| [008](./008-score-display-formatting.md) | Score Display Formatting | Proposed |
-| [009](./009-helm-and-kustomize-handling.md) | Helm and Kustomize Handling | Accepted |
-| [010](./010-monitoring-and-alerting.md) | Monitoring and Alerting | Accepted |
-| [011](./011-multi-tenant-cluster.md) | Multi-Tenant Cluster | Accepted |
-| [012](./012-project-delivery-automation.md) | Project Delivery Automation | Proposed |
-| [013](./013-monitoring-notifications.md) | Monitoring Notifications | Accepted |
-| [015](./015-higher-order-topologies.md) | Higher Order Topologies | Proposed |
-| [016](./016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md) | Harmony Agent and Global Mesh | Proposed |
-| [017-1](./017-1-Nats-Clusters-Interconnection-Topology.md) | NATS Clusters Interconnection Topology | Proposed |
-| [018](./018-Template-Hydration-For-Workload-Deployment.md) | Template Hydration for Workload Deployment | Proposed |
-| [019](./019-Network-bond-setup.md) | Network Bond Setup | Proposed |
-| [020-1](./020-1-zitadel-openbao-secure-config-store.md) | Zitadel + OpenBao Secure Config Store | Accepted |
-| [020](./020-interactive-configuration-crate.md) | Interactive Configuration Crate | Proposed |
-
-## Contributing
-
-When making a significant technical change:
-
-1. **Check existing ADRs** — the decision may already be documented
-2. **Create a new ADR** using the [template](./000-ADR-Template.md) if the change warrants architectural discussion
-3. **Set status to Proposed** and open it for team review
-4. Once accepted and implemented, update the status accordingly

docs/catalogs/README.md

@@ -1,7 +0,0 @@
-# Component Catalogs
-
-This section is the "dictionary" for Harmony. It lists all the reusable components available out-of-the-box.
-
-- [**Scores Catalog**](./scores.md): Discover all available `Scores` (the "what").
-- [**Topologies Catalog**](./topologies.md): A list of all available `Topologies` (the "where").
-- [**Capabilities Catalog**](./capabilities.md): A list of all available `Capabilities` (the "how").

docs/catalogs/capabilities.md

@@ -1,40 +0,0 @@
-# Capabilities Catalog
-
-A `Capability` is a specific feature or API that a `Topology` offers. `Interpret` logic uses these capabilities to execute a `Score`.
-
-This list is primarily for developers **writing new Topologies or Scores**. As a user, you just need to know that the `Topology` you pick (like `K8sAnywhereTopology`) provides the capabilities your `Scores` (like `ApplicationScore`) need.
-
-<!--toc:start-->
-
-- [Capabilities Catalog](#capabilities-catalog)
-  - [Kubernetes & Application](#kubernetes-application)
-  - [Monitoring & Observability](#monitoring-observability)
-  - [Networking (Core Services)](#networking-core-services)
-  - [Networking (Hardware & Host)](#networking-hardware-host)
-
-<!--toc:end-->
-
-## Kubernetes & Application
-
-- **K8sClient**: Provides an authenticated client to interact with a Kubernetes API (create/read/update/delete resources).
-- **HelmCommand**: Provides the ability to execute Helm commands (install, upgrade, template).
-- **TenantManager**: Provides methods for managing tenants in a multi-tenant cluster.
-- **Ingress**: Provides an interface for managing ingress controllers and resources.
-
-## Monitoring & Observability
-
-- **Grafana**: Provides an API for configuring Grafana (datasources, dashboards).
-- **Monitoring**: A general capability for configuring monitoring (e.g., creating Prometheus rules).
-
-## Networking (Core Services)
-
-- **DnsServer**: Provides an interface for creating and managing DNS records.
-- **LoadBalancer**: Provides an interface for configuring a load balancer (e.g., OPNsense, MetalLB).
-- **DhcpServer**: Provides an interface for managing DHCP leases and host bindings.
-- **TftpServer**: Provides an interface for managing files on a TFTP server (e.g., iPXE boot files).
-
-## Networking (Hardware & Host)
-
-- **Router**: Provides an interface for configuring routing rules, typically on a firewall like OPNsense.
-- **Switch**: Provides an interface for configuring a physical network switch (e.g., managing VLANs and port channels).
-- **NetworkManager**: Provides an interface for configuring host-level networking (e.g., creating bonds and bridges on a node).

docs/catalogs/scores.md

@@ -1,102 +0,0 @@
-# Scores Catalog
-
-A `Score` is a declarative description of a desired state. Find the Score you need and add it to your `harmony!` block's `scores` array.
-
-<!--toc:start-->
-
-- [Scores Catalog](#scores-catalog)
-  - [Application Deployment](#application-deployment)
-  - [OKD / Kubernetes Cluster Setup](#okd-kubernetes-cluster-setup)
-  - [Cluster Services & Management](#cluster-services-management)
-  - [Monitoring & Alerting](#monitoring-alerting)
-  - [Infrastructure & Networking (Bare Metal)](#infrastructure-networking-bare-metal)
-  - [Infrastructure & Networking (Cluster)](#infrastructure-networking-cluster)
-  - [Tenant Management](#tenant-management)
-  - [Utility](#utility)
-
-<!--toc:end-->
-
-## Application Deployment
-
-Scores for deploying and managing end-user applications.
-
-- **ApplicationScore**: The primary score for deploying a web application. Describes the application, its framework, and the features it requires (e.g., monitoring, CI/CD).
-- **HelmChartScore**: Deploys a generic Helm chart to a Kubernetes cluster.
-- **ArgoHelmScore**: Deploys an application using an ArgoCD Helm chart.
-- **LAMPScore**: A specialized score for deploying a classic LAMP (Linux, Apache, MySQL, PHP) stack.
-
-## OKD / Kubernetes Cluster Setup
-
-This collection of Scores is used to provision an entire OKD cluster from bare metal. They are typically used in order.
-
-- **OKDSetup01InventoryScore**: Discovers and catalogs the physical hardware.
-- **OKDSetup02BootstrapScore**: Configures the bootstrap node, renders iPXE files, and kicks off the SCOS installation.
-- **OKDSetup03ControlPlaneScore**: Renders iPXE configurations for the control plane nodes.
-- **OKDSetupPersistNetworkBondScore**: Configures network bonds on the nodes and port channels on the switches.
-- **OKDSetup04WorkersScore**: Renders iPXE configurations for the worker nodes.
-- **OKDSetup06InstallationReportScore**: Runs post-installation checks and generates a report.
-- **OKDUpgradeScore**: Manages the upgrade process for an existing OKD cluster.
-
-## Cluster Services & Management
-
-Scores for installing and managing services _inside_ a Kubernetes cluster.
-
-- **K3DInstallationScore**: Installs and configes a local K3D (k3s-in-docker) cluster. Used by `K8sAnywhereTopology`.
-- **CertManagerHelmScore**: Deploys the `cert-manager` Helm chart.
-- **ClusterIssuerScore**: Configures a `ClusterIssuer` for `cert-manager`, (e.g., for Let's Encrypt).
-- **K8sNamespaceScore**: Ensures a Kubernetes namespace exists.
-- **K8sDeploymentScore**: Deploys a generic `Deployment` resource to Kubernetes.
-- **K8sIngressScore**: Configures an `Ingress` resource for a service.
-
-## Monitoring & Alerting
-
-Scores for configuring observability, dashboards, and alerts.
-
-- **ApplicationMonitoringScore**: A generic score to set up monitoring for an application.
-- **ApplicationRHOBMonitoringScore**: A specialized score for setting up monitoring via the Red Hat Observability stack.
-- **HelmPrometheusAlertingScore**: Configures Prometheus alerts via a Helm chart.
-- **K8sPrometheusCRDAlertingScore**: Configures Prometheus alerts using the `PrometheusRule` CRD.
-- **PrometheusAlertScore**: A generic score for creating a Prometheus alert.
-- **RHOBAlertingScore**: Configures alerts specifically for the Red Hat Observability stack.
-- **NtfyScore**: Configures alerts to be sent to a `ntfy.sh` server.
-
-## Infrastructure & Networking (Bare Metal)
-
-Low-level scores for managing physical hardware and network services.
-
-- **DhcpScore**: Configures a DHCP server.
-- **OKDDhcpScore**: A specialized DHCP configuration for the OKD bootstrap process.
-- **OKDBootstrapDhcpScore**: Configures DHCP specifically for the bootstrap node.
-- **DhcpHostBindingScore**: Creates a specific MAC-to-IP binding in the DHCP server.
-- **DnsScore**: Configures a DNS server.
-- **OKDDnsScore**: A specialized DNS configuration for the OKD cluster (e.g., `api.*`, `*.apps.*`).
-- **StaticFilesHttpScore**: Serves a directory of static files (e.g., a documentation site) over HTTP.
-- **TftpScore**: Configures a TFTP server, typically for serving iPXE boot files.
-- **IPxeMacBootFileScore**: Assigns a specific iPXE boot file to a MAC address in the TFTP server.
-- **OKDIpxeScore**: A specialized score for generating the iPXE boot scripts for OKD.
-- **OPNsenseShellCommandScore**: Executes a shell command on an OPNsense firewall.
-
-## Infrastructure & Networking (Cluster)
-
-Network services that run inside the cluster or as part of the topology.
-
-- **LoadBalancerScore**: Configures a general-purpose load balancer.
-- **OKDLoadBalancerScore**: Configures the high-availability load balancers for the OKD API and ingress.
-- **OKDBootstrapLoadBalancerScore**: Configures the load balancer specifically for the bootstrap-time API endpoint.
-- **K8sIngressScore**: Configures an Ingress controller or resource.
-- **HighAvailabilityHostNetworkScore**: Configures network bonds on a host and the corresponding port-channels on the switch stack for high-availability.
-
-## Tenant Management
-
-Scores for managing multi-tenancy within a cluster.
-
-- **TenantScore**: Creates a new tenant (e.g., a namespace, quotas, network policies).
-- **TenantCredentialScore**: Generates and provisions credentials for a new tenant.
-
-## Utility
-
-Helper scores for discovery and inspection.
-
-- **LaunchDiscoverInventoryAgentScore**: Launches the agent responsible for the `OKDSetup01InventoryScore`.
-- **DiscoverHostForRoleScore**: A utility score to find a host matching a specific role in the inventory.
-- **InspectInventoryScore**: Dumps the discovered inventory for inspection.

docs/catalogs/topologies.md

@@ -1,59 +0,0 @@
-# Topologies Catalog
-
-A `Topology` is the logical representation of your infrastructure and its `Capabilities`. You select a `Topology` in your Harmony project to define _where_ your `Scores` will be applied.
-
-<!--toc:start-->
-
-- [Topologies Catalog](#topologies-catalog)
-  - [HAClusterTopology](#haclustertopology)
-  - [K8sAnywhereTopology](#k8sanywheretopology)
-
-<!--toc:end-->
-
-### HAClusterTopology
-
-- **`HAClusterTopology::autoload()`**
-
-This `Topology` represents a high-availability, bare-metal cluster. It is designed for production-grade deployments like OKD.
-
-It models an environment consisting of:
-
-- At least 3 cluster nodes (for control plane/workers)
-- 2 redundant firewalls (e.g., OPNsense)
-- 2 redundant network switches
-
-**Provided Capabilities:**
-This topology provides a rich set of capabilities required for bare-metal provisioning and cluster management, including:
-
-- `K8sClient` (once the cluster is bootstrapped)
-- `DnsServer`
-- `LoadBalancer`
-- `DhcpServer`
-- `TftpServer`
-- `Router` (via the firewalls)
-- `Switch`
-- `NetworkManager` (for host-level network config)
-
----
-
-### K8sAnywhereTopology
-
-- **`K8sAnywhereTopology::from_env()`**
-
-This `Topology` is designed for development and application deployment. It provides a simple, abstract way to deploy to _any_ Kubernetes cluster.
-
-**How it works:**
-
-1. By default (`from_env()` with no env vars), it automatically provisions a **local K3D (k3s-in-docker) cluster** on your machine. This is perfect for local development and testing.
-2. If you provide a `KUBECONFIG` environment variable, it will instead connect to that **existing Kubernetes cluster** (e.g., your staging or production OKD cluster).
-
-This allows you to use the _exact same code_ to deploy your application locally as you do to deploy it to production.
-
-**Provided Capabilities:**
-
-- `K8sClient`
-- `HelmCommand`
-- `TenantManager`
-- `Ingress`
-- `Monitoring`
-- ...and more.

docs/coding-guide.md

@@ -1,229 +0,0 @@
-# Harmony Coding Guide
-
-Harmony is an infrastructure automation framework. It is **code-first and code-only**: operators write Rust programs to declare and drive infrastructure, rather than YAML files or DSL configs. Good code here means a good operator experience.
-
-### Concrete context
-
-We use here the context of the KVM module to explain the coding style. This will make it very easy to understand and should translate quite well to other modules/contexts managed by Harmony like OPNSense and Kubernetes.
-
-## Core Philosophy
-
-### High-level functions over raw primitives
-
-Callers should not need to know about underlying protocols, XML schemas, or API quirks. A function that deploys a VM should accept meaningful parameters like CPU count, memory, and network name — not XML strings.
-
-```rust
-// Bad: caller constructs XML and passes it to a thin wrapper
-let xml = format!(r#"<domain type='kvm'>...</domain>"#, name, memory_kb, ...);
-executor.create_vm(&xml).await?;
-
-// Good: caller describes intent, the module handles representation
-executor.define_vm(&VmConfig::builder("my-vm")
-    .cpu(4)
-    .memory_gb(8)
-    .disk(DiskConfig::new(50))
-    .network(NetworkRef::named("mylan"))
-    .boot_order([BootDevice::Network, BootDevice::Disk])
-    .build())
-    .await?;
-```
-
-The module owns the XML, the virsh invocations, the API calls — not the caller.
-
-### Use the right abstraction layer
-
-Prefer native library bindings over shelling out to CLI tools. The `virt` crate provides direct libvirt bindings and should be used instead of spawning `virsh` subprocesses.
-
-- CLI subprocess calls are fragile: stdout/stderr parsing, exit codes, quoting, PATH differences
-- Native bindings give typed errors, no temp files, no shell escaping
-- `virt::connect::Connect` opens a connection; `virt::domain::Domain` manages VMs; `virt::network::Network` manages virtual networks
-
-### Keep functions small and well-named
-
-Each function should do one thing. If a function is doing two conceptually separate things, split it. Function names should read like plain English: `ensure_network_active`, `define_vm`, `vm_is_running`.
-
-### Prefer short modules over large files
-
-Group related types and functions by concept. A module that handles one resource (e.g., network, domain, storage) is better than a single file for everything.
-
----
-
-## Error Handling
-
-### Use `thiserror` for all error types
-
-Define error types with `thiserror::Error`. This removes the boilerplate of implementing `Display` and `std::error::Error` by hand, keeps error messages close to their variants, and makes types easy to extend.
-
-```rust
-// Bad: hand-rolled Display + std::error::Error
-#[derive(Debug)]
-pub enum KVMError {
-    ConnectionError(String),
-    VMNotFound(String),
-}
-
-impl std::fmt::Display for KVMError { ... }
-impl std::error::Error for KVMError {}
-
-// Good: derive Display via thiserror
-#[derive(thiserror::Error, Debug)]
-pub enum KVMError {
-    #[error("connection failed: {0}")]
-    ConnectionFailed(String),
-    #[error("VM not found: {name}")]
-    VmNotFound { name: String },
-}
-```
-
-### Make bubbling errors easy with `?` and `From`
-
-`?` works on any error type for which there is a `From` impl. Add `From` conversions from lower-level errors into your module's error type so callers can use `?` without boilerplate.
-
-With `thiserror`, wrapping a foreign error is one line:
-
-```rust
-#[derive(thiserror::Error, Debug)]
-pub enum KVMError {
-    #[error("libvirt error: {0}")]
-    Libvirt(#[from] virt::error::Error),
-
-    #[error("IO error: {0}")]
-    Io(#[from] std::io::Error),
-}
-```
-
-This means a call that returns `virt::error::Error` can be `?`-propagated into a `Result<_, KVMError>` without any `.map_err(...)`.
-
-### Typed errors over stringly-typed errors
-
-Avoid `Box<dyn Error>` or `String` as error return types in library code. Callers need to distinguish errors programmatically — `KVMError::VmAlreadyExists` is actionable, `"VM already exists: foo"` as a `String` is not.
-
-At binary entry points (e.g., `main`) it is acceptable to convert to `String` or `anyhow::Error` for display.
-
----
-
-## Logging
-
-### Use the `log` crate macros
-
-All log output must go through the `log` crate. Never use `println!`, `eprintln!`, or `dbg!` in library code. This makes output compatible with any logging backend (env_logger, tracing, structured logging, etc.).
-
-```rust
-// Bad
-println!("Creating VM: {}", name);
-
-// Good
-use log::{info, debug, warn};
-info!("Creating VM: {name}");
-debug!("VM XML:\n{xml}");
-warn!("Network already active, skipping creation");
-```
-
-Use the right level:
-
-| Level   | When to use |
-|---------|-------------|
-| `error` | Unrecoverable failures (before returning Err) |
-| `warn`  | Recoverable issues, skipped steps |
-| `info`  | High-level progress events visible in normal operation |
-| `debug` | Detailed operational info useful for debugging |
-| `trace` | Very granular, per-iteration or per-call data |
-
-Log before significant operations and after unexpected conditions. Do not log inside tight loops at `info` level.
-
----
-
-## Types and Builders
-
-### Derive `Serialize` on all public domain types
-
-All public structs and enums that represent configuration or state should derive `serde::Serialize`. Add `Deserialize` when round-trip serialization is needed.
-
-### Builder pattern for complex configs
-
-When a type has more than three fields or optional fields, provide a builder. The builder pattern allows named, incremental construction without positional arguments.
-
-```rust
-let config = VmConfig::builder("bootstrap")
-    .cpu(4)
-    .memory_gb(8)
-    .disk(DiskConfig::new(50).labeled("os"))
-    .disk(DiskConfig::new(100).labeled("data"))
-    .network(NetworkRef::named("harmonylan"))
-    .boot_order([BootDevice::Network, BootDevice::Disk])
-    .build();
-```
-
-### Avoid `pub` fields on config structs
-
-Expose data through methods or the builder, not raw field access. This preserves the ability to validate, rename, or change representation without breaking callers.
-
----
-
-## Async
-
-### Use `tokio` for all async runtime needs
-
-All async code runs on tokio. Use `tokio::spawn`, `tokio::time`, etc. Use `#[async_trait]` for traits with async methods.
-
-### No blocking in async context
-
-Never call blocking I/O (file I/O, network, process spawn) directly in an async function. Use `tokio::fs`, `tokio::process`, or `tokio::task::spawn_blocking` as appropriate.
-
----
-
-## Module Structure
-
-### Follow the `Score` / `Interpret` pattern
-
-Modules that represent deployable infrastructure should implement `Score<T: Topology>` and `Interpret<T>`:
-
-- `Score` is the serializable, clonable configuration declaring *what* to deploy
-- `Interpret` does the actual work when `execute()` is called
-
-```rust
-pub struct KvmScore {
-    network: NetworkConfig,
-    vms: Vec<VmConfig>,
-}
-
-impl<T: Topology + KvmHost> Score<T> for KvmScore {
-    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(KvmInterpret::new(self.clone()))
-    }
-    fn name(&self) -> String { "KvmScore".to_string() }
-}
-```
-
-### Flatten the public API in `mod.rs`
-
-Internal submodules are implementation detail. Re-export what callers need at the module root:
-
-```rust
-// modules/kvm/mod.rs
-mod connection;
-mod domain;
-mod network;
-mod error;
-mod xml;
-
-pub use connection::KvmConnection;
-pub use domain::{VmConfig, VmConfigBuilder, VmStatus, DiskConfig, BootDevice};
-pub use error::KvmError;
-pub use network::NetworkConfig;
-```
-
----
-
-## Commit Style
-
-Follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/):
-
-```
-feat(kvm): add network isolation support
-fix(kvm): correct memory unit conversion for libvirt
-refactor(kvm): replace virsh subprocess calls with virt crate bindings
-docs: add coding guide
-```
-
-Keep pull requests small and single-purpose (under ~200 lines excluding generated code). Do not mix refactoring, bug fixes, and new features in one PR.

docs/concepts.md

@@ -1,45 +0,0 @@
-# Core Concepts
-
-Harmony's design is based on a few key concepts. Understanding them is the key to unlocking the framework's power.
-
-### 1. Score
-
-- **What it is:** A **Score** is a declarative description of a desired state. It's a "resource" that defines _what_ you want to achieve, not _how_ to do it.
-- **Example:** `ApplicationScore` declares "I want this web application to be running and monitored."
-
-### 2. Topology
-
-- **What it is:** A **Topology** is the logical representation of your infrastructure and its abilities. It's the "where" your Scores will be applied.
-- **Key Job:** A Topology's most important job is to expose which `Capabilities` it supports.
-- **Example:** `HAClusterTopology` represents a bare-metal cluster and exposes `Capabilities` like `NetworkManager` and `Switch`. `K8sAnywhereTopology` represents a Kubernetes cluster and exposes the `K8sClient` `Capability`.
-
-### 3. Capability
-
-- **What it is:** A **Capability** is a specific feature or API that a `Topology` offers. It's the "how" a `Topology` can fulfill a `Score`'s request.
-- **Example:** The `K8sClient` capability offers a way to interact with a Kubernetes API. The `Switch` capability offers a way to configure a physical network switch.
-
-### 4. Interpret
-
-- **What it is:** An **Interpret** is the execution logic that makes a `Score` a reality. It's the "glue" that connects the _desired state_ (`Score`) to the _environment's abilities_ (`Topology`'s `Capabilities`).
-- **How it works:** When you apply a `Score`, Harmony finds the matching `Interpret` for your `Topology`. This `Interpret` then uses the `Capabilities` provided by the `Topology` to execute the necessary steps.
-
-### 5. Inventory
-
-- **What it is:** An **Inventory** is the physical material (the "what") used in a cluster. This is most relevant for bare-metal or on-premise topologies.
-- **Example:** A list of nodes with their roles (control plane, worker), CPU, RAM, and network interfaces. For the `K8sAnywhereTopology`, the inventory might be empty or autoloaded, as the infrastructure is more abstract.
-
-### 6. Configuration & Secrets
-
-- **What it is:** Configuration represents the runtime data required to deploy your `Scores`. This includes both non-sensitive state (like cluster hostnames, deployment profiles) and sensitive secrets (like API keys, database passwords).
-- **How it works:** See the [Configuration Concept Guide](./concepts/configuration.md) to understand Harmony's unified approach to managing schema in Git and state in OpenBao.
-
----
-
-### How They Work Together (The Compile-Time Check)
-
-1. You **write a `Score`** (e.g., `ApplicationScore`).
-2. Your `Score`'s `Interpret` logic requires certain **`Capabilities`** (e.g., `K8sClient` and `Ingress`).
-3. You choose a **`Topology`** to run it on (e.g., `HAClusterTopology`).
-4. **At compile-time**, Harmony checks: "Does `HAClusterTopology` provide the `K8sClient` and `Ingress` capabilities that `ApplicationScore` needs?"
-   - **If Yes:** Your code compiles. You can be confident it will run.
-   - **If No:** The compiler gives you an error. You've just prevented a "config-is-valid-but-platform-is-wrong" runtime error before you even deployed.

docs/concepts/configuration.md

@@ -1,107 +0,0 @@
-# Configuration and Secrets
-
-Harmony treats configuration and secrets as a single concern. Developers use one crate, `harmony_config`, to declare, store, and retrieve all runtime data — whether it is a public hostname or a database password.
-
-## The mental model: schema in Git, state in the store
-
-### Schema
-
-In Harmony, the Rust code is the configuration schema. You declare what your module needs by defining a struct:
-
-```rust
-#[derive(Config, Serialize, Deserialize, JsonSchema, InteractiveParse)]
-struct PostgresConfig {
-    pub host: String,
-    pub port: u16,
-    #[config(secret)]
-    pub password: String,
-}
-```
-
-This struct is tracked in Git. When a branch adds a new field, Git tracks that the branch requires a new value. When a branch removes a field, the old value in the store becomes irrelevant. The struct is always authoritative.
-
-### State
-
-The actual values live in a config store — by default, OpenBao. No `.env` files, no JSON, no YAML in the repository.
-
-When you run your code, Harmony reads the struct (schema) and resolves values from the store (state):
-
-- If the store has the value, it is injected seamlessly.
-- If the store does not have it, Harmony prompts you in the terminal. Your answer is pushed back to the store automatically.
-- When a teammate runs the same code, they are not prompted — you already provided the value.
-
-### How branch switching works
-
-Because the schema is just Rust code tracked in Git, branch switching works naturally:
-
-1. You check out `feat/redis`. The code now requires `RedisConfig`.
-2. You run `cargo run`. Harmony detects that `RedisConfig` has no value in the store. It prompts you.
-3. You provide the values. Harmony pushes them to OpenBao.
-4. Your teammate checks out `feat/redis` and runs `cargo run`. No prompt — the values are already in the store.
-5. You switch back to `main`. `RedisConfig` does not exist in that branch's code. The store entry is ignored.
-
-## Secrets vs. standard configuration
-
-From your application code, there is no difference. You always call `harmony_config::get_or_prompt::<T>()`.
-
-The difference is in the struct definition:
-
-```rust
-// Standard config — stored in plaintext, displayed during prompting.
-#[derive(Config)]
-struct ClusterConfig {
-    pub api_url: String,
-    pub namespace: String,
-}
-
-// Contains a secret field — the entire struct is stored encrypted,
-// and the password field is masked during terminal prompting.
-#[derive(Config)]
-struct DatabaseConfig {
-    pub host: String,
-    #[config(secret)]
-    pub password: String,
-}
-```
-
-If a struct contains any `#[config(secret)]` field, Harmony elevates the entire struct to `ConfigClass::Secret`. The storage backend decides what that means in practice — in the case of OpenBao, it may route the data to a path with stricter ACLs or audit policies.
-
-## Authentication and team sharing
-
-Harmony uses Zitadel (hosted at `sso.nationtech.io`) for identity and OpenBao (hosted at `secrets.nationtech.io`) for storage.
-
-**First run on a new machine:**
-
-1. Harmony detects that you are not logged in.
-2. It prints a short code and URL to your terminal, and opens your browser if possible.
-3. You log in with your corporate identity (Google, GitHub, or Microsoft Entra ID / Azure AD).
-4. Harmony receives an OIDC token, exchanges it for an OpenBao token, and caches the session locally.
-
-**Subsequent runs:**
-
-- Harmony silently refreshes your tokens in the background. You do not need to log in again for up to 90 days of active use.
-- If you are inactive for 30 days, or if an administrator revokes your access in Zitadel, you will be prompted to re-authenticate.
-
-**Offboarding:**
-
-Revoking a user in Zitadel immediately invalidates their ability to refresh tokens or obtain new ones. No manual secret rotation is required.
-
-## Resolution chain
-
-When Harmony resolves a config value, it tries sources in order:
-
-1. **Environment variable** (`HARMONY_CONFIG_{KEY}`) — highest priority. Use this in CI/CD to override any value without touching the store.
-2. **Config store** (OpenBao for teams, local file for solo/offline use) — the primary source for shared team state.
-3. **Interactive prompt** — last resort. Prompts the developer and persists the answer back to the store.
-
-## Schema versioning
-
-The Rust struct is the single source of truth for what configuration looks like. If a developer renames or removes a field on a branch, the store may still contain data shaped for the old version of the struct. When another developer who does not have that change runs the code, deserialization will fail.
-
-In the current implementation, this is handled gracefully: a deserialization failure is treated as a miss, and Harmony re-prompts. The new answer overwrites the stale entry.
-
-A compile-time migration mechanism is planned for a future release to handle this more rigorously at scale.
-
-## Offline and local development
-
-If you are working offline or evaluating Harmony without a team OpenBao instance, the `StoreSource` falls back to a local file store at `~/.local/share/harmony/config/`. The developer experience is identical — prompting, caching, and resolution all work the same way. The only difference is that the state is local to your machine and not shared with teammates.

docs/doc-clone-and-restore-coreos.md

@@ -1,133 +0,0 @@
-## Working procedure to clone and restore CoreOS disk from OKD Cluster
-
-### **Step 1 - take a backup**
-```
-sudo dd if=/dev/old of=/dev/backup status=progress 
-```
-
-### **Step 2 - clone beginning of old disk to new**
-```
-sudo dd if=/dev/old of=/dev/backup status=progress count=1000 bs=1M
-```
-
-### **Step 3 - verify and modify disk partitions**
-list disk partitions
-```
-sgdisk -p /dev/new
-```
-if new disk is smaller than old disk and there is space on the xfs partition of the old disk, modify partitions of new disk
-```
-gdisk /dev/new
-```
-inside of gdisk commands
-```
--v -> verify table
--p -> print table
--d -> select partition to delete partition
--n -> recreate partition with same partition number as deleted partition
-```
-For end sector, either specify the new end or just press Enter for maximum available 
-When asked about partition type, enter the same type code (it will show the old one) 
-```
-p - >to verify 
-w -> to write
-```
-make xfs file system for new partition <new4>
-```
-sudo mkfs.xfs -f /dev/new4
-```
-
-### **Step 4 - copy old PARTUUID **
-
-**careful here**
-get old patuuid:
-```
-sgdisk -i <partition_number> /dev/old_disk  # Note the "Partition unique GUID"
-```
-get labels
-```
-sgdisk -p /dev/old_disk  # Shows partition names in the table
-
-blkid /dev/old_disk*  # Shows PARTUUIDs and labels for all partitions
-```
-set it on new disk
-```
-sgdisk -u <partition_number>:<old_partuuid> /dev/sdc
-```
-partition name:
-```
-sgdisk -c <partition_number>:"<old_name>" /dev/sdc
-```
-verify all:
-```
-lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/old_disk
-```
-
-### **Step 5 - Mount disks and copy files from old to new disk**
-
-mount files before copy:
-
-```
-mkdir -p /mnt/new
-mkdir -p /mnt/old
-mount /dev/old4 /mnt/old
-mount /dev/new4 /mnt/new
-```
-copy:
-
-with -n flag can run as dry-run
-```
-rsync -aAXHvn --numeric-ids /source/ /destination/
-```
-
-```
-rsync -aAXHv --numeric-ids /source/ /destination/
-```
-
-### **Step 6 - Set correct UUID for new partition 4**
-to set uuid with xfs_admin you must unmount first
-
-unmount old devices
-```
-umount /mnt/new
-umount /mnt/old
-```
-
-to set correct uuid for partition 4
-```
-blkid /dev/old4
-```
-```
-xfs_admin -U <old_uuid> /dev/new_partition
-```
-to set labels
-get it 
-```
-sgdisk -i 4 /dev/sda | grep "Partition name"
-```
-set it
-```
-sgdisk -c 4:"<label_name>" /dev/sdc
-
-or
-
-(check existing with xfs_admin -l /dev/old_partition)
-Use xfs_admin -L <label> /dev/new_partition 
-```
-
-### **Step 7 - Verify**
-
-verify everything:
-```
-sgdisk -p /dev/sda  # Old disk
-sgdisk -p /dev/sdc  # New disk
-```
-```
-lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/sda
-lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/sdc
-```
-```
-blkid /dev/sda* | grep UUID=
-blkid /dev/sdc* | grep UUID=
-```
-

docs/doc-remove-worker-flag.md

@@ -1,56 +0,0 @@
-## **Remove Worker flag from OKD Control Planes** 
-
-### **Context**
-On OKD user provisioned infrastructure the control plane nodes can have the flag node-role.kubernetes.io/worker which allows non critical workloads to be scheduled on the control-planes
-
-### **Observed Symptoms**
-- After adding HAProxy servers to the backend each back end appears down 
-- Traffic is redirected to the control planes instead of workers
-- The pods router-default are incorrectly applied on the control planes rather than on the workers
-- Pods are being scheduled on the control planes causing cluster instability
-
-```
-  ss -tlnp | grep 80
-```
-- shows process haproxy  is listening at 0.0.0.0:80 on cps
-- same problem for port 443
-- In namespace rook-ceph certain pods are deploted on cps rather than on worker nodes
-
- ### **Cause**
- - when intalling UPI, the roles (master, worker) are not managed by the Machine Config operator and the cps are made schedulable by default.
-
- ### **Diagnostic**
-check node labels:
-```
-   oc get nodes --show-labels | grep control-plane
-```
-Inspecter kubelet configuration:
-
-```
-cat /etc/systemd/system/kubelet.service
-```
-
-find the line:
-```
-   --node-labels=node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master,node-role.kubernetes.io/worker
-```
-   → presence of label worker confirms the problem.
-
-Verify the flag doesnt come from MCO
-```
-   oc get machineconfig | grep rendered-master
-```
-
-**Solution:**
-To make the control planes non schedulable you must patch the cluster scheduler resource
-
-```	
-oc patch scheduler cluster --type merge -p '{"spec":{"mastersSchedulable":false}}'
-```
-after the patch is applied the workloads can be deplaced by draining the nodes
-
-```
-oc adm cordon <cp-node>
-oc adm drain <cp-node> --ignore-daemonsets –delete-emptydir-data
-```
-

docs/guides/adding-capabilities.md

@@ -1,135 +0,0 @@
-# Adding Capabilities
-
-`Capabilities` are trait methods that a `Topology` exposes to Scores. They are the "how" — the specific APIs and features that let a Score translate intent into infrastructure actions.
-
-## How Capabilities Work
-
-When a Score declares it needs certain Capabilities:
-
-```rust
-impl<T: Topology + K8sclient + HelmCommand> Score<T> for MyScore {
-    // ...
-}
-```
-
-The compiler verifies that the target `Topology` implements both `K8sclient` and `HelmCommand`. If it doesn't, compilation fails. This is the compile-time safety check that prevents invalid configurations from reaching production.
-
-## Built-in Capabilities
-
-Harmony provides a set of standard Capabilities:
-
-| Capability | What it provides |
-|------------|------------------|
-| `K8sclient` | A Kubernetes API client |
-| `HelmCommand` | A configured `helm` CLI invocation |
-| `TlsRouter` | TLS certificate management |
-| `NetworkManager` | Host network configuration |
-| `SwitchClient` | Network switch configuration |
-| `CertificateManagement` | Certificate issuance via cert-manager |
-
-## Implementing a Capability
-
-Capabilities are implemented as trait methods on your Topology:
-
-```rust
-use std::sync::Arc;
-use harmony_k8s::K8sClient;
-use harmony::topology::K8sclient;
-
-pub struct MyTopology {
-    kubeconfig: Option<String>,
-}
-
-#[async_trait]
-impl K8sclient for MyTopology {
-    async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        let client = match &self.kubeconfig {
-            Some(path) => K8sClient::from_kubeconfig(path).await?,
-            None => K8sClient::try_default().await?,
-        };
-        Ok(Arc::new(client))
-    }
-}
-```
-
-## Adding a Custom Capability
-
-For specialized infrastructure needs, add your own Capability as a trait:
-
-```rust
-use async_trait::async_trait;
-use crate::executors::ExecutorError;
-
-/// A capability for configuring network switches
-#[async_trait]
-pub trait SwitchClient: Send + Sync {
-    async fn configure_port(
-        &self,
-        switch: &str,
-        port: &str,
-        vlan: u16,
-    ) -> Result<(), ExecutorError>;
-
-    async fn configure_port_channel(
-        &self,
-        switch: &str,
-        name: &str,
-        ports: &[&str],
-    ) -> Result<(), ExecutorError>;
-}
-```
-
-Then implement it on your Topology:
-
-```rust
-use harmony_infra::brocade::BrocadeClient;
-
-pub struct MyTopology {
-    switch_client: Arc<dyn SwitchClient>,
-}
-
-impl SwitchClient for MyTopology {
-    async fn configure_port(&self, switch: &str, port: &str, vlan: u16) -> Result<(), ExecutorError> {
-        self.switch_client.configure_port(switch, port, vlan).await
-    }
-
-    async fn configure_port_channel(&self, switch: &str, name: &str, ports: &[&str]) -> Result<(), ExecutorError> {
-        self.switch_client.configure_port_channel(switch, name, ports).await
-    }
-}
-```
-
-Now Scores that need `SwitchClient` can run on `MyTopology`.
-
-## Capability Composition
-
-Topologies often compose multiple Capabilities to support complex Scores:
-
-```rust
-pub struct HAClusterTopology {
-    pub kubeconfig: Option<String>,
-    pub router: Arc<dyn Router>,
-    pub load_balancer: Arc<dyn LoadBalancer>,
-    pub switch_client: Arc<dyn SwitchClient>,
-    pub dhcp_server: Arc<dyn DhcpServer>,
-    pub dns_server: Arc<dyn DnsServer>,
-    // ...
-}
-
-impl K8sclient for HAClusterTopology { ... }
-impl HelmCommand for HAClusterTopology { ... }
-impl SwitchClient for HAClusterTopology { ... }
-impl DhcpServer for HAClusterTopology { ... }
-impl DnsServer for HAClusterTopology { ... }
-impl Router for HAClusterTopology { ... }
-impl LoadBalancer for HAClusterTopology { ... }
-```
-
-A Score that needs all of these can run on `HAClusterTopology` because the Topology provides all of them.
-
-## Best Practices
-
-- **Keep Capabilities focused** — one Capability per concern (Kubernetes client, Helm, switch config)
-- **Return meaningful errors** — use specific error types so Scores can handle failures appropriately
-- **Make Capabilities optional where sensible** — not every Topology needs every Capability; use `Option<T>` or a separate trait for optional features
-- **Document preconditions** — if a Capability requires the infrastructure to be in a specific state, document it in the trait doc comments

docs/guides/developer-guide.md

@@ -1,40 +0,0 @@
-# Developer Guide
-
-This section covers how to extend Harmony by building your own `Score`, `Topology`, and `Capability` implementations.
-
-## Writing a Score
-
-A `Score` is a declarative description of desired state. To create your own:
-
-1. Define a struct that represents your desired state
-2. Implement the `Score<T>` trait, where `T` is your target `Topology`
-3. Implement the `Interpret<T>` trait to define how the Score translates to infrastructure actions
-
-See the [Writing a Score](./writing-a-score.md) guide for a step-by-step walkthrough.
-
-## Writing a Topology
-
-A `Topology` models your infrastructure environment. To create your own:
-
-1. Define a struct that holds your infrastructure configuration
-2. Implement the `Topology` trait
-3. Implement the `Capability` traits your Score needs
-
-See the [Writing a Topology](./writing-a-topology.md) guide for details.
-
-## Adding Capabilities
-
-`Capabilities` are the specific APIs or features a `Topology` exposes. They are the bridge between Scores and the actual infrastructure.
-
-See the [Adding Capabilities](./adding-capabilities.md) guide for details on implementing and exposing Capabilities.
-
-## Core Traits Reference
-
-| Trait | Purpose |
-|-------|---------|
-| `Score<T>` | Declares desired state ("what") |
-| `Topology` | Represents infrastructure ("where") |
-| `Interpret<T>` | Execution logic ("how") |
-| `Capability` | A feature exposed by a Topology |
-
-See [Core Concepts](../concepts.md) for the conceptual foundation.

docs/guides/getting-started.md

@@ -1,230 +0,0 @@
-# Getting Started Guide
-
-This guide walks you through deploying your first application with Harmony — a PostgreSQL cluster on a local Kubernetes cluster (K3D). By the end, you'll understand the core workflow: compile a Score, run it through the Harmony CLI, and verify the result.
-
-## What you'll deploy
-
-A fully functional PostgreSQL cluster running in a local K3D cluster, managed by the CloudNativePG operator. This demonstrates the full Harmony pattern:
-
-1. Provision a local Kubernetes cluster (K3D)
-2. Install the required operator (CloudNativePG)
-3. Create a PostgreSQL cluster
-4. Expose it as a Kubernetes Service
-
-## Prerequisites
-
-Before you begin, install the following tools:
-
-- **Rust & Cargo:** [Install Rust](https://rust-lang.org/tools/install) (edition 2024)
-- **Docker:** [Install Docker](https://docs.docker.com/get-docker/) (required for the local K3D cluster)
-- **kubectl:** [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (optional, for inspecting the cluster)
-
-## Step 1: Clone and build
-
-```bash
-# Clone the repository
-git clone https://git.nationtech.io/nationtech/harmony
-cd harmony
-
-# Build the project (this may take a few minutes on first run)
-cargo build --release
-```
-
-## Step 2: Run the PostgreSQL example
-
-```bash
-cargo run -p example-postgresql
-```
-
-Harmony will output its progress as it:
-
-1. **Creates a K3D cluster** named `harmony-postgres-example` (first run only)
-2. **Installs the CloudNativePG operator** into the cluster
-3. **Creates a PostgreSQL cluster** with 1 instance and 1 GiB of storage
-4. **Prints connection details** for your new database
-
-Expected output (abbreviated):
-
-```
-[+] Cluster created
-[+] Installing CloudNativePG operator
-[+] Creating PostgreSQL cluster
-[+] PostgreSQL cluster is ready
-    Namespace: harmony-postgres-example
-    Service: harmony-postgres-example-rw
-    Username: postgres
-    Password: <stored in secret harmony-postgres-example-db-user>
-```
-
-## Step 3: Verify the deployment
-
-Check that the PostgreSQL pods are running:
-
-```bash
-kubectl get pods -n harmony-postgres-example
-```
-
-You should see something like:
-
-```
-NAME                                    READY   STATUS    RESTARTS   AGE
-harmony-postgres-example-1              1/1     Running   0          2m
-```
-
-Get the database password:
-
-```bash
-kubectl get secret -n harmony-postgres-example harmony-postgres-example-db-user -o jsonpath='{.data.password}' | base64 -d
-```
-
-## Step 4: Connect to the database
-
-Forward the PostgreSQL port to your local machine:
-
-```bash
-kubectl port-forward -n harmony-postgres-example svc/harmony-postgres-example-rw 5432:5432
-```
-
-In another terminal, connect with `psql`:
-
-```bash
-psql -h localhost -p 5432 -U postgres
-# Enter the password from Step 4 when prompted
-```
-
-Try a simple query:
-
-```sql
-SELECT version();
-```
-
-## Step 5: Clean up
-
-To delete the PostgreSQL cluster and the local K3D cluster:
-
-```bash
-k3d cluster delete harmony-postgres-example
-```
-
-Alternatively, just delete the PostgreSQL cluster without removing K3D:
-
-```bash
-kubectl delete namespace harmony-postgres-example
-```
-
-## How it works
-
-The example code (`examples/postgresql/src/main.rs`) is straightforward:
-
-```rust
-use harmony::{
-    inventory::Inventory,
-    modules::postgresql::{PostgreSQLScore, capability::PostgreSQLConfig},
-    topology::K8sAnywhereTopology,
-};
-
-#[tokio::main]
-async fn main() {
-    let postgres = PostgreSQLScore {
-        config: PostgreSQLConfig {
-            cluster_name: "harmony-postgres-example".to_string(),
-            namespace: "harmony-postgres-example".to_string(),
-            ..Default::default()
-        },
-    };
-
-    harmony_cli::run(
-        Inventory::autoload(),
-        K8sAnywhereTopology::from_env(),
-        vec![Box::new(postgres)],
-        None,
-    )
-    .await
-    .unwrap();
-}
-```
-
-- **`Inventory::autoload()`** discovers the local environment (or uses an existing inventory)
-- **`K8sAnywhereTopology::from_env()`** connects to K3D if `HARMONY_AUTOINSTALL=true` (the default), or to any Kubernetes cluster via `KUBECONFIG`
-- **`harmony_cli::run(...)`** executes the Score against the Topology, managing the full lifecycle
-
-## Connecting to an existing cluster
-
-By default, Harmony provisions a local K3D cluster. To use an existing Kubernetes cluster instead:
-
-```bash
-export KUBECONFIG=/path/to/your/kubeconfig
-export HARMONY_USE_LOCAL_K3D=false
-export HARMONY_AUTOINSTALL=false
-
-cargo run -p example-postgresql
-```
-
-## Troubleshooting
-
-### Docker is not running
-
-```
-Error: could not create cluster: docker is not running
-```
-
-Start Docker and try again.
-
-### K3D cluster creation fails
-
-```
-Error: failed to create k3d cluster
-```
-
-Ensure you have at least 2 CPU cores and 4 GiB of RAM available for Docker.
-
-### `kubectl` cannot connect to the cluster
-
-```
-error: unable to connect to a kubernetes cluster
-```
-
-After Harmony creates the cluster, it writes the kubeconfig to `~/.kube/config` or to the path in `KUBECONFIG`. Verify:
-
-```bash
-kubectl cluster-info --context k3d-harmony-postgres-example
-```
-
-### Port forward fails
-
-```
-error: unable to forward port
-```
-
-Make sure no other process is using port 5432, or use a different local port:
-
-```bash
-kubectl port-forward -n harmony-postgres-example svc/harmony-postgres-example-rw 15432:5432
-psql -h localhost -p 15432 -U postgres
-```
-
-## Next steps
-
-- [Explore the Scores Catalog](../catalogs/scores.md): See what other Scores are available
-- [Explore the Topologies Catalog](../catalogs/topologies.md): See what infrastructure Topologies are supported
-- [Read the Core Concepts](../concepts.md): Understand the Score / Topology / Interpret pattern in depth
-- [OKD on Bare Metal](../use-cases/okd-on-bare-metal.md): See a complete bare-metal deployment example
-
-## Advanced examples
-
-Once you're comfortable with the basics, these examples demonstrate more advanced use cases. Note that some require specific infrastructure (existing Kubernetes clusters, bare-metal hardware, or multi-cluster environments):
-
-| Example | Description | Prerequisites |
-|---------|-------------|---------------|
-| `monitoring` | Deploy Prometheus alerting with Discord webhooks | Existing K8s cluster |
-| `ntfy` | Deploy ntfy notification server | Existing K8s cluster |
-| `tenant` | Create a multi-tenant namespace with quotas | Existing K8s cluster |
-| `cert_manager` | Provision TLS certificates | Existing K8s cluster |
-| `validate_ceph_cluster_health` | Check Ceph cluster health | Existing Rook/Ceph cluster |
-| `okd_pxe` / `okd_installation` | Provision OKD on bare metal | HAClusterTopology, bare-metal hardware |
-
-To run any example:
-
-```bash
-cargo run -p example-<example_name>
-```

docs/guides/kubernetes-ingress.md

@@ -1,158 +0,0 @@
-# Ingress Resources in Harmony
-
-Harmony generates standard Kubernetes `networking.k8s.io/v1` Ingress resources. This ensures your deployments are portable across any Kubernetes distribution (vanilla K8s, OKD/OpenShift, K3s, etc.) without requiring vendor-specific configurations.
-
-By default, Harmony does **not** set `spec.ingressClassName`. This allows the cluster's default ingress controller to automatically claim the resource, which is the correct approach for most single-controller clusters.
-
----
-
-## TLS Configurations
-
-There are two portable TLS modes for Ingress resources. Use only these in your Harmony deployments.
-
-### 1. Plain HTTP (No TLS)
-
-Omit the `tls` block entirely. The Ingress serves traffic over plain HTTP. Use this for local development or when TLS is terminated elsewhere (e.g., by a service mesh or external load balancer).
-
-```yaml
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: my-app
-  namespace: my-ns
-spec:
-  rules:
-  - host: app.example.com
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: my-app
-            port:
-              number: 8080
-```
-
-### 2. HTTPS with a Named TLS Secret
-
-Provide a `tls` block with both `hosts` and a `secretName`. The ingress controller will use that Secret for TLS termination. The Secret must be a `kubernetes.io/tls` type in the same namespace as the Ingress.
-
-There are two ways to provide this Secret.
-
-#### Option A: Manual Secret
-
-Create the TLS Secret yourself before deploying the Ingress. This is suitable when certificates are issued outside the cluster or managed by another system.
-
-```yaml
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: my-app
-  namespace: my-ns
-spec:
-  rules:
-  - host: app.example.com
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: my-app
-            port:
-              number: 8080
-  tls:
-  - hosts:
-    - app.example.com
-    secretName: app-example-com-tls
-```
-
-#### Option B: Automated via cert-manager (Recommended)
-
-Add the `cert-manager.io/cluster-issuer` annotation to the Ingress. cert-manager will automatically perform the ACME challenge, generate the certificate, store it in the named Secret, and handle renewal. You do not create the Secret yourself.
-
-```yaml
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: my-app
-  namespace: my-ns
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-spec:
-  rules:
-  - host: app.example.com
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: my-app
-            port:
-              number: 8080
-  tls:
-  - hosts:
-    - app.example.com
-    secretName: app-example-com-tls
-```
-
-If you use a namespace-scoped `Issuer` instead of a `ClusterIssuer`, replace the annotation with `cert-manager.io/issuer: <name>`.
-
----
-
-## Do Not Use: TLS Without `secretName`
-
-Avoid TLS entries that omit `secretName`:
-
-```yaml
-# ⚠️ Non-portable — do not use
-tls:
-- hosts:
-  - app.example.com
-```
-
-Behavior for this pattern is **controller-specific and not portable**. On OKD/OpenShift, the ingress-to-route translation rejects it as incomplete. On other controllers, it may silently serve a self-signed fallback or fail in unpredictable ways. Harmony does not support this pattern.
-
----
-
-## Prerequisites for cert-manager
-
-To use automated certificates (Option B above):
-
-1. **cert-manager** must be installed on the cluster.
-2. A `ClusterIssuer` or `Issuer` must exist. A typical Let's Encrypt production issuer:
-
-```yaml
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: team@example.com
-    privateKeySecretRef:
-      name: letsencrypt-prod-account-key
-    solvers:
-    - http01:
-        ingress: {}
-```
-
-3. **DNS must already resolve** to the cluster's ingress endpoint before the Ingress is created. The HTTP01 challenge requires this routing to be active.
-
-For wildcard certificates (e.g. `*.example.com`), HTTP01 cannot be used — configure a DNS01 solver with credentials for your DNS provider instead.
-
----
-
-## OKD / OpenShift Notes
-
-On OKD, standard Ingress resources are automatically translated into OpenShift `Route` objects. The default TLS termination mode is `edge`, which is correct for most HTTP applications. To control this explicitly, add:
-
-```yaml
-annotations:
-  route.openshift.io/termination: edge  # or passthrough / reencrypt
-```
-
-This annotation is ignored on non-OpenShift clusters and is safe to include unconditionally.

docs/guides/writing-a-score.md

@@ -1,164 +0,0 @@
-# Writing a Score
-
-A `Score` declares _what_ you want to achieve. It is decoupled from _how_ it is achieved — that logic lives in an `Interpret`.
-
-## The Pattern
-
-A Score consists of two parts:
-
-1. **A struct** — holds the configuration for your desired state
-2. **A `Score<T>` implementation** — returns an `Interpret` that knows how to execute
-
-An `Interpret` contains the actual execution logic and connects your Score to the capabilities exposed by a `Topology`.
-
-## Example: A Simple Score
-
-Here's a simplified version of `NtfyScore` from the `ntfy` module:
-
-```rust
-use async_trait::async_trait;
-use harmony::{
-    interpret::{Interpret, InterpretError, Outcome},
-    inventory::Inventory,
-    score::Score,
-    topology::{HelmCommand, K8sclient, Topology},
-};
-
-/// MyScore declares "I want to install the ntfy server"
-#[derive(Debug, Clone)]
-pub struct MyScore {
-    pub namespace: String,
-    pub host: String,
-}
-
-impl<T: Topology + HelmCommand + K8sclient> Score<T> for MyScore {
-    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(MyInterpret { score: self.clone() })
-    }
-
-    fn name(&self) -> String {
-        "ntfy [MyScore]".into()
-    }
-}
-
-/// MyInterpret knows _how_ to install ntfy using the Topology's capabilities
-#[derive(Debug)]
-pub struct MyInterpret {
-    pub score: MyScore,
-}
-
-#[async_trait]
-impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for MyInterpret {
-    async fn execute(
-        &self,
-        inventory: &Inventory,
-        topology: &T,
-    ) -> Result<Outcome, InterpretError> {
-        // 1. Get a Kubernetes client from the Topology
-        let client = topology.k8s_client().await?;
-
-        // 2. Use Helm to install the ntfy chart
-        // (via topology's HelmCommand capability)
-
-        // 3. Wait for the deployment to be ready
-        client
-            .wait_until_deployment_ready("ntfy", Some(&self.score.namespace), None)
-            .await?;
-
-        Ok(Outcome::success("ntfy installed".to_string()))
-    }
-}
-```
-
-## The Compile-Time Safety Check
-
-The generic `Score<T>` trait is bounded by `T: Topology`. This means the compiler enforces that your Score only runs on Topologies that expose the capabilities your Interpret needs:
-
-```rust
-// This only compiles if K8sAnywhereTopology (or any T)
-// implements HelmCommand and K8sclient
-impl<T: Topology + HelmCommand + K8sclient> Score<T> for MyScore { ... }
-```
-
-If you try to run this Score against a Topology that doesn't expose `HelmCommand`, you get a compile error — before any code runs.
-
-## Using Your Score
-
-Once defined, your Score integrates with the Harmony CLI:
-
-```rust
-use harmony::{
-    inventory::Inventory,
-    topology::K8sAnywhereTopology,
-};
-
-#[tokio::main]
-async fn main() {
-    let my_score = MyScore {
-        namespace: "monitoring".to_string(),
-        host: "ntfy.example.com".to_string(),
-    };
-
-    harmony_cli::run(
-        Inventory::autoload(),
-        K8sAnywhereTopology::from_env(),
-        vec![Box::new(my_score)],
-        None,
-    )
-    .await
-    .unwrap();
-}
-```
-
-## Key Patterns
-
-### Composing Scores
-
-Scores can include other Scores via features:
-
-```rust
-let app = ApplicationScore {
-    features: vec![
-        Box::new(PackagingDeployment { application: app.clone() }),
-        Box::new(Monitoring { application: app.clone(), alert_receiver: vec![] }),
-    ],
-    application: app,
-};
-```
-
-### Reusing Interpret Logic
-
-Many Scores delegate to shared `Interpret` implementations. For example, `HelmChartScore` provides a reusable Interpret for any Helm-based deployment. Your Score can wrap it:
-
-```rust
-impl<T: Topology + HelmCommand> Score<T> for MyScore {
-    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(HelmChartInterpret { /* your config */ })
-    }
-}
-```
-
-### Accessing Topology Capabilities
-
-Your Interpret accesses infrastructure through Capabilities exposed by the Topology:
-
-```rust
-// Via the Topology trait directly
-let k8s_client = topology.k8s_client().await?;
-let helm = topology.get_helm_command();
-
-// Or via Capability traits
-impl<T: Topology + K8sclient> Interpret<T> for MyInterpret {
-    async fn execute(...) {
-        let client = topology.k8s_client().await?;
-        // use client...
-    }
-}
-```
-
-## Best Practices
-
-- **Keep Scores focused** — one Score per concern (deployment, monitoring, networking)
-- **Use `..Default::default()`** for optional fields so callers only need to specify what they care about
-- **Return `Outcome`** — use `Outcome::success`, `Outcome::failure`, or `Outcome::success_with_details` to communicate results clearly
-- **Handle errors gracefully** — return meaningful `InterpretError` messages that help operators debug issues

docs/guides/writing-a-topology.md

@@ -1,176 +0,0 @@
-# Writing a Topology
-
-A `Topology` models your infrastructure environment and exposes `Capability` traits that Scores use to interact with it. Where a Score declares _what_ you want, a Topology exposes _what_ it can do.
-
-## The Minimum Implementation
-
-At minimum, a Topology needs:
-
-```rust
-use async_trait::async_trait;
-use harmony::{
-    topology::{PreparationError, PreparationOutcome, Topology},
-};
-
-#[derive(Debug, Clone)]
-pub struct MyTopology {
-    pub name: String,
-}
-
-#[async_trait]
-impl Topology for MyTopology {
-    fn name(&self) -> &str {
-        "MyTopology"
-    }
-
-    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
-        // Verify the infrastructure is accessible and ready
-        Ok(PreparationOutcome::Success { details: "ready".to_string() })
-    }
-}
-```
-
-## Implementing Capabilities
-
-Scores express dependencies on Capabilities through trait bounds. For example, if your Topology should support Scores that deploy Helm charts, implement `HelmCommand`:
-
-```rust
-use std::process::Command;
-use harmony::topology::HelmCommand;
-
-impl HelmCommand for MyTopology {
-    fn get_helm_command(&self) -> Command {
-        let mut cmd = Command::new("helm");
-        if let Some(kubeconfig) = &self.kubeconfig {
-            cmd.arg("--kubeconfig").arg(kubeconfig);
-        }
-        cmd
-    }
-}
-```
-
-For Scores that need a Kubernetes client, implement `K8sclient`:
-
-```rust
-use std::sync::Arc;
-use harmony_k8s::K8sClient;
-use harmony::topology::K8sclient;
-
-#[async_trait]
-impl K8sclient for MyTopology {
-    async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        let client = if let Some(kubeconfig) = &self.kubeconfig {
-            K8sClient::from_kubeconfig(kubeconfig).await?
-        } else {
-            K8sClient::try_default().await?
-        };
-        Ok(Arc::new(client))
-    }
-}
-```
-
-## Loading Topology from Environment
-
-For flexibility, implement `from_env()` to read configuration from environment variables:
-
-```rust
-impl MyTopology {
-    pub fn from_env() -> Self {
-        Self {
-            name: std::env::var("MY_TOPOLOGY_NAME")
-                .unwrap_or_else(|_| "default".to_string()),
-            kubeconfig: std::env::var("KUBECONFIG").ok(),
-        }
-    }
-}
-```
-
-This pattern lets operators switch between environments without recompiling:
-
-```bash
-export KUBECONFIG=/path/to/prod-cluster.kubeconfig
-cargo run --example my_example
-```
-
-## Complete Example: K8sAnywhereTopology
-
-The `K8sAnywhereTopology` is the most commonly used Topology and handles both local (K3D) and remote Kubernetes clusters:
-
-```rust
-pub struct K8sAnywhereTopology {
-    pub k8s_state: Arc<OnceCell<K8sState>>,
-    pub tenant_manager: Arc<OnceCell<TenantManager>>,
-    pub config: Arc<K8sAnywhereConfig>,
-}
-
-#[async_trait]
-impl Topology for K8sAnywhereTopology {
-    fn name(&self) -> &str {
-        "K8sAnywhereTopology"
-    }
-
-    async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
-        // 1. If autoinstall is enabled and no cluster exists, provision K3D
-        // 2. Verify kubectl connectivity
-        // 3. Optionally wait for cluster operators to be ready
-        Ok(PreparationOutcome::Success { details: "cluster ready".to_string() })
-    }
-}
-```
-
-## Key Patterns
-
-### Lazy Initialization
-
-Use `OnceCell` for expensive resources like Kubernetes clients:
-
-```rust
-pub struct K8sAnywhereTopology {
-    k8s_state: Arc<OnceCell<K8sState>>,
-}
-```
-
-### Multi-Target Topologies
-
-For Scores that span multiple clusters (like NATS supercluster), implement `MultiTargetTopology`:
-
-```rust
-pub trait MultiTargetTopology: Topology {
-    fn current_target(&self) -> &str;
-    fn set_target(&mut self, target: &str);
-}
-```
-
-### Composing Topologies
-
-Complex topologies combine multiple infrastructure components:
-
-```rust
-pub struct HAClusterTopology {
-    pub router: Arc<dyn Router>,
-    pub load_balancer: Arc<dyn LoadBalancer>,
-    pub firewall: Arc<dyn Firewall>,
-    pub dhcp_server: Arc<dyn DhcpServer>,
-    pub dns_server: Arc<dyn DnsServer>,
-    pub kubeconfig: Option<String>,
-    // ...
-}
-```
-
-## Testing Your Topology
-
-Test Topologies in isolation by implementing them against mock infrastructure:
-
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_topology_ensure_ready() {
-        let topo = MyTopology::from_env();
-        let result = topo.ensure_ready().await;
-        assert!(result.is_ok());
-    }
-}
-```

docs/modules/Multisite_PostgreSQL.md

@@ -1,105 +0,0 @@
-# Design Document: Harmony PostgreSQL Module
-
-**Status:** Draft
-**Last Updated:** 2025-12-01
-**Context:** Multi-site Data Replication & Orchestration
-
-## 1. Overview
-
-The Harmony PostgreSQL Module provides a high-level abstraction for deploying and managing high-availability PostgreSQL clusters across geographically distributed Kubernetes/OKD sites.
-
-Instead of manually configuring complex replication slots, firewalls, and operator settings on each cluster, users define a single intent (a **Score**), and Harmony orchestrates the underlying infrastructure (the **Arrangement**) to establish a Primary-Replica architecture.
-
-Currently, the implementation relies on the **CloudNativePG (CNPG)** operator as the backing engine.
-
-## 2. Architecture
-
-### 2.1 The Abstraction Model
-Following **ADR 003 (Infrastructure Abstraction)**, Harmony separates the *intent* from the *implementation*.
-
-1.  **The Score (Intent):** The user defines a `MultisitePostgreSQL` resource. This describes *what* is needed (e.g., "A Postgres 15 cluster with 10GB storage, Primary on Site A, Replica on Site B").
-2.  **The Interpret (Action):** Harmony MultisitePostgreSQLInterpret processes this Score and orchestrates the deployment on both sites to reach the state defined in the Score.
-3.  **The Capability (Implementation):** The PostgreSQL Capability is implemented by the K8sTopology and the interpret can deploy it, configure it and fetch information about it. The concrete implementation will rely on the mature CloudnativePG operator to manage all the Kubernetes resources required.
-
-### 2.2 Network Connectivity (TLS Passthrough)
-
-One of the critical challenges in multi-site orchestration is secure connectivity between clusters that may have dynamic IPs or strict firewalls.
-
-To solve this, we utilize **OKD/OpenShift Routes with TLS Passthrough**.
-
-*   **Mechanism:** The Primary site exposes a `Route` configured for `termination: passthrough`.
-*   **Routing:** The OpenShift HAProxy router inspects the **SNI (Server Name Indication)** header of the incoming TCP connection to route traffic to the correct PostgreSQL Pod.
-*   **Security:** SSL is **not** terminated at the ingress router. The encrypted stream is passed directly to the PostgreSQL instance. Mutual TLS (mTLS) authentication is handled natively by CNPG between the Primary and Replica instances.
-*   **Dynamic IPs:** Because connections are established via DNS hostnames (the Route URL), this architecture is resilient to dynamic IP changes at the Primary site.
-
-#### Traffic Flow Diagram
-
-```text
-[ Site B: Replica ]                 [ Site A: Primary ]
-      |                                     |
-(CNPG Instance) --[Encrypted TCP]--> (OKD HAProxy Router)
-      |           (Port 443)                |
-      |                                     |
-      |                            [SNI Inspection]
-      |                                     |
-      |                                     v
-      |                            (PostgreSQL Primary Pod)
-      |                                   (Port 5432)
-```
-
-## 3. Design Decisions
-
-### Why CloudNativePG?
-We selected CloudNativePG because it relies exclusively on standard Kubernetes primitives and uses the native PostgreSQL replication protocol (WAL shipping/Streaming). This aligns with Harmony's goal of being "K8s Native."
-
-### Why TLS Passthrough instead of VPN/NodePort?
-*   **NodePort:** Requires static IPs and opening non-standard ports on the firewall, which violates our security constraints.
-*   **VPN (e.g., Wireguard/Tailscale):** While secure, it introduces significant complexity (sidecars, key management) and external dependencies.
-*   **TLS Passthrough:** Leverages the existing Ingress/Router infrastructure already present in OKD. It requires zero additional software and respects multi-tenancy (Routes are namespaced).
-
-### Configuration Philosophy (YAGNI)
-The current design exposes a **generic configuration surface**. Users can configure standard parameters (Storage size, CPU/Memory requests, Postgres version).
-
-**We explicitly do not expose advanced CNPG or PostgreSQL configurations at this stage.**
-
-*   **Reasoning:** We aim to keep the API surface small and manageable.
-*   **Future Path:** We plan to implement a "pass-through" mechanism to allow sending raw config maps or custom parameters to the underlying engine (CNPG) *only when a concrete use case arises*. Until then, we adhere to the **YAGNI (You Ain't Gonna Need It)** principle to avoid premature optimization and API bloat.
-
-## 4. Usage Guide
-
-To deploy a multi-site cluster, apply the `MultisitePostgreSQL` resource to the Harmony Control Plane.
-
-### Example Manifest
-
-```yaml
-apiVersion: harmony.io/v1alpha1
-kind: MultisitePostgreSQL
-metadata:
-  name: finance-db
-  namespace: tenant-a
-spec:
-  version: "15"
-  storage: "10Gi"
-  resources:
-    requests:
-      cpu: "500m"
-      memory: "1Gi"
-  
-  # Topology Definition
-  topology:
-    primary:
-      site: "site-paris" # The name of the cluster in Harmony
-    replicas:
-      - site: "site-newyork"
-```
-
-### What happens next?
-1.  Harmony detects the CR.
-2.  **On Site Paris:** It deploys a CNPG Cluster (Primary) and creates a Passthrough Route `postgres-finance-db.apps.site-paris.example.com`.
-3.  **On Site New York:** It deploys a CNPG Cluster (Replica) configured with `externalClusters` pointing to the Paris Route.
-4.  Data begins replicating immediately over the encrypted channel.
-
-## 5. Troubleshooting
-
-*   **Connection Refused:** Ensure the Primary site's Route is successfully admitted by the Ingress Controller.
-*   **Certificate Errors:** CNPG manages mTLS automatically. If errors persist, ensure the CA secrets were correctly propagated by Harmony from Primary to Replica namespaces.

docs/one_liners.md

@@ -1,16 +0,0 @@
-# Handy one liners for infrastructure management
-
-### Delete all evicted pods from a cluster
-
-```sh
- kubectl get po -A | grep Evic | awk '{ print "-n " $1 " " $2 }' | xargs -L 1 kubectl delete po
-```
-> Pods are evicted when the node they are running on lacks the ressources to keep them going. The most common case is when ephemeral storage becomes too full because of something like a log file getting too big.
->
-> It could also happen because of memory or cpu pressure due to unpredictable workloads.
-> 
-> This means it is generally ok to delete them.
->
-> However, in a perfectly configured deployment and cluster, pods should rarely, if ever, get evicted. For example, a log file getting too big should be reconfigured not to use too much space, or the deployment should be configured to reserve the correct amount of ephemeral storage space.
->
-> Note that deleting evicted pods do not solve the underlying issue, make sure to understand why the pod was evicted in the first place and put the proper solution in place.

docs/use-cases/README.md

@@ -1,17 +0,0 @@
-# Use Cases
-
-Real-world scenarios demonstrating Harmony in action.
-
-## Available Use Cases
-
-### [PostgreSQL on Local K3D](./postgresql-on-local-k3d.md)
-
-Deploy a fully functional PostgreSQL cluster on a local K3D cluster in under 10 minutes. The quickest way to see Harmony in action.
-
-### [OKD on Bare Metal](./okd-on-bare-metal.md)
-
-A complete walkthrough of bootstrapping a high-availability OKD cluster from physical hardware. Covers inventory discovery, bootstrap, control plane, and worker provisioning.
-
----
-
-_These use cases are community-tested scenarios. For questions or contributions, open an issue on the [Harmony repository](https://git.nationtech.io/NationTech/harmony/issues)._

docs/use-cases/okd-on-bare-metal.md

@@ -1,159 +0,0 @@
-# Use Case: OKD on Bare Metal
-
-Provision a production-grade OKD (OpenShift Kubernetes Distribution) cluster from physical hardware using Harmony. This use case covers the full lifecycle: hardware discovery, bootstrap, control plane, workers, and post-install validation.
-
-## What you'll have at the end
-
-A highly-available OKD cluster with:
-- 3 control plane nodes
-- 2+ worker nodes
-- Network bonding configured on nodes and switches
-- Load balancer routing API and ingress traffic
-- DNS and DHCP services for the cluster
-- Post-install health validation
-
-## Target hardware model
-
-This setup assumes a typical lab environment:
-
-```
-┌─────────────────────────────────────────────────────────┐
-│  Network 192.168.x.0/24 (flat, DHCP + PXE capable)     │
-│                                                         │
-│  ┌──────────┐  ┌──────────┐  ┌──────────┐            │
-│  │  cp0     │  │  cp1     │  │  cp2     │  (control) │
-│  └──────────┘  └──────────┘  └──────────┘            │
-│  ┌──────────┐  ┌──────────┐                            │
-│  │  wk0     │  │  wk1     │  ...        (workers)    │
-│  └──────────┘  └──────────┘                            │
-│  ┌──────────┐                                          │
-│  │ bootstrap│  (temporary, can be repurposed)          │
-│  └──────────┘                                          │
-│                                                         │
-│  ┌──────────┐  ┌──────────┐                            │
-│  │ firewall │  │  switch  │  (OPNsense + Brocade)     │
-│  └──────────┘  └──────────┘                            │
-└─────────────────────────────────────────────────────────┘
-```
-
-## Required infrastructure
-
-Harmony models this as an `HAClusterTopology`, which requires these capabilities:
-
-| Capability | Implementation |
-|------------|---------------|
-| **Router** | OPNsense firewall |
-| **Load Balancer** | OPNsense HAProxy |
-| **Firewall** | OPNsense |
-| **DHCP Server** | OPNsense |
-| **TFTP Server** | OPNsense |
-| **HTTP Server** | OPNsense |
-| **DNS Server** | OPNsense |
-| **Node Exporter** | Prometheus node_exporter on OPNsense |
-| **Switch Client** | Brocade SNMP |
-
-See `examples/okd_installation/` for a reference topology implementation.
-
-## The Provisioning Pipeline
-
-Harmony orchestrates OKD installation in ordered stages:
-
-### Stage 1: Inventory Discovery (`OKDSetup01InventoryScore`)
-
-Harmony boots all nodes via PXE into a CentOS Stream live environment, runs an inventory agent on each, and collects:
-- MAC addresses and NIC details
-- IP addresses assigned by DHCP
-- Hardware profile (CPU, RAM, storage)
-
-This is the "discovery-first" approach: no pre-configuration required on nodes.
-
-### Stage 2: Bootstrap Node (`OKDSetup02BootstrapScore`)
-
-The user selects one discovered node to serve as the bootstrap node. Harmony:
-- Renders per-MAC iPXE boot configuration with OKD 4.19 SCOS live assets + ignition
-- Reboots the bootstrap node via SSH
-- Waits for the bootstrap process to complete (API server becomes available)
-
-### Stage 3: Control Plane (`OKDSetup03ControlPlaneScore`)
-
-With bootstrap complete, Harmony provisions the control plane nodes:
-- Renders per-MAC iPXE for each control plane node
-- Reboots via SSH and waits for node to join the cluster
-- Applies network bond configuration via NMState MachineConfig where relevant
-
-### Stage 4: Network Bonding (`OKDSetupPersistNetworkBondScore`)
-
-Configures LACP bonds on nodes and corresponding port-channels on the switch stack for high-availability.
-
-### Stage 5: Worker Nodes (`OKDSetup04WorkersScore`)
-
-Provisions worker nodes similarly to control plane, joining them to the cluster.
-
-### Stage 6: Sanity Check (`OKDSetup05SanityCheckScore`)
-
-Validates:
-- API server is reachable
-- Ingress controller is operational
-- Cluster operators are healthy
-- SDN (software-defined networking) is functional
-
-### Stage 7: Installation Report (`OKDSetup06InstallationReportScore`)
-
-Produces a machine-readable JSON report and human-readable summary of the installation.
-
-## Network notes
-
-**During discovery:** Ports must be in access mode (no LACP). DHCP succeeds; iPXE loads CentOS Stream live with Kickstart and starts the inventory endpoint.
-
-**During provisioning:** After SCOS is on disk and Ignition/MachineConfig can be applied, bonds are set persistently. This avoids the PXE/DHCP recovery race condition that occurs if bonding is configured too early.
-
-**PXE limitation:** The generic discovery path cannot use bonded networks for PXE boot because the DHCP recovery process conflicts with bond formation.
-
-## Configuration knobs
-
-When using `OKDInstallationPipeline`, configure these domains:
-
-| Parameter | Example | Description |
-|-----------|---------|-------------|
-| `public_domain` | `apps.example.com` | Wildcard domain for application ingress |
-| `internal_domain` | `cluster.local` | Internal cluster DNS domain |
-
-## Running the example
-
-See `examples/okd_installation/` for a complete reference. The topology must be configured with your infrastructure details:
-
-```bash
-# Configure the example with your hardware/network specifics
-# See examples/okd_installation/src/topology.rs
-
-cargo run -p example-okd_installation
-```
-
-This example requires:
-- Physical hardware configured as described above
-- OPNsense firewall with SSH access
-- Brocade switch with SNMP access
-- All nodes connected to the same Layer 2 network
-
-## Post-install
-
-After the cluster is bootstrapped, `~/.kube/config` is updated with the cluster credentials. Verify:
-
-```bash
-kubectl get nodes
-kubectl get pods -n openshift-monitoring
-oc get routes -n openshift-console
-```
-
-## Next steps
-
-- Enable monitoring with `PrometheusAlertScore` or `OpenshiftClusterAlertScore`
-- Configure TLS certificates with `CertManagerHelmScore`
-- Add storage with Rook Ceph
-- Scale workers with `OKDSetup04WorkersScore`
-
-## Further reading
-
-- [OKD Installation Module](../../harmony/src/modules/okd/installation.rs) — source of truth for pipeline stages
-- [HAClusterTopology](../../harmony/src/domain/topology/ha_cluster.rs) — infrastructure capability model
-- [Scores Catalog](../catalogs/scores.md) — all available Scores including OKD-specific ones

docs/use-cases/postgresql-on-local-k3d.md

@@ -1,115 +0,0 @@
-# Use Case: PostgreSQL on Local K3D
-
-Deploy a production-grade PostgreSQL cluster on a local Kubernetes cluster (K3D) using Harmony. This is the fastest way to get started with Harmony and requires no external infrastructure.
-
-## What you'll have at the end
-
-A fully operational PostgreSQL cluster with:
-- 1 primary instance with 1 GiB of storage
-- CloudNativePG operator managing the cluster lifecycle
-- Automatic failover support (foundation for high-availability)
-- Exposed as a Kubernetes Service for easy connection
-
-## Prerequisites
-
-- Rust 2024 edition
-- Docker running locally
-- ~5 minutes
-
-## The Score
-
-The entire deployment is expressed in ~20 lines of Rust:
-
-```rust
-use harmony::{
-    inventory::Inventory,
-    modules::postgresql::{PostgreSQLScore, capability::PostgreSQLConfig},
-    topology::K8sAnywhereTopology,
-};
-
-#[tokio::main]
-async fn main() {
-    let postgres = PostgreSQLScore {
-        config: PostgreSQLConfig {
-            cluster_name: "harmony-postgres-example".to_string(),
-            namespace: "harmony-postgres-example".to_string(),
-            ..Default::default()
-        },
-    };
-
-    harmony_cli::run(
-        Inventory::autoload(),
-        K8sAnywhereTopology::from_env(),
-        vec![Box::new(postgres)],
-        None,
-    )
-    .await
-    .unwrap();
-}
-```
-
-## What Harmony does
-
-When you run this, Harmony:
-
-1. **Connects to K8sAnywhereTopology** — this auto-provisions a K3D cluster if none exists
-2. **Installs the CloudNativePG operator** — one-time setup that enables PostgreSQL cluster management in Kubernetes
-3. **Creates a PostgreSQL cluster** — Harmony translates the Score into a `Cluster` CRD and applies it
-4. **Exposes the database** — creates a Kubernetes Service for the PostgreSQL primary
-
-## Running it
-
-```bash
-cargo run -p example-postgresql
-```
-
-## Verifying the deployment
-
-```bash
-# Check pods
-kubectl get pods -n harmony-postgres-example
-
-# Get the password
-PASSWORD=$(kubectl get secret -n harmony-postgres-example \
-  harmony-postgres-example-db-user \
-  -o jsonpath='{.data.password}' | base64 -d)
-
-# Connect via port-forward
-kubectl port-forward -n harmony-postgres-example svc/harmony-postgres-example-rw 5432:5432
-psql -h localhost -p 5432 -U postgres -W "$PASSWORD"
-```
-
-## Customizing the deployment
-
-The `PostgreSQLConfig` struct supports:
-
-| Field | Default | Description |
-|-------|---------|-------------|
-| `cluster_name` | — | Name of the PostgreSQL cluster |
-| `namespace` | — | Kubernetes namespace to deploy to |
-| `instances` | `1` | Number of instances |
-| `storage_size` | `1Gi` | Persistent storage size per instance |
-
-Example with custom settings:
-
-```rust
-let postgres = PostgreSQLScore {
-    config: PostgreSQLConfig {
-        cluster_name: "my-prod-db".to_string(),
-        namespace: "database".to_string(),
-        instances: 3,
-        storage_size: "10Gi".to_string().into(),
-        ..Default::default()
-    },
-};
-```
-
-## Extending the pattern
-
-This pattern extends to any Kubernetes-native workload:
-
-- Add **monitoring** by including a `Monitoring` feature alongside your Score
-- Add **TLS certificates** by including a `CertificateScore`
-- Add **tenant isolation** by wrapping in a `TenantScore`
-
-See [Scores Catalog](../catalogs/scores.md) for the full list.

examples/README.md

@@ -1,127 +0,0 @@
-# Examples
-
-This directory contains runnable examples demonstrating Harmony's capabilities. Each example is a self-contained program that can be run with `cargo run -p example-<name>`.
-
-## Quick Reference
-
-| Example | Description | Local K3D | Existing Cluster | Hardware Needed |
-|---------|-------------|:---------:|:----------------:|:---------------:|
-| `postgresql` | Deploy a PostgreSQL cluster | ✅ | ✅ | — |
-| `ntfy` | Deploy ntfy notification server | ✅ | ✅ | — |
-| `tenant` | Create a multi-tenant namespace | ✅ | ✅ | — |
-| `cert_manager` | Provision TLS certificates | ✅ | ✅ | — |
-| `node_health` | Check Kubernetes node health | ✅ | ✅ | — |
-| `monitoring` | Deploy Prometheus alerting | ✅ | ✅ | — |
-| `monitoring_with_tenant` | Monitoring + tenant isolation | ✅ | ✅ | — |
-| `operatorhub_catalog` | Install OperatorHub catalog | ✅ | ✅ | — |
-| `validate_ceph_cluster_health` | Verify Ceph cluster health | — | ✅ | Rook/Ceph |
-| `remove_rook_osd` | Remove a Rook OSD | — | ✅ | Rook/Ceph |
-| `brocade_snmp_server` | Configure Brocade switch SNMP | — | ✅ | Brocade switch |
-| `opnsense_node_exporter` | Node exporter on OPNsense | — | ✅ | OPNsense firewall |
-| `okd_pxe` | PXE boot configuration for OKD | — | — | ✅ |
-| `okd_installation` | Full OKD bare-metal install | — | — | ✅ |
-| `okd_cluster_alerts` | OKD cluster monitoring alerts | — | ✅ | OKD cluster |
-| `multisite_postgres` | Multi-site PostgreSQL failover | — | ✅ | Multi-cluster |
-| `nats` | Deploy NATS messaging | — | ✅ | Multi-cluster |
-| `nats-supercluster` | NATS supercluster across sites | — | ✅ | Multi-cluster |
-| `lamp` | LAMP stack deployment | ✅ | ✅ | — |
-| `openbao` | Deploy OpenBao vault | ✅ | ✅ | — |
-| `zitadel` | Deploy Zitadel identity provider | ✅ | ✅ | — |
-| `try_rust_webapp` | Rust webapp with packaging | ✅ | ✅ | Submodule |
-| `rust` | Rust webapp with full monitoring | ✅ | ✅ | — |
-| `rhob_application_monitoring` | RHOB monitoring setup | ✅ | ✅ | — |
-| `sttest` | Full OKD stack test | — | — | ✅ |
-| `application_monitoring_with_tenant` | App monitoring + tenant | — | ✅ | OKD cluster |
-| `kube-rs` | Direct kube-rs client usage | ✅ | ✅ | — |
-| `k8s_drain_node` | Drain a Kubernetes node | ✅ | ✅ | — |
-| `k8s_write_file_on_node` | Write files to K8s nodes | ✅ | ✅ | — |
-| `harmony_inventory_builder` | Discover hosts via subnet scan | ✅ | — | — |
-| `cli` | CLI tool with inventory discovery | ✅ | — | — |
-| `tui` | Terminal UI demonstration | ✅ | — | — |
-
-## Status Legend
-
-| Symbol | Meaning |
-|--------|---------|
-| ✅ | Works out-of-the-box |
-| — | Not applicable or requires specific setup |
-
-## By Category
-
-### Data Services
-- **`postgresql`** — Deploy a PostgreSQL cluster via CloudNativePG
-- **`multisite_postgres`** — Multi-site PostgreSQL with failover
-- **`public_postgres`** — Public-facing PostgreSQL (⚠️ uses NationTech DNS)
-
-### Kubernetes Utilities
-- **`node_health`** — Check node health in a cluster
-- **`k8s_drain_node`** — Drain and reboot a node
-- **`k8s_write_file_on_node`** — Write files to nodes
-- **`validate_ceph_cluster_health`** — Verify Ceph/Rook cluster health
-- **`remove_rook_osd`** — Remove an OSD from Rook/Ceph
-- **`kube-rs`** — Direct Kubernetes client usage demo
-
-### Monitoring & Alerting
-- **`monitoring`** — Deploy Prometheus alerting with Discord webhooks
-- **`monitoring_with_tenant`** — Monitoring with tenant isolation
-- **`ntfy`** — Deploy ntfy notification server
-- **`okd_cluster_alerts`** — OKD-specific cluster alerts
-
-### Application Deployment
-- **`try_rust_webapp`** — Deploy a Rust webapp with packaging (⚠️ requires `tryrust.org` submodule)
-- **`rust`** — Rust webapp with full monitoring features
-- **`rhob_application_monitoring`** — Red Hat Observability Stack monitoring
-- **`lamp`** — LAMP stack deployment (⚠️ uses NationTech DNS)
-- **`application_monitoring_with_tenant`** — App monitoring with tenant isolation
-
-### Infrastructure & Bare Metal
-- **`okd_installation`** — Full OKD cluster from scratch
-- **`okd_pxe`** — PXE boot configuration for OKD
-- **`sttest`** — Full OKD stack test with specific hardware
-- **`brocade_snmp_server`** — Configure Brocade switch via SNMP
-- **`opnsense_node_exporter`** — Node exporter on OPNsense firewall
-
-### Multi-Cluster
-- **`nats`** — NATS deployment on a cluster
-- **`nats-supercluster`** — NATS supercluster across multiple sites
-- **`multisite_postgres`** — PostgreSQL with multi-site failover
-
-### Identity & Secrets
-- **`openbao`** — Deploy OpenBao vault (⚠️ uses NationTech DNS)
-- **`zitadel`** — Deploy Zitadel identity provider (⚠️ uses NationTech DNS)
-
-### Cluster Services
-- **`cert_manager`** — Provision TLS certificates
-- **`tenant`** — Create a multi-tenant namespace
-- **`operatorhub_catalog`** — Install OperatorHub catalog sources
-
-### Development & Testing
-- **`cli`** — CLI tool with inventory discovery
-- **`tui`** — Terminal UI demonstration
-- **`harmony_inventory_builder`** — Host discovery via subnet scan
-
-## Running Examples
-
-```bash
-# Build first
-cargo build --release
-
-# Run any example
-cargo run -p example-postgresql
-cargo run -p example-ntfy
-cargo run -p example-tenant
-```
-
-For examples that need an existing Kubernetes cluster:
-
-```bash
-export KUBECONFIG=/path/to/your/kubeconfig
-export HARMONY_USE_LOCAL_K3D=false
-export HARMONY_AUTOINSTALL=false
-
-cargo run -p example-monitoring
-```
-
-## Notes on Private Infrastructure
-
-Some examples use NationTech-hosted infrastructure by default (DNS domains like `*.nationtech.io`, `*.harmony.mcd`). These are not suitable for public use without modification. See the [Getting Started Guide](../docs/guides/getting-started.md) for the recommended public examples.