adr: draft ADR proposing harmony agent and nats-jetstram for decentralized workload management

Reviewed-on: #191

.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-C", "link-arg=/STACK:8000000"]
+
+[target.x86_64-pc-windows-gnu]
+rustflags = ["-C", "link-arg=-Wl,--stack,8000000"]

.dockerignore

@@ -0,0 +1,2 @@
+target/
+Dockerfile

.gitattributes

@@ -2,3 +2,5 @@ bootx64.efi filter=lfs diff=lfs merge=lfs -text
 grubx64.efi filter=lfs diff=lfs merge=lfs -text
 initrd filter=lfs diff=lfs merge=lfs -text
 linux filter=lfs diff=lfs merge=lfs -text
+data/okd/bin/* filter=lfs diff=lfs merge=lfs -text
+data/okd/installer_image/* filter=lfs diff=lfs merge=lfs -text

.gitea/workflows/check.yml

@@ -0,0 +1,18 @@
+name: Run Check Script
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  check:
+    runs-on: docker
+    container:
+      image: hub.nationtech.io/harmony/harmony_composer:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run check script
+        run: bash check.sh

.gitea/workflows/harmony_composer.yaml

@@ -0,0 +1,95 @@
+name: Compile and package harmony_composer
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  package_harmony_composer:
+    container:
+      image: hub.nationtech.io/harmony/harmony_composer:latest
+    runs-on: dind
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build for Linux x86_64
+        run: cargo build --release --bin harmony_composer --target x86_64-unknown-linux-gnu
+
+      - name: Build for Windows x86_64 GNU
+        run: cargo build --release --bin harmony_composer --target x86_64-pc-windows-gnu
+
+      - name: Setup log into hub.nationtech.io
+        uses: docker/login-action@v3
+        with:
+          registry: hub.nationtech.io
+          username: ${{ secrets.HUB_BOT_USER }}
+          password: ${{ secrets.HUB_BOT_PASSWORD }}
+
+      # TODO: build ARM images and MacOS binaries (or other targets) too
+
+      - name: Update snapshot-latest tag
+        run: |
+          git config user.name "Gitea CI"
+          git config user.email "ci@nationtech.io"
+          git tag -f snapshot-latest
+          git push origin snapshot-latest --force
+
+      - name: Install jq
+        run: apt install -y jq # The current image includes apt lists so we don't have to apt update and rm /var/lib/apt... every time. But if the image is optimized it won't work anymore
+
+      - name: Create or update release
+        run: |
+          # First, check if release exists and delete it if it does
+          RELEASE_ID=$(curl -s -X GET \
+            -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
+            "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/tags/snapshot-latest" \
+            | jq -r '.id // empty')
+
+          if [ -n "$RELEASE_ID" ]; then
+            # Delete existing release
+            curl -X DELETE \
+              -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
+              "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/$RELEASE_ID"
+          fi
+
+          # Create new release
+          RESPONSE=$(curl -X POST \
+            -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "tag_name": "snapshot-latest",
+              "name": "Latest Snapshot",
+              "body": "Automated snapshot build from master branch",
+              "draft": false,
+              "prerelease": true
+            }' \
+            "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases")
+
+          echo "RELEASE_ID=$(echo $RESPONSE | jq -r '.id')" >> $GITHUB_ENV
+
+      - name: Upload Linux binary
+        run: |
+          curl -X POST \
+            -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
+            -H "Content-Type: application/octet-stream" \
+            --data-binary "@target/x86_64-unknown-linux-gnu/release/harmony_composer" \
+            "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/${{ env.RELEASE_ID }}/assets?name=harmony_composer"
+
+      - name: Upload Windows binary
+        run: |
+          curl -X POST \
+            -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
+            -H "Content-Type: application/octet-stream" \
+            --data-binary "@target/x86_64-pc-windows-gnu/release/harmony_composer.exe" \
+            "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/${{ env.RELEASE_ID }}/assets?name=harmony_composer.exe"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: hub.nationtech.io/harmony/harmony_composer:latest

.gitignore

@@ -1,3 +1,26 @@
-target
-private_repos
-log/
+### General ###
+private_repos/
+
+### Harmony ###
+harmony.log
+data/okd/installation_files*
+
+### Helm ###
+# Chart dependencies
+**/charts/*.tgz
+
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "examples/try_rust_webapp/tryrust.org"]
+	path = examples/try_rust_webapp/tryrust.org
+	url = https://github.com/rust-dd/tryrust.org.git

.sqlx/query-2ea29df2326f7c84bd4100ad510a3fd4878dc2e217dc83f9bf45a402dfd62a91.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT host_id FROM host_role_mapping WHERE role = ?",
+  "describe": {
+    "columns": [
+      {
+        "name": "host_id",
+        "ordinal": 0,
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "2ea29df2326f7c84bd4100ad510a3fd4878dc2e217dc83f9bf45a402dfd62a91"
+}

.sqlx/query-8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "SQLite",
+  "query": "\n        SELECT\n            p1.id,\n            p1.version_id,\n            p1.data as \"data: Json<PhysicalHost>\"\n        FROM\n            physical_hosts p1\n        INNER JOIN (\n            SELECT\n                id,\n                MAX(version_id) AS max_version\n            FROM\n                physical_hosts\n            GROUP BY\n                id\n        ) p2 ON p1.id = p2.id AND p1.version_id = p2.max_version\n        ",
+  "describe": {
+    "columns": [
+      {
+        "name": "id",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "version_id",
+        "ordinal": 1,
+        "type_info": "Text"
+      },
+      {
+        "name": "data: Json<PhysicalHost>",
+        "ordinal": 2,
+        "type_info": "Blob"
+      }
+    ],
+    "parameters": {
+      "Right": 0
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067"
+}

.sqlx/query-934035c7ca6e064815393e4e049a7934b0a7fac04a4fe4b2a354f0443d630990.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT id, version_id, data as \"data: Json<PhysicalHost>\" FROM physical_hosts WHERE id = ? ORDER BY version_id DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "name": "id",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "version_id",
+        "ordinal": 1,
+        "type_info": "Text"
+      },
+      {
+        "name": "data: Json<PhysicalHost>",
+        "ordinal": 2,
+        "type_info": "Null"
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "934035c7ca6e064815393e4e049a7934b0a7fac04a4fe4b2a354f0443d630990"
+}

.sqlx/query-df7a7c9cfdd0972e2e0ce7ea444ba8bc9d708a4fb89d5593a0be2bbebde62aff.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n        INSERT INTO host_role_mapping (host_id, role)\n        VALUES (?, ?)\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": []
+  },
+  "hash": "df7a7c9cfdd0972e2e0ce7ea444ba8bc9d708a4fb89d5593a0be2bbebde62aff"
+}

.sqlx/query-f10f615ee42129ffa293e46f2f893d65a237d31d24b74a29c6a8d8420d255ab8.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "INSERT INTO physical_hosts (id, version_id, data) VALUES (?, ?, ?)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 3
+    },
+    "nullable": []
+  },
+  "hash": "f10f615ee42129ffa293e46f2f893d65a237d31d24b74a29c6a8d8420d255ab8"
+}

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Contributing to the Harmony project
+
+## Write small P-R
+
+Aim for the smallest piece of work that is mergeable.
+
+Mergeable means that :
+
+- it does not break the build
+- it moves the codebase one step forward
+
+P-Rs can be many things, they do not have to be complete features.
+
+### What a P-R **should** be
+
+- Introduce a new trait : This will be the place to discuss the new trait addition, its design and implementation
+- A new implementation of a trait : a new concrete implementation of the LoadBalancer trait
+- A new CI check : something that improves quality, robustness, ci performance
+- Documentation improvements
+- Refactoring
+- Bugfix
+
+### What a P-R **should not** be
+
+- Large. Anything over 200 lines (excluding generated lines) should have a very good reason to be this large.
+- A mix of refactoring, bug fixes and new features.
+- Introducing multiple new features or ideas at once.
+- Multiple new implementations of a trait/functionnality at once
+
+The general idea is to keep P-Rs small and single purpose.
+
+## Commit message formatting
+
+We follow conventional commits guidelines.
+
+https://www.conventionalcommits.org/en/v1.0.0/

Cargo.toml

@@ -9,6 +9,14 @@ members = [
   "harmony_tui",
   "opnsense-config",
   "opnsense-config-xml",
+  "harmony_cli",
+  "k3d",
+  "harmony_composer",
+  "harmony_inventory_agent",
+  "harmony_secret_derive",
+  "harmony_secret",
+  "adr/agent_discovery/mdns",
+  "brocade",
 ]
 
 [workspace.package]
@@ -17,25 +25,55 @@ readme = "README.md"
 license = "GNU AGPL v3"
 
 [workspace.dependencies]
-log = "0.4.22"
-env_logger = "0.11.5"
-derive-new = "0.7.0"
-async-trait = "0.1.82"
-tokio = { version = "1.40.0", features = ["io-std", "fs"] }
-cidr = "0.2.3"
-russh = "0.45.0"
-russh-keys = "0.45.0"
-rand = "0.8.5"
-url = "2.5.4"
-kube = "0.98.0"
-k8s-openapi = { version = "0.24.0", features = [ "v1_30" ] }
-serde_yaml = "0.9.34"
-http = "1.2.0"
-
-[workspace.dependencies.uuid]
-version = "1.11.0"
-features = [
-    "v4",                # Lets you generate random UUIDs
-    "fast-rng",          # Use a faster (but still sufficiently random) RNG
-    "macro-diagnostics", # Enable better diagnostics for compile-time UUIDs
-]
+log = { version = "0.4", features = ["kv"] }
+env_logger = "0.11"
+derive-new = "0.7"
+async-trait = "0.1"
+tokio = { version = "1.40", features = [
+  "io-std",
+  "fs",
+  "macros",
+  "rt-multi-thread",
+] }
+cidr = { features = ["serde"], version = "0.2" }
+russh = "0.45"
+russh-keys = "0.45"
+rand = "0.9"
+url = "2.5"
+kube = { version = "1.1.0", features = [
+  "config",
+  "client",
+  "runtime",
+  "rustls-tls",
+  "ws",
+  "jsonpatch",
+] }
+k8s-openapi = { version = "0.25", features = ["v1_30"] }
+serde_yaml = "0.9"
+serde-value = "0.7"
+http = "1.2"
+inquire = "0.7"
+convert_case = "0.8"
+chrono = "0.4"
+similar = "2"
+uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] }
+pretty_assertions = "1.4.1"
+tempfile = "3.20.0"
+bollard = "0.19.1"
+base64 = "0.22.1"
+tar = "0.4.44"
+lazy_static = "1.5.0"
+directories = "6.0.0"
+thiserror = "2.0.14"
+serde = { version = "1.0.209", features = ["derive", "rc"] }
+serde_json = "1.0.127"
+askama = "0.14"
+sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
+reqwest = { version = "0.12", features = [
+  "blocking",
+  "stream",
+  "rustls-tls",
+  "http2",
+  "json",
+], default-features = false }
+assertor = "0.0.4"

Dockerfile

@@ -0,0 +1,26 @@
+FROM docker.io/rust:1.89.0 AS build
+
+WORKDIR /app
+
+COPY . .
+
+RUN cargo build --release --bin harmony_composer
+
+FROM docker.io/rust:1.89.0
+
+WORKDIR /app
+
+RUN rustup target add x86_64-pc-windows-gnu
+RUN rustup target add x86_64-unknown-linux-gnu
+RUN rustup component add rustfmt
+RUN rustup component add clippy
+
+RUN apt update
+
+# TODO: Consider adding more supported targets
+# nodejs for checkout action, docker for building containers, mingw for cross-compiling for windows
+RUN apt install -y nodejs docker.io mingw-w64
+
+COPY --from=build /app/target/release/harmony_composer .
+
+ENTRYPOINT ["/app/harmony_composer"]

README.md

@@ -1,9 +1,161 @@
-### Watch the whole repo on every change
+# Harmony : Open-source infrastructure orchestration that treats your platform like first-class code
 
-Due to the current setup being a mix of separate repositories with gitignore and rust workspace, a few options are required for cargo-watch to have the desired behavior :
+_By [NationTech](https://nationtech.io)_
 
-```sh
-RUST_LOG=info cargo watch --ignore-nothing -w harmony -w private_repos/ -x 'run --bin nationtech' 
+[![Build](https://git.nationtech.io/NationTech/harmony/actions/workflows/check.yml/badge.svg)](https://git.nationtech.io/nationtech/harmony)
+[![License](https://img.shields.io/badge/license-AGPLv3-blue?style=flat-square)](LICENSE)
+
+### Unify
+
+- **Project Scaffolding**
+- **Infrastructure Provisioning**
+- **Application Deployment**
+- **Day-2 operations**
+
+All in **one strongly-typed Rust codebase**.
+
+### Deploy anywhere
+
+From a **developer laptop** to a **global production cluster**, a single **source of truth** drives the **full software lifecycle.**
+
+---
+
+## 1 · The Harmony Philosophy
+
+Infrastructure is essential, but it shouldn’t be your core business. Harmony is built on three guiding principles that make modern platforms reliable, repeatable, and easy to reason about.
+
+| Principle                              | What it means for you                                                                                                                                                                |
+| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| **Infrastructure as Resilient Code**   | Replace sprawling YAML and bash scripts with type-safe Rust. Test, refactor, and version your platform just like application code.                                                   |
+| **Prove It Works — Before You Deploy** | Harmony uses the compiler to verify that your application’s needs match the target environment’s capabilities at **compile-time**, eliminating an entire class of runtime outages.   |
+| **One Unified Model**                  | Software and infrastructure are a single system. Harmony models them together, enabling deep automation—from bare-metal servers to Kubernetes workloads—with zero context switching. |
+
+These principles surface as simple, ergonomic Rust APIs that let teams focus on their product while trusting the platform underneath.
+
+---
+
+## 2 · Quick Start
+
+The snippet below spins up a complete **production-grade Rust + Leptos Webapp** with monitoring. Swap it for your own scores to deploy anything from microservices to machine-learning pipelines.
+
+```rust
+use harmony::{
+    inventory::Inventory,
+    modules::{
+        application::{
+            ApplicationScore, RustWebFramework, RustWebapp,
+            features::{PackagingDeployment, rhob_monitoring::Monitoring},
+        },
+        monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
+    },
+    topology::K8sAnywhereTopology,
+};
+use harmony_macros::hurl;
+use std::{path::PathBuf, sync::Arc};
+
+#[tokio::main]
+async fn main() {
+    let application = Arc::new(RustWebapp {
+        name: "harmony-example-leptos".to_string(),
+        project_root: PathBuf::from(".."), // <== Your project root, usually .. if you use the standard `/harmony` folder
+        framework: Some(RustWebFramework::Leptos),
+        service_port: 8080,
+    });
+
+    // Define your Application deployment and the features you want
+    let app = ApplicationScore {
+        features: vec![
+            Box::new(PackagingDeployment {
+                application: application.clone(),
+            }),
+            Box::new(Monitoring {
+                application: application.clone(),
+                alert_receiver: vec![
+                    Box::new(DiscordWebhook {
+                        name: "test-discord".to_string(),
+                        url: hurl!("https://discord.doesnt.exist.com"), // <== Get your discord webhook url
+                    }),
+                ],
+            }),
+        ],
+        application,
+    };
+
+    harmony_cli::run(
+        Inventory::autoload(),
+        K8sAnywhereTopology::from_env(), // <== Deploy to local automatically provisioned local k3d by default or connect to any kubernetes cluster
+        vec![Box::new(app)],
+        None,
+    )
+    .await
+    .unwrap();
+}
 ```
 
-This will run the nationtech bin (likely `private_repos/nationtech/src/main.rs`) on any change in the harmony or private_repos folders.
+Run it:
+
+```bash
+cargo run
+```
+
+Harmony analyses the code, shows an execution plan in a TUI, and applies it once you confirm. Same code, same binary—every environment.
+
+---
+
+## 3 · Core Concepts
+
+| Term             | One-liner                                                                                            |
+| ---------------- | ---------------------------------------------------------------------------------------------------- |
+| **Score<T>**     | Declarative description of the desired state (e.g., `LAMPScore`).                                    |
+| **Interpret<T>** | Imperative logic that realises a `Score` on a specific environment.                                  |
+| **Topology**     | An environment (local k3d, AWS, bare-metal) exposing verified _Capabilities_ (Kubernetes, DNS, …).   |
+| **Maestro**      | Orchestrator that compiles Scores + Topology, ensuring all capabilities line up **at compile-time**. |
+| **Inventory**    | Optional catalogue of physical assets for bare-metal and edge deployments.                           |
+
+A visual overview is in the diagram below.
+
+[Harmony Core Architecture](docs/diagrams/Harmony_Core_Architecture.drawio.svg)
+
+---
+
+## 4 · Install
+
+Prerequisites:
+
+- Rust
+- Docker (if you deploy locally)
+- `kubectl` / `helm` for Kubernetes-based topologies
+
+```bash
+git clone https://git.nationtech.io/nationtech/harmony
+cd harmony
+cargo build --release          # builds the CLI, TUI and libraries
+```
+
+---
+
+## 5 · Learning More
+
+- **Architectural Decision Records** – dive into the rationale
+  - [ADR-001 · Why Rust](adr/001-rust.md)
+  - [ADR-003 · Infrastructure Abstractions](adr/003-infrastructure-abstractions.md)
+  - [ADR-006 · Secret Management](adr/006-secret-management.md)
+  - [ADR-011 · Multi-Tenant Cluster](adr/011-multi-tenant-cluster.md)
+
+- **Extending Harmony** – write new Scores / Interprets, add hardware like OPNsense firewalls, or embed Harmony in your own tooling (`/docs`).
+
+- **Community** – discussions and roadmap live in [GitLab issues](https://git.nationtech.io/nationtech/harmony/-/issues). PRs, ideas, and feedback are welcome!
+
+---
+
+## 6 · License
+
+Harmony is released under the **GNU AGPL v3**.
+
+> We choose a strong copyleft license to ensure the project—and every improvement to it—remains open and benefits the entire community. Fork it, enhance it, even out-innovate us; just keep it open.
+
+See [LICENSE](LICENSE) for the full text.
+
+---
+
+_Made with ❤️ & 🦀 by the NationTech and the Harmony community_

adr/000-ADR-Template.md

@@ -0,0 +1,33 @@
+# Architecture Decision Record: \<Title\>
+
+Initial Author: \<Name\>
+
+Initial Date: \<Date\>
+
+Last Updated Date: \<Date\>
+
+## Status
+
+Proposed/Pending/Accepted/Implemented
+
+## Context
+
+The problem, background, the "why" behind this decision/discussion
+
+## Decision
+
+Proposed solution to the problem
+
+## Rationale
+
+Reasoning behind the decision
+
+## Consequences
+
+Pros/Cons of chosen solution
+
+## Alternatives considered
+
+Pros/Cons of various proposed solutions considered
+
+## Additional Notes

adr/006-secret-management.md

@@ -4,7 +4,7 @@
 
 Proposed
 
-### TODO :
+### TODO [#3](https://git.nationtech.io/NationTech/harmony/issues/3):
 
 Before accepting this proposal we need to run a POC to validate this potential issue :
 

adr/008-score-display-formatting.md

@@ -0,0 +1,62 @@
+## Architecture Decision Record: Data Representation and UI Rendering for Score Types
+
+**Status:** Proposed
+
+**TL;DR:** `Score` types will be serialized (using `serde`) for presentation in UIs. This decouples data definition from presentation, improving scalability and reducing complexity for developers defining `Score` types. New UI types only need to handle existing field types, and new `Score` types don’t require UI changes as long as they use existing field types.  Adding a new field type *does* require updates to all UIs.
+
+**Key benefits:** Scalability, reduced complexity for `Score` authors, decoupling of data and presentation.
+
+**Key trade-off:** Adding new field types requires updating all UIs.
+
+---
+
+**Context:**
+
+Harmony is a pure Rust infrastructure orchestrator focused on compile-time safety and providing a developer-friendly, Ansible-module-like experience for defining infrastructure configurations via "Scores". These Scores (e.g., `LAMPScore`) are Rust structs composed of specific, strongly-typed fields (e.g., `VersionField`, `UrlField`, `PathField`) which are validated at compile-time using macros (`Version!`, `Url!`, etc.).
+
+A key requirement is displaying the configuration defined in these Scores across various user interfaces (Web UI, TUI, potentially Mobile UI, etc.) in a consistent and type-safe manner. As the number of Score types is expected to grow significantly (hundreds or thousands), we need a scalable approach for rendering their data that avoids tightly coupling Score definitions to specific UI implementations.
+
+The primary challenge is preventing the need for every `Score` struct author to implement multiple display traits (e.g., `Display`, `WebDisplay`, `TuiDisplay`) for every potential UI target. This would create an N x M complexity problem (N Scores * M UI types) and place an unreasonable burden on Score developers, hindering scalability and maintainability.
+
+**Decision:**
+
+1.  **Mandatory Serialization:** All `Score` structs *must* implement `serde::Serialize` and `serde::Deserialize`. They *will not* be required to implement `std::fmt::Display` or any custom UI-specific display traits (e.g., `WebDisplay`, `TuiDisplay`).
+2.  **Field-Level Rendering:** Responsibility for rendering data will reside within the UI components. Each UI (Web, TUI, etc.) will implement logic to display *individual field types* (e.g., `UrlField`, `VersionField`, `IpAddressField`, `SecretField`).
+3.  **Data Access via Serialization:** UIs will primarily interact with `Score` data through its serialized representation (e.g., JSON obtained via `serde_json`). This provides a standardized interface for UIs to consume the data structure agnostic of the specific `Score` type. Alternatively, UIs *could* potentially use reflection or specific visitor patterns on the `Score` struct itself, but serialization is the preferred decoupling mechanism.
+
+**Rationale:**
+
+1.  **Decoupling Data from Presentation:** This decision cleanly separates the data definition (`Score` structs and their fields) from the presentation logic (UI rendering). `Score` authors can focus solely on defining the data and its structure, while UI developers focus on how to best present known data *types*.
+2.  **Scalability:** This approach scales significantly better than requiring display trait implementations on Scores:
+    *   Adding a *new Score type* requires *no changes* to existing UI code, provided it uses existing field types.
+    *   Adding a *new UI type* requires implementing rendering logic only for the defined set of *field types*, not for every individual `Score` type. This reduces the N x M complexity to N + M complexity (approximately).
+3.  **Simplicity for Score Authors:** Requiring only `serde::Serialize + Deserialize` (which can often be derived automatically with `#[derive(Serialize, Deserialize)]`) is a much lower burden than implementing custom rendering logic for multiple, potentially unknown, UI targets.
+4.  **Leverages Rust Ecosystem Standards:** `serde` is the de facto standard for serialization and deserialization in Rust. Relying on it aligns with common Rust practices and benefits from its robustness, performance, and extensive tooling.
+5.  **Consistency for UIs:** Serialization provides a consistent, structured format (like JSON) for UIs to consume data, regardless of the underlying `Score` struct's complexity or composition.
+6.  **Flexibility for UI Implementation:** UIs can choose the best way to render each field type based on their capabilities (e.g., a `UrlField` might be a clickable link in a Web UI, plain text in a TUI; a `SecretField` might be masked).
+
+**Consequences:**
+
+**Positive:**
+
+*   Greatly improved scalability for adding new Score types and UI targets.
+*   Strong separation of concerns between data definition and presentation.
+*   Reduced implementation burden and complexity for Score authors.
+*   Consistent mechanism for UIs to access and interpret Score data.
+*   Aligns well with the Hexagonal Architecture (ADR-002) by treating UIs as adapters interacting with the application core via a defined port (the serialized data contract).
+
+**Negative:**
+
+*   Adding a *new field type* (e.g., `EmailField`) requires updates to *all* existing UI implementations to support rendering it.
+*   UI components become dependent on the set of defined field types and need comprehensive logic to handle each one appropriately.
+*   Potential minor overhead of serialization/deserialization compared to direct function calls (though likely negligible for UI purposes).
+*   Requires careful design and management of the standard library of field types.
+
+**Alternatives Considered:**
+
+1.  **`Score` Implements `std::fmt::Display`:**
+    *   _Rejected:_ Too simplistic. Only suitable for basic text rendering, doesn't cater to structured UIs (Web, etc.), and doesn't allow type-specific rendering logic (e.g., masking secrets). Doesn't scale to multiple UI formats.
+2.  **`Score` Implements Multiple Custom Display Traits (`WebDisplay`, `TuiDisplay`, etc.):**
+    *   _Rejected:_ Leads directly to the N x M complexity problem. Tightly couples Score definitions to specific UI implementations. Places an excessive burden on Score authors, hindering adoption and scalability.
+3.  **Generic Display Trait with Context (`Score` implements `DisplayWithContext<UIContext>`):**
+    *   _Rejected:_ More flexible than multiple traits, but still requires Score authors to implement potentially complex rendering logic within the `Score` definition itself. The `Score` would still need awareness of different UI contexts, leading to undesirable coupling. Managing context types adds complexity.

adr/009-helm-and-kustomize-handling.md

@@ -0,0 +1,61 @@
+# Architecture Decision Record: Helm and Kustomize Handling
+
+Initial Author: Taha Hawa
+
+Initial Date: 2025-04-15
+
+Last Updated Date: 2025-04-15
+
+## Status
+
+Proposed
+
+## Context
+
+We need to find a way to handle Helm charts and deploy them to a Kubernetes cluster. Helm has a lot of extra functionality that we may or may not need. Kustomize handles Helm charts by inflating them and applying them as vanilla Kubernetes yaml. How should Harmony handle it?
+
+## Decision
+
+In order to move quickly and efficiently, Harmony should handle Helm charts similarly to how Kustomize does: invoke Helm to inflate/render the charts with the needed inputs, and deploy the rendered artifacts to Kubernetes as if it were vanilla manifests.
+
+## Rationale
+
+A lot of Helm's features aren't strictly necessary and would add unneeded overhead. This is likely the fastest way to go from zero to deployed. Other tools (e.g. Kustomize) already do this. Kustomize has tooling for patching and modifying k8s manifests before deploying, and Harmony should have that power too, even if it's not what Helm typically intends.
+
+Perhaps in future also have a Kustomize resource in Harmony? Which could handle Helm charts for Harmony as well/instead.
+
+## Consequences
+
+**Pros**:
+
+- Much easier (and faster) than implementing all of Helm's featureset
+- Can potentially re-use code from K8sResource already present in Harmony
+- Harmony retains more control over how the deployment goes after rendering (i.e. can act like Kustomize, or leverage Kustomize itself to modify deployments after rendering/inflation)
+- Reduce (unstable) surface of dealing with Helm binary
+
+**Cons**:
+
+- Lose some Helm functionality
+- Potentially lose some compatibility with Helm
+
+## Alternatives considered
+
+- ### Implement Helm resouce/client fully in Harmony
+  - **Pros**:
+    - Retain full compatibility with Helm as a tool
+    - Retain full functionality of Helm
+  - **Cons**:
+    - Longer dev time
+    - More complex integration
+    - Dealing with larger (unstable) surface of Helm as a binary
+- ### Leverage Kustomize to deal with Helm charts
+  - **Pros**:
+    - Already has a good, minimal inflation solution built
+    - Powerful post-processing/patching
+    - Can integrate with `kubectl`
+  - **Cons**:
+    - Unstable binary tool/surface to deal with
+    - Still requires Helm to be installed as well as Kustomize
+    - Not all Helm features supported
+
+## Additional Notes

adr/010-monitoring-alerting/architecture.rs

@@ -0,0 +1,73 @@
+pub trait MonitoringSystem {}
+
+// 1. Modified AlertReceiver trait:
+//    - Removed the problematic `clone` method.
+//    - Added `box_clone` which returns a Box<dyn AlertReceiver>.
+pub trait AlertReceiver {
+    type M: MonitoringSystem;
+    fn install(&self, sender: &Self::M) -> Result<(), String>;
+    // This method allows concrete types to clone themselves into a Box<dyn AlertReceiver>
+    fn box_clone(&self) -> Box<dyn AlertReceiver<M = Self::M>>;
+}
+#[derive(Clone)]
+struct Prometheus{}
+impl MonitoringSystem for Prometheus {}
+
+#[derive(Clone)] // Keep derive(Clone) for DiscordWebhook itself
+struct DiscordWebhook{}
+
+impl AlertReceiver for DiscordWebhook {
+    type M = Prometheus;
+    fn install(&self, sender: &Self::M) -> Result<(), String> {
+        // Placeholder for actual installation logic
+        println!("DiscordWebhook installed for Prometheus monitoring.");
+        Ok(())
+    }
+    // 2. Implement `box_clone` for DiscordWebhook:
+    //    This uses the derived `Clone` for DiscordWebhook to create a new boxed instance.
+    fn box_clone(&self) -> Box<dyn AlertReceiver<M = Self::M>> {
+        Box::new(self.clone())
+    }
+}
+
+// 3. Implement `std::clone::Clone` for `Box<dyn AlertReceiver<M= M>>`:
+//    This allows `Box<dyn AlertReceiver>` to be cloned.
+//    The `+ 'static` lifetime bound is often necessary for trait objects stored in collections,
+//    ensuring they live long enough.
+impl<M: MonitoringSystem + 'static> Clone for Box<dyn AlertReceiver<M= M>> {
+    fn clone(&self) -> Self {
+        self.box_clone() // Call the custom `box_clone` method
+    }
+}
+
+// MonitoringConfig can now derive Clone because its `receivers` field
+// (Vec<Box<dyn AlertReceiver<M = M>>>) is now cloneable.
+#[derive(Clone)]
+struct MonitoringConfig <M: MonitoringSystem + 'static>{
+    receivers: Vec<Box<dyn AlertReceiver<M = M>>>
+}
+
+// Example usage to demonstrate compilation and functionality
+fn main() {
+    let prometheus_instance = Prometheus{};
+    let discord_webhook_instance = DiscordWebhook{};
+
+    let mut config = MonitoringConfig {
+        receivers: Vec::new()
+    };
+
+    // Create a boxed alert receiver
+    let boxed_receiver: Box<dyn AlertReceiver<M = Prometheus>> = Box::new(discord_webhook_instance);
+    config.receivers.push(boxed_receiver);
+
+    // Clone the config, which will now correctly clone the boxed receiver
+    let cloned_config = config.clone();
+
+    println!("Original config has {} receivers.", config.receivers.len());
+    println!("Cloned config has {} receivers.", cloned_config.receivers.len());
+
+    // Example of using the installed receiver
+    if let Some(receiver) = config.receivers.get(0) {
+        let _ = receiver.install(&prometheus_instance);
+    }
+}

adr/010-monitoring-and-alerting.md

@@ -0,0 +1,68 @@
+# Architecture Decision Record: Monitoring and Alerting
+
+Initial Author : Willem Rolleman
+Date : April 28 2025
+
+## Status
+
+Proposed
+
+## Context
+
+A harmony user should be able to initialize a monitoring stack easily, either at the first run of Harmony, or that integrates with existing proects and infra without creating multiple instances of the monitoring stack or overwriting existing alerts/configurations.The user also needs a simple way to configure the stack so that it watches the projects. There should be reasonable defaults configured that are easily customizable for each project    
+
+## Decision
+
+Create MonitoringStack score that creates a maestro to launch the monitoring stack or not if it is already present. 
+The MonitoringStack score can be passed to the maestro in the vec! scores list
+
+## Rationale
+
+Having the score launch a maestro will allow the user to easily create a new monitoring stack and keeps composants grouped together. The MonitoringScore can handle all the logic for adding alerts, ensuring that the stack is running etc. 
+
+## Alerternatives considered
+
+- ### Implement alerting and monitoring stack using existing HelmScore for each project
+    - **Pros**:
+        - Each project can choose to use the monitoring and alerting stack that they choose
+        - Less overhead in terms of care harmony code
+        - can add Box::new(grafana::grafanascore(namespace)) 
+    - **Cons**:
+        - No default solution implemented
+        - Dev needs to chose what they use
+        - Increases complexity of score projects
+        - Each project will create a new monitoring and alerting instance rather than joining the existing one
+
+
+- ### Use OKD grafana and prometheus
+    - **Pros**:
+        - Minimal config to do in Harmony
+    - **Cons**:
+        - relies on OKD so will not working for local testing via k3d
+
+- ### Create a monitoring and alerting crate similar to harmony tui
+    - **Pros**:
+        - Creates a default solution that can be implemented once by harmony
+        - can create a join function that will allow a project to connect to the existing solution
+        - eliminates risk of creating multiple instances of grafana or prometheus
+    - **Cons**:
+        - more complex than using a helm score
+        - management of values files for individual functions becomes more complicated, ie how do you create alerts for one project via helm install that doesnt overwrite the other alerts 
+
+- ### Add monitoring to Maestro struct so whether the monitoring stack is used must be defined 
+    - **Pros**:
+        - less for the user to define
+        - may be easier to set defaults 
+    - **Cons**:
+        - feels counterintuitive 
+        - would need to modify the structure of the maestro and how it operates which seems like a bad idea
+        - unclear how to allow user to pass custom values/configs to the monitoring stack for subsequent projects
+
+- ### Create MonitoringStack score to add to scores vec! which loads a maestro to install stack if not ready or add custom endpoints/alerts to existing stack
+    - **Pros**:
+        - Maestro already accepts a list of scores to initialize
+        - leaving out the monitoring score simply means the user does not want monitoring
+        - if the monitoring stack is already created, the MonitoringStack score doesn't necessarily need to be added to each project
+        - composants of the monitoring stack are bundled together and can be expaned or modified from the same place
+    - **Cons**:
+        - maybe need to create  

adr/011-multi-tenant-cluster.md

@@ -0,0 +1,161 @@
+# Architecture Decision Record: Multi-Tenancy Strategy for Harmony Managed Clusters
+
+Initial Author: Jean-Gabriel Gill-Couture
+
+Initial Date: 2025-05-26
+
+## Status
+
+Proposed
+
+## Context
+
+Harmony manages production OKD/Kubernetes clusters that serve multiple clients with varying trust levels and operational requirements. We need a multi-tenancy strategy that provides:
+
+1. **Strong isolation** between client workloads while maintaining operational simplicity
+2. **Controlled API access** allowing clients self-service capabilities within defined boundaries
+3. **Security-first approach** protecting both the cluster infrastructure and tenant data
+4. **Harmony-native implementation** using our Score/Interpret pattern for automated tenant provisioning
+5. **Scalable management** supporting both small trusted clients and larger enterprise customers
+
+The official Kubernetes multi-tenancy documentation identifies two primary models: namespace-based isolation and virtual control planes per tenant. Given Harmony's focus on operational simplicity, provider-agnostic abstractions (ADR-003), and hexagonal architecture (ADR-002), we must choose an approach that balances security, usability, and maintainability.
+
+Our clients represent a hybrid tenancy model:
+- **Customer multi-tenancy**: Each client operates independently with no cross-tenant trust
+- **Team multi-tenancy**: Individual clients may have multiple team members requiring coordinated access
+- **API access requirement**: Unlike pure SaaS scenarios, clients need controlled Kubernetes API access for self-service operations
+
+The official kubernetes documentation on multi tenancy heavily inspired this ADR : https://kubernetes.io/docs/concepts/security/multi-tenancy/
+
+## Decision
+
+Implement **namespace-based multi-tenancy** with the following architecture:
+
+### 1. Network Security Model
+- **Private cluster access**: Kubernetes API and OpenShift console accessible only via WireGuard VPN
+- **No public exposure**: Control plane endpoints remain internal to prevent unauthorized access attempts
+- **VPN-based authentication**: Initial access control through WireGuard client certificates
+
+### 2. Tenant Isolation Strategy
+- **Dedicated namespace per tenant**: Each client receives an isolated namespace with access limited only to the required resources and operations
+- **Complete network isolation**: NetworkPolicies prevent cross-namespace communication while allowing full egress to public internet
+- **Resource governance**: ResourceQuotas and LimitRanges enforce CPU, memory, and storage consumption limits
+- **Storage access control**: Clients can create PersistentVolumeClaims but cannot directly manipulate PersistentVolumes or access other tenants' storage
+
+### 3. Access Control Framework
+- **Principle of Least Privilege**: RBAC grants only necessary permissions within tenant namespace scope
+- **Namespace-scoped**: Clients can create/modify/delete resources within their namespace
+- **Cluster-level restrictions**: No access to cluster-wide resources, other namespaces, or sensitive cluster operations
+- **Whitelisted operations**: Controlled self-service capabilities for ingress, secrets, configmaps, and workload management
+
+### 4. Identity Management Evolution
+- **Phase 1**: Manual provisioning of VPN access and Kubernetes ServiceAccounts/Users
+- **Phase 2**: Migration to Keycloak-based identity management (aligning with ADR-006) for centralized authentication and lifecycle management
+
+### 5. Harmony Integration
+- **TenantScore implementation**: Declarative tenant provisioning using Harmony's Score/Interpret pattern
+- **Topology abstraction**: Tenant configuration abstracted from underlying Kubernetes implementation details
+- **Automated deployment**: Complete tenant setup automated through Harmony's orchestration capabilities
+
+## Rationale
+
+### Network Security Through VPN Access
+- **Defense in depth**: VPN requirement adds critical security layer preventing unauthorized cluster access
+- **Simplified firewall rules**: No need for complex public endpoint protections or rate limiting
+- **Audit capability**: VPN access provides clear audit trail of cluster connections
+- **Aligns with enterprise practices**: Most enterprise customers already use VPN infrastructure
+
+### Namespace Isolation vs Virtual Control Planes
+Following Kubernetes official guidance, namespace isolation provides:
+- **Lower resource overhead**: Virtual control planes require dedicated etcd, API server, and controller manager per tenant
+- **Operational simplicity**: Single control plane to maintain, upgrade, and monitor
+- **Cross-tenant service integration**: Enables future controlled cross-tenant communication if required
+- **Proven stability**: Namespace-based isolation is well-tested and widely deployed
+- **Cost efficiency**: Significantly lower infrastructure costs compared to dedicated control planes
+
+### Hybrid Tenancy Model Suitability
+Our approach addresses both customer and team multi-tenancy requirements:
+- **Customer isolation**: Strong network and RBAC boundaries prevent cross-tenant interference
+- **Team collaboration**: Multiple team members can share namespace access through group-based RBAC
+- **Self-service balance**: Controlled API access enables client autonomy without compromising security
+
+### Harmony Architecture Alignment
+- **Provider agnostic**: TenantScore abstracts multi-tenancy concepts, enabling future support for other Kubernetes distributions
+- **Hexagonal architecture**: Tenant management becomes an infrastructure capability accessed through well-defined ports
+- **Declarative automation**: Tenant lifecycle fully managed through Harmony's Score execution model
+
+## Consequences
+
+### Positive Consequences
+- **Strong security posture**: VPN + namespace isolation provides robust tenant separation
+- **Operational efficiency**: Single cluster management with automated tenant provisioning
+- **Client autonomy**: Self-service capabilities reduce operational support burden
+- **Scalable architecture**: Can support hundreds of tenants per cluster without architectural changes
+- **Future flexibility**: Foundation supports evolution to more sophisticated multi-tenancy models
+- **Cost optimization**: Shared infrastructure maximizes resource utilization
+
+### Negative Consequences
+- **VPN operational overhead**: Requires VPN infrastructure management
+- **Manual provisioning complexity**: Phase 1 manual user management creates administrative burden
+- **Network policy dependency**: Requires CNI with NetworkPolicy support (OVN-Kubernetes provides this and is the OKD/Openshift default)
+- **Cluster-wide resource limitations**: Some advanced Kubernetes features require cluster-wide access
+- **Single point of failure**: Cluster outage affects all tenants simultaneously
+
+### Migration Challenges
+- **Legacy client integration**: Existing clients may need VPN client setup and credential migration
+- **Monitoring complexity**: Per-tenant observability requires careful metric and log segmentation
+- **Backup considerations**: Tenant data backup must respect isolation boundaries
+
+## Alternatives Considered
+
+### Alternative 1: Virtual Control Plane Per Tenant
+**Pros**: Complete control plane isolation, full Kubernetes API access per tenant
+**Cons**: 3-5x higher resource usage, complex cross-tenant networking, operational complexity scales linearly with tenants
+
+**Rejected**: Resource overhead incompatible with cost-effective multi-tenancy goals
+
+### Alternative 2: Dedicated Clusters Per Tenant
+**Pros**: Maximum isolation, independent upgrade cycles, simplified security model
+**Cons**: Exponential operational complexity, prohibitive costs, resource waste
+
+**Rejected**: Operational overhead makes this approach unsustainable for multiple clients
+
+### Alternative 3: Public API with Advanced Authentication
+**Pros**: No VPN requirement, potentially simpler client access
+**Cons**: Larger attack surface, complex rate limiting and DDoS protection, increased security monitoring requirements
+
+**Rejected**: Risk/benefit analysis favors VPN-based access control
+
+### Alternative 4: Service Mesh Based Isolation
+**Pros**: Fine-grained traffic control, encryption, advanced observability
+**Cons**: Significant operational complexity, performance overhead, steep learning curve
+
+**Rejected**: Complexity overhead outweighs benefits for current requirements; remains option for future enhancement
+
+## Additional Notes
+
+### Implementation Roadmap
+1. **Phase 1**: Implement VPN access and manual tenant provisioning
+2. **Phase 2**: Deploy TenantScore automation for namespace, RBAC, and NetworkPolicy management
+4. **Phase 3**: Work on privilege escalation from pods, audit for weaknesses, enforce security policies on pod runtimes
+3. **Phase 4**: Integrate Keycloak for centralized identity management
+4. **Phase 5**: Add advanced monitoring and per-tenant observability
+
+### TenantScore Structure Preview
+```rust
+pub struct TenantScore {
+    pub tenant_config: TenantConfig,
+    pub resource_quotas: ResourceQuotaConfig,
+    pub network_isolation: NetworkIsolationPolicy,
+    pub storage_access: StorageAccessConfig,
+    pub rbac_config: RBACConfig,
+}
+```
+
+### Future Enhancements
+- **Cross-tenant service mesh**: For approved inter-tenant communication
+- **Advanced monitoring**: Per-tenant Prometheus/Grafana instances
+- **Backup automation**: Tenant-scoped backup policies
+- **Cost allocation**: Detailed per-tenant resource usage tracking
+
+This ADR establishes the foundation for secure, scalable multi-tenancy in Harmony-managed clusters while maintaining operational simplicity and cost effectiveness. A follow-up ADR will detail the Tenant abstraction and user management mechanisms within the Harmony framework.

adr/011-tenant/NetworkPolicy.yaml

@@ -0,0 +1,41 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: tenant-isolation-policy
+  namespace: testtenant
+spec:
+  podSelector: {}  # Selects all pods in the namespace
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  - from:
+    - podSelector: {}  # Allow from all pods in the same namespace
+  egress:
+  - to:
+    - podSelector: {}  # Allow to all pods in the same namespace
+  - to:
+    - podSelector: {}
+      namespaceSelector: 
+        matchLabels:
+          kubernetes.io/metadata.name: openshift-dns # Target the openshift-dns namespace
+    # Note, only opening port 53 is not enough, will have to dig deeper into this one eventually
+    # ports:
+    # - protocol: UDP
+    #   port: 53
+    # - protocol: TCP
+    #   port: 53
+  # Allow egress to public internet only
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+        except:
+        - 10.0.0.0/8      # RFC1918
+        - 172.16.0.0/12   # RFC1918
+        - 192.168.0.0/16  # RFC1918
+        - 169.254.0.0/16  # Link-local
+        - 127.0.0.0/8     # Loopback
+        - 224.0.0.0/4     # Multicast
+        - 240.0.0.0/4     # Reserved
+        - 100.64.0.0/10   # Carrier-grade NAT
+        - 0.0.0.0/8       # Reserved

adr/011-tenant/TestDeployment.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: testtenant
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: testtenant2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-web
+  namespace: testtenant
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: test-web
+  template:
+    metadata:
+      labels:
+        app: test-web
+    spec:
+      containers:
+      - name: nginx
+        image: nginxinc/nginx-unprivileged
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-web
+  namespace: testtenant
+spec:
+  selector:
+    app: test-web
+  ports:
+  - port: 80
+    targetPort: 8080
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-client
+  namespace: testtenant
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: test-client
+  template:
+    metadata:
+      labels:
+        app: test-client
+    spec:
+      containers:
+      - name: curl
+        image: curlimages/curl:latest
+        command: ["/bin/sh", "-c", "sleep 3600"]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-web
+  namespace: testtenant2
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: test-web
+  template:
+    metadata:
+      labels:
+        app: test-web
+    spec:
+      containers:
+      - name: nginx
+        image: nginxinc/nginx-unprivileged
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-web
+  namespace: testtenant2
+spec:
+  selector:
+    app: test-web
+  ports:
+  - port: 80
+    targetPort: 8080

adr/012-project-delivery-automation.md

@@ -0,0 +1,63 @@
+# Architecture Decision Record: \<Title\>
+
+Initial Author: Jean-Gabriel Gill-Couture
+
+Initial Date: 2025-06-04
+
+Last Updated Date: 2025-06-04
+
+## Status
+
+Proposed
+
+## Context
+
+As Harmony's goal is to make software delivery easier, we must provide an easy way for developers to express their app's semantics and dependencies with great abstractions, in a similar fashion to what the score.dev project is doing.
+
+Thus, we started working on ways to package common types of applications such as LAMP, which we started working on with `LAMPScore`.
+
+Now is time for the next step : we want to pave the way towards complete lifecycle automation. To do this, we will start with a way to execute Harmony's modules easily from anywhere, starting with locally and in CI environments.
+
+## Decision
+
+To achieve easy, portable execution of Harmony, we will follow this architecture :
+
+- Host a basic harmony release that is compiled with the CLI by our gitea/github server
+    - This binary will do the following : check if there is a `harmony` folder in the current path
+    - If yes
+        - Check if cargo is available locally and compile the harmony binary, or compile the harmony binary using a rust docker container, if neither cargo or a container runtime is available, output a message explaining the situation
+        - Run the newly compiled binary. (Ideally using pid handoff like exec does but some research around this should be done. I think handing off the process is to help with OS interaction such as terminal apps, signals, exit codes, process handling, etc but there might be some side effects)
+    - If not
+        - Suggest initializing a project by auto detecting what the project looks like
+        - When the project type cannot be auto detected, provide links to Harmony's documentation on how to set up a project, a link to the examples folder, and a ask the user if he wants to initialize an empty Harmony project in the current folder
+            - harmony/Cargo.toml with dependencies set
+            - harmony/src/main.rs with an example LAMPScore setup and ready to run
+- This same binary can be used in a CI environment to run the target project's Harmony module. By default, we provide these opinionated steps :
+    1. **An empty check step.** The purpose of this step is to run all tests and checks against the codebase. For complex projects this could involve a very complex pipeline of test environments setup and execution but this is out of scope for now. This is not handled by harmony. For projects with automatic setup, we can fill this step with something like `cargo fmt --check; cargo test; cargo build` but Harmony is not directly involved in the execution of this step.
+    2. **Package and publish.** Once all checks have passed, the production ready container is built and pushed to a registry. This is done by Harmony.
+    3. **Deploy to staging automatically.** 
+    4. **Run a sanity check on staging.** As Harmony is responsible for deploying, Harmony should have all the knowledge of how to perform a sanity check on the staging environment. This will, most of the time, be a simple verification of the kubernetes health of all deployed components, and a poke on the public endpoint when there is one.
+    5. **Deploy to production automatically.** Many projects will require manual approval here, this can be easily set up in the CI afterwards, but our opinion is that 
+    6. **Run a sanity check on production.** Same check as staging, but on production.
+
+*Note on providing a base pipeline :* Having a complete pipeline set up automatically will encourage development teams to build upon these by adding tests where they belong. The goal here is to provide an opiniated solution that works for most small and large projects. Of course, many orgnizations will need to add steps such as deploying to sandbox environments, requiring more advanced approvals, more complex publication and coordination with other projects. But this here encompasses the basics required to build and deploy software reliably at any scale.
+
+### Environment setup
+
+TBD : For now, environments (tenants) will be set up and configured manually. Harmony will rely on the kubeconfig provided in the environment where it is running to deploy in the namespace.
+
+For the CD tool such as Argo or Flux they will be activated by default by Harmony when using application level Scores such as LAMPScore in a similar way that the container is automatically built. Then, CI deployment steps will be notifying the CD tool using its API of the new release to deploy.
+
+## Rationale
+
+Reasoning behind the decision
+
+## Consequences
+
+Pros/Cons of chosen solution
+
+## Alternatives considered
+
+Pros/Cons of various proposed solutions considered
+
+## Additional Notes

adr/013-monitoring-notifications.md

@@ -0,0 +1,78 @@
+# Architecture Decision Record: Monitoring Notifications
+
+Initial Author: Taha Hawa
+
+Initial Date: 2025-06-26
+
+Last Updated Date: 2025-06-26
+
+## Status
+
+Proposed
+
+## Context
+
+We need to send notifications (typically from AlertManager/Prometheus) and we need to receive said notifications on mobile devices for sure in some way, whether it's push messages, SMS, phone call, email, etc or all of the above.
+
+## Decision
+
+We should go with https://ntfy.sh except host it ourselves.
+
+`ntfy` is an open source solution written in Go that has the features we need.
+
+## Rationale
+
+`ntfy` has pretty much everything we need (push notifications, email forwarding, receives via webhook), and nothing/not much we don't. Good fit, lightweight.
+
+## Consequences
+
+Pros:
+
+- topics, with ACLs
+- lightweight
+- reliable
+- easy to configure
+- mobile app
+  - the mobile app can listen via websocket, poll, or receive via Firebase/GCM on Android, or similar on iOS.
+- Forward to email
+- Text-to-Speech phone call messages using Twilio integration
+- Operates based on simple HTTP requests/Webhooks, easily usable via AlertManager
+
+Cons:
+
+- No SMS pushes
+- SQLite DB, makes it harder to HA/scale
+
+## Alternatives considered
+
+[AWS SNS](https://aws.amazon.com/sns/):
+Pros:
+
+- highly reliable
+- no hosting needed
+
+Cons:
+
+- no control, not self hosted
+- costs (per usage)
+
+[Apprise](https://github.com/caronc/apprise):
+Pros:
+
+- Way more ways of sending notifications
+- Can use ntfy as one of the backends/ways of sending
+
+Cons:
+
+- Way too overkill for what we need in terms of features
+
+[Gotify](https://github.com/gotify/server):
+Pros:
+
+- simple, lightweight, golang, etc
+
+Cons:
+
+- Pushes topics are per-user
+
+## Additional Notes

adr/015-higher-order-topologies.md

@@ -0,0 +1,114 @@
+# Architecture Decision Record: Higher-Order Topologies
+
+**Initial Author:** Jean-Gabriel Gill-Couture  
+**Initial Date:** 2025-12-08  
+**Last Updated Date:** 2025-12-08  
+
+## Status
+
+Implemented
+
+## Context
+
+Harmony models infrastructure as **Topologies** (deployment targets like `K8sAnywhereTopology`, `LinuxHostTopology`) implementing **Capabilities** (tech traits like `PostgreSQL`, `Docker`).
+
+**Higher-Order Topologies** (e.g., `FailoverTopology<T>`) compose/orchestrate capabilities *across* multiple underlying topologies (e.g., primary+replica `T`).
+
+Naive design requires manual `impl Capability for HigherOrderTopology<T>` *per T per capability*, causing:
+- **Impl explosion**: N topologies × M capabilities = N×M boilerplate.
+- **ISP violation**: Topologies forced to impl unrelated capabilities.
+- **Maintenance hell**: New topology needs impls for *all* orchestrated capabilities; new capability needs impls for *all* topologies/higher-order.
+- **Barrier to extension**: Users can't easily add topologies without todos/panics.
+
+This makes scaling Harmony impractical as ecosystem grows.
+
+## Decision
+
+Use **blanket trait impls** on higher-order topologies to *automatically* derive orchestration:
+
+````rust
+/// Higher-Order Topology: Orchestrates capabilities across sub-topologies.
+pub struct FailoverTopology<T> {
+    /// Primary sub-topology.
+    primary: T,
+    /// Replica sub-topology.
+    replica: T,
+}
+
+/// Automatically provides PostgreSQL failover for *any* `T: PostgreSQL`.
+/// Delegates to primary for queries; orchestrates deploy across both.
+#[async_trait]
+impl<T: PostgreSQL> PostgreSQL for FailoverTopology<T> {
+    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String> {
+        // Deploy primary; extract certs/endpoint;
+        // deploy replica with pg_basebackup + TLS passthrough.
+        // (Full impl logged/elaborated.)
+    }
+
+    // Delegate queries to primary.
+    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String> {
+        self.primary.get_replication_certs(cluster_name).await
+    }
+    // ...
+}
+
+/// Similarly for other capabilities.
+#[async_trait]
+impl<T: Docker> Docker for FailoverTopology<T> {
+    // Failover Docker orchestration.
+}
+````
+
+**Key properties:**
+- **Auto-derivation**: `Failover<K8sAnywhere>` gets `PostgreSQL` iff `K8sAnywhere: PostgreSQL`.
+- **No boilerplate**: One blanket impl per capability *per higher-order type*.
+
+## Rationale
+
+- **Composition via generics**: Rust trait solver auto-selects impls; zero runtime cost.
+- **Compile-time safety**: Missing `T: Capability` → compile error (no panics).
+- **Scalable**: O(capabilities) impls per higher-order; new `T` auto-works.
+- **ISP-respecting**: Capabilities only surface if sub-topology provides.
+- **Centralized logic**: Orchestration (e.g., cert propagation) in one place.
+
+**Example usage:**
+````rust
+// ✅ Works: K8sAnywhere: PostgreSQL → Failover provides failover PG
+let pg_failover: FailoverTopology<K8sAnywhereTopology> = ...;
+pg_failover.deploy_pg(config).await;
+
+// ✅ Works: LinuxHost: Docker → Failover provides failover Docker
+let docker_failover: FailoverTopology<LinuxHostTopology> = ...;
+docker_failover.deploy_docker(...).await;
+
+// ❌ Compile fail: K8sAnywhere !: Docker
+let invalid: FailoverTopology<K8sAnywhereTopology>;
+invalid.deploy_docker(...); // `T: Docker` bound unsatisfied
+````
+
+## Consequences
+
+**Pros:**
+- **Extensible**: New topology `AWSTopology: PostgreSQL` → instant `Failover<AWSTopology>: PostgreSQL`.
+- **Lean**: No useless impls (e.g., no `K8sAnywhere: Docker`).
+- **Observable**: Logs trace every step.
+
+**Cons:**
+- **Monomorphization**: Generics generate code per T (mitigated: few Ts).
+- **Delegation opacity**: Relies on rustdoc/logs for internals.
+
+## Alternatives considered
+
+| Approach | Pros | Cons |
+|----------|------|------|
+| **Manual per-T impls**<br>`impl PG for Failover<K8s> {..}`<br>`impl PG for Failover<Linux> {..}` | Explicit control | N×M explosion; violates ISP; hard to extend. |
+| **Dynamic trait objects**<br>`Box<dyn AnyCapability>` | Runtime flex | Perf hit; type erasure; error-prone dispatch. |
+| **Mega-topology trait**<br>All-in-one `OrchestratedTopology` | Simple wiring | Monolithic; poor composition. |
+| **Registry dispatch**<br>Runtime capability lookup | Decoupled | Complex; no compile safety; perf/debug overhead. |
+
+**Selected**: Blanket impls leverage Rust generics for safe, zero-cost composition.
+
+## Additional Notes
+
+- Applies to `MultisiteTopology<T>`, `ShardedTopology<T>`, etc.
+- `FailoverTopology` in `failover.rs` is first implementation.

adr/015-higher-order-topologies/example.rs

@@ -0,0 +1,153 @@
+//! Example of Higher-Order Topologies in Harmony.
+//! Demonstrates how `FailoverTopology<T>` automatically provides failover for *any* capability
+//! supported by a sub-topology `T` via blanket trait impls.
+//! 
+//! Key insight: No manual impls per T or capability -- scales effortlessly.
+//! Users can:
+//! - Write new `Topology` (impl capabilities on a struct).
+//! - Compose with `FailoverTopology` (gets capabilities if T has them).
+//! - Compile fails if capability missing (safety).
+
+use async_trait::async_trait;
+use tokio;
+
+/// Capability trait: Deploy and manage PostgreSQL.
+#[async_trait]
+pub trait PostgreSQL {
+    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String>;
+    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String>;
+}
+
+/// Capability trait: Deploy Docker.
+#[async_trait]
+pub trait Docker {
+    async fn deploy_docker(&self) -> Result<String, String>;
+}
+
+/// Configuration for PostgreSQL deployments.
+#[derive(Clone)]
+pub struct PostgreSQLConfig;
+
+/// Replication certificates.
+#[derive(Clone)]
+pub struct ReplicationCerts;
+
+/// Concrete topology: Kubernetes Anywhere (supports PostgreSQL).
+#[derive(Clone)]
+pub struct K8sAnywhereTopology;
+
+#[async_trait]
+impl PostgreSQL for K8sAnywhereTopology {
+    async fn deploy(&self, _config: &PostgreSQLConfig) -> Result<String, String> {
+        // Real impl: Use k8s helm chart, operator, etc.
+        Ok("K8sAnywhere PostgreSQL deployed".to_string())
+    }
+
+    async fn get_replication_certs(&self, _cluster_name: &str) -> Result<ReplicationCerts, String> {
+        Ok(ReplicationCerts)
+    }
+}
+
+/// Concrete topology: Linux Host (supports Docker).
+#[derive(Clone)]
+pub struct LinuxHostTopology;
+
+#[async_trait]
+impl Docker for LinuxHostTopology {
+    async fn deploy_docker(&self) -> Result<String, String> {
+        // Real impl: Install/configure Docker on host.
+        Ok("LinuxHost Docker deployed".to_string())
+    }
+}
+
+/// Higher-Order Topology: Composes multiple sub-topologies (primary + replica).
+/// Automatically derives *all* capabilities of `T` with failover orchestration.
+/// 
+/// - If `T: PostgreSQL`, then `FailoverTopology<T>: PostgreSQL` (blanket impl).
+/// - Same for `Docker`, etc. No boilerplate!
+/// - Compile-time safe: Missing `T: Capability` → error.
+#[derive(Clone)]
+pub struct FailoverTopology<T> {
+    /// Primary sub-topology.
+    pub primary: T,
+    /// Replica sub-topology.
+    pub replica: T,
+}
+
+/// Blanket impl: Failover PostgreSQL if T provides PostgreSQL.
+/// Delegates reads to primary; deploys to both.
+#[async_trait]
+impl<T: PostgreSQL + Send + Sync + Clone> PostgreSQL for FailoverTopology<T> {
+    async fn deploy(&self, config: &PostgreSQLConfig) -> Result<String, String> {
+        // Orchestrate: Deploy primary first, then replica (e.g., via pg_basebackup).
+        let primary_result = self.primary.deploy(config).await?;
+        let replica_result = self.replica.deploy(config).await?;
+        Ok(format!("Failover PG deployed: {} | {}", primary_result, replica_result))
+    }
+
+    async fn get_replication_certs(&self, cluster_name: &str) -> Result<ReplicationCerts, String> {
+        // Delegate to primary (replica follows).
+        self.primary.get_replication_certs(cluster_name).await
+    }
+}
+
+/// Blanket impl: Failover Docker if T provides Docker.
+#[async_trait]
+impl<T: Docker + Send + Sync + Clone> Docker for FailoverTopology<T> {
+    async fn deploy_docker(&self) -> Result<String, String> {
+        // Orchestrate across primary + replica.
+        let primary_result = self.primary.deploy_docker().await?;
+        let replica_result = self.replica.deploy_docker().await?;
+        Ok(format!("Failover Docker deployed: {} | {}", primary_result, replica_result))
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    let config = PostgreSQLConfig;
+
+    println!("=== ✅ PostgreSQL Failover (K8sAnywhere supports PG) ===");
+    let pg_failover = FailoverTopology {
+        primary: K8sAnywhereTopology,
+        replica: K8sAnywhereTopology,
+    };
+    let result = pg_failover.deploy(&config).await.unwrap();
+    println!("Result: {}", result);
+
+    println!("\n=== ✅ Docker Failover (LinuxHost supports Docker) ===");
+    let docker_failover = FailoverTopology {
+        primary: LinuxHostTopology,
+        replica: LinuxHostTopology,
+    };
+    let result = docker_failover.deploy_docker().await.unwrap();
+    println!("Result: {}", result);
+
+    println!("\n=== ❌ Would fail to compile (K8sAnywhere !: Docker) ===");
+    // let invalid = FailoverTopology {
+    //     primary: K8sAnywhereTopology,
+    //     replica: K8sAnywhereTopology,
+    // };
+    // invalid.deploy_docker().await.unwrap(); // Error: `K8sAnywhereTopology: Docker` not satisfied!
+    // Very clear error message :
+    // error[E0599]: the method `deploy_docker` exists for struct `FailoverTopology<K8sAnywhereTopology>`, but its trait bounds were not satisfied
+    //   --> src/main.rs:90:9
+    //    |
+    //  4 | pub struct FailoverTopology<T> {
+    //    | ------------------------------ method `deploy_docker` not found for this struct because it doesn't satisfy `FailoverTopology<K8sAnywhereTopology>: Docker`
+    // ...
+    // 37 | struct K8sAnywhereTopology;
+    //    | -------------------------- doesn't satisfy `K8sAnywhereTopology: Docker`
+    // ...
+    // 90 | invalid.deploy_docker(); // `T: Docker` bound unsatisfied
+    //    |         ^^^^^^^^^^^^^ method cannot be called on `FailoverTopology<K8sAnywhereTopology>` due to unsatisfied trait bounds
+    //    |
+    // note: trait bound `K8sAnywhereTopology: Docker` was not satisfied
+    //   --> src/main.rs:61:9
+    //    |
+    // 61 | impl<T: Docker + Send + Sync> Docker for FailoverTopology<T> {
+    //    |         ^^^^^^                ------     -------------------
+    //    |         |
+    //    |         unsatisfied trait bound introduced here
+    // note: the trait `Docker` must be implemented
+}
+

adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md

@@ -0,0 +1,90 @@
+# Architecture Decision Record: Global Orchestration Mesh & The Harmony Agent
+
+**Status:** Proposed
+**Date:** 2025-12-19
+
+## Context
+
+Harmony is designed to enable a truly decentralized infrastructure where independent clusters—owned by different organizations or running on diverse hardware—can collaborate reliably. This vision combines the decentralization of Web3 with the performance and capabilities of Web2.
+
+Currently, Harmony operates as a stateless CLI tool, invoked manually or via CI runners. While effective for deployment, this model presents a critical limitation: **a CLI cannot react to real-time events.**
+
+To achieve automated failover and dynamic workload management, we need a system that is "always on." Relying on manual intervention or scheduled CI jobs to recover from a cluster failure creates unacceptable latency and prevents us from scaling to thousands of nodes.
+
+Furthermore, we face a challenge in serving diverse workloads:
+*   **Financial workloads** require absolute consistency (CP - Consistency/Partition Tolerance).
+*   **AI/Inference workloads** require maximum availability (AP - Availability/Partition Tolerance).
+
+There are many more use cases, but those are the two extremes.
+
+We need a unified architecture that automates cluster coordination and supports both consistency models without requiring a complete re-architecture in the future.
+
+## Decision
+
+We propose a fundamental architectural evolution. It has been clear since the start of Harmony that it would be necessary to transition Harmony from a purely ephemeral CLI tool to a system that includes a persistent **Harmony Agent**. This Agent will connect to a **Global Orchestration Mesh** based on a strongly consistent protocol.
+
+The proposal consists of four key pillars:
+
+### 1. The Harmony Agent (New Component)
+We will develop a long-running process (Daemon/Agent) to be deployed alongside workloads.
+*   **Shift from CLI:** Unlike the CLI, which applies configuration and exits, the Agent maintains a persistent connection to the mesh.
+*   **Responsibility:** It actively monitors cluster health, participates in consensus, and executes lifecycle commands (start/stop/fence) instantly when the mesh dictates a state change.
+
+### 2. The Technology: NATS JetStream
+We will utilize **NATS JetStream** as the underlying transport and consensus layer for the Agent and the Mesh.
+*   **Why not raw Raft?** Implementing a raw Raft library requires building and maintaining the transport layer, log compaction, snapshotting, and peer discovery manually. NATS JetStream provides a battle-tested, distributed log and Key-Value store (based on Raft) out of the box, along with a high-performance pub/sub system for event propagation.
+*   **Role:** It will act as the "source of truth" for the cluster state.
+
+### 3. Strong Consistency at the Mesh Layer
+The mesh will operate with **Strong Consistency** by default.
+*   All critical cluster state changes (topology updates, lease acquisitions, leadership elections) will require consensus among the Agents.
+*   This ensures that in the event of a network partition, we have a mathematical guarantee of which side holds the valid state, preventing data corruption.
+
+### 4. Public UX: The `FailoverStrategy` Abstraction
+To keep the user experience stable and simple, we will expose the complexity of the mesh through a high-level configuration API, tentatively called `FailoverStrategy`.
+
+The user defines the *intent* in their config, and the Harmony Agent automates the *execution*:
+
+*   **`FailoverStrategy::AbsoluteConsistency`**:
+    *   *Use Case:* Banking, Transactional DBs.
+    *   *Behavior:* If the mesh detects a partition, the Agent on the minority side immediately halts workloads. No split-brain is ever allowed.
+*   **`FailoverStrategy::SplitBrainAllowed`**:
+    *   *Use Case:* LLM Inference, Stateless Web Servers.
+    *   *Behavior:* If a partition occurs, the Agent keeps workloads running to maximize uptime. State is reconciled when connectivity returns.
+
+## Rationale
+
+**The Necessity of an Agent**
+You cannot automate what you do not monitor. Moving to an Agent-based model is the only way to achieve sub-second reaction times to infrastructure failures. It transforms Harmony from a deployment tool into a self-healing platform.
+
+**Scaling & Decentralization**
+To allow independent clusters to collaborate, they need a shared language. A strongly consistent mesh allows Cluster A (Organization X) and Cluster B (Organization Y) to agree on workload placement without a central authority.
+
+**Why Strong Consistency First?**
+It is technically feasible to relax a strongly consistent system to allow for "Split Brain" behavior (AP) when the user requests it. However, it is nearly impossible to take an eventually consistent system and force it to be strongly consistent (CP) later. By starting with strict constraints, we cover the hardest use cases (Finance) immediately.
+
+**Future Topologies**
+While our immediate need is `FailoverTopology` (Multi-site), this architecture supports any future topology logic:
+*   **`CostTopology`**: Agents negotiate to route workloads to the cluster with the cheapest spot instances.
+*   **`HorizontalTopology`**: Spreading a single workload across 100 clusters for massive scale.
+*   **`GeoTopology`**: Ensuring data stays within specific legal jurisdictions.
+
+The mesh provides the *capability* (consensus and messaging); the topology provides the *logic*.
+
+## Consequences
+
+**Positive**
+*   **Automation:** Eliminates manual failover, enabling massive scale.
+*   **Reliability:** Guarantees data safety for critical workloads by default.
+*   **Flexibility:** A single codebase serves both high-frequency trading and AI inference.
+*   **Stability:** The public API remains abstract, allowing us to optimize the mesh internals without breaking user code.
+
+**Negative**
+*   **Deployment Complexity:** Users must now deploy and maintain a running service (the Agent) rather than just downloading a binary.
+*   **Engineering Complexity:** Integrating NATS JetStream and handling distributed state machines is significantly more complex than the current CLI logic.
+
+## Implementation Plan (Short Term)
+1.  **Agent Bootstrap:** Create the initial scaffold for the Harmony Agent (daemon).
+2.  **Mesh Integration:** Prototype NATS JetStream embedding within the Agent.
+3.  **Strategy Implementation:** Add `FailoverStrategy` to the configuration schema and implement the logic in the Agent to read and act on it.
+4.  **Migration:** Transition the current manual failover scripts into event-driven logic handled by the Agent.

adr/agent_discovery/mdns/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "mdns"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+mdns-sd = "0.14"
+tokio = { version = "1", features = ["full"] }
+futures = "0.3"
+dmidecode = "0.2" # For getting the motherboard ID on the agent
+log.workspace=true
+env_logger.workspace=true
+clap = { version = "4.5.46", features = ["derive"] }
+get_if_addrs = "0.5.3"
+local-ip-address = "0.6.5"

adr/agent_discovery/mdns/src/advertise.rs

@@ -0,0 +1,60 @@
+// harmony-agent/src/main.rs
+
+use log::info;
+use mdns_sd::{ServiceDaemon, ServiceInfo};
+use std::collections::HashMap;
+
+use crate::SERVICE_TYPE;
+
+// The service we are advertising.
+const SERVICE_PORT: u16 = 43210; // A port for the service. It needs one, even if unused.
+
+pub async fn advertise() {
+    info!("Starting Harmony Agent...");
+
+    // Get a unique ID for this machine.
+    let motherboard_id = "some motherboard id";
+    let instance_name = format!("harmony-agent-{}", motherboard_id);
+    info!("This agent's instance name: {}", instance_name);
+    info!("Advertising with ID: {}", motherboard_id);
+
+    // Create a new mDNS daemon.
+    let mdns = ServiceDaemon::new().expect("Failed to create mDNS daemon");
+
+    // Create a TXT record HashMap to hold our metadata.
+    let mut properties = HashMap::new();
+    properties.insert("id".to_string(), motherboard_id.to_string());
+    properties.insert("version".to_string(), "1.0".to_string());
+
+    // Create the service information.
+    // The instance name should be unique on the network.
+    let local_ip = local_ip_address::local_ip().unwrap();
+    let service_info = ServiceInfo::new(
+        SERVICE_TYPE,
+        &instance_name,
+        "harmony-host.local.", // A hostname for the service
+        local_ip,
+        // "0.0.0.0",
+        SERVICE_PORT,
+        Some(properties),
+    )
+    .expect("Failed to create service info");
+
+    // Register our service with the daemon.
+    mdns.register(service_info)
+        .expect("Failed to register service");
+
+    info!(
+        "Service '{}' registered and now being advertised.",
+        instance_name
+    );
+    info!("Agent is running. Press Ctrl+C to exit.");
+
+    for iface in get_if_addrs::get_if_addrs().unwrap() {
+        println!("{:#?}", iface);
+    }
+
+    // Keep the agent running indefinitely.
+    tokio::signal::ctrl_c().await.unwrap();
+    info!("Shutting down agent.");
+}

adr/agent_discovery/mdns/src/discover.rs

@@ -0,0 +1,109 @@
+use mdns_sd::{ServiceDaemon, ServiceEvent};
+
+use crate::SERVICE_TYPE;
+
+pub async fn discover() {
+    println!("Starting Harmony Master and browsing for agents...");
+
+    // Create a new mDNS daemon.
+    let mdns = ServiceDaemon::new().expect("Failed to create mDNS daemon");
+
+    // Start browsing for the service type.
+    // The receiver will be a stream of events.
+    let receiver = mdns.browse(SERVICE_TYPE).expect("Failed to browse");
+
+    println!(
+        "Listening for mDNS events for '{}'. Press Ctrl+C to exit.",
+        SERVICE_TYPE
+    );
+
+    std::thread::spawn(move || {
+        while let Ok(event) = receiver.recv() {
+            match event {
+                ServiceEvent::ServiceData(resolved) => {
+                    println!("Resolved a new service: {}", resolved.fullname);
+                }
+                other_event => {
+                    println!("Received other event: {:?}", &other_event);
+                }
+            }
+        }
+    });
+
+    // Gracefully shutdown the daemon.
+    std::thread::sleep(std::time::Duration::from_secs(1000000));
+    mdns.shutdown().unwrap();
+
+    // Process events as they come in.
+    // while let Ok(event) = receiver.recv_async().await {
+    //     debug!("Received event {event:?}");
+    //     // match event {
+    //     //     ServiceEvent::ServiceFound(svc_type, fullname) => {
+    //     //         println!("\n--- Agent Discovered ---");
+    //     //         println!("  Service Name: {}", fullname());
+    //     //         // You can now resolve this service to get its IP, port, and TXT records
+    //     //         // The resolve operation is a separate network call.
+    //     //         let receiver = mdns.browse(info.get_fullname()).unwrap();
+    //     //         if let Ok(resolve_event) = receiver.recv_timeout(Duration::from_secs(2)) {
+    //     //              if let ServiceEvent::ServiceResolved(info) = resolve_event {
+    //     //                 let ip = info.get_addresses().iter().next().unwrap();
+    //     //                 let port = info.get_port();
+    //     //                 let motherboard_id = info.get_property("id").map_or("N/A", |v| v.val_str());
+    //     //
+    //     //                 println!("  IP: {}:{}", ip, port);
+    //     //                 println!("  Motherboard ID: {}", motherboard_id);
+    //     //                 println!("------------------------");
+    //     //
+    //     //                 // TODO: Add this agent to your central list of discovered hosts.
+    //     //              }
+    //     //         } else {
+    //     //             println!("Could not resolve service '{}' in time.", info.get_fullname());
+    //     //         }
+    //     //     }
+    //     //     ServiceEvent::ServiceRemoved(info) => {
+    //     //         println!("\n--- Agent Removed ---");
+    //     //         println!("  Service Name: {}", info.get_fullname());
+    //     //         println!("---------------------");
+    //     //         // TODO: Remove this agent from your list.
+    //     //     }
+    //     //     _ => {
+    //     //         // We don't care about other event types for this example
+    //     //     }
+    //     // }
+    // }
+}
+
+async fn _discover_example() {
+    use mdns_sd::{ServiceDaemon, ServiceEvent};
+
+    // Create a daemon
+    let mdns = ServiceDaemon::new().expect("Failed to create daemon");
+
+    // Use recently added `ServiceEvent::ServiceData`.
+    mdns.use_service_data(true)
+        .expect("Failed to use ServiceData");
+
+    // Browse for a service type.
+    let service_type = "_mdns-sd-my-test._udp.local.";
+    let receiver = mdns.browse(service_type).expect("Failed to browse");
+
+    // Receive the browse events in sync or async. Here is
+    // an example of using a thread. Users can call `receiver.recv_async().await`
+    // if running in async environment.
+    std::thread::spawn(move || {
+        while let Ok(event) = receiver.recv() {
+            match event {
+                ServiceEvent::ServiceData(resolved) => {
+                    println!("Resolved a new service: {}", resolved.fullname);
+                }
+                other_event => {
+                    println!("Received other event: {:?}", &other_event);
+                }
+            }
+        }
+    });
+
+    // Gracefully shutdown the daemon.
+    std::thread::sleep(std::time::Duration::from_secs(1));
+    mdns.shutdown().unwrap();
+}

adr/agent_discovery/mdns/src/main.rs

@@ -0,0 +1,31 @@
+use clap::{Parser, ValueEnum};
+
+mod advertise;
+mod discover;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Args {
+    #[arg(value_enum)]
+    profile: Profiles,
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
+enum Profiles {
+    Advertise,
+    Discover,
+}
+
+// The service type we are looking for.
+const SERVICE_TYPE: &str = "_harmony._tcp.local.";
+
+#[tokio::main]
+async fn main() {
+    env_logger::init();
+    let args = Args::parse();
+
+    match args.profile {
+        Profiles::Advertise => advertise::advertise().await,
+        Profiles::Discover => discover::discover().await,
+    }
+}

brocade/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "brocade"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+async-trait.workspace = true
+harmony_types = { path = "../harmony_types" }
+russh.workspace = true
+russh-keys.workspace = true
+tokio.workspace = true
+log.workspace = true
+env_logger.workspace = true
+regex = "1.11.3"
+harmony_secret = { path = "../harmony_secret" }
+serde.workspace = true

brocade/examples/main.rs

@@ -0,0 +1,70 @@
+use std::net::{IpAddr, Ipv4Addr};
+
+use brocade::BrocadeOptions;
+use harmony_secret::{Secret, SecretManager};
+use harmony_types::switch::PortLocation;
+use serde::{Deserialize, Serialize};
+
+#[derive(Secret, Clone, Debug, Serialize, Deserialize)]
+struct BrocadeSwitchAuth {
+    username: String,
+    password: String,
+}
+
+#[tokio::main]
+async fn main() {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    // let ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 250)); // old brocade @ ianlet
+    let ip = IpAddr::V4(Ipv4Addr::new(192, 168, 55, 101)); // brocade @ sto1
+    // let ip = IpAddr::V4(Ipv4Addr::new(192, 168, 4, 11)); // brocade @ st
+    let switch_addresses = vec![ip];
+
+    let config = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .unwrap();
+
+    let brocade = brocade::init(
+        &switch_addresses,
+        22,
+        &config.username,
+        &config.password,
+        Some(BrocadeOptions {
+            dry_run: true,
+            ..Default::default()
+        }),
+    )
+    .await
+    .expect("Brocade client failed to connect");
+
+    let entries = brocade.get_stack_topology().await.unwrap();
+    println!("Stack topology: {entries:#?}");
+
+    let entries = brocade.get_interfaces().await.unwrap();
+    println!("Interfaces: {entries:#?}");
+
+    let version = brocade.version().await.unwrap();
+    println!("Version: {version:?}");
+
+    println!("--------------");
+    let mac_adddresses = brocade.get_mac_address_table().await.unwrap();
+    println!("VLAN\tMAC\t\t\tPORT");
+    for mac in mac_adddresses {
+        println!("{}\t{}\t{}", mac.vlan, mac.mac_address, mac.port);
+    }
+
+    println!("--------------");
+    let channel_name = "1";
+    brocade.clear_port_channel(channel_name).await.unwrap();
+
+    println!("--------------");
+    let channel_id = brocade.find_available_channel_id().await.unwrap();
+
+    println!("--------------");
+    let channel_name = "HARMONY_LAG";
+    let ports = [PortLocation(2, 0, 35)];
+    brocade
+        .create_port_channel(channel_id, channel_name, &ports)
+        .await
+        .unwrap();
+}

brocade/src/fast_iron.rs

@@ -0,0 +1,212 @@
+use super::BrocadeClient;
+use crate::{
+    BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, MacAddressEntry,
+    PortChannelId, PortOperatingMode, parse_brocade_mac_address, shell::BrocadeShell,
+};
+
+use async_trait::async_trait;
+use harmony_types::switch::{PortDeclaration, PortLocation};
+use log::{debug, info};
+use regex::Regex;
+use std::{collections::HashSet, str::FromStr};
+
+#[derive(Debug)]
+pub struct FastIronClient {
+    shell: BrocadeShell,
+    version: BrocadeInfo,
+}
+
+impl FastIronClient {
+    pub fn init(mut shell: BrocadeShell, version_info: BrocadeInfo) -> Self {
+        shell.before_all(vec!["skip-page-display".into()]);
+        shell.after_all(vec!["page".into()]);
+
+        Self {
+            shell,
+            version: version_info,
+        }
+    }
+
+    fn parse_mac_entry(&self, line: &str) -> Option<Result<MacAddressEntry, Error>> {
+        debug!("[Brocade] Parsing mac address entry: {line}");
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() < 3 {
+            return None;
+        }
+
+        let (vlan, mac_address, port) = match parts.len() {
+            3 => (
+                u16::from_str(parts[0]).ok()?,
+                parse_brocade_mac_address(parts[1]).ok()?,
+                parts[2].to_string(),
+            ),
+            _ => (
+                1,
+                parse_brocade_mac_address(parts[0]).ok()?,
+                parts[1].to_string(),
+            ),
+        };
+
+        let port =
+            PortDeclaration::parse(&port).map_err(|e| Error::UnexpectedError(format!("{e}")));
+
+        match port {
+            Ok(p) => Some(Ok(MacAddressEntry {
+                vlan,
+                mac_address,
+                port: p,
+            })),
+            Err(e) => Some(Err(e)),
+        }
+    }
+
+    fn parse_stack_port_entry(&self, line: &str) -> Option<Result<InterSwitchLink, Error>> {
+        debug!("[Brocade] Parsing stack port entry: {line}");
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() < 10 {
+            return None;
+        }
+
+        let local_port = PortLocation::from_str(parts[0]).ok()?;
+
+        Some(Ok(InterSwitchLink {
+            local_port,
+            remote_port: None,
+        }))
+    }
+
+    fn build_port_channel_commands(
+        &self,
+        channel_id: PortChannelId,
+        channel_name: &str,
+        ports: &[PortLocation],
+    ) -> Vec<String> {
+        let mut commands = vec![
+            "configure terminal".to_string(),
+            format!("lag {channel_name} static id {channel_id}"),
+        ];
+
+        for port in ports {
+            commands.push(format!("ports ethernet {port}"));
+        }
+
+        commands.push(format!("primary-port {}", ports[0]));
+        commands.push("deploy".into());
+        commands.push("exit".into());
+        commands.push("write memory".into());
+        commands.push("exit".into());
+
+        commands
+    }
+}
+
+#[async_trait]
+impl BrocadeClient for FastIronClient {
+    async fn version(&self) -> Result<BrocadeInfo, Error> {
+        Ok(self.version.clone())
+    }
+
+    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error> {
+        info!("[Brocade] Showing MAC address table...");
+
+        let output = self
+            .shell
+            .run_command("show mac-address", ExecutionMode::Regular)
+            .await?;
+
+        output
+            .lines()
+            .skip(2)
+            .filter_map(|line| self.parse_mac_entry(line))
+            .collect()
+    }
+
+    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error> {
+        let output = self
+            .shell
+            .run_command("show interface stack-ports", crate::ExecutionMode::Regular)
+            .await?;
+
+        output
+            .lines()
+            .skip(1)
+            .filter_map(|line| self.parse_stack_port_entry(line))
+            .collect()
+    }
+
+    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error> {
+        todo!()
+    }
+
+    async fn configure_interfaces(
+        &self,
+        _interfaces: Vec<(String, PortOperatingMode)>,
+    ) -> Result<(), Error> {
+        todo!()
+    }
+
+    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error> {
+        info!("[Brocade] Finding next available channel id...");
+
+        let output = self
+            .shell
+            .run_command("show lag", ExecutionMode::Regular)
+            .await?;
+        let re = Regex::new(r"=== LAG .* ID\s+(\d+)").expect("Invalid regex");
+
+        let used_ids: HashSet<u8> = output
+            .lines()
+            .filter_map(|line| {
+                re.captures(line)
+                    .and_then(|c| c.get(1))
+                    .and_then(|id_match| id_match.as_str().parse().ok())
+            })
+            .collect();
+
+        let mut next_id: u8 = 1;
+        loop {
+            if !used_ids.contains(&next_id) {
+                break;
+            }
+            next_id += 1;
+        }
+
+        info!("[Brocade] Found channel id: {next_id}");
+        Ok(next_id)
+    }
+
+    async fn create_port_channel(
+        &self,
+        channel_id: PortChannelId,
+        channel_name: &str,
+        ports: &[PortLocation],
+    ) -> Result<(), Error> {
+        info!(
+            "[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}"
+        );
+
+        let commands = self.build_port_channel_commands(channel_id, channel_name, ports);
+        self.shell
+            .run_commands(commands, ExecutionMode::Privileged)
+            .await?;
+
+        info!("[Brocade] Port-channel '{channel_name}' configured.");
+        Ok(())
+    }
+
+    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error> {
+        info!("[Brocade] Clearing port-channel: {channel_name}");
+
+        let commands = vec![
+            "configure terminal".to_string(),
+            format!("no lag {channel_name}"),
+            "write memory".to_string(),
+        ];
+        self.shell
+            .run_commands(commands, ExecutionMode::Privileged)
+            .await?;
+
+        info!("[Brocade] Port-channel '{channel_name}' cleared.");
+        Ok(())
+    }
+}

brocade/src/lib.rs

@@ -0,0 +1,338 @@
+use std::net::IpAddr;
+use std::{
+    fmt::{self, Display},
+    time::Duration,
+};
+
+use crate::network_operating_system::NetworkOperatingSystemClient;
+use crate::{
+    fast_iron::FastIronClient,
+    shell::{BrocadeSession, BrocadeShell},
+};
+
+use async_trait::async_trait;
+use harmony_types::net::MacAddress;
+use harmony_types::switch::{PortDeclaration, PortLocation};
+use regex::Regex;
+
+mod fast_iron;
+mod network_operating_system;
+mod shell;
+mod ssh;
+
+#[derive(Default, Clone, Debug)]
+pub struct BrocadeOptions {
+    pub dry_run: bool,
+    pub ssh: ssh::SshOptions,
+    pub timeouts: TimeoutConfig,
+}
+
+#[derive(Clone, Debug)]
+pub struct TimeoutConfig {
+    pub shell_ready: Duration,
+    pub command_execution: Duration,
+    pub command_output: Duration,
+    pub cleanup: Duration,
+    pub message_wait: Duration,
+}
+
+impl Default for TimeoutConfig {
+    fn default() -> Self {
+        Self {
+            shell_ready: Duration::from_secs(10),
+            command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while
+            command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output"
+            cleanup: Duration::from_secs(10),
+            message_wait: Duration::from_millis(500),
+        }
+    }
+}
+
+enum ExecutionMode {
+    Regular,
+    Privileged,
+}
+
+#[derive(Clone, Debug)]
+pub struct BrocadeInfo {
+    os: BrocadeOs,
+    version: String,
+}
+
+#[derive(Clone, Debug)]
+pub enum BrocadeOs {
+    NetworkOperatingSystem,
+    FastIron,
+    Unknown,
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub struct MacAddressEntry {
+    pub vlan: u16,
+    pub mac_address: MacAddress,
+    pub port: PortDeclaration,
+}
+
+pub type PortChannelId = u8;
+
+/// Represents a single physical or logical link connecting two switches within a stack or fabric.
+///
+/// This structure provides a standardized view of the topology regardless of the
+/// underlying Brocade OS configuration (stacking vs. fabric).
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct InterSwitchLink {
+    /// The local port on the switch where the topology command was run.
+    pub local_port: PortLocation,
+    /// The port on the directly connected neighboring switch.
+    pub remote_port: Option<PortLocation>,
+}
+
+/// Represents the key running configuration status of a single switch interface.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct InterfaceInfo {
+    /// The full configuration name (e.g., "TenGigabitEthernet 1/0/1", "FortyGigabitEthernet 2/0/2").
+    pub name: String,
+    /// The physical location of the interface.
+    pub port_location: PortLocation,
+    /// The parsed type and name prefix of the interface.
+    pub interface_type: InterfaceType,
+    /// The primary configuration mode defining the interface's behavior (L2, L3, Fabric).
+    pub operating_mode: Option<PortOperatingMode>,
+    /// Indicates the current state of the interface.
+    pub status: InterfaceStatus,
+}
+
+/// Categorizes the functional type of a switch interface.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum InterfaceType {
+    /// Physical or virtual Ethernet interface (e.g., TenGigabitEthernet, FortyGigabitEthernet).
+    Ethernet(String),
+}
+
+impl fmt::Display for InterfaceType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            InterfaceType::Ethernet(name) => write!(f, "{name}"),
+        }
+    }
+}
+
+/// Defines the primary configuration mode of a switch interface, representing mutually exclusive roles.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum PortOperatingMode {
+    /// The interface is explicitly configured for Brocade fabric roles (ISL or Trunk enabled).
+    Fabric,
+    /// The interface is configured for standard Layer 2 switching as Trunk port (`switchport mode trunk`).
+    Trunk,
+    /// The interface is configured for standard Layer 2 switching as Access port (`switchport` without trunk mode).
+    Access,
+}
+
+/// Defines the possible status of an interface.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum InterfaceStatus {
+    /// The interface is connected.
+    Connected,
+    /// The interface is not connected and is not expected to be.
+    NotConnected,
+    /// The interface is not connected but is expected to be (configured with `no shutdown`).
+    SfpAbsent,
+}
+
+pub async fn init(
+    ip_addresses: &[IpAddr],
+    port: u16,
+    username: &str,
+    password: &str,
+    options: Option<BrocadeOptions>,
+) -> Result<Box<dyn BrocadeClient + Send + Sync>, Error> {
+    let shell = BrocadeShell::init(ip_addresses, port, username, password, options).await?;
+
+    let version_info = shell
+        .with_session(ExecutionMode::Regular, |session| {
+            Box::pin(get_brocade_info(session))
+        })
+        .await?;
+
+    Ok(match version_info.os {
+        BrocadeOs::FastIron => Box::new(FastIronClient::init(shell, version_info)),
+        BrocadeOs::NetworkOperatingSystem => {
+            Box::new(NetworkOperatingSystemClient::init(shell, version_info))
+        }
+        BrocadeOs::Unknown => todo!(),
+    })
+}
+
+#[async_trait]
+pub trait BrocadeClient: std::fmt::Debug {
+    /// Retrieves the operating system and version details from the connected Brocade switch.
+    ///
+    /// This is typically the first call made after establishing a connection to determine
+    /// the switch OS family (e.g., FastIron, NOS) for feature compatibility.
+    ///
+    /// # Returns
+    ///
+    /// A `BrocadeInfo` structure containing parsed OS type and version string.
+    async fn version(&self) -> Result<BrocadeInfo, Error>;
+
+    /// Retrieves the dynamically learned MAC address table from the switch.
+    ///
+    /// This is crucial for discovering where specific network endpoints (MAC addresses)
+    /// are currently located on the physical ports.
+    ///
+    /// # Returns
+    ///
+    /// A vector of `MacAddressEntry`, where each entry typically contains VLAN, MAC address,
+    /// and the associated port name/index.
+    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error>;
+
+    /// Derives the physical connections used to link multiple switches together
+    /// to form a single logical entity (stack, fabric, etc.).
+    ///
+    /// This abstracts the underlying configuration (e.g., stack ports, fabric ports)
+    /// to return a standardized view of the topology.
+    ///
+    /// # Returns
+    ///
+    /// A vector of `InterSwitchLink` structs detailing which ports are used for stacking/fabric.
+    /// If the switch is not stacked, returns an empty vector.
+    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error>;
+
+    /// Retrieves the status for all interfaces
+    ///
+    /// # Returns
+    ///
+    /// A vector of `InterfaceInfo` structures.
+    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error>;
+
+    /// Configures a set of interfaces to be operated with a specified mode (access ports, ISL, etc.).
+    async fn configure_interfaces(
+        &self,
+        interfaces: Vec<(String, PortOperatingMode)>,
+    ) -> Result<(), Error>;
+
+    /// Scans the existing configuration to find the next available (unused)
+    /// Port-Channel ID (`lag` or `trunk`) for assignment.
+    ///
+    /// # Returns
+    ///
+    /// The smallest, unassigned `PortChannelId` within the supported range.
+    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error>;
+
+    /// Creates and configures a new Port-Channel (Link Aggregation Group or LAG)
+    /// using the specified channel ID and ports.
+    ///
+    /// The resulting configuration must be persistent (saved to startup-config).
+    /// Assumes a static LAG configuration mode unless specified otherwise by the implementation.
+    ///
+    /// # Parameters
+    ///
+    /// * `channel_id`: The ID (e.g., 1-128) for the logical port channel.
+    /// * `channel_name`: A descriptive name for the LAG (used in configuration context).
+    /// * `ports`: A slice of `PortLocation` structs defining the physical member ports.
+    async fn create_port_channel(
+        &self,
+        channel_id: PortChannelId,
+        channel_name: &str,
+        ports: &[PortLocation],
+    ) -> Result<(), Error>;
+
+    /// Removes all configuration associated with the specified Port-Channel name.
+    ///
+    /// This operation should be idempotent; attempting to clear a non-existent
+    /// channel should succeed (or return a benign error).
+    ///
+    /// # Parameters
+    ///
+    /// * `channel_name`: The name of the Port-Channel (LAG) to delete.
+    ///
+    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error>;
+}
+
+async fn get_brocade_info(session: &mut BrocadeSession) -> Result<BrocadeInfo, Error> {
+    let output = session.run_command("show version").await?;
+
+    if output.contains("Network Operating System") {
+        let re = Regex::new(r"Network Operating System Version:\s*(?P<version>[a-zA-Z0-9.\-]+)")
+            .expect("Invalid regex");
+        let version = re
+            .captures(&output)
+            .and_then(|cap| cap.name("version"))
+            .map(|m| m.as_str().to_string())
+            .unwrap_or_default();
+
+        return Ok(BrocadeInfo {
+            os: BrocadeOs::NetworkOperatingSystem,
+            version,
+        });
+    } else if output.contains("ICX") {
+        let re = Regex::new(r"(?m)^\s*SW: Version\s*(?P<version>[a-zA-Z0-9.\-]+)")
+            .expect("Invalid regex");
+        let version = re
+            .captures(&output)
+            .and_then(|cap| cap.name("version"))
+            .map(|m| m.as_str().to_string())
+            .unwrap_or_default();
+
+        return Ok(BrocadeInfo {
+            os: BrocadeOs::FastIron,
+            version,
+        });
+    }
+
+    Err(Error::UnexpectedError("Unknown Brocade OS version".into()))
+}
+
+fn parse_brocade_mac_address(value: &str) -> Result<MacAddress, String> {
+    let cleaned_mac = value.replace('.', "");
+
+    if cleaned_mac.len() != 12 {
+        return Err(format!("Invalid MAC address: {value}"));
+    }
+
+    let mut bytes = [0u8; 6];
+    for (i, pair) in cleaned_mac.as_bytes().chunks(2).enumerate() {
+        let byte_str = std::str::from_utf8(pair).map_err(|_| "Invalid UTF-8")?;
+        bytes[i] =
+            u8::from_str_radix(byte_str, 16).map_err(|_| format!("Invalid hex in MAC: {value}"))?;
+    }
+
+    Ok(MacAddress(bytes))
+}
+
+#[derive(Debug)]
+pub enum Error {
+    NetworkError(String),
+    AuthenticationError(String),
+    ConfigurationError(String),
+    TimeoutError(String),
+    UnexpectedError(String),
+    CommandError(String),
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::NetworkError(msg) => write!(f, "Network error: {msg}"),
+            Error::AuthenticationError(msg) => write!(f, "Authentication error: {msg}"),
+            Error::ConfigurationError(msg) => write!(f, "Configuration error: {msg}"),
+            Error::TimeoutError(msg) => write!(f, "Timeout error: {msg}"),
+            Error::UnexpectedError(msg) => write!(f, "Unexpected error: {msg}"),
+            Error::CommandError(msg) => write!(f, "{msg}"),
+        }
+    }
+}
+
+impl From<Error> for String {
+    fn from(val: Error) -> Self {
+        format!("{val}")
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl From<russh::Error> for Error {
+    fn from(value: russh::Error) -> Self {
+        Error::NetworkError(format!("Russh client error: {value}"))
+    }
+}

brocade/src/network_operating_system.rs

@@ -0,0 +1,333 @@
+use std::str::FromStr;
+
+use async_trait::async_trait;
+use harmony_types::switch::{PortDeclaration, PortLocation};
+use log::{debug, info};
+use regex::Regex;
+
+use crate::{
+    BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo,
+    InterfaceStatus, InterfaceType, MacAddressEntry, PortChannelId, PortOperatingMode,
+    parse_brocade_mac_address, shell::BrocadeShell,
+};
+
+#[derive(Debug)]
+pub struct NetworkOperatingSystemClient {
+    shell: BrocadeShell,
+    version: BrocadeInfo,
+}
+
+impl NetworkOperatingSystemClient {
+    pub fn init(mut shell: BrocadeShell, version_info: BrocadeInfo) -> Self {
+        shell.before_all(vec!["terminal length 0".into()]);
+
+        Self {
+            shell,
+            version: version_info,
+        }
+    }
+
+    fn parse_mac_entry(&self, line: &str) -> Option<Result<MacAddressEntry, Error>> {
+        debug!("[Brocade] Parsing mac address entry: {line}");
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() < 5 {
+            return None;
+        }
+
+        let (vlan, mac_address, port) = match parts.len() {
+            5 => (
+                u16::from_str(parts[0]).ok()?,
+                parse_brocade_mac_address(parts[1]).ok()?,
+                parts[4].to_string(),
+            ),
+            _ => (
+                u16::from_str(parts[0]).ok()?,
+                parse_brocade_mac_address(parts[1]).ok()?,
+                parts[5].to_string(),
+            ),
+        };
+
+        let port =
+            PortDeclaration::parse(&port).map_err(|e| Error::UnexpectedError(format!("{e}")));
+
+        match port {
+            Ok(p) => Some(Ok(MacAddressEntry {
+                vlan,
+                mac_address,
+                port: p,
+            })),
+            Err(e) => Some(Err(e)),
+        }
+    }
+
+    fn parse_inter_switch_link_entry(&self, line: &str) -> Option<Result<InterSwitchLink, Error>> {
+        debug!("[Brocade] Parsing inter switch link entry: {line}");
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() < 10 {
+            return None;
+        }
+
+        let local_port = PortLocation::from_str(parts[2]).ok()?;
+        let remote_port = PortLocation::from_str(parts[5]).ok()?;
+
+        Some(Ok(InterSwitchLink {
+            local_port,
+            remote_port: Some(remote_port),
+        }))
+    }
+
+    fn parse_interface_status_entry(&self, line: &str) -> Option<Result<InterfaceInfo, Error>> {
+        debug!("[Brocade] Parsing interface status entry: {line}");
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        if parts.len() < 6 {
+            return None;
+        }
+
+        let interface_type = match parts[0] {
+            "Fo" => InterfaceType::Ethernet("FortyGigabitEthernet".to_string()),
+            "Te" => InterfaceType::Ethernet("TenGigabitEthernet".to_string()),
+            _ => return None,
+        };
+        let port_location = PortLocation::from_str(parts[1]).ok()?;
+        let status = match parts[2] {
+            "connected" => InterfaceStatus::Connected,
+            "notconnected" => InterfaceStatus::NotConnected,
+            "sfpAbsent" => InterfaceStatus::SfpAbsent,
+            _ => return None,
+        };
+        let operating_mode = match parts[3] {
+            "ISL" => Some(PortOperatingMode::Fabric),
+            "Trunk" => Some(PortOperatingMode::Trunk),
+            "Access" => Some(PortOperatingMode::Access),
+            "--" => None,
+            _ => return None,
+        };
+
+        Some(Ok(InterfaceInfo {
+            name: format!("{interface_type} {port_location}"),
+            port_location,
+            interface_type,
+            operating_mode,
+            status,
+        }))
+    }
+
+    fn map_configure_interfaces_error(&self, err: Error) -> Error {
+        debug!("[Brocade] {err}");
+
+        if let Error::CommandError(message) = &err {
+            if message.contains("switchport")
+                && message.contains("Cannot configure aggregator member")
+            {
+                let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap();
+
+                if let Some(caps) = re.captures(message) {
+                    let interface_type = &caps[1];
+                    let port_location = &caps[2];
+                    let interface = format!("{interface_type} {port_location}");
+
+                    return Error::CommandError(format!(
+                        "Cannot configure interface '{interface}', it is a member of a port-channel (LAG)"
+                    ));
+                }
+            }
+        }
+
+        err
+    }
+}
+
+#[async_trait]
+impl BrocadeClient for NetworkOperatingSystemClient {
+    async fn version(&self) -> Result<BrocadeInfo, Error> {
+        Ok(self.version.clone())
+    }
+
+    async fn get_mac_address_table(&self) -> Result<Vec<MacAddressEntry>, Error> {
+        let output = self
+            .shell
+            .run_command("show mac-address-table", ExecutionMode::Regular)
+            .await?;
+
+        output
+            .lines()
+            .skip(1)
+            .filter_map(|line| self.parse_mac_entry(line))
+            .collect()
+    }
+
+    async fn get_stack_topology(&self) -> Result<Vec<InterSwitchLink>, Error> {
+        let output = self
+            .shell
+            .run_command("show fabric isl", ExecutionMode::Regular)
+            .await?;
+
+        output
+            .lines()
+            .skip(6)
+            .filter_map(|line| self.parse_inter_switch_link_entry(line))
+            .collect()
+    }
+
+    async fn get_interfaces(&self) -> Result<Vec<InterfaceInfo>, Error> {
+        let output = self
+            .shell
+            .run_command(
+                "show interface status rbridge-id all",
+                ExecutionMode::Regular,
+            )
+            .await?;
+
+        output
+            .lines()
+            .skip(2)
+            .filter_map(|line| self.parse_interface_status_entry(line))
+            .collect()
+    }
+
+    async fn configure_interfaces(
+        &self,
+        interfaces: Vec<(String, PortOperatingMode)>,
+    ) -> Result<(), Error> {
+        info!("[Brocade] Configuring {} interface(s)...", interfaces.len());
+
+        let mut commands = vec!["configure terminal".to_string()];
+
+        for interface in interfaces {
+            commands.push(format!("interface {}", interface.0));
+
+            match interface.1 {
+                PortOperatingMode::Fabric => {
+                    commands.push("fabric isl enable".into());
+                    commands.push("fabric trunk enable".into());
+                }
+                PortOperatingMode::Trunk => {
+                    commands.push("switchport".into());
+                    commands.push("switchport mode trunk".into());
+                    commands.push("no spanning-tree shutdown".into());
+                    commands.push("no fabric isl enable".into());
+                    commands.push("no fabric trunk enable".into());
+                }
+                PortOperatingMode::Access => {
+                    commands.push("switchport".into());
+                    commands.push("switchport mode access".into());
+                    commands.push("switchport access vlan 1".into());
+                    commands.push("no spanning-tree shutdown".into());
+                    commands.push("no fabric isl enable".into());
+                    commands.push("no fabric trunk enable".into());
+                }
+            }
+
+            commands.push("no shutdown".into());
+            commands.push("exit".into());
+        }
+
+        self.shell
+            .run_commands(commands, ExecutionMode::Regular)
+            .await
+            .map_err(|err| self.map_configure_interfaces_error(err))?;
+
+        info!("[Brocade] Interfaces configured.");
+
+        Ok(())
+    }
+
+    async fn find_available_channel_id(&self) -> Result<PortChannelId, Error> {
+        info!("[Brocade] Finding next available channel id...");
+
+        let output = self
+            .shell
+            .run_command("show port-channel summary", ExecutionMode::Regular)
+            .await?;
+
+        let used_ids: Vec<u8> = output
+            .lines()
+            .skip(6)
+            .filter_map(|line| {
+                let parts: Vec<&str> = line.split_whitespace().collect();
+                if parts.len() < 8 {
+                    return None;
+                }
+
+                u8::from_str(parts[0]).ok()
+            })
+            .collect();
+
+        let mut next_id: u8 = 1;
+        loop {
+            if !used_ids.contains(&next_id) {
+                break;
+            }
+            next_id += 1;
+        }
+
+        info!("[Brocade] Found channel id: {next_id}");
+        Ok(next_id)
+    }
+
+    async fn create_port_channel(
+        &self,
+        channel_id: PortChannelId,
+        channel_name: &str,
+        ports: &[PortLocation],
+    ) -> Result<(), Error> {
+        info!(
+            "[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}",
+            ports
+                .iter()
+                .map(|p| format!("{p}"))
+                .collect::<Vec<String>>()
+                .join(", ")
+        );
+
+        let interfaces = self.get_interfaces().await?;
+
+        let mut commands = vec![
+            "configure terminal".into(),
+            format!("interface port-channel {}", channel_id),
+            "no shutdown".into(),
+            "exit".into(),
+        ];
+
+        for port in ports {
+            let interface = interfaces.iter().find(|i| i.port_location == *port);
+            let Some(interface) = interface else {
+                continue;
+            };
+
+            commands.push(format!("interface {}", interface.name));
+            commands.push("no switchport".into());
+            commands.push("no ip address".into());
+            commands.push("no fabric isl enable".into());
+            commands.push("no fabric trunk enable".into());
+            commands.push(format!("channel-group {channel_id} mode active"));
+            commands.push("no shutdown".into());
+            commands.push("exit".into());
+        }
+
+        self.shell
+            .run_commands(commands, ExecutionMode::Regular)
+            .await?;
+
+        info!("[Brocade] Port-channel '{channel_name}' configured.");
+
+        Ok(())
+    }
+
+    async fn clear_port_channel(&self, channel_name: &str) -> Result<(), Error> {
+        info!("[Brocade] Clearing port-channel: {channel_name}");
+
+        let commands = vec![
+            "configure terminal".into(),
+            format!("no interface port-channel {}", channel_name),
+            "exit".into(),
+        ];
+
+        self.shell
+            .run_commands(commands, ExecutionMode::Regular)
+            .await?;
+
+        info!("[Brocade] Port-channel '{channel_name}' cleared.");
+        Ok(())
+    }
+}

brocade/src/shell.rs

@@ -0,0 +1,370 @@
+use std::net::IpAddr;
+use std::time::Duration;
+use std::time::Instant;
+
+use crate::BrocadeOptions;
+use crate::Error;
+use crate::ExecutionMode;
+use crate::TimeoutConfig;
+use crate::ssh;
+
+use log::debug;
+use log::info;
+use russh::ChannelMsg;
+use tokio::time::timeout;
+
+#[derive(Debug)]
+pub struct BrocadeShell {
+    ip: IpAddr,
+    port: u16,
+    username: String,
+    password: String,
+    options: BrocadeOptions,
+    before_all_commands: Vec<String>,
+    after_all_commands: Vec<String>,
+}
+
+impl BrocadeShell {
+    pub async fn init(
+        ip_addresses: &[IpAddr],
+        port: u16,
+        username: &str,
+        password: &str,
+        options: Option<BrocadeOptions>,
+    ) -> Result<Self, Error> {
+        let ip = ip_addresses
+            .first()
+            .ok_or_else(|| Error::ConfigurationError("No IP addresses provided".to_string()))?;
+
+        let base_options = options.unwrap_or_default();
+        let options = ssh::try_init_client(username, password, ip, base_options).await?;
+
+        Ok(Self {
+            ip: *ip,
+            port,
+            username: username.to_string(),
+            password: password.to_string(),
+            before_all_commands: vec![],
+            after_all_commands: vec![],
+            options,
+        })
+    }
+
+    pub async fn open_session(&self, mode: ExecutionMode) -> Result<BrocadeSession, Error> {
+        BrocadeSession::open(
+            self.ip,
+            self.port,
+            &self.username,
+            &self.password,
+            self.options.clone(),
+            mode,
+        )
+        .await
+    }
+
+    pub async fn with_session<F, R>(&self, mode: ExecutionMode, callback: F) -> Result<R, Error>
+    where
+        F: FnOnce(
+            &mut BrocadeSession,
+        ) -> std::pin::Pin<
+            Box<dyn std::future::Future<Output = Result<R, Error>> + Send + '_>,
+        >,
+    {
+        let mut session = self.open_session(mode).await?;
+
+        let _ = session.run_commands(self.before_all_commands.clone()).await;
+        let result = callback(&mut session).await;
+        let _ = session.run_commands(self.after_all_commands.clone()).await;
+
+        session.close().await?;
+        result
+    }
+
+    pub async fn run_command(&self, command: &str, mode: ExecutionMode) -> Result<String, Error> {
+        let mut session = self.open_session(mode).await?;
+
+        let _ = session.run_commands(self.before_all_commands.clone()).await;
+        let result = session.run_command(command).await;
+        let _ = session.run_commands(self.after_all_commands.clone()).await;
+
+        session.close().await?;
+        result
+    }
+
+    pub async fn run_commands(
+        &self,
+        commands: Vec<String>,
+        mode: ExecutionMode,
+    ) -> Result<(), Error> {
+        let mut session = self.open_session(mode).await?;
+
+        let _ = session.run_commands(self.before_all_commands.clone()).await;
+        let result = session.run_commands(commands).await;
+        let _ = session.run_commands(self.after_all_commands.clone()).await;
+
+        session.close().await?;
+        result
+    }
+
+    pub fn before_all(&mut self, commands: Vec<String>) {
+        self.before_all_commands = commands;
+    }
+
+    pub fn after_all(&mut self, commands: Vec<String>) {
+        self.after_all_commands = commands;
+    }
+}
+
+pub struct BrocadeSession {
+    pub channel: russh::Channel<russh::client::Msg>,
+    pub mode: ExecutionMode,
+    pub options: BrocadeOptions,
+}
+
+impl BrocadeSession {
+    pub async fn open(
+        ip: IpAddr,
+        port: u16,
+        username: &str,
+        password: &str,
+        options: BrocadeOptions,
+        mode: ExecutionMode,
+    ) -> Result<Self, Error> {
+        let client = ssh::create_client(ip, port, username, password, &options).await?;
+        let mut channel = client.channel_open_session().await?;
+
+        channel
+            .request_pty(false, "vt100", 80, 24, 0, 0, &[])
+            .await?;
+        channel.request_shell(false).await?;
+
+        wait_for_shell_ready(&mut channel, &options.timeouts).await?;
+
+        if let ExecutionMode::Privileged = mode {
+            try_elevate_session(&mut channel, username, password, &options.timeouts).await?;
+        }
+
+        Ok(Self {
+            channel,
+            mode,
+            options,
+        })
+    }
+
+    pub async fn close(&mut self) -> Result<(), Error> {
+        debug!("[Brocade] Closing session...");
+
+        self.channel.data(&b"exit\n"[..]).await?;
+        if let ExecutionMode::Privileged = self.mode {
+            self.channel.data(&b"exit\n"[..]).await?;
+        }
+
+        let start = Instant::now();
+        while start.elapsed() < self.options.timeouts.cleanup {
+            match timeout(self.options.timeouts.message_wait, self.channel.wait()).await {
+                Ok(Some(ChannelMsg::Close)) => break,
+                Ok(Some(_)) => continue,
+                Ok(None) | Err(_) => break,
+            }
+        }
+
+        debug!("[Brocade] Session closed.");
+        Ok(())
+    }
+
+    pub async fn run_command(&mut self, command: &str) -> Result<String, Error> {
+        if self.should_skip_command(command) {
+            return Ok(String::new());
+        }
+
+        debug!("[Brocade] Running command: '{command}'...");
+
+        self.channel
+            .data(format!("{}\n", command).as_bytes())
+            .await?;
+        tokio::time::sleep(Duration::from_millis(100)).await;
+
+        let output = self.collect_command_output().await?;
+        let output = String::from_utf8(output)
+            .map_err(|_| Error::UnexpectedError("Invalid UTF-8 in command output".to_string()))?;
+
+        self.check_for_command_errors(&output, command)?;
+        Ok(output)
+    }
+
+    pub async fn run_commands(&mut self, commands: Vec<String>) -> Result<(), Error> {
+        for command in commands {
+            self.run_command(&command).await?;
+        }
+        Ok(())
+    }
+
+    fn should_skip_command(&self, command: &str) -> bool {
+        if (command.starts_with("write") || command.starts_with("deploy")) && self.options.dry_run {
+            info!("[Brocade] Dry-run mode enabled, skipping command: {command}");
+            return true;
+        }
+        false
+    }
+
+    async fn collect_command_output(&mut self) -> Result<Vec<u8>, Error> {
+        let mut output = Vec::new();
+        let start = Instant::now();
+        let read_timeout = Duration::from_millis(500);
+        let log_interval = Duration::from_secs(5);
+        let mut last_log = Instant::now();
+
+        loop {
+            if start.elapsed() > self.options.timeouts.command_execution {
+                return Err(Error::TimeoutError(
+                    "Timeout waiting for command completion.".into(),
+                ));
+            }
+
+            if start.elapsed() > self.options.timeouts.command_output
+                && last_log.elapsed() > log_interval
+            {
+                info!("[Brocade] Waiting for command output...");
+                last_log = Instant::now();
+            }
+
+            match timeout(read_timeout, self.channel.wait()).await {
+                Ok(Some(ChannelMsg::Data { data } | ChannelMsg::ExtendedData { data, .. })) => {
+                    output.extend_from_slice(&data);
+                    let current_output = String::from_utf8_lossy(&output);
+                    if current_output.contains('>') || current_output.contains('#') {
+                        return Ok(output);
+                    }
+                }
+                Ok(Some(ChannelMsg::Eof | ChannelMsg::Close)) => return Ok(output),
+                Ok(Some(ChannelMsg::ExitStatus { exit_status })) => {
+                    debug!("[Brocade] Command exit status: {exit_status}");
+                }
+                Ok(Some(_)) => continue,
+                Ok(None) | Err(_) => {
+                    if output.is_empty() {
+                        if let Ok(None) = timeout(read_timeout, self.channel.wait()).await {
+                            break;
+                        }
+                        continue;
+                    }
+
+                    tokio::time::sleep(Duration::from_millis(100)).await;
+                    let current_output = String::from_utf8_lossy(&output);
+                    if current_output.contains('>') || current_output.contains('#') {
+                        return Ok(output);
+                    }
+                }
+            }
+        }
+
+        Ok(output)
+    }
+
+    fn check_for_command_errors(&self, output: &str, command: &str) -> Result<(), Error> {
+        const ERROR_PATTERNS: &[&str] = &[
+            "invalid input",
+            "syntax error",
+            "command not found",
+            "unknown command",
+            "permission denied",
+            "access denied",
+            "authentication failed",
+            "configuration error",
+            "failed to",
+            "error:",
+        ];
+
+        let output_lower = output.to_lowercase();
+        if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) {
+            return Err(Error::CommandError(format!(
+                "Command error: {}",
+                output.trim()
+            )));
+        }
+
+        if !command.starts_with("show") && output.trim().is_empty() {
+            return Err(Error::CommandError(format!(
+                "Command '{command}' produced no output"
+            )));
+        }
+
+        Ok(())
+    }
+}
+
+async fn wait_for_shell_ready(
+    channel: &mut russh::Channel<russh::client::Msg>,
+    timeouts: &TimeoutConfig,
+) -> Result<(), Error> {
+    let mut buffer = Vec::new();
+    let start = Instant::now();
+
+    while start.elapsed() < timeouts.shell_ready {
+        match timeout(timeouts.message_wait, channel.wait()).await {
+            Ok(Some(ChannelMsg::Data { data })) => {
+                buffer.extend_from_slice(&data);
+                let output = String::from_utf8_lossy(&buffer);
+                let output = output.trim();
+                if output.ends_with('>') || output.ends_with('#') {
+                    debug!("[Brocade] Shell ready");
+                    return Ok(());
+                }
+            }
+            Ok(Some(_)) => continue,
+            Ok(None) => break,
+            Err(_) => continue,
+        }
+    }
+    Ok(())
+}
+
+async fn try_elevate_session(
+    channel: &mut russh::Channel<russh::client::Msg>,
+    username: &str,
+    password: &str,
+    timeouts: &TimeoutConfig,
+) -> Result<(), Error> {
+    channel.data(&b"enable\n"[..]).await?;
+    let start = Instant::now();
+    let mut buffer = Vec::new();
+
+    while start.elapsed() < timeouts.shell_ready {
+        match timeout(timeouts.message_wait, channel.wait()).await {
+            Ok(Some(ChannelMsg::Data { data })) => {
+                buffer.extend_from_slice(&data);
+                let output = String::from_utf8_lossy(&buffer);
+
+                if output.ends_with('#') {
+                    debug!("[Brocade] Privileged mode established");
+                    return Ok(());
+                }
+
+                if output.contains("User Name:") {
+                    channel.data(format!("{}\n", username).as_bytes()).await?;
+                    buffer.clear();
+                } else if output.contains("Password:") {
+                    channel.data(format!("{}\n", password).as_bytes()).await?;
+                    buffer.clear();
+                } else if output.contains('>') {
+                    return Err(Error::AuthenticationError(
+                        "Enable authentication failed".into(),
+                    ));
+                }
+            }
+            Ok(Some(_)) => continue,
+            Ok(None) => break,
+            Err(_) => continue,
+        }
+    }
+
+    let output = String::from_utf8_lossy(&buffer);
+    if output.ends_with('#') {
+        debug!("[Brocade] Privileged mode established");
+        Ok(())
+    } else {
+        Err(Error::AuthenticationError(format!(
+            "Enable failed. Output:\n{output}"
+        )))
+    }
+}

brocade/src/ssh.rs

@@ -0,0 +1,113 @@
+use std::borrow::Cow;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use russh::client::Handler;
+use russh::kex::DH_G1_SHA1;
+use russh::kex::ECDH_SHA2_NISTP256;
+use russh_keys::key::SSH_RSA;
+
+use super::BrocadeOptions;
+use super::Error;
+
+#[derive(Default, Clone, Debug)]
+pub struct SshOptions {
+    pub preferred_algorithms: russh::Preferred,
+}
+
+impl SshOptions {
+    fn ecdhsa_sha2_nistp256() -> Self {
+        Self {
+            preferred_algorithms: russh::Preferred {
+                kex: Cow::Borrowed(&[ECDH_SHA2_NISTP256]),
+                key: Cow::Borrowed(&[SSH_RSA]),
+                ..Default::default()
+            },
+        }
+    }
+
+    fn legacy() -> Self {
+        Self {
+            preferred_algorithms: russh::Preferred {
+                kex: Cow::Borrowed(&[DH_G1_SHA1]),
+                key: Cow::Borrowed(&[SSH_RSA]),
+                ..Default::default()
+            },
+        }
+    }
+}
+
+pub struct Client;
+
+#[async_trait]
+impl Handler for Client {
+    type Error = Error;
+
+    async fn check_server_key(
+        &mut self,
+        _server_public_key: &russh_keys::key::PublicKey,
+    ) -> Result<bool, Self::Error> {
+        Ok(true)
+    }
+}
+
+pub async fn try_init_client(
+    username: &str,
+    password: &str,
+    ip: &std::net::IpAddr,
+    base_options: BrocadeOptions,
+) -> Result<BrocadeOptions, Error> {
+    let ssh_options = vec![
+        SshOptions::default(),
+        SshOptions::ecdhsa_sha2_nistp256(),
+        SshOptions::legacy(),
+    ];
+
+    for ssh in ssh_options {
+        let opts = BrocadeOptions {
+            ssh,
+            ..base_options.clone()
+        };
+        let client = create_client(*ip, 22, username, password, &opts).await;
+
+        match client {
+            Ok(_) => {
+                return Ok(opts);
+            }
+            Err(e) => match e {
+                Error::NetworkError(e) => {
+                    if e.contains("No common key exchange algorithm") {
+                        continue;
+                    } else {
+                        return Err(Error::NetworkError(e));
+                    }
+                }
+                _ => return Err(e),
+            },
+        }
+    }
+
+    Err(Error::NetworkError(
+        "Could not establish ssh connection: wrong key exchange algorithm)".to_string(),
+    ))
+}
+
+pub async fn create_client(
+    ip: std::net::IpAddr,
+    port: u16,
+    username: &str,
+    password: &str,
+    options: &BrocadeOptions,
+) -> Result<russh::client::Handle<Client>, Error> {
+    let config = russh::client::Config {
+        preferred: options.ssh.preferred_algorithms.clone(),
+        ..Default::default()
+    };
+    let mut client = russh::client::connect(Arc::new(config), (ip, port), Client {}).await?;
+    if !client.authenticate_password(username, password).await? {
+        return Err(Error::AuthenticationError(
+            "ssh authentication failed".to_string(),
+        ));
+    }
+    Ok(client)
+}

check.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+rustc --version
+cargo check --all-targets --all-features --keep-going
+cargo fmt --check
+cargo clippy
+cargo test

data/okd/installer_image/scos-live-initramfs.x86_64.img

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-initramfs.x86_64.img

data/okd/installer_image/scos-live-kernel.x86_64

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-kernel.x86_64

data/okd/installer_image/scos-live-rootfs.x86_64.img

@@ -0,0 +1 @@
+scos-9.0.20250510-0-live-rootfs.x86_64.img

data/pxe/okd/README.md

@@ -0,0 +1,8 @@
+Here lies all the data files required for an OKD cluster PXE boot setup.
+
+This inclues ISO files, binary boot files, ipxe, etc.
+
+TODO as of august 2025 :
+
+- `harmony_inventory_agent` should be downloaded from official releases, this embedded version is practical for now though
+- The cluster ssh key should be generated and handled by harmony with the private key saved in a secret store

data/pxe/okd/http_files/.gitattributes

@@ -0,0 +1,9 @@
+harmony_inventory_agent filter=lfs diff=lfs merge=lfs -text
+os filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9 filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/initrd.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/vmlinuz filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/efiboot.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/install.img filter=lfs diff=lfs merge=lfs -text
+os/centos-stream-9/images/pxeboot filter=lfs diff=lfs merge=lfs -text

data/pxe/okd/http_files/cluster_ssh_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBx6bDylvC68cVpjKfEFtLQJ/dOFi6PVS2vsIOqPDJIc jeangab@liliane2

data/watchguard/pxe-http-files/.gitattributes

@@ -0,0 +1 @@
+slitaz/* filter=lfs diff=lfs merge=lfs -text

data/watchguard/pxe-http-files/boot.ipxe

@@ -0,0 +1,6 @@
+#!ipxe
+
+set base-url http://192.168.33.1:8080
+set hostfile ${base-url}/byMAC/01-${mac:hexhyp}.ipxe
+
+chain ${hostfile} || chain ${base-url}/default.ipxe

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-01-bc-68.ipxe

@@ -0,0 +1,35 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Boot to Slitaz - old linux for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/nvme0n1
+set ignition-file ncd0/master.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-02-60-fa.ipxe

@@ -0,0 +1,35 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Boot to Slitaz - old linux for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/nvme0n1
+set ignition-file ncd0/master.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-02-61-0f.ipxe

@@ -0,0 +1,35 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Slitaz - an old linux image for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/sda
+set ignition-file ncd0/worker.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-02-61-1a.ipxe

@@ -0,0 +1,35 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Boot to Slitaz - old linux for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/nvme0n1
+set ignition-file ncd0/master.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-02-61-26.ipxe

@@ -0,0 +1,35 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Slitaz - an old linux image for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/sda
+set ignition-file ncd0/worker.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/byMAC/01-c4-62-37-02-61-70.ipxe

@@ -0,0 +1,37 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item okdinstallation  Install OKD
+item slitaz  Slitaz - an old linux image for debugging
+choose selected
+
+goto ${selected}
+
+:local
+exit
+# This is the bootstrap node
+# it will become wk2
+
+#################################
+# okdinstallation
+#################################
+:okdinstallation
+set base-url http://192.168.33.1:8080
+set kernel-image fcos/fedora-coreos-39.20231101.3.0-live-kernel-x86_64
+set live-rootfs fcos/fedora-coreos-39.20231101.3.0-live-rootfs.x86_64.img
+set live-initramfs fcos/fedora-coreos-39.20231101.3.0-live-initramfs.x86_64.img
+set install-disk /dev/sda
+set ignition-file ncd0/worker.ign
+
+kernel ${base-url}/${kernel-image} initrd=main coreos.live.rootfs_url=${base-url}/${live-rootfs} coreos.inst.install_dev=${install-disk} coreos.inst.ignition_url=${base-url}/${ignition-file}  ip=enp1s0:dhcp
+initrd --name main ${base-url}/${live-initramfs}
+boot
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot

data/watchguard/pxe-http-files/default.ipxe

@@ -0,0 +1,71 @@
+#!ipxe
+menu PXE Boot Menu - [${mac}]
+item local  Boot from Hard Disk
+item slitaz Boot slitaz live environment [tux|root:root]
+#item ubuntu-server Ubuntu 24.04.1 live server
+#item ubuntu-desktop Ubuntu 24.04.1 desktop
+#item systemrescue System Rescue 11.03
+item memtest memtest
+#choose --default local --timeout 5000 selected
+choose selected
+
+goto ${selected}
+
+:local
+exit
+
+#################################
+# slitaz
+#################################
+:slitaz
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/slitaz
+kernel ${base_url}/vmlinuz-2.6.37-slitaz rw root=/dev/null vga=788 initrd=rootfs.gz
+initrd ${base_url}/rootfs.gz
+boot
+
+#################################
+# Ubuntu Server
+#################################
+:ubuntu-server
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/ubuntu/live-server-24.04.1
+
+kernel ${base_url}/vmlinuz ip=dhcp url=${base_url}/ubuntu-24.04.1-live-server-amd64.iso autoinstall ds=nocloud
+initrd ${base_url}/initrd
+boot
+
+#################################
+# Ubuntu Desktop
+#################################
+:ubuntu-desktop
+set server_ip 192.168.33.1:8080
+set base_url http://${server_ip}/ubuntu/desktop-24.04.1
+
+kernel ${base_url}/vmlinuz ip=dhcp url=${base_url}/ubuntu-24.04.1-desktop-amd64.iso autoinstall ds=nocloud
+initrd ${base_url}/initrd
+boot
+
+#################################
+# System Rescue
+#################################
+:systemrescue
+set base-url http://192.168.33.1:8080/systemrescue
+
+kernel ${base-url}/vmlinuz initrd=sysresccd.img boot=systemrescue docache
+initrd ${base-url}/sysresccd.img
+boot
+
+#################################
+# MemTest86 (BIOS/UEFI)
+#################################
+:memtest
+iseq ${platform} efi && goto memtest_efi || goto memtest_bios
+
+:memtest_efi
+kernel http://192.168.33.1:8080/memtest/memtest64.efi
+boot
+
+:memtest_bios
+kernel http://192.168.33.1:8080/memtest/memtest64.bin
+boot

data/watchguard/pxe-http-files/paul

@@ -1 +0,0 @@
-hey i am paul