use new harmony_cli::run

Co-authored-by: Ian Letourneau <letourneau.ian@gmail.com>
Reviewed-on: #93

.gitignore

@@ -1,4 +1,25 @@
-target
+### General ###
-private_repos
+private_repos/
-log/
+
-*.tgz
+### Harmony ###
+harmony.log
+
+### Helm ###
+# Chart dependencies
+**/charts/*.tgz
+
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb

Cargo.lock

@@ -96,6 +96,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ansi_term"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b3568b48b7cefa6b8ce125f9bb4989e52fbcc29ebea88df04cc7c5f12f70455"
+
 [[package]]
 name = "anstream"
 version = "0.6.19"
@@ -650,6 +656,19 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "console"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width 0.2.0",
+ "windows-sys 0.60.2",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -1137,6 +1156,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.35"
@@ -1240,6 +1265,18 @@ dependencies = [
 name = "example"
 version = "0.0.0"
 
+[[package]]
+name = "example-application-monitoring-with-tenant"
+version = "0.1.0"
+dependencies = [
+ "env_logger",
+ "harmony",
+ "harmony_cli",
+ "logging",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "example-cli"
 version = "0.1.0"
@@ -1355,6 +1392,7 @@ dependencies = [
 name = "example-rust"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "env_logger",
  "harmony",
  "harmony_cli",
@@ -1427,6 +1465,18 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
 
+[[package]]
+name = "filetime"
+version = "0.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "libredox",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "flate2"
 version = "1.1.2"
@@ -1726,6 +1776,8 @@ name = "harmony"
 version = "0.1.0"
 dependencies = [
  "async-trait",
+ "base64 0.22.1",
+ "bollard",
  "chrono",
  "cidr",
  "convert_case",
@@ -1742,27 +1794,32 @@ dependencies = [
  "helm-wrapper-rs",
  "hex",
  "http 1.3.1",
- "inquire",
  "k3d-rs",
  "k8s-openapi",
  "kube",
+ "kube-derive",
  "lazy_static",
  "libredfish",
  "log",
  "non-blank-string-rs",
+ "once_cell",
  "opnsense-config",
  "opnsense-config-xml",
+ "pretty_assertions",
  "rand 0.9.1",
  "reqwest 0.11.27",
  "russh",
  "rust-ipmi",
+ "schemars 0.8.22",
  "semver",
  "serde",
  "serde-value",
  "serde_json",
+ "serde_with",
  "serde_yaml",
  "similar",
  "strum 0.27.1",
+ "tar",
  "temp-dir",
  "temp-file",
  "tempfile",
@@ -1778,10 +1835,15 @@ version = "0.1.0"
 dependencies = [
  "assert_cmd",
  "clap",
+ "console",
  "env_logger",
  "harmony",
  "harmony_tui",
+ "indicatif",
+ "indicatif-log-bridge",
  "inquire",
+ "lazy_static",
+ "log",
  "tokio",
 ]
 
@@ -1792,10 +1854,16 @@ dependencies = [
  "bollard",
  "cargo_metadata",
  "clap",
+ "console",
  "current_platform",
  "env_logger",
  "futures-util",
+ "harmony_cli",
+ "indicatif",
+ "indicatif-log-bridge",
+ "lazy_static",
  "log",
+ "once_cell",
  "serde_json",
  "tokio",
 ]
@@ -2391,6 +2459,29 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd"
+dependencies = [
+ "console",
+ "portable-atomic",
+ "unicode-width 0.2.0",
+ "unit-prefix",
+ "web-time",
+]
+
+[[package]]
+name = "indicatif-log-bridge"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63703cf9069b85dbe6fe26e1c5230d013dee99d3559cd3d02ba39e099ef7ab02"
+dependencies = [
+ "indicatif",
+ "log",
+]
+
 [[package]]
 name = "indoc"
 version = "2.0.6"
@@ -2598,6 +2689,7 @@ dependencies = [
  "k8s-openapi",
  "kube-client",
  "kube-core",
+ "kube-derive",
  "kube-runtime",
 ]
 
@@ -2651,12 +2743,27 @@ dependencies = [
  "http 1.3.1",
  "json-patch",
  "k8s-openapi",
+ "schemars 0.8.22",
  "serde",
  "serde-value",
  "serde_json",
  "thiserror 2.0.12",
 ]
 
+[[package]]
+name = "kube-derive"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "079fc8c1c397538628309cfdee20696ebdcc26745f9fb17f89b78782205bd995"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn",
+]
+
 [[package]]
 name = "kube-runtime"
 version = "1.1.0"
@@ -2726,6 +2833,7 @@ checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
 dependencies = [
  "bitflags 2.9.1",
  "libc",
+ "redox_syscall",
 ]
 
 [[package]]
@@ -2771,6 +2879,15 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "logging"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "461a8beca676e8ab1bd468c92e9b4436d6368e11e96ae038209e520cfe665e46"
+dependencies = [
+ "ansi_term",
+]
+
 [[package]]
 name = "lru"
 version = "0.12.5"
@@ -4068,6 +4185,18 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "schemars"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
+dependencies = [
+ "dyn-clone",
+ "schemars_derive",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "schemars"
 version = "0.9.0"
@@ -4080,6 +4209,30 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "schemars"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
+dependencies = [
+ "dyn-clone",
+ "ref-cast",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars_derive"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -4212,6 +4365,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_derive_internals"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.140"
@@ -4280,22 +4444,36 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.13.0"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf65a400f8f66fb7b0552869ad70157166676db75ed8181f8104ea91cf9d0b42"
+checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5"
 dependencies = [
  "base64 0.22.1",
  "chrono",
  "hex",
  "indexmap 1.9.3",
  "indexmap 2.10.0",
- "schemars",
+ "schemars 0.9.0",
+ "schemars 1.0.4",
  "serde",
  "serde_derive",
  "serde_json",
+ "serde_with_macros",
  "time",
 ]
 
+[[package]]
+name = "serde_with_macros"
+version = "3.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.9.34+deprecated"
@@ -4668,6 +4846,17 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
+[[package]]
+name = "tar"
+version = "0.4.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
+dependencies = [
+ "filetime",
+ "libc",
+ "xattr",
+]
+
 [[package]]
 name = "temp-dir"
 version = "0.1.16"
@@ -5125,6 +5314,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817"
+
 [[package]]
 name = "universal-hash"
 version = "0.5.1"
@@ -5713,6 +5908,16 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "xattr"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909"
+dependencies = [
+ "libc",
+ "rustix 1.0.7",
+]
+
 [[package]]
 name = "xml-rs"
 version = "0.8.26"

Cargo.toml

@@ -52,3 +52,7 @@ convert_case = "0.8"
 chrono = "0.4"
 similar = "2"
 uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] }
+pretty_assertions = "1.4.1"
+bollard = "0.19.1"
+base64 = "0.22.1"
+tar = "0.4.44"

README.md

@@ -1,5 +1,6 @@
-# Harmony : Open-source infrastructure orchestration that treats your platform like first-class code.
+# Harmony : Open-source infrastructure orchestration that treats your platform like first-class code
-*By [NationTech](https://nationtech.io)*
+
+_By [NationTech](https://nationtech.io)_
 
 [![Build](https://git.nationtech.io/NationTech/harmony/actions/workflows/check.yml/badge.svg)](https://git.nationtech.io/nationtech/harmony)
 [![License](https://img.shields.io/badge/license-AGPLv3-blue?style=flat-square)](LICENSE)
@@ -24,7 +25,7 @@ From a **developer laptop** to a **global production cluster**, a single **sourc
 Infrastructure is essential, but it shouldn’t be your core business. Harmony is built on three guiding principles that make modern platforms reliable, repeatable, and easy to reason about.
 
 | Principle                              | What it means for you                                                                                                                                                                |
-|-----------|-----------------------|
+| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | **Infrastructure as Resilient Code**   | Replace sprawling YAML and bash scripts with type-safe Rust. Test, refactor, and version your platform just like application code.                                                   |
 | **Prove It Works — Before You Deploy** | Harmony uses the compiler to verify that your application’s needs match the target environment’s capabilities at **compile-time**, eliminating an entire class of runtime outages.   |
 | **One Unified Model**                  | Software and infrastructure are a single system. Harmony models them together, enabling deep automation—from bare-metal servers to Kubernetes workloads—with zero context switching. |
@@ -63,22 +64,20 @@ async fn main() {
         },
     };
 
-    // 2. Pick where it should run
+    // 2. Enhance with extra scores (monitoring, CI/CD, …)
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
-        Inventory::autoload(),                // auto-detect hardware / kube-config
-        K8sAnywhereTopology::from_env(),      // local k3d, CI, staging, prod…
-    )
-    .await
-    .unwrap();
-
-    // 3. Enhance with extra scores (monitoring, CI/CD, …)
     let mut monitoring = MonitoringAlertingStackScore::new();
     monitoring.namespace = Some(lamp_stack.config.namespace.clone());
 
-    maestro.register_all(vec![Box::new(lamp_stack), Box::new(monitoring)]);
+    // 3. Run your scores on the desired topology & inventory
-
+    harmony_cli::run(
-    // 4. Launch an interactive CLI / TUI
+        Inventory::autoload(),                // auto-detect hardware / kube-config
-    harmony_cli::init(maestro, None).await.unwrap();
+        K8sAnywhereTopology::from_env(),      // local k3d, CI, staging, prod…
+        vec![
+          Box::new(lamp_stack),
+          Box::new(monitoring)
+        ],
+        None
+    ).await.unwrap();
 }
 ```
 
@@ -95,10 +94,10 @@ Harmony analyses the code, shows an execution plan in a TUI, and applies it once
 ## 3 · Core Concepts
 
 | Term             | One-liner                                                                                            |
-|------|-----------|
+| ---------------- | ---------------------------------------------------------------------------------------------------- |
 | **Score<T>**     | Declarative description of the desired state (e.g., `LAMPScore`).                                    |
 | **Interpret<T>** | Imperative logic that realises a `Score` on a specific environment.                                  |
-| **Topology** | An environment (local k3d, AWS, bare-metal) exposing verified *Capabilities* (Kubernetes, DNS, …). |
+| **Topology**     | An environment (local k3d, AWS, bare-metal) exposing verified _Capabilities_ (Kubernetes, DNS, …).   |
 | **Maestro**      | Orchestrator that compiles Scores + Topology, ensuring all capabilities line up **at compile-time**. |
 | **Inventory**    | Optional catalogue of physical assets for bare-metal and edge deployments.                           |
 
@@ -112,9 +111,9 @@ A visual overview is in the diagram below.
 
 Prerequisites:
 
-* Rust 
+- Rust
-* Docker (if you deploy locally)
+- Docker (if you deploy locally)
-* `kubectl` / `helm` for Kubernetes-based topologies
+- `kubectl` / `helm` for Kubernetes-based topologies
 
 ```bash
 git clone https://git.nationtech.io/nationtech/harmony
@@ -126,15 +125,15 @@ cargo build --release          # builds the CLI, TUI and libraries
 
 ## 5 · Learning More
 
-* **Architectural Decision Records** – dive into the rationale  
+- **Architectural Decision Records** – dive into the rationale
   - [ADR-001 · Why Rust](adr/001-rust.md)
   - [ADR-003 · Infrastructure Abstractions](adr/003-infrastructure-abstractions.md)
   - [ADR-006 · Secret Management](adr/006-secret-management.md)
   - [ADR-011 · Multi-Tenant Cluster](adr/011-multi-tenant-cluster.md)
 
-* **Extending Harmony** – write new Scores / Interprets, add hardware like OPNsense firewalls, or embed Harmony in your own tooling (`/docs`).
+- **Extending Harmony** – write new Scores / Interprets, add hardware like OPNsense firewalls, or embed Harmony in your own tooling (`/docs`).
 
-* **Community** – discussions and roadmap live in [GitLab issues](https://git.nationtech.io/nationtech/harmony/-/issues). PRs, ideas, and feedback are welcome!
+- **Community** – discussions and roadmap live in [GitLab issues](https://git.nationtech.io/nationtech/harmony/-/issues). PRs, ideas, and feedback are welcome!
 
 ---
 
@@ -148,4 +147,4 @@ See [LICENSE](LICENSE) for the full text.
 
 ---
 
-*Made with ❤️ & 🦀 by the NationTech and the Harmony community*
+_Made with ❤️ & 🦀 by the NationTech and the Harmony community_

examples/application_monitoring_with_tenant/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "example-application-monitoring-with-tenant"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+env_logger.workspace = true
+harmony = { version = "0.1.0", path = "../../harmony" }
+harmony_cli = { version = "0.1.0", path = "../../harmony_cli" }
+logging = "0.1.0"
+tokio.workspace = true
+url.workspace = true

examples/application_monitoring_with_tenant/src/main.rs

@@ -0,0 +1,61 @@
+use std::{path::PathBuf, sync::Arc};
+
+use harmony::{
+    data::Id,
+    inventory::Inventory,
+    maestro::Maestro,
+    modules::{
+        application::{
+            ApplicationScore, RustWebFramework, RustWebapp,
+            features::{ContinuousDelivery, Monitoring},
+        },
+        monitoring::alert_channel::{
+            discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
+        },
+        tenant::TenantScore,
+    },
+    topology::{K8sAnywhereTopology, Url, tenant::TenantConfig},
+};
+
+#[tokio::main]
+async fn main() {
+    //TODO there is a bug where the application is deployed into the namespace matching the
+    //application name and the tenant is created in the namesapce matching the tenant name
+    //in order for the application to be deployed in the tenant namespace the application.name and
+    //the TenantConfig.name must match
+    let tenant = TenantScore {
+        config: TenantConfig {
+            id: Id::from_str("test-tenant-id"),
+            name: "example-monitoring".to_string(),
+            ..Default::default()
+        },
+    };
+    let application = Arc::new(RustWebapp {
+        name: "example-monitoring".to_string(),
+        domain: Url::Url(url::Url::parse("https://rustapp.harmony.example.com").unwrap()),
+        project_root: PathBuf::from("./examples/rust/webapp"),
+        framework: Some(RustWebFramework::Leptos),
+    });
+
+    let webhook_receiver = WebhookReceiver {
+        name: "sample-webhook-receiver".to_string(),
+        url: Url::Url(url::Url::parse("https://webhook-doesnt-exist.com").unwrap()),
+    };
+
+    let app = ApplicationScore {
+        features: vec![Box::new(Monitoring {
+            alert_receiver: vec![Box::new(webhook_receiver)],
+            application: application.clone(),
+        })],
+        application,
+    };
+
+    harmony_cli::run(
+        Inventory::autoload(),
+        K8sAnywhereTopology::from_env(),
+        vec![Box::new(tenant), Box::new(app)],
+        None,
+    )
+    .await
+    .unwrap();
+}

examples/cli/src/main.rs

@@ -1,20 +1,21 @@
 use harmony::{
     inventory::Inventory,
-    maestro::Maestro,
     modules::dummy::{ErrorScore, PanicScore, SuccessScore},
     topology::LocalhostTopology,
 };
 
 #[tokio::main]
 async fn main() {
-    let inventory = Inventory::autoload();
+    harmony_cli::run(
-    let topology = LocalhostTopology::new();
+        Inventory::autoload(),
-    let mut maestro = Maestro::initialize(inventory, topology).await.unwrap();
+        LocalhostTopology::new(),
-
+        vec![
-    maestro.register_all(vec![
             Box::new(SuccessScore {}),
             Box::new(ErrorScore {}),
             Box::new(PanicScore {}),
-    ]);
+        ],
-    harmony_cli::init(maestro, None).await.unwrap();
+        None,
+    )
+    .await
+    .unwrap();
 }

examples/lamp/src/main.rs

@@ -1,7 +1,6 @@
 use harmony::{
     data::Version,
     inventory::Inventory,
-    maestro::Maestro,
     modules::lamp::{LAMPConfig, LAMPScore},
     topology::{K8sAnywhereTopology, Url},
 };
@@ -43,15 +42,13 @@ async fn main() {
     // K8sAnywhereTopology as it is the most automatic one that enables you to easily deploy
     // locally, to development environment from a CI, to staging, and to production with settings
     // that automatically adapt to each environment grade.
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
+    harmony_cli::run(
         Inventory::autoload(),
         K8sAnywhereTopology::from_env(),
+        vec![Box::new(lamp_stack)],
+        None,
     )
     .await
     .unwrap();
-
-    maestro.register_all(vec![Box::new(lamp_stack)]);
-    // Here we bootstrap the CLI, this gives some nice features if you need them
-    harmony_cli::init(maestro, None).await.unwrap();
 }
 // That's it, end of the infra as code.

examples/monitoring/src/main.rs

@@ -2,7 +2,6 @@ use std::collections::HashMap;
 
 use harmony::{
     inventory::Inventory,
-    maestro::Maestro,
     modules::{
         monitoring::{
             alert_channel::discord_alert_channel::DiscordWebhook,
@@ -51,8 +50,8 @@ async fn main() {
 
     let service_monitor_endpoint = ServiceMonitorEndpoint {
         port: Some("80".to_string()),
-        path: "/metrics".to_string(),
+        path: Some("/metrics".to_string()),
-        scheme: HTTPScheme::HTTP,
+        scheme: Some(HTTPScheme::HTTP),
         ..Default::default()
     };
 
@@ -74,13 +73,13 @@ async fn main() {
         rules: vec![Box::new(additional_rules), Box::new(additional_rules2)],
         service_monitors: vec![service_monitor],
     };
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
+
+    harmony_cli::run(
         Inventory::autoload(),
         K8sAnywhereTopology::from_env(),
+        vec![Box::new(alerting_score)],
+        None,
     )
     .await
     .unwrap();
-
-    maestro.register_all(vec![Box::new(alerting_score)]);
-    harmony_cli::init(maestro, None).await.unwrap();
 }

examples/monitoring_with_tenant/src/main.rs

@@ -3,7 +3,6 @@ use std::collections::HashMap;
 use harmony::{
     data::Id,
     inventory::Inventory,
-    maestro::Maestro,
     modules::{
         monitoring::{
             alert_channel::discord_alert_channel::DiscordWebhook,
@@ -54,8 +53,8 @@ async fn main() {
 
     let service_monitor_endpoint = ServiceMonitorEndpoint {
         port: Some("80".to_string()),
-        path: "/metrics".to_string(),
+        path: Some("/metrics".to_string()),
-        scheme: HTTPScheme::HTTP,
+        scheme: Some(HTTPScheme::HTTP),
         ..Default::default()
     };
 
@@ -78,13 +77,13 @@ async fn main() {
         rules: vec![Box::new(additional_rules)],
         service_monitors: vec![service_monitor],
     };
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
+
+    harmony_cli::run(
         Inventory::autoload(),
         K8sAnywhereTopology::from_env(),
+        vec![Box::new(tenant), Box::new(alerting_score)],
+        None,
     )
     .await
     .unwrap();
-
-    maestro.register_all(vec![Box::new(tenant), Box::new(alerting_score)]);
-    harmony_cli::init(maestro, None).await.unwrap();
 }

examples/nanodc/src/main.rs

@@ -10,7 +10,7 @@ use harmony::{
     inventory::Inventory,
     maestro::Maestro,
     modules::{
-        http::HttpScore,
+        http::StaticFilesHttpScore,
         ipxe::IpxeScore,
         okd::{
             bootstrap_dhcp::OKDBootstrapDhcpScore,
@@ -126,7 +126,7 @@ async fn main() {
         harmony::modules::okd::load_balancer::OKDLoadBalancerScore::new(&topology);
 
     let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    let http_score = HttpScore::new(Url::LocalFolder(
+    let http_score = StaticFilesHttpScore::new(Url::LocalFolder(
         "./data/watchguard/pxe-http-files".to_string(),
     ));
     let ipxe_score = IpxeScore::new();

examples/ntfy/src/main.rs

@@ -1,19 +1,18 @@
 use harmony::{
-    inventory::Inventory, maestro::Maestro, modules::monitoring::ntfy::ntfy::NtfyScore,
+    inventory::Inventory, modules::monitoring::ntfy::ntfy::NtfyScore, topology::K8sAnywhereTopology,
-    topology::K8sAnywhereTopology,
 };
 
 #[tokio::main]
 async fn main() {
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
+    harmony_cli::run(
         Inventory::autoload(),
         K8sAnywhereTopology::from_env(),
+        vec![Box::new(NtfyScore {
+            namespace: "monitoring".to_string(),
+            host: "localhost".to_string(),
+        })],
+        None,
     )
     .await
     .unwrap();
-
-    maestro.register_all(vec![Box::new(NtfyScore {
-        namespace: "monitoring".to_string(),
-    })]);
-    harmony_cli::init(maestro, None).await.unwrap();
 }

examples/opnsense/src/main.rs

@@ -11,7 +11,7 @@ use harmony::{
     maestro::Maestro,
     modules::{
         dummy::{ErrorScore, PanicScore, SuccessScore},
-        http::HttpScore,
+        http::StaticFilesHttpScore,
         okd::{dhcp::OKDDhcpScore, dns::OKDDnsScore, load_balancer::OKDLoadBalancerScore},
         opnsense::OPNsenseShellCommandScore,
         tftp::TftpScore,
@@ -81,7 +81,7 @@ async fn main() {
     let load_balancer_score = OKDLoadBalancerScore::new(&topology);
 
     let tftp_score = TftpScore::new(Url::LocalFolder("./data/watchguard/tftpboot".to_string()));
-    let http_score = HttpScore::new(Url::LocalFolder(
+    let http_score = StaticFilesHttpScore::new(Url::LocalFolder(
         "./data/watchguard/pxe-http-files".to_string(),
     ));
     let mut maestro = Maestro::initialize(inventory, topology).await.unwrap();

examples/rust/.gitignore

@@ -0,0 +1,3 @@
+Dockerfile.harmony
+.harmony_generated
+harmony

examples/rust/Cargo.toml

@@ -12,3 +12,4 @@ tokio = { workspace = true }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
+base64.workspace = true

examples/rust/src/main.rs

@@ -2,39 +2,57 @@ use std::{path::PathBuf, sync::Arc};
 
 use harmony::{
     inventory::Inventory,
-    maestro::Maestro,
+    modules::{
-    modules::application::{
+        application::{
             ApplicationScore, RustWebFramework, RustWebapp,
             features::{ContinuousDelivery, Monitoring},
         },
+        monitoring::alert_channel::{
+            discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
+        },
+    },
     topology::{K8sAnywhereTopology, Url},
 };
 
 #[tokio::main]
 async fn main() {
-    env_logger::init();
     let application = Arc::new(RustWebapp {
         name: "harmony-example-rust-webapp".to_string(),
         domain: Url::Url(url::Url::parse("https://rustapp.harmony.example.com").unwrap()),
-        project_root: PathBuf::from("./examples/rust/webapp"),
+        project_root: PathBuf::from("./webapp"), // Relative from 'harmony-path' param
         framework: Some(RustWebFramework::Leptos),
     });
 
+    let discord_receiver = DiscordWebhook {
+        name: "test-discord".to_string(),
+        url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
+    };
+
+    let webhook_receiver = WebhookReceiver {
+        name: "sample-webhook-receiver".to_string(),
+        url: Url::Url(url::Url::parse("https://webhook-doesnt-exist.com").unwrap()),
+    };
+
     let app = ApplicationScore {
         features: vec![
             Box::new(ContinuousDelivery {
                 application: application.clone(),
             }),
-            Box::new(Monitoring {}),
+            Box::new(Monitoring {
-            // TODO add monitoring, backups, multisite ha, etc
+                application: application.clone(),
+                alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
+            }),
+            // TODO add backups, multisite ha, etc
         ],
         application,
     };
 
-    let topology = K8sAnywhereTopology::from_env();
+    harmony_cli::run(
-    let mut maestro = Maestro::initialize(Inventory::autoload(), topology)
+        Inventory::autoload(),
+        K8sAnywhereTopology::from_env(),
+        vec![Box::new(app)],
+        None,
+    )
     .await
     .unwrap();
-    maestro.register_all(vec![Box::new(app)]);
-    harmony_cli::init(maestro, None).await.unwrap();
 }

examples/rust/webapp/Dockerfile.harmony

@@ -1,16 +0,0 @@
-FROM rust:bookworm as builder
-RUN apt-get update && apt-get install -y --no-install-recommends clang wget && wget https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz && tar -xvf cargo-binstall-x86_64-unknown-linux-musl.tgz && cp cargo-binstall /usr/local/cargo/bin && rm cargo-binstall-x86_64-unknown-linux-musl.tgz cargo-binstall && apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN cargo binstall cargo-leptos -y
-RUN rustup target add wasm32-unknown-unknown
-WORKDIR /app
-COPY . .
-RUN cargo leptos build --release -vv
-FROM debian:bookworm-slim
-RUN groupadd -r appgroup && useradd -r -s /bin/false -g appgroup appuser
-ENV LEPTOS_SITE_ADDR=0.0.0.0:3000
-EXPOSE 3000/tcp
-WORKDIR /home/appuser
-COPY --from=builder /app/target/site/pkg /home/appuser/pkg
-COPY --from=builder /app/target/release/harmony-example-rust-webapp /home/appuser/harmony-example-rust-webapp
-USER appuser
-CMD /home/appuser/harmony-example-rust-webapp

examples/tenant/src/main.rs

@@ -1,7 +1,6 @@
 use harmony::{
     data::Id,
     inventory::Inventory,
-    maestro::Maestro,
     modules::tenant::TenantScore,
     topology::{K8sAnywhereTopology, tenant::TenantConfig},
 };
@@ -16,15 +15,14 @@ async fn main() {
         },
     };
 
-    let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
+    harmony_cli::run(
         Inventory::autoload(),
         K8sAnywhereTopology::from_env(),
+        vec![Box::new(tenant)],
+        None,
     )
     .await
     .unwrap();
-
-    maestro.register_all(vec![Box::new(tenant)]);
-    harmony_cli::init(maestro, None).await.unwrap();
 }
 
 // TODO write tests

harmony/Cargo.toml

@@ -27,12 +27,11 @@ harmony_macros = { path = "../harmony_macros" }
 harmony_types = { path = "../harmony_types" }
 uuid.workspace = true
 url.workspace = true
-kube.workspace = true
+kube = { workspace = true, features = ["derive"] }
 k8s-openapi.workspace = true
 serde_yaml.workspace = true
 http.workspace = true
 serde-value.workspace = true
-inquire.workspace = true
 helm-wrapper-rs = "0.4.0"
 non-blank-string-rs = "1.0.4"
 k3d-rs = { path = "../k3d" }
@@ -58,3 +57,13 @@ futures-util = "0.3.31"
 tokio-util = "0.7.15"
 strum = { version = "0.27.1", features = ["derive"] }
 tempfile = "3.20.0"
+serde_with = "3.14.0"
+schemars = "0.8.22"
+kube-derive = "1.1.0"
+bollard.workspace = true
+tar.workspace = true
+base64.workspace = true
+once_cell = "1.21.3"
+
+[dev-dependencies]
+pretty_assertions.workspace = true

harmony/src/domain/instrumentation.rs

@@ -0,0 +1,63 @@
+use log::debug;
+use once_cell::sync::Lazy;
+use tokio::sync::broadcast;
+
+use super::interpret::{InterpretError, Outcome};
+
+#[derive(Debug, Clone)]
+pub enum HarmonyEvent {
+    HarmonyStarted,
+    PrepareTopologyStarted {
+        topology: String,
+    },
+    TopologyPrepared {
+        topology: String,
+        outcome: Outcome,
+    },
+    InterpretExecutionStarted {
+        topology: String,
+        interpret: String,
+        message: String,
+    },
+    InterpretExecutionFinished {
+        topology: String,
+        interpret: String,
+        outcome: Result<Outcome, InterpretError>,
+    },
+}
+
+static HARMONY_EVENT_BUS: Lazy<broadcast::Sender<HarmonyEvent>> = Lazy::new(|| {
+    // TODO: Adjust channel capacity
+    let (tx, _rx) = broadcast::channel(100);
+    tx
+});
+
+pub fn instrument(event: HarmonyEvent) -> Result<(), &'static str> {
+    match HARMONY_EVENT_BUS.send(event) {
+        Ok(_) => Ok(()),
+        Err(_) => Err("send error: no subscribers"),
+    }
+}
+
+pub async fn subscribe<F, Fut>(name: &str, mut handler: F)
+where
+    F: FnMut(HarmonyEvent) -> Fut + Send + 'static,
+    Fut: Future<Output = bool> + Send,
+{
+    let mut rx = HARMONY_EVENT_BUS.subscribe();
+    debug!("[{name}] Service started. Listening for events...");
+    loop {
+        match rx.recv().await {
+            Ok(event) => {
+                if !handler(event).await {
+                    debug!("[{name}] Handler requested exit.");
+                    break;
+                }
+            }
+            Err(broadcast::error::RecvError::Lagged(n)) => {
+                debug!("[{name}] Lagged behind by {n} messages.");
+            }
+            Err(_) => break,
+        }
+    }
+}

harmony/src/domain/maestro/mod.rs

@@ -1,6 +1,8 @@
 use std::sync::{Arc, Mutex, RwLock};
 
-use log::{info, warn};
+use log::{debug, warn};
+
+use crate::instrumentation::{self, HarmonyEvent};
 
 use super::{
     interpret::{InterpretError, InterpretStatus, Outcome},
@@ -40,13 +42,18 @@ impl<T: Topology> Maestro<T> {
     /// Ensures the associated Topology is ready for operations.
     /// Delegates the readiness check and potential setup actions to the Topology.
     pub async fn prepare_topology(&self) -> Result<Outcome, InterpretError> {
-        info!("Ensuring topology '{}' is ready...", self.topology.name());
+        instrumentation::instrument(HarmonyEvent::PrepareTopologyStarted {
+            topology: self.topology.name().to_string(),
+        })
+        .unwrap();
+
         let outcome = self.topology.ensure_ready().await?;
-        info!(
+
-            "Topology '{}' readiness check complete: {}",
+        instrumentation::instrument(HarmonyEvent::TopologyPrepared {
-            self.topology.name(),
+            topology: self.topology.name().to_string(),
-            outcome.status
+            outcome: outcome.clone(),
-        );
+        })
+        .unwrap();
 
         self.topology_preparation_result
             .lock()
@@ -80,11 +87,11 @@ impl<T: Topology> Maestro<T> {
                 self.topology.name(),
             );
         }
-        info!("Running score {score:?}");
+        debug!("Running score {score:?}");
         let interpret = score.create_interpret();
-        info!("Launching interpret {interpret:?}");
+        debug!("Launching interpret {interpret:?}");
         let result = interpret.execute(&self.inventory, &self.topology).await;
-        info!("Got result {result:?}");
+        debug!("Got result {result:?}");
         result
     }
 

harmony/src/domain/mod.rs

@@ -3,6 +3,7 @@ pub mod data;
 pub mod executors;
 pub mod filter;
 pub mod hardware;
+pub mod instrumentation;
 pub mod interpret;
 pub mod inventory;
 pub mod maestro;

harmony/src/domain/topology/k8s.rs

@@ -17,7 +17,7 @@ use kube::{
     runtime::wait::await_condition,
 };
 use log::{debug, error, trace};
-use serde::de::DeserializeOwned;
+use serde::{Serialize, de::DeserializeOwned};
 use similar::{DiffableStr, TextDiff};
 
 #[derive(new, Clone)]
@@ -25,6 +25,15 @@ pub struct K8sClient {
     client: Client,
 }
 
+impl Serialize for K8sClient {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}
+
 impl std::fmt::Debug for K8sClient {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         // This is a poor man's debug implementation for now as kube::Client does not provide much
@@ -260,17 +269,33 @@ impl K8sClient {
     ) -> Result<(), Error> {
         let obj: DynamicObject = serde_yaml::from_value(yaml.clone()).expect("TODO do not unwrap");
         let name = obj.metadata.name.as_ref().expect("YAML must have a name");
-        let namespace = obj
+
+        let api_version = yaml
+            .get("apiVersion")
+            .expect("couldn't get apiVersion from YAML")
+            .as_str()
+            .expect("couldn't get apiVersion as str");
+        let kind = yaml
+            .get("kind")
+            .expect("couldn't get kind from YAML")
+            .as_str()
+            .expect("couldn't get kind as str");
+
+        let split: Vec<&str> = api_version.splitn(2, "/").collect();
+        let g = split[0];
+        let v = split[1];
+
+        let gvk = GroupVersionKind::gvk(g, v, kind);
+        let api_resource = ApiResource::from_gvk(&gvk);
+
+        let namespace = match ns {
+            Some(n) => n,
+            None => obj
                 .metadata
                 .namespace
                 .as_ref()
-            .expect("YAML must have a namespace");
+                .expect("YAML must have a namespace"),
-
+        };
-        // 4. Define the API resource type using the GVK from the object.
-        //    The plural name 'applications' is taken from your CRD definition.
-        error!("This only supports argocd application harcoded, very rrrong");
-        let gvk = GroupVersionKind::gvk("argoproj.io", "v1alpha1", "Application");
-        let api_resource = ApiResource::from_gvk_with_plural(&gvk, "applications");
 
         // 5. Create a dynamic API client for this resource type.
         let api: Api<DynamicObject> =

harmony/src/domain/topology/k8s_anywhere.rs

@@ -1,30 +1,39 @@
 use std::{process::Command, sync::Arc};
 
 use async_trait::async_trait;
-use inquire::Confirm;
 use log::{debug, info, warn};
 use serde::Serialize;
 use tokio::sync::OnceCell;
 
 use crate::{
     executors::ExecutorError,
-    interpret::{InterpretError, Outcome},
+    interpret::{InterpretError, InterpretStatus, Outcome},
     inventory::Inventory,
-    maestro::Maestro,
+    modules::{
-    modules::k3d::K3DInstallationScore,
+        k3d::K3DInstallationScore,
-    topology::LocalhostTopology,
+        monitoring::kube_prometheus::crd::{
+            crd_alertmanager_config::CRDPrometheus,
+            prometheus_operator::prometheus_operator_helm_chart_score,
+        },
+        prometheus::{
+            k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
+            prometheus::PrometheusApplicationMonitoring,
+        },
+    },
+    score::Score,
 };
 
 use super::{
     DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology,
     k8s::K8sClient,
+    oberservability::monitoring::AlertReceiver,
     tenant::{TenantConfig, TenantManager, k8s::K8sTenantManager},
 };
 
 #[derive(Clone, Debug)]
 struct K8sState {
     client: Arc<K8sClient>,
-    _source: K8sSource,
+    source: K8sSource,
     message: String,
 }
 
@@ -58,8 +67,32 @@ impl K8sclient for K8sAnywhereTopology {
     }
 }
 
+#[async_trait]
+impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+    async fn install_prometheus(
+        &self,
+        sender: &CRDPrometheus,
+        inventory: &Inventory,
+        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> Result<Outcome, InterpretError> {
+        let po_result = self.ensure_prometheus_operator(sender).await?;
+
+        if po_result.status == InterpretStatus::NOOP {
+            debug!("Skipping Prometheus CR installation due to missing operator.");
+            return Ok(Outcome::noop());
+        }
+        self.get_k8s_prometheus_application_score(sender.clone(), receivers)
+            .await
+            .create_interpret()
+            .execute(inventory, self)
+            .await?;
+
+        Ok(Outcome::success(format!("No action, working on cluster  ")))
+    }
+}
+
 impl Serialize for K8sAnywhereTopology {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
     where
         S: serde::Serializer,
     {
@@ -84,6 +117,19 @@ impl K8sAnywhereTopology {
         }
     }
 
+    async fn get_k8s_prometheus_application_score(
+        &self,
+        sender: CRDPrometheus,
+        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> K8sPrometheusCRDAlertingScore {
+        K8sPrometheusCRDAlertingScore {
+            sender,
+            receivers: receivers.unwrap_or_else(Vec::new),
+            service_monitors: vec![],
+            prometheus_rules: vec![],
+        }
+    }
+
     fn is_helm_available(&self) -> Result<(), String> {
         let version_result = Command::new("helm")
             .arg("version")
@@ -94,9 +140,8 @@ impl K8sAnywhereTopology {
             return Err("Failed to run 'helm -version'".to_string());
         }
 
-        // Print the version output
         let version_output = String::from_utf8_lossy(&version_result.stdout);
-        println!("Helm version: {}", version_output.trim());
+        debug!("Helm version: {}", version_output.trim());
 
         Ok(())
     }
@@ -114,9 +159,10 @@ impl K8sAnywhereTopology {
     }
 
     async fn try_install_k3d(&self) -> Result<(), InterpretError> {
-        let maestro = Maestro::initialize(Inventory::autoload(), LocalhostTopology::new()).await?;
+        self.get_k3d_installation_score()
-        let k3d_score = self.get_k3d_installation_score();
+            .create_interpret()
-        maestro.interpret(Box::new(k3d_score)).await?;
+            .execute(&Inventory::empty(), self)
+            .await?;
         Ok(())
     }
 
@@ -126,7 +172,7 @@ impl K8sAnywhereTopology {
         // TODO this deserves some refactoring, it is becoming a bit hard to figure out
         // be careful when making modifications here
         if k8s_anywhere_config.use_local_k3d {
-            info!("Using local k3d cluster because of use_local_k3d set to true");
+            debug!("Using local k3d cluster because of use_local_k3d set to true");
         } else {
             if let Some(kubeconfig) = &k8s_anywhere_config.kubeconfig {
                 debug!("Loading kubeconfig {kubeconfig}");
@@ -134,7 +180,7 @@ impl K8sAnywhereTopology {
                     Some(client) => {
                         return Ok(Some(K8sState {
                             client: Arc::new(client),
-                            _source: K8sSource::Kubeconfig,
+                            source: K8sSource::Kubeconfig,
                             message: format!("Loaded k8s client from kubeconfig {kubeconfig}"),
                         }));
                     }
@@ -158,22 +204,13 @@ impl K8sAnywhereTopology {
         }
 
         if !k8s_anywhere_config.autoinstall {
-            debug!("Autoinstall confirmation prompt");
-            let confirmation = Confirm::new( "Harmony autoinstallation is not activated, do you wish to launch autoinstallation? : ")
-                .with_default(false)
-                .prompt()
-                .expect("Unexpected prompt error");
-            debug!("Autoinstall confirmation {confirmation}");
-
-            if !confirmation {
             warn!(
                 "Installation cancelled, K8sAnywhere could not initialize a valid Kubernetes client"
             );
             return Ok(None);
         }
-        }
 
-        info!("Starting K8sAnywhere installation");
+        debug!("Starting K8sAnywhere installation");
         self.try_install_k3d().await?;
         let k3d_score = self.get_k3d_installation_score();
         // I feel like having to rely on the k3d_rs crate here is a smell
@@ -185,8 +222,8 @@ impl K8sAnywhereTopology {
         let state = match k3d.get_client().await {
             Ok(client) => K8sState {
                 client: Arc::new(K8sClient::new(client)),
-                _source: K8sSource::LocalK3d,
+                source: K8sSource::LocalK3d,
-                message: "Successfully installed K3D cluster and acquired client".to_string(),
+                message: "K8s client ready".to_string(),
             },
             Err(_) => todo!(),
         };
@@ -201,6 +238,7 @@ impl K8sAnywhereTopology {
 
         self.tenant_manager
             .get_or_try_init(async || -> Result<K8sTenantManager, String> {
+                // TOOD: checker si K8s ou K3d/s tenant manager (ref. issue https://git.nationtech.io/NationTech/harmony/issues/94)
                 let k8s_client = self.k8s_client().await?;
                 Ok(K8sTenantManager::new(k8s_client))
             })
@@ -217,6 +255,48 @@ impl K8sAnywhereTopology {
             )),
         }
     }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &CRDPrometheus,
+    ) -> Result<Outcome, InterpretError> {
+        let status = Command::new("sh")
+            .args(["-c", "kubectl get crd -A | grep -i prometheuses"])
+            .status()
+            .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e)))?;
+
+        if !status.success() {
+            if let Some(Some(k8s_state)) = self.k8s_state.get() {
+                match k8s_state.source {
+                    K8sSource::LocalK3d => {
+                        debug!("installing prometheus operator");
+                        let op_score =
+                            prometheus_operator_helm_chart_score(sender.namespace.clone());
+                        op_score
+                            .create_interpret()
+                            .execute(&Inventory::empty(), self)
+                            .await?;
+                        return Ok(Outcome::success(
+                            "installed prometheus operator".to_string(),
+                        ));
+                    }
+                    K8sSource::Kubeconfig => {
+                        debug!("unable to install prometheus operator, contact cluster admin");
+                        return Ok(Outcome::noop());
+                    }
+                }
+            } else {
+                warn!("Unable to detect k8s_state. Skipping Prometheus Operator install.");
+                return Ok(Outcome::noop());
+            }
+        }
+
+        debug!("Prometheus operator is already present, skipping install");
+
+        Ok(Outcome::success(
+            "prometheus operator present in cluster".to_string(),
+        ))
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -237,7 +317,7 @@ pub struct K8sAnywhereConfig {
     ///
     /// When enabled, autoinstall will setup a K3D cluster on the localhost. https://k3d.io/stable/
     ///
-    /// Default: false
+    /// Default: true
     pub autoinstall: bool,
 
     /// Whether to use local k3d cluster.
@@ -256,7 +336,7 @@ impl K8sAnywhereConfig {
             use_system_kubeconfig: std::env::var("HARMONY_USE_SYSTEM_KUBECONFIG")
                 .map_or_else(|_| false, |v| v.parse().ok().unwrap_or(false)),
             autoinstall: std::env::var("HARMONY_AUTOINSTALL")
-                .map_or_else(|_| false, |v| v.parse().ok().unwrap_or(false)),
+                .map_or_else(|_| true, |v| v.parse().ok().unwrap_or(false)),
             // TODO harmony_profile should be managed at a more core level than this
             harmony_profile: std::env::var("HARMONY_PROFILE").map_or_else(
                 |_| "dev".to_string(),

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -1,3 +1,5 @@
+use std::any::Any;
+
 use async_trait::async_trait;
 use log::debug;
 
@@ -62,7 +64,9 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
 #[async_trait]
 pub trait AlertReceiver<S: AlertSender>: std::fmt::Debug + Send + Sync {
     async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
+    fn name(&self) -> String;
     fn clone_box(&self) -> Box<dyn AlertReceiver<S>>;
+    fn as_any(&self) -> &dyn Any;
 }
 
 #[async_trait]
@@ -72,6 +76,6 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 }
 
 #[async_trait]
-pub trait ScrapeTarger<S: AlertSender> {
+pub trait ScrapeTarget<S: AlertSender> {
     async fn install(&self, sender: &S) -> Result<(), InterpretError>;
 }

harmony/src/domain/topology/tenant/k8s.rs

@@ -231,8 +231,13 @@ impl K8sTenantManager {
               {
                 "to": [
                   {
+                      //TODO this ip is from the docker network that k3d is running on
+                      //since k3d does not deploy kube-api-server as a pod it needs to ahve the ip
+                      //address opened up
+                      //need to find a way to automatically detect the ip address from the docker
+                      //network
                 "ipBlock": {
-                    "cidr": "172.23.0.0/16",
+                    "cidr": "172.24.0.0/16",
                     }
                   }
                 ]

harmony/src/modules/application/features/argo_types.rs

@@ -1,13 +1,11 @@
-use std::{backtrace, collections::HashMap};
-
-use k8s_openapi::{Metadata, NamespaceResourceScope, Resource};
 use log::debug;
 use serde::Serialize;
+use serde_with::skip_serializing_none;
 use serde_yaml::Value;
-use url::Url;
 
 use crate::modules::application::features::CDApplicationConfig;
 
+#[skip_serializing_none]
 #[derive(Clone, Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Helm {
@@ -27,13 +25,18 @@ pub struct Helm {
     pub namespace: Option<String>,
 }
 
+#[skip_serializing_none]
 #[derive(Clone, Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Source {
-    pub repo_url: Url,
+    // Using string for this because URL enforces a URL scheme at the beginning but Helm, ArgoCD, etc do not, and it can be counterproductive,
+    // as the only way I've found to get OCI working isn't by using oci:// but rather no scheme at all
+    #[serde(rename = "repoURL")]
+    pub repo_url: String,
     pub target_revision: Option<String>,
     pub chart: String,
     pub helm: Helm,
+    pub path: String,
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -67,6 +70,7 @@ pub struct SyncPolicy {
     pub retry: Retry,
 }
 
+#[skip_serializing_none]
 #[derive(Clone, Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct ArgoApplication {
@@ -85,7 +89,7 @@ impl Default for ArgoApplication {
             namespace: Default::default(),
             project: Default::default(),
             source: Source {
-                repo_url: Url::parse("http://asdf").expect("Couldn't parse to URL"),
+                repo_url: "http://asdf".to_string(),
                 target_revision: None,
                 chart: "".to_string(),
                 helm: Helm {
@@ -104,6 +108,7 @@ impl Default for ArgoApplication {
                     api_versions: vec![],
                     namespace: None,
                 },
+                path: "".to_string(),
             },
             sync_policy: SyncPolicy {
                 automated: Automated {
@@ -133,10 +138,10 @@ impl From<CDApplicationConfig> for ArgoApplication {
             namespace: Some(value.namespace),
             project: "default".to_string(),
             source: Source {
-                repo_url: Url::parse(value.helm_chart_repo_url.to_string().as_str())
+                repo_url: value.helm_chart_repo_url,
-                    .expect("couldn't convert to URL"),
+                target_revision: Some(value.version.to_string()),
-                target_revision: None,
+                chart: value.helm_chart_name.clone(),
-                chart: value.helm_chart_name,
+                path: value.helm_chart_name,
                 helm: Helm {
                     pass_credentials: None,
                     parameters: vec![],
@@ -145,7 +150,7 @@ impl From<CDApplicationConfig> for ArgoApplication {
                     value_files: vec![],
                     ignore_missing_value_files: None,
                     values: None,
-                    values_object: Some(value.values_overrides),
+                    values_object: value.values_overrides,
                     skip_crds: None,
                     skip_schema_validation: None,
                     version: None,
@@ -213,7 +218,7 @@ spec:
         let mut yaml_value: Value =
             serde_yaml::from_str(yaml_str.as_str()).expect("couldn't parse string to YAML");
 
-        let mut spec = yaml_value
+        let spec = yaml_value
             .get_mut("spec")
             .expect("couldn't get spec from yaml")
             .as_mapping_mut()
@@ -252,7 +257,7 @@ spec:
 
 #[cfg(test)]
 mod tests {
-    use url::Url;
+    use pretty_assertions::assert_eq;
 
     use crate::modules::application::features::{
         ArgoApplication, Automated, Backoff, Helm, Retry, Source, SyncPolicy,
@@ -265,7 +270,7 @@ mod tests {
             namespace: Some("test-ns".to_string()),
             project: "test-project".to_string(),
             source: Source {
-                repo_url: Url::parse("http://test").unwrap(),
+                repo_url: "http://test".to_string(),
                 target_revision: None,
                 chart: "test-chart".to_string(),
                 helm: Helm {
@@ -284,6 +289,7 @@ mod tests {
                     api_versions: vec![],
                     namespace: None,
                 },
+                path: "".to_string(),
             },
             sync_policy: SyncPolicy {
                 automated: Automated {
@@ -315,24 +321,15 @@ spec:
     server: https://kubernetes.default.svc
     namespace: test-ns
   source:
-    repoUrl: http://test/
+    repoURL: http://test
-    targetRevision: null
     chart: test-chart
     helm:
-      passCredentials: null
       parameters: []
       fileParameters: []
       releaseName: test-release-neame
       valueFiles: []
-      ignoreMissingValueFiles: null
-      values: null
-      valuesObject: null
-      skipCrds: null
-      skipSchemaValidation: null
-      version: null
-      kubeVersion: null
       apiVersions: []
-      namespace: null
+    path: ''
   syncPolicy:
     automated:
       prune: false

harmony/src/modules/application/features/continuous_delivery.rs

@@ -1,7 +1,7 @@
 use std::{io::Write, process::Command, sync::Arc};
 
 use async_trait::async_trait;
-use log::{error, info};
+use log::{debug, error};
 use serde_yaml::Value;
 use tempfile::NamedTempFile;
 
@@ -9,15 +9,12 @@ use crate::{
     config::HARMONY_DATA_DIR,
     data::Version,
     inventory::Inventory,
-    modules::{
+    modules::application::{
-        application::{
         Application, ApplicationFeature, HelmPackage, OCICompliant,
         features::{ArgoApplication, ArgoHelmScore},
     },
-        helm::chart::HelmChartScore,
-    },
     score::Score,
-    topology::{DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology, Url},
+    topology::{DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology},
 };
 
 /// ContinuousDelivery in Harmony provides this functionality :
@@ -66,7 +63,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         error!("TODO hardcoded k3d bin path is wrong");
         let k3d_bin_path = (*HARMONY_DATA_DIR).join("k3d").join("k3d");
         // --- 1. Import the container image into the k3d cluster ---
-        info!(
+        debug!(
             "Importing image '{}' into k3d cluster 'harmony'",
             image_name
         );
@@ -83,7 +80,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         }
 
         // --- 2. Get the kubeconfig for the k3d cluster and write it to a temp file ---
-        info!("Retrieving kubeconfig for k3d cluster 'harmony'");
+        debug!("Retrieving kubeconfig for k3d cluster 'harmony'");
         let kubeconfig_output = Command::new(&k3d_bin_path)
             .args(["kubeconfig", "get", "harmony"])
             .output()
@@ -104,7 +101,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         let kubeconfig_path = temp_kubeconfig.path().to_str().unwrap();
 
         // --- 3. Install or upgrade the Helm chart in the cluster ---
-        info!(
+        debug!(
             "Deploying Helm chart '{}' to namespace '{}'",
             chart_url, app_name
         );
@@ -134,7 +131,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
             ));
         }
 
-        info!("Successfully deployed '{}' to local k3d cluster.", app_name);
+        debug!("Successfully deployed '{}' to local k3d cluster.", app_name);
         Ok(())
     }
 }
@@ -148,23 +145,19 @@ impl<
     async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
         let image = self.application.image_name();
 
-        // TODO
-        error!(
-            "TODO reverse helm chart packaging and docker image build. I put helm package first for faster iterations"
-        );
-
         // TODO Write CI/CD workflow files
         // we can autotedect the CI type using the remote url (default to github action for github
         // url, etc..)
         // Or ask for it when unknown
 
         let helm_chart = self.application.build_push_helm_package(&image).await?;
-        info!("Pushed new helm chart {helm_chart}");
+        debug!("Pushed new helm chart {helm_chart}");
 
+        error!("TODO Make building image configurable/skippable if image already exists (prompt)");
         let image = self.application.build_push_oci_image().await?;
-        info!("Pushed new docker image {image}");
+        debug!("Pushed new docker image {image}");
 
-        info!("Installing ContinuousDelivery feature");
+        debug!("Installing ContinuousDelivery feature");
         // TODO this is a temporary hack for demo purposes, the deployment target should be driven
         // by the topology only and we should not have to know how to perform tasks like this for
         // which the topology should be responsible.
@@ -184,17 +177,17 @@ impl<
                     .await?;
             }
             target => {
-                info!("Deploying to target {target:?}");
+                debug!("Deploying to target {target:?}");
                 let score = ArgoHelmScore {
                     namespace: "harmonydemo-staging".to_string(),
-                    openshift: true,
+                    openshift: false,
                     domain: "argo.harmonydemo.apps.st.mcd".to_string(),
                     argo_apps: vec![ArgoApplication::from(CDApplicationConfig {
-                        // helm pull oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart/harmony-example-rust-webapp-chart --version 0.1.0
+                        // helm pull oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart --version 0.1.0
                         version: Version::from("0.1.0").unwrap(),
-                        helm_chart_repo_url: Url::Url(url::Url::parse("oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart/harmony-example-rust-webapp-chart").unwrap()),
+                        helm_chart_repo_url: "hub.nationtech.io/harmony".to_string(),
                         helm_chart_name: "harmony-example-rust-webapp-chart".to_string(),
-                        values_overrides: Value::Null,
+                        values_overrides: None,
                         name: "harmony-demo-rust-webapp".to_string(),
                         namespace: "harmonydemo-staging".to_string(),
                     })],
@@ -206,14 +199,7 @@ impl<
                     .unwrap();
             }
         };
-
+        Ok(())
-        todo!("1. Create ArgoCD score that installs argo using helm chart, see if Taha's already done it
-            - [X] Package app (docker image, helm chart)
-            - [X] Push to registry
-            - [X] Push only if staging or prod
-            - [X] Deploy to local k3d when target is local
-            - [ ] Poke Argo
-            - [ ] Ensure app is up")
     }
     fn name(&self) -> String {
         "ContinuousDelivery".to_string()
@@ -224,9 +210,9 @@ impl<
 /// more CD systems
 pub struct CDApplicationConfig {
     pub version: Version,
-    pub helm_chart_repo_url: Url,
+    pub helm_chart_repo_url: String,
     pub helm_chart_name: String,
-    pub values_overrides: Value,
+    pub values_overrides: Option<Value>,
     pub name: String,
     pub namespace: String,
 }

harmony/src/modules/application/features/helm_argocd_score.rs

@@ -1,5 +1,5 @@
 use async_trait::async_trait;
-use k8s_openapi::Resource;
+use log::error;
 use non_blank_string_rs::NonBlankString;
 use serde::Serialize;
 use std::str::FromStr;
@@ -50,6 +50,7 @@ impl<T: Topology + K8sclient + HelmCommand> Interpret<T> for ArgoInterpret {
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        error!("Uncomment below, only disabled for debugging");
         self.score
             .create_interpret()
             .execute(inventory, topology)
@@ -645,7 +646,7 @@ server:
   # Argo CD server ingress configuration
   ingress:
     # -- Enable an ingress resource for the Argo CD server
-    enabled: false
+    enabled: true
     # -- Specific implementation for ingress controller. One of `generic`, `aws` or `gke`
     ## Additional configuration might be required in related configuration sections
     controller: generic

harmony/src/modules/application/features/monitoring.rs

@@ -1,40 +1,99 @@
-use async_trait::async_trait;
+use std::sync::Arc;
-use log::info;
+
+use crate::modules::application::{Application, ApplicationFeature};
+use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
 
 use crate::{
     inventory::Inventory,
-    modules::{
+    modules::monitoring::{
-        application::{Application, ApplicationFeature},
+        alert_channel::webhook_receiver::WebhookReceiver, ntfy::ntfy::NtfyScore,
-        monitoring::{
-            application_monitoring::k8s_application_monitoring_score::ApplicationPrometheusMonitoringScore,
-            kube_prometheus::{
-                helm_prometheus_alert_score::HelmPrometheusAlertingScore,
-                types::{NamespaceSelector, ServiceMonitor},
-            },
-        },
     },
     score::Score,
-    topology::{HelmCommand, Topology, tenant::TenantManager},
+    topology::{HelmCommand, K8sclient, Topology, Url, tenant::TenantManager},
 };
+use crate::{
+    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    topology::oberservability::monitoring::AlertReceiver,
+};
+use async_trait::async_trait;
+use base64::{Engine as _, engine::general_purpose};
+use log::{debug, info};
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Clone)]
-pub struct Monitoring {}
+pub struct Monitoring {
+    pub application: Arc<dyn Application>,
+    pub alert_receiver: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+}
 
 #[async_trait]
-impl<T: Topology + HelmCommand + 'static + TenantManager> ApplicationFeature<T> for Monitoring {
+impl<
+    T: Topology
+        + HelmCommand
+        + 'static
+        + TenantManager
+        + K8sclient
+        + std::fmt::Debug
+        + PrometheusApplicationMonitoring<CRDPrometheus>,
+> ApplicationFeature<T> for Monitoring
+{
     async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
         info!("Ensuring monitoring is available for application");
-        let mut service_monitor = ServiceMonitor::default();
+        let namespace = topology
-        service_monitor.namespace_selector = Some(NamespaceSelector {
+            .get_tenant_config()
-            any: true,
+            .await
-            match_names: vec![],
+            .map(|ns| ns.name.clone())
-        });
+            .unwrap_or_else(|| self.application.name());
-        let alerting_score = ApplicationPrometheusMonitoringScore {
+
-            receivers: vec![],
+        let mut alerting_score = ApplicationMonitoringScore {
-            rules: vec![],
+            sender: CRDPrometheus {
-            service_monitors: vec![service_monitor],
+                namespace: namespace.clone(),
+                client: topology.k8s_client().await.unwrap(),
+            },
+            application: self.application.clone(),
+            receivers: self.alert_receiver.clone(),
+        };
+        let ntfy = NtfyScore {
+            namespace: namespace.clone(),
+            host: "localhost".to_string(),
+        };
+        ntfy.create_interpret()
+            .execute(&Inventory::empty(), topology)
+            .await
+            .expect("couldn't create interpret for ntfy");
+
+        let ntfy_default_auth_username = "harmony";
+        let ntfy_default_auth_password = "harmony";
+        let ntfy_default_auth_header = format!(
+            "Basic {}",
+            general_purpose::STANDARD.encode(format!(
+                "{ntfy_default_auth_username}:{ntfy_default_auth_password}"
+            ))
+        );
+
+        debug!("ntfy_default_auth_header: {ntfy_default_auth_header}");
+
+        let ntfy_default_auth_param = general_purpose::STANDARD
+            .encode(ntfy_default_auth_header)
+            .replace("=", "");
+
+        debug!("ntfy_default_auth_param: {ntfy_default_auth_param}");
+
+        let ntfy_receiver = WebhookReceiver {
+            name: "ntfy-webhook".to_string(),
+            url: Url::Url(
+                url::Url::parse(
+                    format!(
+                        "http://ntfy.{}.svc.cluster.local/rust-web-app?auth={ntfy_default_auth_param}",
+                        namespace.clone()
+                    )
+                    .as_str(),
+                )
+                .unwrap(),
+            ),
         };
 
+        alerting_score.receivers.push(Box::new(ntfy_receiver));
         alerting_score
             .create_interpret()
             .execute(&Inventory::empty(), topology)

harmony/src/modules/application/mod.rs

@@ -5,11 +5,12 @@ mod rust;
 use std::sync::Arc;
 
 pub use feature::*;
-use log::info;
+use log::debug;
 pub use oci::*;
 pub use rust::*;
 
 use async_trait::async_trait;
+use serde::Serialize;
 
 use crate::{
     data::{Id, Version},
@@ -36,7 +37,7 @@ impl<A: Application, T: Topology + std::fmt::Debug> Interpret<T> for Application
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
         let app_name = self.application.name();
-        info!(
+        debug!(
             "Preparing {} features [{}] for application {app_name}",
             self.features.len(),
             self.features
@@ -46,7 +47,7 @@ impl<A: Application, T: Topology + std::fmt::Debug> Interpret<T> for Application
                 .join(", ")
         );
         for feature in self.features.iter() {
-            info!(
+            debug!(
                 "Installing feature {} for application {app_name}",
                 feature.name()
             );
@@ -59,9 +60,7 @@ impl<A: Application, T: Topology + std::fmt::Debug> Interpret<T> for Application
                 }
             };
         }
-        todo!(
+        Ok(Outcome::success("successfully created app".to_string()))
-            "Do I need to do anything more than this here?? I feel like the Application trait itself should expose something like ensure_ready but its becoming redundant. We'll see as this evolves."
-        )
     }
 
     fn get_name(&self) -> InterpretName {
@@ -80,3 +79,12 @@ impl<A: Application, T: Topology + std::fmt::Debug> Interpret<T> for Application
         todo!()
     }
 }
+
+impl Serialize for dyn Application {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}

harmony/src/modules/application/rust.rs

@@ -4,11 +4,15 @@ use std::process;
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use bollard::query_parameters::PushImageOptionsBuilder;
+use bollard::{Docker, body_full};
 use dockerfile_builder::Dockerfile;
 use dockerfile_builder::instruction::{CMD, COPY, ENV, EXPOSE, FROM, RUN, USER, WORKDIR};
 use dockerfile_builder::instruction_builder::CopyBuilder;
-use log::{debug, error, info};
+use futures_util::StreamExt;
+use log::{debug, error, log_enabled};
 use serde::Serialize;
+use tar::Archive;
 
 use crate::config::{REGISTRY_PROJECT, REGISTRY_URL};
 use crate::{
@@ -69,19 +73,19 @@ impl Application for RustWebapp {
 #[async_trait]
 impl HelmPackage for RustWebapp {
     async fn build_push_helm_package(&self, image_url: &str) -> Result<String, String> {
-        info!("Starting Helm chart build and push for '{}'", self.name);
+        debug!("Starting Helm chart build and push for '{}'", self.name);
 
         // 1. Create the Helm chart files on disk.
         let chart_dir = self
             .create_helm_chart_files(image_url)
             .map_err(|e| format!("Failed to create Helm chart files: {}", e))?;
-        info!("Successfully created Helm chart files in {:?}", chart_dir);
+        debug!("Successfully created Helm chart files in {:?}", chart_dir);
 
         // 2. Package the chart into a .tgz archive.
         let packaged_chart_path = self
             .package_helm_chart(&chart_dir)
             .map_err(|e| format!("Failed to package Helm chart: {}", e))?;
-        info!(
+        debug!(
             "Successfully packaged Helm chart: {}",
             packaged_chart_path.to_string_lossy()
         );
@@ -90,7 +94,7 @@ impl HelmPackage for RustWebapp {
         let oci_chart_url = self
             .push_helm_chart(&packaged_chart_path)
             .map_err(|e| format!("Failed to push Helm chart: {}", e))?;
-        info!("Successfully pushed Helm chart to: {}", oci_chart_url);
+        debug!("Successfully pushed Helm chart to: {}", oci_chart_url);
 
         Ok(oci_chart_url)
     }
@@ -103,24 +107,22 @@ impl OCICompliant for RustWebapp {
     async fn build_push_oci_image(&self) -> Result<String, String> {
         // This function orchestrates the build and push process.
         // It's async to match the trait definition, though the underlying docker commands are blocking.
-        info!("Starting OCI image build and push for '{}'", self.name);
+        debug!("Starting OCI image build and push for '{}'", self.name);
 
-        // 1. Build the local image by calling the synchronous helper function.
+        // 1. Build the image by calling the synchronous helper function.
-        let local_image_name = self.local_image_name();
+        let image_tag = self.image_name();
-        self.build_docker_image(&local_image_name)
+        self.build_docker_image(&image_tag)
+            .await
             .map_err(|e| format!("Failed to build Docker image: {}", e))?;
-        info!(
+        debug!("Successfully built Docker image: {}", image_tag);
-            "Successfully built local Docker image: {}",
-            local_image_name
-        );
 
-        let remote_image_name = self.image_name();
         // 2. Push the image to the registry.
-        self.push_docker_image(&local_image_name, &remote_image_name)
+        self.push_docker_image(&image_tag)
+            .await
             .map_err(|e| format!("Failed to push Docker image: {}", e))?;
-        info!("Successfully pushed Docker image to: {}", remote_image_name);
+        debug!("Successfully pushed Docker image to: {}", image_tag);
 
-        Ok(remote_image_name)
+        Ok(image_tag)
     }
 
     fn local_image_name(&self) -> String {
@@ -153,68 +155,74 @@ impl RustWebapp {
     }
 
     /// Builds the Docker image using the generated Dockerfile.
-    pub fn build_docker_image(
+    pub async fn build_docker_image(
         &self,
         image_name: &str,
     ) -> Result<String, Box<dyn std::error::Error>> {
-        info!("Generating Dockerfile for '{}'", self.name);
+        debug!("Generating Dockerfile for '{}'", self.name);
-        let dockerfile_path = self.build_dockerfile()?;
+        let _dockerfile_path = self.build_dockerfile()?;
 
-        info!(
+        let docker = Docker::connect_with_socket_defaults().unwrap();
-            "Building Docker image with file {} from root {}",
+
-            dockerfile_path.to_string_lossy(),
+        let quiet = !log_enabled!(log::Level::Debug);
-            self.project_root.to_string_lossy()
+
+        let build_image_options = bollard::query_parameters::BuildImageOptionsBuilder::default()
+            .dockerfile("Dockerfile.harmony")
+            .t(image_name)
+            .q(quiet)
+            .version(bollard::query_parameters::BuilderVersion::BuilderV1)
+            .platform("linux/x86_64");
+
+        let mut temp_tar_builder = tar::Builder::new(Vec::new());
+        let _ = temp_tar_builder
+            .append_dir_all("", self.project_root.clone())
+            .unwrap();
+        let archive = temp_tar_builder
+            .into_inner()
+            .expect("couldn't finish creating tar");
+        let archived_files = Archive::new(archive.as_slice())
+            .entries()
+            .unwrap()
+            .map(|entry| entry.unwrap().path().unwrap().into_owned())
+            .collect::<Vec<_>>();
+
+        debug!("files in docker tar: {:#?}", archived_files);
+
+        let mut image_build_stream = docker.build_image(
+            build_image_options.build(),
+            None,
+            Some(body_full(archive.into())),
         );
-        let output = process::Command::new("docker")
-            .args([
-                "build",
-                "--file",
-                dockerfile_path.to_str().unwrap(),
-                "-t",
-                &image_name,
-                self.project_root.to_str().unwrap(),
-            ])
-            .spawn()?
-            .wait_with_output()?;
 
-        self.check_output(&output, "Failed to build Docker image")?;
+        while let Some(msg) = image_build_stream.next().await {
+            println!("Message: {msg:?}");
+        }
 
         Ok(image_name.to_string())
     }
 
     /// Tags and pushes a Docker image to the configured remote registry.
-    fn push_docker_image(
+    async fn push_docker_image(
         &self,
-        image_name: &str,
+        image_tag: &str,
-        full_tag: &str,
     ) -> Result<String, Box<dyn std::error::Error>> {
-        info!("Pushing docker image {full_tag}");
+        debug!("Pushing docker image {image_tag}");
 
-        // Tag the image for the remote registry.
+        let docker = Docker::connect_with_socket_defaults().unwrap();
-        let output = process::Command::new("docker")
+
-            .args(["tag", image_name, &full_tag])
+        // let push_options = PushImageOptionsBuilder::new().tag(tag);
-            .spawn()?
+
-            .wait_with_output()?;
+        let mut push_image_stream = docker.push_image(
-        self.check_output(&output, "Tagging docker image failed")?;
+            image_tag,
-        debug!(
+            Some(PushImageOptionsBuilder::new().build()),
-            "docker tag output: stdout: {}, stderr: {}",
+            None,
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
         );
 
-        // Push the image.
+        while let Some(msg) = push_image_stream.next().await {
-        let output = process::Command::new("docker")
+            println!("Message: {msg:?}");
-            .args(["push", &full_tag])
+        }
-            .spawn()?
-            .wait_with_output()?;
-        self.check_output(&output, "Pushing docker image failed")?;
-        debug!(
-            "docker push output: stdout: {}, stderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        );
 
-        Ok(full_tag.to_string())
+        Ok(image_tag.to_string())
     }
 
     /// Checks the output of a process command for success.
@@ -349,7 +357,11 @@ impl RustWebapp {
         image_url: &str,
     ) -> Result<PathBuf, Box<dyn std::error::Error>> {
         let chart_name = format!("{}-chart", self.name);
-        let chart_dir = self.project_root.join("helm").join(&chart_name);
+        let chart_dir = self
+            .project_root
+            .join(".harmony_generated")
+            .join("helm")
+            .join(&chart_name);
         let templates_dir = chart_dir.join("templates");
         fs::create_dir_all(&templates_dir)?;
 
@@ -416,7 +428,7 @@ ingress:
 Expand the name of the chart.
 */}}
 {{- define "chart.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- default .Chart.Name $.Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
@@ -424,7 +436,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 */}}
 {{- define "chart.fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- $name := default .Chart.Name $.Values.nameOverride }}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
 {{- end }}
 "#;
@@ -437,12 +449,12 @@ kind: Service
 metadata:
   name: {{ include "chart.fullname" . }}
 spec:
-  type: {{ .Values.service.type }}
+  type: {{ $.Values.service.type }}
   ports:
-    - port: {{ .Values.service.port }}
+    - name: main
-      targetPort: 3000
+      port: {{ $.Values.service.port | default 3000 }}
+      targetPort: {{ $.Values.service.port | default 3000 }}
       protocol: TCP
-      name: http
   selector:
     app: {{ include "chart.name" . }}
 "#;
@@ -455,7 +467,7 @@ kind: Deployment
 metadata:
   name: {{ include "chart.fullname" . }}
 spec:
-  replicas: {{ .Values.replicaCount }}
+  replicas: {{ $.Values.replicaCount }}
   selector:
     matchLabels:
       app: {{ include "chart.name" . }}
@@ -466,28 +478,28 @@ spec:
     spec:
       containers:
         - name: {{ .Chart.Name }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ $.Values.image.pullPolicy }}
           ports:
-            - name: http
+            - name: main
-              containerPort: 3000
+              containerPort: {{ $.Values.service.port | default 3000 }}
               protocol: TCP
 "#;
         fs::write(templates_dir.join("deployment.yaml"), deployment_yaml)?;
 
         // Create templates/ingress.yaml
         let ingress_yaml = r#"
-{{- if .Values.ingress.enabled -}}
+{{- if $.Values.ingress.enabled -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ include "chart.fullname" . }}
   annotations:
-    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+    {{- toYaml $.Values.ingress.annotations | nindent 4 }}
 spec:
-  {{- if .Values.ingress.tls }}
+  {{- if $.Values.ingress.tls }}
   tls:
-    {{- range .Values.ingress.tls }}
+    {{- range $.Values.ingress.tls }}
     - hosts:
         {{- range .hosts }}
         - {{ . | quote }}
@@ -496,7 +508,7 @@ spec:
     {{- end }}
   {{- end }}
   rules:
-    {{- range .Values.ingress.hosts }}
+    {{- range $.Values.ingress.hosts }}
     - host: {{ .host | quote }}
       http:
         paths:
@@ -507,7 +519,7 @@ spec:
               service:
                 name: {{ include "chart.fullname" $ }}
                 port:
-                  number: 3000
+                  number: {{ $.Values.service.port | default 3000 }}
           {{- end }}
     {{- end }}
 {{- end }}
@@ -523,14 +535,18 @@ spec:
         chart_dir: &PathBuf,
     ) -> Result<PathBuf, Box<dyn std::error::Error>> {
         let chart_dirname = chart_dir.file_name().expect("Should find a chart dirname");
-        info!(
+        debug!(
             "Launching `helm package {}` cli with CWD {}",
             chart_dirname.to_string_lossy(),
-            &self.project_root.join("helm").to_string_lossy()
+            &self
+                .project_root
+                .join(".harmony_generated")
+                .join("helm")
+                .to_string_lossy()
         );
         let output = process::Command::new("helm")
             .args(["package", chart_dirname.to_str().unwrap()])
-            .current_dir(&self.project_root.join("helm")) // Run package from the parent dir
+            .current_dir(&self.project_root.join(".harmony_generated").join("helm")) // Run package from the parent dir
             .output()?;
 
         self.check_output(&output, "Failed to package Helm chart")?;
@@ -547,7 +563,11 @@ spec:
         }
 
         // The output from helm is relative, so we join it with the execution directory.
-        Ok(self.project_root.join("helm").join(tgz_name))
+        Ok(self
+            .project_root
+            .join(".harmony_generated")
+            .join("helm")
+            .join(tgz_name))
     }
 
     /// Pushes a packaged Helm chart to an OCI registry.
@@ -560,7 +580,7 @@ spec:
         let oci_push_url = format!("oci://{}/{}", *REGISTRY_URL, *REGISTRY_PROJECT);
         let oci_pull_url = format!("{oci_push_url}/{}-chart", self.name);
 
-        info!(
+        debug!(
             "Pushing Helm chart {} to {}",
             packaged_chart_path.to_string_lossy(),
             oci_push_url

harmony/src/modules/http.rs

@@ -10,14 +10,25 @@ use crate::{
     topology::{HttpServer, Topology, Url},
 };
 
+/// Configure an HTTP server that is provided by the Topology
+///
+/// This Score will let you easily specify a file path to be served by the HTTP server
+///
+/// For example, if you have a folder of assets at `/var/www/assets` simply do :
+///
+/// ```rust,ignore
+/// StaticFilesHttpScore {
+///   files_to_serve: url!("file:///var/www/assets"),
+/// }
+/// ```
 #[derive(Debug, new, Clone, Serialize)]
-pub struct HttpScore {
+pub struct StaticFilesHttpScore {
     files_to_serve: Url,
 }
 
-impl<T: Topology + HttpServer> Score<T> for HttpScore {
+impl<T: Topology + HttpServer> Score<T> for StaticFilesHttpScore {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(HttpInterpret::new(self.clone()))
+        Box::new(StaticFilesHttpInterpret::new(self.clone()))
     }
 
     fn name(&self) -> String {
@@ -26,12 +37,12 @@ impl<T: Topology + HttpServer> Score<T> for HttpScore {
 }
 
 #[derive(Debug, new, Clone)]
-pub struct HttpInterpret {
+pub struct StaticFilesHttpInterpret {
-    score: HttpScore,
+    score: StaticFilesHttpScore,
 }
 
 #[async_trait]
-impl<T: Topology + HttpServer> Interpret<T> for HttpInterpret {
+impl<T: Topology + HttpServer> Interpret<T> for StaticFilesHttpInterpret {
     async fn execute(
         &self,
         _inventory: &Inventory,

harmony/src/modules/k3d/install.rs

@@ -1,12 +1,13 @@
 use std::path::PathBuf;
 
 use async_trait::async_trait;
-use log::info;
+use log::{debug, info};
 use serde::Serialize;
 
 use crate::{
     config::HARMONY_DATA_DIR,
     data::{Id, Version},
+    instrumentation::{self, HarmonyEvent},
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
     score::Score,
@@ -50,22 +51,38 @@ impl<T: Topology> Interpret<T> for K3dInstallationInterpret {
     async fn execute(
         &self,
         _inventory: &Inventory,
-        _topology: &T,
+        topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        instrumentation::instrument(HarmonyEvent::InterpretExecutionStarted {
+            topology: topology.name().into(),
+            interpret: "k3d-installation".into(),
+            message: "installing k3d...".into(),
+        })
+        .unwrap();
+
         let k3d = k3d_rs::K3d::new(
             self.score.installation_path.clone(),
             Some(self.score.cluster_name.clone()),
         );
-        match k3d.ensure_installed().await {
+        let outcome = match k3d.ensure_installed().await {
             Ok(_client) => {
-                let msg = format!("k3d cluster {} is installed ", self.score.cluster_name);
+                let msg = format!("k3d cluster '{}' installed ", self.score.cluster_name);
-                info!("{msg}");
+                debug!("{msg}");
                 Ok(Outcome::success(msg))
             }
             Err(msg) => Err(InterpretError::new(format!(
-                "K3dInstallationInterpret failed to ensure k3d is installed : {msg}"
+                "failed to ensure k3d is installed : {msg}"
             ))),
-        }
+        };
+
+        instrumentation::instrument(HarmonyEvent::InterpretExecutionFinished {
+            topology: topology.name().into(),
+            interpret: "k3d-installation".into(),
+            outcome: outcome.clone(),
+        })
+        .unwrap();
+
+        outcome
     }
     fn get_name(&self) -> InterpretName {
         InterpretName::K3dInstallation

harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs

@@ -1,7 +1,16 @@
+use std::any::Any;
+use std::collections::BTreeMap;
+
 use async_trait::async_trait;
+use k8s_openapi::api::core::v1::Secret;
+use kube::api::ObjectMeta;
 use serde::Serialize;
+use serde_json::json;
 use serde_yaml::{Mapping, Value};
 
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
+    AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
+};
 use crate::{
     interpret::{InterpretError, Outcome},
     modules::monitoring::{
@@ -20,14 +29,98 @@ pub struct DiscordWebhook {
     pub url: Url,
 }
 
+#[async_trait]
+impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+        let ns = sender.namespace.clone();
+        let secret_name = format!("{}-secret", self.name.clone());
+        let webhook_key = format!("{}", self.url.clone());
+
+        let mut string_data = BTreeMap::new();
+        string_data.insert("webhook-url".to_string(), webhook_key.clone());
+
+        let secret = Secret {
+            metadata: kube::core::ObjectMeta {
+                name: Some(secret_name.clone()),
+                ..Default::default()
+            },
+            string_data: Some(string_data),
+            type_: Some("Opaque".to_string()),
+            ..Default::default()
+        };
+
+        let _ = sender.client.apply(&secret, Some(&ns)).await;
+
+        let spec = AlertmanagerConfigSpec {
+            data: json!({
+                "route": {
+                    "receiver": self.name,
+                },
+                "receivers": [
+                    {
+                        "name": self.name,
+                        "discordConfigs": [
+                            {
+                            "apiURL": {
+                                "name": secret_name,
+                                "key":  "webhook-url",
+                            },
+                            "title": "{{ template \"discord.default.title\" . }}",
+                            "message": "{{ template \"discord.default.message\" . }}"
+                            }
+                        ]
+                    }
+                ]
+            }),
+        };
+
+        let alertmanager_configs = AlertmanagerConfig {
+            metadata: ObjectMeta {
+                name: Some(self.name.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(ns),
+                ..Default::default()
+            },
+            spec,
+        };
+
+        sender
+            .client
+            .apply(&alertmanager_configs, Some(&sender.namespace))
+            .await?;
+        Ok(Outcome::success(format!(
+            "installed crd-alertmanagerconfigs for {}",
+            self.name
+        )))
+    }
+    fn name(&self) -> String {
+        "discord-webhook".to_string()
+    }
+    fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> {
+        Box::new(self.clone())
+    }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
 #[async_trait]
 impl AlertReceiver<Prometheus> for DiscordWebhook {
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
+    fn name(&self) -> String {
+        "discord-webhook".to_string()
+    }
     fn clone_box(&self) -> Box<dyn AlertReceiver<Prometheus>> {
         Box::new(self.clone())
     }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait]
@@ -48,6 +141,12 @@ impl AlertReceiver<KubePrometheus> for DiscordWebhook {
     fn clone_box(&self) -> Box<dyn AlertReceiver<KubePrometheus>> {
         Box::new(self.clone())
     }
+    fn name(&self) -> String {
+        "discord-webhook".to_string()
+    }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait]

harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs

@@ -1,11 +1,19 @@
+use std::any::Any;
+
 use async_trait::async_trait;
+use kube::api::ObjectMeta;
+use log::debug;
 use serde::Serialize;
+use serde_json::json;
 use serde_yaml::{Mapping, Value};
 
 use crate::{
     interpret::{InterpretError, Outcome},
     modules::monitoring::{
         kube_prometheus::{
+            crd::crd_alertmanager_config::{
+                AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
+            },
             prometheus::{KubePrometheus, KubePrometheusReceiver},
             types::{AlertChannelConfig, AlertManagerChannelConfig},
         },
@@ -20,14 +28,81 @@ pub struct WebhookReceiver {
     pub url: Url,
 }
 
+#[async_trait]
+impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+        let spec = AlertmanagerConfigSpec {
+            data: json!({
+                "route": {
+                    "receiver": self.name,
+                },
+                "receivers": [
+                    {
+                        "name": self.name,
+                        "webhookConfigs": [
+                            {
+                            "url": self.url,
+                            }
+                        ]
+                    }
+                ]
+            }),
+        };
+
+        let alertmanager_configs = AlertmanagerConfig {
+            metadata: ObjectMeta {
+                name: Some(self.name.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec,
+        };
+        debug!(
+            "alert manager configs: \n{:#?}",
+            alertmanager_configs.clone()
+        );
+
+        sender
+            .client
+            .apply(&alertmanager_configs, Some(&sender.namespace))
+            .await?;
+        Ok(Outcome::success(format!(
+            "installed crd-alertmanagerconfigs for {}",
+            self.name
+        )))
+    }
+
+    fn name(&self) -> String {
+        "webhook-receiver".to_string()
+    }
+
+    fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> {
+        Box::new(self.clone())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
 #[async_trait]
 impl AlertReceiver<Prometheus> for WebhookReceiver {
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
+    fn name(&self) -> String {
+        "webhook-receiver".to_string()
+    }
     fn clone_box(&self) -> Box<dyn AlertReceiver<Prometheus>> {
         Box::new(self.clone())
     }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait]
@@ -44,9 +119,15 @@ impl AlertReceiver<KubePrometheus> for WebhookReceiver {
     async fn install(&self, sender: &KubePrometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
+    fn name(&self) -> String {
+        "webhook-receiver".to_string()
+    }
     fn clone_box(&self) -> Box<dyn AlertReceiver<KubePrometheus>> {
         Box::new(self.clone())
     }
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
 
 #[async_trait]

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -0,0 +1,78 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use serde::Serialize;
+
+use crate::{
+    data::{Id, Version},
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    modules::{
+        application::Application,
+        monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+        prometheus::prometheus::PrometheusApplicationMonitoring,
+    },
+    score::Score,
+    topology::{Topology, oberservability::monitoring::AlertReceiver},
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct ApplicationMonitoringScore {
+    pub sender: CRDPrometheus,
+    pub application: Arc<dyn Application>,
+    pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+}
+
+impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+    for ApplicationMonitoringScore
+{
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(ApplicationMonitoringInterpret {
+            score: self.clone(),
+        })
+    }
+
+    fn name(&self) -> String {
+        "ApplicationMonitoringScore".to_string()
+    }
+}
+
+#[derive(Debug)]
+pub struct ApplicationMonitoringInterpret {
+    score: ApplicationMonitoringScore,
+}
+
+#[async_trait]
+impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
+    for ApplicationMonitoringInterpret
+{
+    async fn execute(
+        &self,
+        inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        topology
+            .install_prometheus(
+                &self.score.sender,
+                inventory,
+                Some(self.score.receivers.clone()),
+            )
+            .await
+    }
+
+    fn get_name(&self) -> InterpretName {
+        todo!()
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}

harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs

@@ -1,41 +0,0 @@
-use std::sync::{Arc, Mutex};
-
-use serde::Serialize;
-
-use crate::{
-    modules::monitoring::{
-        kube_prometheus::types::ServiceMonitor,
-        prometheus::{prometheus::Prometheus, prometheus_config::PrometheusConfig},
-    },
-    score::Score,
-    topology::{
-        HelmCommand, Topology,
-        oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret},
-        tenant::TenantManager,
-    },
-};
-
-#[derive(Clone, Debug, Serialize)]
-pub struct ApplicationPrometheusMonitoringScore {
-    pub receivers: Vec<Box<dyn AlertReceiver<Prometheus>>>,
-    pub rules: Vec<Box<dyn AlertRule<Prometheus>>>,
-    pub service_monitors: Vec<ServiceMonitor>,
-}
-
-impl<T: Topology + HelmCommand + TenantManager> Score<T> for ApplicationPrometheusMonitoringScore {
-    fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
-        let config = Arc::new(Mutex::new(PrometheusConfig::new()));
-        config
-            .try_lock()
-            .expect("couldn't lock config")
-            .additional_service_monitors = self.service_monitors.clone();
-        Box::new(AlertingInterpret {
-            sender: Prometheus::new(),
-            receivers: self.receivers.clone(),
-            rules: self.rules.clone(),
-        })
-    }
-    fn name(&self) -> String {
-        "ApplicationPrometheusMonitoringScore".to_string()
-    }
-}

harmony/src/modules/monitoring/application_monitoring/mod.rs

@@ -1 +1 @@
-pub mod k8s_application_monitoring_score;
+pub mod application_monitoring_score;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs

@@ -0,0 +1,50 @@
+use std::sync::Arc;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::topology::{
+    k8s::K8sClient,
+    oberservability::monitoring::{AlertReceiver, AlertSender},
+};
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1alpha1",
+    kind = "AlertmanagerConfig",
+    plural = "alertmanagerconfigs",
+    namespaced
+)]
+pub struct AlertmanagerConfigSpec {
+    #[serde(flatten)]
+    pub data: serde_json::Value,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CRDPrometheus {
+    pub namespace: String,
+    pub client: Arc<K8sClient>,
+}
+
+impl AlertSender for CRDPrometheus {
+    fn name(&self) -> String {
+        "CRDAlertManager".to_string()
+    }
+}
+
+impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
+impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanagers.rs

@@ -0,0 +1,53 @@
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+use super::crd_prometheuses::LabelSelector;
+
+/// Rust CRD for `Alertmanager` from Prometheus Operator
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "Alertmanager",
+    plural = "alertmanagers",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct AlertmanagerSpec {
+    /// Number of replicas for HA
+    pub replicas: i32,
+
+    /// Selectors for AlertmanagerConfig CRDs
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alertmanager_config_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alertmanager_config_namespace_selector: Option<LabelSelector>,
+
+    /// Optional pod template metadata (annotations, labels)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pod_metadata: Option<LabelSelector>,
+
+    /// Optional topology spread settings
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub version: Option<String>,
+}
+
+impl Default for AlertmanagerSpec {
+    fn default() -> Self {
+        AlertmanagerSpec {
+            replicas: 1,
+
+            // Match all AlertmanagerConfigs in the same namespace
+            alertmanager_config_namespace_selector: None,
+
+            // Empty selector matches all AlertmanagerConfigs in that namespace
+            alertmanager_config_selector: Some(LabelSelector::default()),
+
+            pod_metadata: None,
+            version: None,
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs

@@ -0,0 +1,30 @@
+use std::collections::BTreeMap;
+
+use crate::modules::{
+    monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule,
+    prometheus::alerts::k8s::{
+        deployment::alert_deployment_unavailable,
+        pod::{alert_container_restarting, alert_pod_not_ready, pod_failed},
+        pvc::high_pvc_fill_rate_over_two_days,
+        service::alert_service_down,
+    },
+};
+
+use super::crd_prometheus_rules::Rule;
+
+pub fn build_default_application_rules() -> Vec<Rule> {
+    let pod_failed: Rule = pod_failed().into();
+    let container_restarting: Rule = alert_container_restarting().into();
+    let pod_not_ready: Rule = alert_pod_not_ready().into();
+    let service_down: Rule = alert_service_down().into();
+    let deployment_unavailable: Rule = alert_deployment_unavailable().into();
+    let high_pvc_fill_rate: Rule = high_pvc_fill_rate_over_two_days().into();
+    vec![
+        pod_failed,
+        container_restarting,
+        pod_not_ready,
+        service_down,
+        deployment_unavailable,
+        high_pvc_fill_rate,
+    ]
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs

@@ -0,0 +1,153 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use super::crd_prometheuses::LabelSelector;
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "Grafana",
+    plural = "grafanas",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSpec {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub config: Option<GrafanaConfig>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_user: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_password: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ingress: Option<GrafanaIngress>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub persistence: Option<GrafanaPersistence>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resources: Option<ResourceRequirements>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub log: Option<GrafanaLogConfig>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub security: Option<GrafanaSecurityConfig>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaLogConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub mode: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub level: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSecurityConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_user: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_password: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaIngress {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hosts: Option<Vec<String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaPersistence {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub storage_class_name: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub size: Option<String>,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "GrafanaDashboard",
+    plural = "grafanadashboards",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDashboardSpec {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resync_period: Option<String>,
+
+    pub instance_selector: LabelSelector,
+
+    pub json: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "GrafanaDatasource",
+    plural = "grafanadatasources",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceSpec {
+    pub instance_selector: LabelSelector,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub allow_cross_namespace_import: Option<bool>,
+
+    pub datasource: GrafanaDatasourceConfig,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceConfig {
+    pub access: String,
+    pub database: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json_data: Option<BTreeMap<String, String>>,
+    pub name: String,
+    pub r#type: String,
+    pub url: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct ResourceRequirements {
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub limits: BTreeMap<String, String>,
+
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub requests: BTreeMap<String, String>,
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs

@@ -0,0 +1,59 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule;
+
+use super::crd_default_rules::build_default_application_rules;
+
+#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "PrometheusRule",
+    plural = "prometheusrules",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct PrometheusRuleSpec {
+    pub groups: Vec<RuleGroup>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct RuleGroup {
+    pub name: String,
+    pub rules: Vec<Rule>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Rule {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alert: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub expr: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub for_: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub labels: Option<std::collections::BTreeMap<String, String>>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub annotations: Option<std::collections::BTreeMap<String, String>>,
+}
+
+impl From<PrometheusAlertRule> for Rule {
+    fn from(value: PrometheusAlertRule) -> Self {
+        Rule {
+            alert: Some(value.alert),
+            expr: Some(value.expr),
+            for_: value.r#for,
+            labels: Some(value.labels.into_iter().collect::<BTreeMap<_, _>>()),
+            annotations: Some(value.annotations.into_iter().collect::<BTreeMap<_, _>>()),
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheuses.rs

@@ -0,0 +1,118 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::modules::monitoring::kube_prometheus::types::Operator;
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "Prometheus",
+    plural = "prometheuses",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct PrometheusSpec {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alerting: Option<PrometheusSpecAlerting>,
+
+    pub service_account_name: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_monitor_namespace_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_monitor_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_discovery_role: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pod_monitor_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rule_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rule_namespace_selector: Option<LabelSelector>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct NamespaceSelector {
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub match_names: Vec<String>,
+}
+
+/// Contains alerting configuration, specifically Alertmanager endpoints.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+pub struct PrometheusSpecAlerting {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub alertmanagers: Option<Vec<AlertmanagerEndpoints>>,
+}
+
+/// Represents an Alertmanager endpoint configuration used by Prometheus.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+pub struct AlertmanagerEndpoints {
+    /// Name of the Alertmanager Service.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+
+    /// Namespace of the Alertmanager Service.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub namespace: Option<String>,
+
+    /// Port to access on the Alertmanager Service (e.g. "web").
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub port: Option<String>,
+
+    /// Scheme to use for connecting (e.g. "http").
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub scheme: Option<String>,
+    // Other fields like `tls_config`, `path_prefix`, etc., can be added if needed.
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelector {
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub match_labels: BTreeMap<String, String>,
+
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub match_expressions: Vec<LabelSelectorRequirement>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelectorRequirement {
+    pub key: String,
+    pub operator: Operator,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub values: Vec<String>,
+}
+
+impl Default for PrometheusSpec {
+    fn default() -> Self {
+        PrometheusSpec {
+            alerting: None,
+
+            service_account_name: "prometheus".into(),
+
+            // null means "only my namespace"
+            service_monitor_namespace_selector: None,
+
+            // empty selector means match all ServiceMonitors in that namespace
+            service_monitor_selector: Some(LabelSelector::default()),
+
+            service_discovery_role: Some("Endpoints".into()),
+
+            pod_monitor_selector: None,
+
+            rule_selector: None,
+
+            rule_namespace_selector: Some(LabelSelector::default()),
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/grafana_default_dashboard.rs

@@ -0,0 +1,203 @@
+pub fn build_default_dashboard(namespace: &str) -> String {
+    let dashboard = format!(
+        r#"{{
+  "annotations": {{
+    "list": []
+  }},
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 171105,
+  "panels": [
+    {{
+      "datasource": "$datasource",
+      "fieldConfig": {{
+        "defaults": {{
+          "unit": "short"
+        }},
+        "overrides": []
+      }},
+      "gridPos": {{
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      }},
+      "id": 1,
+      "options": {{
+        "reduceOptions": {{
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }}
+      }},
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {{
+          "expr": "sum(kube_pod_status_phase{{namespace=\"{namespace}\",  phase=\"Running\"}})",
+          "legendFormat": "",
+          "refId": "A"
+        }}
+      ],
+      "title": "Pods in Namespace",
+      "type": "stat"
+    }},
+    {{
+      "datasource": "$datasource",
+      "fieldConfig": {{
+        "defaults": {{
+          "unit": "short"
+        }},
+        "overrides": []
+      }},
+      "gridPos": {{
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      }},
+      "id": 2,
+      "options": {{
+        "reduceOptions": {{
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }}
+      }},
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {{
+          "expr": "sum(kube_pod_status_phase{{phase=\"Failed\", namespace=\"{namespace}\"}})",
+          "legendFormat": "",
+          "refId": "A"
+        }}
+      ],
+      "title": "Pods in Failed State",
+      "type": "stat"
+    }},
+    {{
+      "datasource": "$datasource",
+      "fieldConfig": {{
+        "defaults": {{
+          "unit": "percentunit"
+        }},
+        "overrides": []
+      }},
+      "gridPos": {{
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      }},
+      "id": 3,
+      "options": {{
+        "reduceOptions": {{
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }}
+      }},
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {{
+          "expr": "sum(kube_deployment_status_replicas_available{{namespace=\"{namespace}\"}}) / sum(kube_deployment_spec_replicas{{namespace=\"{namespace}\"}})",
+          "legendFormat": "",
+          "refId": "A"
+        }}
+      ],
+      "title": "Deployment Health (Available / Desired)",
+      "type": "stat"
+    }},
+    {{
+      "datasource": "$datasource",
+      "fieldConfig": {{
+        "defaults": {{
+          "unit": "short"
+        }},
+        "overrides": []
+      }},
+      "gridPos": {{
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      }},
+      "id": 4,
+      "options": {{
+        "reduceOptions": {{
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }}
+      }},
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {{
+          "expr": "sum by(pod) (rate(kube_pod_container_status_restarts_total{{namespace=\"{namespace}\"}}[5m]))",
+          "legendFormat": "{{{{pod}}}}",
+          "refId": "A"
+        }}
+      ],
+      "title": "Container Restarts (per pod)",
+      "type": "timeseries"
+    }},
+    {{
+      "datasource": "$datasource",
+      "fieldConfig": {{
+        "defaults": {{
+          "unit": "short"
+        }},
+        "overrides": []
+      }},
+      "gridPos": {{
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      }},
+      "id": 5,
+      "options": {{
+        "reduceOptions": {{
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }}
+      }},
+      "pluginVersion": "9.0.0",
+      "targets": [
+        {{
+          "expr": "sum(ALERTS{{alertstate=\"firing\", namespace=\"{namespace}\"}}) or vector(0)",
+          "legendFormat": "",
+          "refId": "A"
+        }}
+      ],
+      "title": "Firing Alerts in Namespace",
+      "type": "stat"
+    }}
+  ],
+  "schemaVersion": 36,
+  "templating": {{
+    "list": [
+      {{
+        "name": "datasource",
+        "type": "datasource",
+        "pluginId": "prometheus",
+        "label": "Prometheus",
+        "query": "prometheus",
+        "refresh": 1,
+        "hide": 0,
+        "current": {{
+          "selected": true,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        }}
+      }}
+    ]
+  }},
+  "title": "Tenant Namespace Overview",
+  "version": 1
+}}"#
+    );
+    dashboard
+}

harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs

@@ -0,0 +1,20 @@
+use std::str::FromStr;
+
+use non_blank_string_rs::NonBlankString;
+
+use crate::modules::helm::chart::HelmChartScore;
+
+pub fn grafana_operator_helm_chart_score(ns: String) -> HelmChartScore {
+    HelmChartScore {
+        namespace: Some(NonBlankString::from_str(&ns).unwrap()),
+        release_name: NonBlankString::from_str("grafana_operator").unwrap(),
+        chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana-operator")
+            .unwrap(),
+        chart_version: None,
+        values_overrides: None,
+        values_yaml: None,
+        create_namespace: true,
+        install_only: true,
+        repository: None,
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs

@@ -0,0 +1,11 @@
+pub mod crd_alertmanager_config;
+pub mod crd_alertmanagers;
+pub mod crd_default_rules;
+pub mod crd_grafana;
+pub mod crd_prometheus_rules;
+pub mod crd_prometheuses;
+pub mod grafana_default_dashboard;
+pub mod grafana_operator;
+pub mod prometheus_operator;
+pub mod role;
+pub mod service_monitor;

harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs

@@ -0,0 +1,22 @@
+use std::str::FromStr;
+
+use non_blank_string_rs::NonBlankString;
+
+use crate::modules::helm::chart::HelmChartScore;
+
+pub fn prometheus_operator_helm_chart_score(ns: String) -> HelmChartScore {
+    HelmChartScore {
+        namespace: Some(NonBlankString::from_str(&ns).unwrap()),
+        release_name: NonBlankString::from_str("prometheus-operator").unwrap(),
+        chart_name: NonBlankString::from_str(
+            "oci://hub.nationtech.io/harmony/nt-prometheus-operator",
+        )
+        .unwrap(),
+        chart_version: None,
+        values_overrides: None,
+        values_yaml: None,
+        create_namespace: true,
+        install_only: true,
+        repository: None,
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/role.rs

@@ -0,0 +1,62 @@
+use k8s_openapi::api::{
+    core::v1::ServiceAccount,
+    rbac::v1::{PolicyRule, Role, RoleBinding, RoleRef, Subject},
+};
+use kube::api::ObjectMeta;
+
+pub fn build_prom_role(role_name: String, namespace: String) -> Role {
+    Role {
+        metadata: ObjectMeta {
+            name: Some(role_name),
+            namespace: Some(namespace),
+            ..Default::default()
+        },
+        rules: Some(vec![PolicyRule {
+            api_groups: Some(vec!["".into()]), // core API group
+            resources: Some(vec!["services".into(), "endpoints".into(), "pods".into()]),
+            verbs: vec!["get".into(), "list".into(), "watch".into()],
+            ..Default::default()
+        }]),
+    }
+}
+
+pub fn build_prom_rolebinding(
+    role_name: String,
+    namespace: String,
+    service_account_name: String,
+) -> RoleBinding {
+    RoleBinding {
+        metadata: ObjectMeta {
+            name: Some(format!("{}-rolebinding", role_name)),
+            namespace: Some(namespace.clone()),
+            ..Default::default()
+        },
+        role_ref: RoleRef {
+            api_group: "rbac.authorization.k8s.io".into(),
+            kind: "Role".into(),
+            name: role_name,
+        },
+        subjects: Some(vec![Subject {
+            kind: "ServiceAccount".into(),
+            name: service_account_name,
+            namespace: Some(namespace.clone()),
+            ..Default::default()
+        }]),
+    }
+}
+
+pub fn build_prom_service_account(
+    service_account_name: String,
+    namespace: String,
+) -> ServiceAccount {
+    ServiceAccount {
+        automount_service_account_token: None,
+        image_pull_secrets: None,
+        metadata: ObjectMeta {
+            name: Some(service_account_name),
+            namespace: Some(namespace),
+            ..Default::default()
+        },
+        secrets: None,
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/service_monitor.rs

@@ -0,0 +1,89 @@
+use std::collections::{BTreeMap, HashMap};
+
+use kube::{CustomResource, Resource, api::ObjectMeta};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::interpret::InterpretError;
+
+use crate::modules::monitoring::kube_prometheus::types::{
+    HTTPScheme, MatchExpression, NamespaceSelector, Operator, Selector,
+    ServiceMonitor as KubeServiceMonitor, ServiceMonitorEndpoint,
+};
+
+/// This is the top-level struct for the ServiceMonitor Custom Resource.
+/// The `#[derive(CustomResource)]` macro handles all the boilerplate for you,
+/// including the `impl Resource`.
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "ServiceMonitor",
+    plural = "servicemonitors",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ServiceMonitorSpec {
+    /// A label selector to select services to monitor.
+    pub selector: Selector,
+
+    /// A list of endpoints on the selected services to be monitored.
+    pub endpoints: Vec<ServiceMonitorEndpoint>,
+
+    /// Selector to select which namespaces the Kubernetes Endpoints objects
+    /// are discovered from.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub namespace_selector: Option<NamespaceSelector>,
+
+    /// The label to use to retrieve the job name from.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub job_label: Option<String>,
+
+    /// Pod-based target labels to transfer from the Kubernetes Pod onto the target.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub pod_target_labels: Vec<String>,
+
+    /// TargetLabels transfers labels on the Kubernetes Service object to the target.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub target_labels: Vec<String>,
+}
+
+impl Default for ServiceMonitorSpec {
+    fn default() -> Self {
+        let mut labels = HashMap::new();
+        Self {
+            selector: Selector {
+                match_labels: { labels },
+                match_expressions: vec![MatchExpression {
+                    key: "app.kubernetes.io/name".into(),
+                    operator: Operator::Exists,
+                    values: vec![],
+                }],
+            },
+            endpoints: vec![ServiceMonitorEndpoint {
+                port: Some("http".to_string()),
+                path: Some("/metrics".into()),
+                interval: Some("30s".into()),
+                scheme: Some(HTTPScheme::HTTP),
+                ..Default::default()
+            }],
+            namespace_selector: None, // only the same namespace
+            job_label: Some("app".into()),
+            pod_target_labels: vec![],
+            target_labels: vec![],
+        }
+    }
+}
+
+impl From<KubeServiceMonitor> for ServiceMonitorSpec {
+    fn from(value: KubeServiceMonitor) -> Self {
+        Self {
+            selector: value.selector,
+            endpoints: value.endpoints,
+            namespace_selector: value.namespace_selector,
+            job_label: value.job_label,
+            pod_target_labels: value.pod_target_labels,
+            target_labels: value.target_labels,
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/helm/config.rs

@@ -35,18 +35,18 @@ impl KubePrometheusConfig {
             windows_monitoring: false,
             alert_manager: true,
             grafana: true,
-            node_exporter: false,
+            node_exporter: true,
             prometheus: true,
             kubernetes_service_monitors: true,
-            kubernetes_api_server: false,
+            kubernetes_api_server: true,
             kubelet: true,
-            kube_controller_manager: false,
+            kube_controller_manager: true,
-            kube_etcd: false,
+            kube_etcd: true,
-            kube_proxy: false,
+            kube_proxy: true,
             kube_state_metrics: true,
             prometheus_operator: true,
-            core_dns: false,
+            core_dns: true,
-            kube_scheduler: false,
+            kube_scheduler: true,
             alert_receiver_configs: vec![],
             alert_rules: vec![],
             additional_service_monitors: vec![],

harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs

@@ -12,8 +12,8 @@ use crate::modules::{
     helm::chart::HelmChartScore,
     monitoring::kube_prometheus::types::{
         AlertGroup, AlertManager, AlertManagerAdditionalPromRules, AlertManagerConfig,
-        AlertManagerRoute, AlertManagerSpec, AlertManagerValues, ConfigReloader, Limits,
+        AlertManagerConfigSelector, AlertManagerRoute, AlertManagerSpec, AlertManagerValues,
-        PrometheusConfig, Requests, Resources,
+        ConfigReloader, Limits, PrometheusConfig, Requests, Resources,
     },
 };
 
@@ -70,7 +70,7 @@ pub fn kube_prometheus_helm_chart_score(
         r#"
 global:
   rbac:
-    create: false
+    create: true
 prometheus:
   enabled: {prometheus}
   prometheusSpec:
@@ -245,7 +245,7 @@ prometheus-node-exporter:
       cpu: 200m
       memory: 250Mi
 prometheusOperator:
-  enabled: false
+  enabled: true
   resources:
     requests:
       cpu: 100m
@@ -332,6 +332,11 @@ prometheusOperator:
             .push(receiver.channel_receiver.clone());
     }
 
+    let mut labels = BTreeMap::new();
+    labels.insert("alertmanagerConfig".to_string(), "enabled".to_string());
+    let alert_manager_config_selector = AlertManagerConfigSelector {
+        match_labels: labels,
+    };
     let alert_manager_values = AlertManagerValues {
         alertmanager: AlertManager {
             enabled: config.alert_manager,
@@ -347,6 +352,8 @@ prometheusOperator:
                         cpu: "100m".to_string(),
                     },
                 },
+                alert_manager_config_selector,
+                replicas: 2,
             },
             init_config_reloader: ConfigReloader {
                 resources: Resources {

harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs

@@ -28,7 +28,7 @@ impl<T: Topology + HelmCommand + TenantManager> Score<T> for HelmPrometheusAlert
             .expect("couldn't lock config")
             .additional_service_monitors = self.service_monitors.clone();
         Box::new(AlertingInterpret {
-            sender: KubePrometheus::new(),
+            sender: KubePrometheus { config },
             receivers: self.receivers.clone(),
             rules: self.rules.clone(),
         })

harmony/src/modules/monitoring/kube_prometheus/mod.rs

@@ -1,3 +1,4 @@
+pub mod crd;
 pub mod helm;
 pub mod helm_prometheus_alert_score;
 pub mod prometheus;

harmony/src/modules/monitoring/kube_prometheus/types.rs

@@ -1,7 +1,8 @@
 use std::collections::{BTreeMap, HashMap};
 
 use async_trait::async_trait;
-use serde::Serialize;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
 use serde_yaml::{Mapping, Sequence, Value};
 
 use crate::modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup;
@@ -55,6 +56,14 @@ pub struct AlertManagerChannelConfig {
 #[serde(rename_all = "camelCase")]
 pub struct AlertManagerSpec {
     pub(crate) resources: Resources,
+    pub replicas: u32,
+    pub alert_manager_config_selector: AlertManagerConfigSelector,
+}
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AlertManagerConfigSelector {
+    pub match_labels: BTreeMap<String, String>,
 }
 
 #[derive(Debug, Clone, Serialize)]
@@ -86,7 +95,7 @@ pub struct AlertGroup {
     pub groups: Vec<AlertManagerRuleGroup>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub enum HTTPScheme {
     #[serde(rename = "http")]
     HTTP,
@@ -94,7 +103,7 @@ pub enum HTTPScheme {
     HTTPS,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub enum Operator {
     In,
     NotIn,
@@ -139,74 +148,83 @@ pub struct ServiceMonitorTLSConfig {
     pub server_name: Option<String>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct ServiceMonitorEndpoint {
-    // ## Name of the endpoint's service port
+    /// Name of the service port this endpoint refers to.
-    // ## Mutually exclusive with targetPort
     pub port: Option<String>,
 
-    // ## Name or number of the endpoint's target port
+    /// Interval at which metrics should be scraped.
-    // ## Mutually exclusive with port
+    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub target_port: Option<String>,
-
-    // ## File containing bearer token to be used when scraping targets
-    // ##
-    pub bearer_token_file: Option<String>,
-
-    // ## Interval at which metrics should be scraped
-    // ##
     pub interval: Option<String>,
 
-    // ## HTTP path to scrape for metrics
+    /// The HTTP path to scrape for metrics.
-    // ##
+    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub path: String,
+    pub path: Option<String>,
 
-    // ## HTTP scheme to use for scraping
+    /// HTTP scheme to use for scraping.
-    // ##
+    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub scheme: HTTPScheme,
+    pub scheme: Option<HTTPScheme>,
 
-    // ## TLS configuration to use when scraping the endpoint
+    /// Relabelings to apply to samples before scraping.
-    // ##
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub tls_config: Option<ServiceMonitorTLSConfig>,
+    pub relabelings: Vec<RelabelConfig>,
 
-    // ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
+    /// MetricRelabelings to apply to samples after scraping, but before ingestion.
-    // ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    // ##
+    pub metric_relabelings: Vec<RelabelConfig>,
-    // # - action: keep
-    // #   regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+'
-    // #   sourceLabels: [__name__]
-    pub metric_relabelings: Vec<Mapping>,
-
-    // ## RelabelConfigs to apply to samples before scraping
-    // ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
-    // ##
-    // # - sourceLabels: [__meta_kubernetes_pod_node_name]
-    // #   separator: ;
-    // #   regex: ^(.*)$
-    // #   targetLabel: nodename
-    // #   replacement: $1
-    // #   action: replace
-    pub relabelings: Vec<Mapping>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct RelabelConfig {
+    /// The action to perform based on the regex matching.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub action: Option<String>,
+
+    /// A list of labels from which to extract values.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub source_labels: Vec<String>,
+
+    /// Separator to be used when concatenating source_labels.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub separator: Option<String>,
+
+    /// The label to which the resulting value is written.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub target_label: Option<String>,
+
+    /// A regular expression to match against the concatenated source label values.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub regex: Option<String>,
+
+    /// The replacement value to use.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub replacement: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct MatchExpression {
     pub key: String,
-    pub operator: Operator,
+    pub operator: Operator, // "In", "NotIn", "Exists", "DoesNotExist"
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub values: Vec<String>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct Selector {
-    //   # label selector for services
+    /// A map of key-value pairs to match.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub match_labels: HashMap<String, String>,
+
+    /// A list of label selector requirements.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub match_expressions: Vec<MatchExpression>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct ServiceMonitor {
     pub name: String,
@@ -250,10 +268,15 @@ pub struct ServiceMonitor {
     pub fallback_scrape_protocol: Option<String>,
 }
 
-#[derive(Debug, Serialize, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct NamespaceSelector {
+    /// Select all namespaces.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub any: bool,
+
+    /// List of namespace names to select from.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub match_names: Vec<String>,
 }
 
@@ -275,19 +298,3 @@ impl Default for ServiceMonitor {
         }
     }
 }
-
-impl Default for ServiceMonitorEndpoint {
-    fn default() -> Self {
-        Self {
-            port: Some("80".to_string()),
-            target_port: Default::default(),
-            bearer_token_file: Default::default(),
-            interval: Default::default(),
-            path: "/metrics".to_string(),
-            scheme: HTTPScheme::HTTP,
-            tls_config: Default::default(),
-            metric_relabelings: Default::default(),
-            relabelings: Default::default(),
-        }
-    }
-}

harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs

@@ -3,7 +3,7 @@ use std::str::FromStr;
 
 use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
 
-pub fn ntfy_helm_chart_score(namespace: String) -> HelmChartScore {
+pub fn ntfy_helm_chart_score(namespace: String, host: String) -> HelmChartScore {
     let values = format!(
         r#"
 replicaCount: 1
@@ -28,12 +28,12 @@ service:
   port: 80
 
 ingress:
-  enabled: false
+  enabled: true
 #  annotations:
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
   hosts:
-    - host: ntfy.host.com
+    - host: {host}
       paths:
         - path: /
           pathType: ImplementationSpecific
@@ -58,6 +58,7 @@ config:
     # web-root: "disable"
     enable-signup: false
     enable-login: "true"
+    enable-metrics: "true"
 
 persistence:
   enabled: true

harmony/src/modules/monitoring/ntfy/ntfy.rs

@@ -17,6 +17,7 @@ use crate::{
 #[derive(Debug, Clone, Serialize)]
 pub struct NtfyScore {
     pub namespace: String,
+    pub host: String,
 }
 
 impl<T: Topology + HelmCommand + K8sclient> Score<T> for NtfyScore {
@@ -126,7 +127,7 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for NtfyInterpret {
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
-        ntfy_helm_chart_score(self.score.namespace.clone())
+        ntfy_helm_chart_score(self.score.namespace.clone(), self.score.host.clone())
             .create_interpret()
             .execute(inventory, topology)
             .await?;

harmony/src/modules/prometheus/alerts/k8s/deployment.rs

@@ -0,0 +1,23 @@
+use std::collections::HashMap;
+
+use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule;
+
+pub fn alert_deployment_unavailable() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "DeploymentUnavailable".into(),
+        expr: "kube_deployment_status_replicas_unavailable > 0".into(),
+        r#for: Some("2m".into()),
+        labels: HashMap::from([("severity".into(), "warning".into())]),
+        annotations: HashMap::from([
+            (
+                "summary".into(),
+                "Deployment has unavailable replicas".into(),
+            ),
+            (
+                "description".into(),
+                "A deployment in this namespace has unavailable replicas for over 2 minutes."
+                    .into(),
+            ),
+        ]),
+    }
+}

harmony/src/modules/prometheus/alerts/k8s/memory_usage.rs

@@ -0,0 +1,37 @@
+use std::collections::HashMap;
+
+use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule;
+
+pub fn alert_high_memory_usage() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "HighMemoryUsage".into(),
+        expr: "container_memory_working_set_bytes{container!=\"\",namespace!=\"\"} > 500000000"
+            .into(),
+        r#for: Some("2m".into()),
+        labels: HashMap::from([("severity".into(), "warning".into())]),
+        annotations: HashMap::from([
+            ("summary".into(), "Pod is using high memory".into()),
+            (
+                "description".into(),
+                "A pod is consuming more than 500Mi of memory.".into(),
+            ),
+        ]),
+    }
+}
+
+pub fn alert_high_cpu_usage() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "HighCPUUsage".into(),
+        expr: "rate(container_cpu_usage_seconds_total{container!=\"\",namespace!=\"\"}[1m]) > 0.9"
+            .into(),
+        r#for: Some("1m".into()),
+        labels: HashMap::from([("severity".into(), "warning".into())]),
+        annotations: HashMap::from([
+            ("summary".into(), "Pod is using high CPU".into()),
+            (
+                "description".into(),
+                "A pod is using more than 90% of a core over 1 minute.".into(),
+            ),
+        ]),
+    }
+}

harmony/src/modules/prometheus/alerts/k8s/mod.rs

@@ -1 +1,5 @@
+pub mod deployment;
+pub mod memory_usage;
+pub mod pod;
 pub mod pvc;
+pub mod service;

harmony/src/modules/prometheus/alerts/k8s/pod.rs

@@ -0,0 +1,55 @@
+use std::collections::HashMap;
+
+use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule;
+
+pub fn pod_failed() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "PodFailed".into(),
+        expr: "kube_pod_status_phase{phase=\"Failed\"} > 2".into(),
+        r#for: Some("2m".into()),
+        labels: HashMap::from([("severity".into(), "critical".into())]),
+        annotations: HashMap::from([
+            ("summary".into(), "A pod has failed".into()),
+            (
+                "description".into(),
+                "One or more pods are in Failed phase.".into(),
+            ),
+        ]),
+    }
+}
+
+pub fn alert_container_restarting() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "ContainerRestarting".into(),
+        expr: "increase(kube_pod_container_status_restarts_total[5m]) > 3".into(),
+        r#for: Some("5m".into()),
+        labels: HashMap::from([("severity".into(), "warning".into())]),
+        annotations: HashMap::from([
+            (
+                "summary".into(),
+                "Container is restarting frequently".into(),
+            ),
+            (
+                "description".into(),
+                "A container in this namespace has restarted more than 3 times in 5 minutes."
+                    .into(),
+            ),
+        ]),
+    }
+}
+
+pub fn alert_pod_not_ready() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "PodNotReady".into(),
+        expr: "kube_pod_status_ready{condition=\"true\"} == 0".into(),
+        r#for: Some("2m".into()),
+        labels: HashMap::from([("severity".into(), "warning".into())]),
+        annotations: HashMap::from([
+            ("summary".into(), "Pod is not ready".into()),
+            (
+                "description".into(),
+                "A pod in the namespace is not reporting Ready status.".into(),
+            ),
+        ]),
+    }
+}

harmony/src/modules/prometheus/alerts/k8s/service.rs

@@ -0,0 +1,19 @@
+use std::collections::HashMap;
+
+use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule;
+
+pub fn alert_service_down() -> PrometheusAlertRule {
+    PrometheusAlertRule {
+        alert: "ServiceDown".into(),
+        expr: "up == 0".into(),
+        r#for: Some("1m".into()),
+        labels: HashMap::from([("severity".into(), "critical".into())]),
+        annotations: HashMap::from([
+            ("summary".into(), "Service is down".into()),
+            (
+                "description".into(),
+                "A target service in the namespace is not responding to Prometheus scrapes.".into(),
+            ),
+        ]),
+    }
+}

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -0,0 +1,569 @@
+use std::fs;
+use std::{collections::BTreeMap, sync::Arc};
+use tempfile::tempdir;
+
+use async_trait::async_trait;
+use kube::api::ObjectMeta;
+use log::{debug, info};
+use serde::Serialize;
+use std::process::Command;
+
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
+use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
+    Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
+    GrafanaDatasourceSpec, GrafanaSpec,
+};
+use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
+    PrometheusRule, PrometheusRuleSpec, RuleGroup,
+};
+use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard::build_default_dashboard;
+use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
+use crate::topology::oberservability::monitoring::AlertReceiver;
+use crate::topology::{K8sclient, Topology, k8s::K8sClient};
+use crate::{
+    data::{Id, Version},
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanagers::{Alertmanager, AlertmanagerSpec},
+        crd_prometheuses::{
+            AlertmanagerEndpoints, LabelSelector, Prometheus, PrometheusSpec,
+            PrometheusSpecAlerting,
+        },
+        role::{build_prom_role, build_prom_rolebinding, build_prom_service_account},
+    },
+    score::Score,
+};
+
+use super::prometheus::PrometheusApplicationMonitoring;
+
+#[derive(Clone, Debug, Serialize)]
+pub struct K8sPrometheusCRDAlertingScore {
+    pub sender: CRDPrometheus,
+    pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
+}
+
+impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+    for K8sPrometheusCRDAlertingScore
+{
+    fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
+        Box::new(K8sPrometheusCRDAlertingInterpret {
+            sender: self.sender.clone(),
+            receivers: self.receivers.clone(),
+            service_monitors: self.service_monitors.clone(),
+            prometheus_rules: self.prometheus_rules.clone(),
+        })
+    }
+
+    fn name(&self) -> String {
+        "CRDApplicationAlertingScore".into()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct K8sPrometheusCRDAlertingInterpret {
+    pub sender: CRDPrometheus,
+    pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
+    for K8sPrometheusCRDAlertingInterpret
+{
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await.unwrap();
+        self.ensure_grafana_operator().await?;
+        self.install_prometheus(&client).await?;
+        self.install_alert_manager(&client).await?;
+        self.install_client_kube_metrics().await?;
+        self.install_grafana(&client).await?;
+        self.install_receivers(&self.sender, &self.receivers)
+            .await?;
+        self.install_rules(&self.prometheus_rules, &client).await?;
+        self.install_monitors(self.service_monitors.clone(), &client)
+            .await?;
+        Ok(Outcome::success(format!(
+            "deployed application monitoring composants"
+        )))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        todo!()
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl K8sPrometheusCRDAlertingInterpret {
+    async fn crd_exists(&self, crd: &str) -> bool {
+        let status = Command::new("sh")
+            .args(["-c", "kubectl get crd -A | grep -i", crd])
+            .status()
+            .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e)))
+            .unwrap();
+
+        status.success()
+    }
+
+    async fn install_chart(
+        &self,
+        chart_path: String,
+        chart_name: String,
+    ) -> Result<(), InterpretError> {
+        let temp_dir =
+            tempdir().map_err(|e| InterpretError::new(format!("Tempdir error: {}", e)))?;
+        let temp_path = temp_dir.path().to_path_buf();
+        debug!("Using temp directory: {}", temp_path.display());
+        let chart = format!("{}/{}", chart_path, chart_name);
+        let pull_output = Command::new("helm")
+            .args(["pull", &chart, "--destination", temp_path.to_str().unwrap()])
+            .output()
+            .map_err(|e| InterpretError::new(format!("Helm pull error: {}", e)))?;
+
+        if !pull_output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Helm pull failed: {}",
+                String::from_utf8_lossy(&pull_output.stderr)
+            )));
+        }
+
+        let tgz_path = fs::read_dir(&temp_path)
+            .unwrap()
+            .filter_map(|entry| {
+                let entry = entry.ok()?;
+                let path = entry.path();
+                if path.extension()? == "tgz" {
+                    Some(path)
+                } else {
+                    None
+                }
+            })
+            .next()
+            .ok_or_else(|| InterpretError::new("Could not find pulled Helm chart".into()))?;
+
+        debug!("Installing chart from: {}", tgz_path.display());
+
+        let install_output = Command::new("helm")
+            .args([
+                "install",
+                &chart_name,
+                tgz_path.to_str().unwrap(),
+                "--namespace",
+                &self.sender.namespace.clone(),
+                "--create-namespace",
+                "--wait",
+                "--atomic",
+            ])
+            .output()
+            .map_err(|e| InterpretError::new(format!("Helm install error: {}", e)))?;
+
+        if !install_output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Helm install failed: {}",
+                String::from_utf8_lossy(&install_output.stderr)
+            )));
+        }
+
+        debug!(
+            "Installed chart {}/{} in namespace: {}",
+            &chart_path,
+            &chart_name,
+            self.sender.namespace.clone()
+        );
+        Ok(())
+    }
+
+    async fn ensure_grafana_operator(&self) -> Result<Outcome, InterpretError> {
+        if self.crd_exists("grafanas.grafana.integreatly.org").await {
+            debug!("grafana CRDs already exist — skipping install.");
+            return Ok(Outcome::success("Grafana CRDs already exist".to_string()));
+        }
+
+        let _ = Command::new("helm")
+            .args([
+                "repo",
+                "add",
+                "grafana-operator",
+                "https://grafana.github.io/helm-charts",
+            ])
+            .output()
+            .unwrap();
+
+        let _ = Command::new("helm")
+            .args(["repo", "update"])
+            .output()
+            .unwrap();
+
+        let output = Command::new("helm")
+            .args([
+                "install",
+                "grafana-operator",
+                "grafana-operator/grafana-operator",
+                "--namespace",
+                &self.sender.namespace.clone(),
+                "--create-namespace",
+                "--set",
+                "namespaceScope=true",
+            ])
+            .output()
+            .unwrap();
+
+        if !output.status.success() {
+            return Err(InterpretError::new(format!(
+                "helm install failed:\nstdout: {}\nstderr: {}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            )));
+        }
+
+        Ok(Outcome::success(format!(
+            "installed grafana operator in ns {}",
+            self.sender.namespace.clone()
+        )))
+    }
+
+    async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        debug!(
+            "installing crd-prometheuses in namespace {}",
+            self.sender.namespace.clone()
+        );
+        debug!("building role/rolebinding/serviceaccount for crd-prometheus");
+        let rolename = format!("{}-prom", self.sender.namespace.clone());
+        let sa_name = format!("{}-prom-sa", self.sender.namespace.clone());
+        let role = build_prom_role(rolename.clone(), self.sender.namespace.clone());
+        let rolebinding = build_prom_rolebinding(
+            rolename.clone(),
+            self.sender.namespace.clone(),
+            sa_name.clone(),
+        );
+        let sa = build_prom_service_account(sa_name.clone(), self.sender.namespace.clone());
+        let prom_spec = PrometheusSpec {
+            alerting: Some(PrometheusSpecAlerting {
+                alertmanagers: Some(vec![AlertmanagerEndpoints {
+                    name: Some("alertmanager-operated".into()),
+                    namespace: Some(self.sender.namespace.clone()),
+                    port: Some("web".into()),
+                    scheme: Some("http".into()),
+                }]),
+            }),
+            service_account_name: sa_name.clone(),
+            service_monitor_namespace_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([(
+                    "kubernetes.io/metadata.name".to_string(),
+                    self.sender.namespace.clone(),
+                )]),
+                match_expressions: vec![],
+            }),
+            service_monitor_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]),
+                ..Default::default()
+            }),
+
+            service_discovery_role: Some("Endpoints".into()),
+
+            pod_monitor_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]),
+                ..Default::default()
+            }),
+
+            rule_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("role".to_string(), "prometheus-rule".to_string())]),
+                ..Default::default()
+            }),
+
+            rule_namespace_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([(
+                    "kubernetes.io/metadata.name".to_string(),
+                    self.sender.namespace.clone(),
+                )]),
+                match_expressions: vec![],
+            }),
+        };
+        let prom = Prometheus {
+            metadata: ObjectMeta {
+                name: Some(self.sender.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("client".to_string(), "prometheus".to_string()),
+                ])),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_spec,
+        };
+        client
+            .apply(&role, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus role: {:#?} in ns {:#?}",
+            role.metadata.name.unwrap(),
+            role.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&rolebinding, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus rolebinding: {:#?} in ns {:#?}",
+            rolebinding.metadata.name.unwrap(),
+            rolebinding.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&sa, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus service account: {:#?} in ns {:#?}",
+            sa.metadata.name.unwrap(),
+            sa.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&prom, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus: {:#?} in ns {:#?}",
+            &prom.metadata.name.clone().unwrap(),
+            &prom.metadata.namespace.clone().unwrap()
+        );
+
+        Ok(Outcome::success(format!(
+            "successfully deployed crd-prometheus {:#?}",
+            prom
+        )))
+    }
+
+    async fn install_alert_manager(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let am = Alertmanager {
+            metadata: ObjectMeta {
+                name: Some(self.sender.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: AlertmanagerSpec::default(),
+        };
+        client
+            .apply(&am, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed service monitor {:#?}",
+            am.metadata.name
+        )))
+    }
+    async fn install_monitors(
+        &self,
+        mut monitors: Vec<ServiceMonitor>,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let default_service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.sender.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("client".to_string(), "prometheus".to_string()),
+                    (
+                        "app.kubernetes.io/name".to_string(),
+                        "kube-state-metrics".to_string(),
+                    ),
+                ])),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
+        monitors.push(default_service_monitor);
+        for monitor in monitors.iter() {
+            client
+                .apply(monitor, Some(&self.sender.namespace.clone()))
+                .await
+                .map_err(|e| InterpretError::new(e.to_string()))?;
+        }
+        Ok(Outcome::success(
+            "succesfully deployed service monitors".to_string(),
+        ))
+    }
+
+    async fn install_rules(
+        &self,
+        rules: &Vec<RuleGroup>,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut prom_rule_spec = PrometheusRuleSpec {
+            groups: rules.clone(),
+        };
+
+        let default_rules_group = RuleGroup {
+            name: format!("default-rules"),
+            rules: build_default_application_rules(),
+        };
+
+        prom_rule_spec.groups.push(default_rules_group);
+        let prom_rules = PrometheusRule {
+            metadata: ObjectMeta {
+                name: Some(self.sender.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("role".to_string(), "prometheus-rule".to_string()),
+                ])),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_rule_spec,
+        };
+        client
+            .apply(&prom_rules, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed rules {:#?}",
+            prom_rules.metadata.name
+        )))
+    }
+
+    async fn install_client_kube_metrics(&self) -> Result<Outcome, InterpretError> {
+        self.install_chart(
+            "oci://hub.nationtech.io/harmony".to_string(),
+            "nt-kube-metrics".to_string(),
+        )
+        .await?;
+        Ok(Outcome::success(format!(
+            "Installed client kube metrics in ns {}",
+            &self.sender.namespace.clone()
+        )))
+    }
+
+    async fn install_grafana(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        let mut label = BTreeMap::new();
+        label.insert("dashboards".to_string(), "grafana".to_string());
+        let labels = LabelSelector {
+            match_labels: label.clone(),
+            match_expressions: vec![],
+        };
+        let mut json_data = BTreeMap::new();
+        json_data.insert("timeInterval".to_string(), "5s".to_string());
+        let namespace = self.sender.namespace.clone();
+
+        let json = build_default_dashboard(&namespace);
+
+        let graf_data_source = GrafanaDatasource {
+            metadata: ObjectMeta {
+                name: Some(format!(
+                    "grafana-datasource-{}",
+                    self.sender.namespace.clone()
+                )),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaDatasourceSpec {
+                instance_selector: labels.clone(),
+                allow_cross_namespace_import: Some(false),
+                datasource: GrafanaDatasourceConfig {
+                    access: "proxy".to_string(),
+                    database: Some("prometheus".to_string()),
+                    json_data: Some(json_data),
+                    //this is fragile
+                    name: format!("prometheus-{}-0", self.sender.namespace.clone()),
+                    r#type: "prometheus".to_string(),
+                    url: format!(
+                        "http://prometheus-operated.{}.svc.cluster.local:9090",
+                        self.sender.namespace.clone()
+                    ),
+                },
+            },
+        };
+
+        client
+            .apply(&graf_data_source, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+
+        let graf_dashboard = GrafanaDashboard {
+            metadata: ObjectMeta {
+                name: Some(format!(
+                    "grafana-dashboard-{}",
+                    self.sender.namespace.clone()
+                )),
+                namespace: Some(self.sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaDashboardSpec {
+                resync_period: Some("30s".to_string()),
+                instance_selector: labels.clone(),
+                json,
+            },
+        };
+
+        client
+            .apply(&graf_dashboard, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+
+        let grafana = Grafana {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-{}", self.sender.namespace.clone())),
+                namespace: Some(self.sender.namespace.clone()),
+                labels: Some(label.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaSpec {
+                config: None,
+                admin_user: None,
+                admin_password: None,
+                ingress: None,
+                persistence: None,
+                resources: None,
+            },
+        };
+        client
+            .apply(&grafana, Some(&self.sender.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed grafana instance {:#?}",
+            grafana.metadata.name
+        )))
+    }
+
+    async fn install_receivers(
+        &self,
+        sender: &CRDPrometheus,
+        receivers: &Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+    ) -> Result<Outcome, InterpretError> {
+        for receiver in receivers.iter() {
+            receiver.install(sender).await.map_err(|err| {
+                InterpretError::new(format!("failed to install receiver: {}", err))
+            })?;
+        }
+        Ok(Outcome::success("successfully deployed receivers".into()))
+    }
+}

harmony/src/modules/prometheus/mod.rs

@@ -1 +1,3 @@
 pub mod alerts;
+pub mod k8s_prometheus_alerting_score;
+pub mod prometheus;

harmony/src/modules/prometheus/prometheus.rs

@@ -0,0 +1,17 @@
+use async_trait::async_trait;
+
+use crate::{
+    interpret::{InterpretError, Outcome},
+    inventory::Inventory,
+    topology::oberservability::monitoring::{AlertReceiver, AlertSender},
+};
+
+#[async_trait]
+pub trait PrometheusApplicationMonitoring<S: AlertSender> {
+    async fn install_prometheus(
+        &self,
+        sender: &S,
+        inventory: &Inventory,
+        receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
+    ) -> Result<Outcome, InterpretError>;
+}

harmony_cli/Cargo.toml

@@ -13,6 +13,11 @@ harmony_tui = { path = "../harmony_tui", optional = true }
 inquire.workspace = true
 tokio.workspace = true
 env_logger.workspace = true
+console = "0.16.0"
+indicatif = "0.18.0"
+lazy_static = "1.5.0"
+log.workspace = true
+indicatif-log-bridge = "0.2.3"
 
 
 [features]

harmony_cli/src/cli_logger.rs

@@ -0,0 +1,116 @@
+use harmony::instrumentation::{self, HarmonyEvent};
+use indicatif::{MultiProgress, ProgressBar};
+use indicatif_log_bridge::LogWrapper;
+use std::{
+    collections::{HashMap, hash_map},
+    sync::{Arc, Mutex},
+};
+
+use crate::progress;
+
+pub fn init() -> tokio::task::JoinHandle<()> {
+    configure_logger();
+    let handle = tokio::spawn(handle_events());
+
+    loop {
+        if instrumentation::instrument(HarmonyEvent::HarmonyStarted).is_ok() {
+            break;
+        }
+    }
+
+    handle
+}
+
+fn configure_logger() {
+    let logger =
+        env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).build();
+    let level = logger.filter();
+    let multi = MultiProgress::new();
+    LogWrapper::new(multi.clone(), logger).try_init().unwrap();
+    log::set_max_level(level);
+}
+
+async fn handle_events() {
+    instrumentation::subscribe("Harmony CLI Logger", {
+        let sections: Arc<Mutex<HashMap<String, MultiProgress>>> =
+            Arc::new(Mutex::new(HashMap::new()));
+        let progress_bars: Arc<Mutex<HashMap<String, ProgressBar>>> =
+            Arc::new(Mutex::new(HashMap::new()));
+
+        move |event| {
+            let sections_clone = Arc::clone(&sections);
+            let progress_bars_clone = Arc::clone(&progress_bars);
+
+            async move {
+                let mut sections = sections_clone.lock().unwrap();
+                let mut progress_bars = progress_bars_clone.lock().unwrap();
+
+                match event {
+                    HarmonyEvent::HarmonyStarted => {}
+                    HarmonyEvent::PrepareTopologyStarted { topology: name } => {
+                        let section = progress::new_section(format!(
+                            "{} Preparing environment: {name}...",
+                            crate::theme::EMOJI_TOPOLOGY,
+                        ));
+                        (*sections).insert(name, section);
+                    }
+                    HarmonyEvent::TopologyPrepared {
+                        topology: name,
+                        outcome,
+                    } => {
+                        let section = (*sections).get(&name).unwrap();
+                        let progress = progress::add_spinner(section, "".into());
+
+                        match outcome.status {
+                            harmony::interpret::InterpretStatus::SUCCESS => {
+                                progress::success(section, Some(progress), outcome.message);
+                            }
+                            harmony::interpret::InterpretStatus::FAILURE => {
+                                progress::error(section, Some(progress), outcome.message);
+                            }
+                            harmony::interpret::InterpretStatus::RUNNING => todo!(),
+                            harmony::interpret::InterpretStatus::QUEUED => todo!(),
+                            harmony::interpret::InterpretStatus::BLOCKED => todo!(),
+                            harmony::interpret::InterpretStatus::NOOP => {
+                                progress::skip(section, Some(progress), outcome.message);
+                            }
+                        }
+                    }
+                    HarmonyEvent::InterpretExecutionStarted {
+                        interpret: name,
+                        topology,
+                        message,
+                    } => {
+                        let section = (*sections).get(&topology).unwrap();
+                        let progress_bar = progress::add_spinner(section, message);
+
+                        (*progress_bars).insert(name, progress_bar);
+                    }
+                    HarmonyEvent::InterpretExecutionFinished {
+                        topology,
+                        interpret: name,
+                        outcome,
+                    } => {
+                        let section = (*sections).get(&topology).unwrap();
+                        let progress_bar = (*progress_bars).get(&name).cloned();
+
+                        let _ = section.clear();
+
+                        match outcome {
+                            Ok(outcome) => {
+                                progress::success(section, progress_bar, outcome.message);
+                            }
+                            Err(err) => {
+                                progress::error(section, progress_bar, err.to_string());
+                            }
+                        }
+
+                        (*progress_bars).remove(&name);
+                    }
+                }
+                true
+            }
+        }
+    })
+    .await;
+}

harmony_cli/src/lib.rs

@@ -1,8 +1,14 @@
 use clap::Parser;
 use clap::builder::ArgPredicate;
-use harmony;
+use harmony::inventory::Inventory;
+use harmony::maestro::Maestro;
 use harmony::{score::Score, topology::Topology};
 use inquire::Confirm;
+use log::debug;
+
+pub mod cli_logger; // FIXME: Don't make me pub
+pub mod progress;
+pub mod theme;
 
 #[cfg(feature = "tui")]
 use harmony_tui;
@@ -80,7 +86,24 @@ fn list_scores_with_index<T: Topology>(scores_vec: &Vec<Box<dyn Score<T>>>) -> S
     return display_str;
 }
 
-pub async fn init<T: Topology + Send + Sync + 'static>(
+pub async fn run<T: Topology + Send + Sync + 'static>(
+    inventory: Inventory,
+    topology: T,
+    scores: Vec<Box<dyn Score<T>>>,
+    args_struct: Option<Args>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let cli_logger_handle = cli_logger::init();
+
+    let mut maestro = Maestro::initialize(inventory, topology).await.unwrap();
+    maestro.register_all(scores);
+
+    let result = init(maestro, args_struct).await;
+
+    let _ = tokio::try_join!(cli_logger_handle);
+    result
+}
+
+async fn init<T: Topology + Send + Sync + 'static>(
     maestro: harmony::maestro::Maestro<T>,
     args_struct: Option<Args>,
 ) -> Result<(), Box<dyn std::error::Error>> {
@@ -134,7 +157,7 @@ pub async fn init<T: Topology + Send + Sync + 'static>(
 
     // Run filtered scores
     for s in scores_vec {
-        println!("Running: {}", s.name());
+        debug!("Running: {}", s.name());
         maestro.interpret(s).await?;
     }
 

harmony_cli/src/progress.rs

@@ -0,0 +1,50 @@
+use std::time::Duration;
+
+use indicatif::{MultiProgress, ProgressBar};
+
+pub fn new_section(title: String) -> MultiProgress {
+    let multi_progress = MultiProgress::new();
+    let _ = multi_progress.println(title);
+
+    multi_progress
+}
+
+pub fn add_spinner(multi_progress: &MultiProgress, message: String) -> ProgressBar {
+    let progress = multi_progress.add(ProgressBar::new_spinner());
+
+    progress.set_style(crate::theme::SPINNER_STYLE.clone());
+    progress.set_message(message);
+    progress.enable_steady_tick(Duration::from_millis(100));
+
+    progress
+}
+
+pub fn success(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
+    if let Some(progress) = progress {
+        multi_progress.remove(&progress)
+    }
+
+    let progress = multi_progress.add(ProgressBar::new_spinner());
+    progress.set_style(crate::theme::SUCCESS_SPINNER_STYLE.clone());
+    progress.finish_with_message(message);
+}
+
+pub fn error(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
+    if let Some(progress) = progress {
+        multi_progress.remove(&progress)
+    }
+
+    let progress = multi_progress.add(ProgressBar::new_spinner());
+    progress.set_style(crate::theme::ERROR_SPINNER_STYLE.clone());
+    progress.finish_with_message(message);
+}
+
+pub fn skip(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
+    if let Some(progress) = progress {
+        multi_progress.remove(&progress)
+    }
+
+    let progress = multi_progress.add(ProgressBar::new_spinner());
+    progress.set_style(crate::theme::SKIP_SPINNER_STYLE.clone());
+    progress.finish_with_message(message);
+}

harmony_cli/src/theme.rs

@@ -0,0 +1,26 @@
+use console::Emoji;
+use indicatif::ProgressStyle;
+use lazy_static::lazy_static;
+
+pub static EMOJI_HARMONY: Emoji<'_, '_> = Emoji("🎼", "");
+pub static EMOJI_SUCCESS: Emoji<'_, '_> = Emoji("✅", "");
+pub static EMOJI_SKIP: Emoji<'_, '_> = Emoji("⏭️", "");
+pub static EMOJI_ERROR: Emoji<'_, '_> = Emoji("⚠️", "");
+pub static EMOJI_DEPLOY: Emoji<'_, '_> = Emoji("🚀", "");
+pub static EMOJI_TOPOLOGY: Emoji<'_, '_> = Emoji("📦", "");
+
+lazy_static! {
+    pub static ref SPINNER_STYLE: ProgressStyle = ProgressStyle::default_spinner()
+        .template("    {spinner:.green} {msg}")
+        .unwrap()
+        .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]);
+    pub static ref SUCCESS_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
+        .clone()
+        .tick_strings(&[format!("{}", EMOJI_SUCCESS).as_str()]);
+    pub static ref SKIP_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
+        .clone()
+        .tick_strings(&[format!("{}", EMOJI_SKIP).as_str()]);
+    pub static ref ERROR_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
+        .clone()
+        .tick_strings(&[format!("{}", EMOJI_ERROR).as_str()]);
+}

harmony_composer/Cargo.toml

@@ -15,3 +15,9 @@ current_platform = "0.2.0"
 futures-util = "0.3.31"
 serde_json = "1.0.140"
 cargo_metadata = "0.20.0"
+indicatif = "0.18.0"
+console = "0.16.0"
+lazy_static = "1.5.0"
+once_cell = "1.21.3"
+harmony_cli = { path = "../harmony_cli" }
+indicatif-log-bridge = "0.2.3"

harmony_composer/src/harmony_composer_logger.rs

@@ -0,0 +1,112 @@
+use indicatif::{MultiProgress, ProgressBar};
+use indicatif_log_bridge::LogWrapper;
+use log::error;
+use std::{
+    collections::HashMap,
+    sync::{Arc, Mutex},
+};
+
+use crate::instrumentation::{self, HarmonyComposerEvent};
+
+pub fn init() -> tokio::task::JoinHandle<()> {
+    configure_logger();
+    let handle = tokio::spawn(handle_events());
+
+    loop {
+        if instrumentation::instrument(HarmonyComposerEvent::HarmonyComposerStarted).is_ok() {
+            break;
+        }
+    }
+
+    handle
+}
+
+fn configure_logger() {
+    let logger =
+        env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).build();
+    let level = logger.filter();
+    let multi = MultiProgress::new();
+    LogWrapper::new(multi.clone(), logger).try_init().unwrap();
+    log::set_max_level(level);
+}
+
+pub async fn handle_events() {
+    const PROGRESS_SETUP: &str = "project-initialization";
+    const PROGRESS_DEPLOYMENT: &str = "deployment";
+
+    instrumentation::subscribe("Harmony Composer Logger", {
+        let progresses: Arc<Mutex<HashMap<String, MultiProgress>>> =
+            Arc::new(Mutex::new(HashMap::new()));
+        let compilation_progress = Arc::new(Mutex::new(None::<ProgressBar>));
+
+        move |event| {
+            let progresses_clone = Arc::clone(&progresses);
+            let compilation_progress_clone = Arc::clone(&compilation_progress);
+
+            async move {
+                let mut progresses_guard = progresses_clone.lock().unwrap();
+                let mut compilation_progress_guard = compilation_progress_clone.lock().unwrap();
+
+                match event {
+                    HarmonyComposerEvent::HarmonyComposerStarted => {}
+                    HarmonyComposerEvent::ProjectInitializationStarted => {
+                        let multi_progress = harmony_cli::progress::new_section(format!(
+                            "{} Initializing Harmony project...",
+                            harmony_cli::theme::EMOJI_HARMONY,
+                        ));
+                        (*progresses_guard).insert(PROGRESS_SETUP.to_string(), multi_progress);
+                    }
+                    HarmonyComposerEvent::ProjectInitialized => println!("\n"),
+                    HarmonyComposerEvent::ProjectCompilationStarted { details } => {
+                        let initialization_progress =
+                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
+                        let _ = initialization_progress.clear();
+
+                        let progress =
+                            harmony_cli::progress::add_spinner(initialization_progress, details);
+                        *compilation_progress_guard = Some(progress);
+                    }
+                    HarmonyComposerEvent::ProjectCompiled => {
+                        let initialization_progress =
+                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
+
+                        harmony_cli::progress::success(
+                            initialization_progress,
+                            (*compilation_progress_guard).take(),
+                            "project compiled".to_string(),
+                        );
+                    }
+                    HarmonyComposerEvent::ProjectCompilationFailed { details } => {
+                        let initialization_progress =
+                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
+
+                        harmony_cli::progress::error(
+                            initialization_progress,
+                            (*compilation_progress_guard).take(),
+                            "failed to compile project".to_string(),
+                        );
+
+                        error!("{details}");
+                    }
+                    HarmonyComposerEvent::DeploymentStarted { target } => {
+                        let multi_progress = harmony_cli::progress::new_section(format!(
+                            "{} Starting deployment to {target}...\n\n",
+                            harmony_cli::theme::EMOJI_DEPLOY
+                        ));
+                        (*progresses_guard).insert(PROGRESS_DEPLOYMENT.to_string(), multi_progress);
+                    }
+                    HarmonyComposerEvent::DeploymentCompleted { details } => println!("\n"),
+                    HarmonyComposerEvent::Shutdown => {
+                        for (_, progresses) in (*progresses_guard).iter() {
+                            progresses.clear().unwrap();
+                        }
+
+                        return false;
+                    }
+                }
+                true
+            }
+        }
+    })
+    .await
+}

harmony_composer/src/instrumentation.rs

@@ -0,0 +1,53 @@
+use log::debug;
+use once_cell::sync::Lazy;
+use tokio::sync::broadcast;
+
+#[derive(Debug, Clone)]
+pub enum HarmonyComposerEvent {
+    HarmonyComposerStarted,
+    ProjectInitializationStarted,
+    ProjectInitialized,
+    ProjectCompilationStarted { details: String },
+    ProjectCompiled,
+    ProjectCompilationFailed { details: String },
+    DeploymentStarted { target: String },
+    DeploymentCompleted { details: String },
+    Shutdown,
+}
+
+static HARMONY_COMPOSER_EVENT_BUS: Lazy<broadcast::Sender<HarmonyComposerEvent>> =
+    Lazy::new(|| {
+        // TODO: Adjust channel capacity
+        let (tx, _rx) = broadcast::channel(16);
+        tx
+    });
+
+pub fn instrument(event: HarmonyComposerEvent) -> Result<(), &'static str> {
+    match HARMONY_COMPOSER_EVENT_BUS.send(event) {
+        Ok(_) => Ok(()),
+        Err(_) => Err("send error: no subscribers"),
+    }
+}
+
+pub async fn subscribe<F, Fut>(name: &str, mut handler: F)
+where
+    F: FnMut(HarmonyComposerEvent) -> Fut + Send + 'static,
+    Fut: Future<Output = bool> + Send,
+{
+    let mut rx = HARMONY_COMPOSER_EVENT_BUS.subscribe();
+    debug!("[{name}] Service started. Listening for events...");
+    loop {
+        match rx.recv().await {
+            Ok(event) => {
+                if !handler(event).await {
+                    debug!("[{name}] Handler requested exit.");
+                    break;
+                }
+            }
+            Err(broadcast::error::RecvError::Lagged(n)) => {
+                debug!("[{name}] Lagged behind by {n} messages.");
+            }
+            Err(_) => break,
+        }
+    }
+}

harmony_composer/src/main.rs

@@ -7,12 +7,16 @@ use bollard::secret::HostConfig;
 use cargo_metadata::{Artifact, Message, MetadataCommand};
 use clap::{Args, Parser, Subcommand};
 use futures_util::StreamExt;
-use log::info;
+use instrumentation::HarmonyComposerEvent;
+use log::{debug, info, log_enabled};
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use tokio::fs;
 
+mod harmony_composer_logger;
+mod instrumentation;
+
 #[derive(Parser)]
 #[command(version, about, long_about = None, flatten_help = true, propagate_version = true)]
 struct GlobalArgs {
@@ -66,13 +70,15 @@ struct AllArgs {
 
 #[tokio::main]
 async fn main() {
-    env_logger::init();
+    let hc_logger_handle = harmony_composer_logger::init();
     let cli_args = GlobalArgs::parse();
 
     let harmony_path = Path::new(&cli_args.harmony_path)
         .try_exists()
         .expect("couldn't check if path exists");
 
+    instrumentation::instrument(HarmonyComposerEvent::ProjectInitializationStarted).unwrap();
+
     let harmony_bin_path: PathBuf = match harmony_path {
         true => {
             compile_harmony(
@@ -85,6 +91,8 @@ async fn main() {
         false => todo!("implement autodetect code"),
     };
 
+    instrumentation::instrument(HarmonyComposerEvent::ProjectInitialized).unwrap();
+
     match cli_args.command {
         Some(command) => match command {
             Commands::Check(args) => {
@@ -98,7 +106,9 @@ async fn main() {
                     .expect("couldn't check if path exists")
                 {
                     true => (),
-                    false => todo!("implement couldn't find path logic"),
+                    false => todo!(
+                        "path {check_script_str} not found. Other paths currently unsupported."
+                    ),
                 };
 
                 let check_output = Command::new(check_script)
@@ -114,19 +124,31 @@ async fn main() {
             }
             Commands::Deploy(args) => {
                 let deploy = if args.staging {
+                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
+                        target: "staging".to_string(),
+                    })
+                    .unwrap();
                     todo!("implement staging deployment")
                 } else if args.prod {
+                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
+                        target: "prod".to_string(),
+                    })
+                    .unwrap();
                     todo!("implement prod deployment")
                 } else {
+                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
+                        target: "dev".to_string(),
+                    })
+                    .unwrap();
                     Command::new(harmony_bin_path).arg("-y").arg("-a").spawn()
                 }
                 .expect("failed to run harmony deploy");
 
                 let deploy_output = deploy.wait_with_output().unwrap();
-                println!(
+                instrumentation::instrument(HarmonyComposerEvent::DeploymentCompleted {
-                    "deploy output: {}",
+                    details: String::from_utf8(deploy_output.stdout).unwrap(),
-                    String::from_utf8(deploy_output.stdout).expect("couldn't parse from utf8")
+                })
-                );
+                .unwrap();
             }
             Commands::All(_args) => todo!(
                 "take all previous match arms and turn them into separate functions, and call them all one after the other"
@@ -135,6 +157,10 @@ async fn main() {
         },
         None => todo!("run interactively, ask for info on CLI"),
     }
+
+    instrumentation::instrument(HarmonyComposerEvent::Shutdown).unwrap();
+
+    let _ = tokio::try_join!(hc_logger_handle);
 }
 
 #[derive(Clone, Debug, clap::ValueEnum)]
@@ -155,6 +181,7 @@ async fn compile_harmony(
 
     let cargo_exists = Command::new("which")
         .arg("cargo")
+        .stdout(Stdio::null())
         .status()
         .expect("couldn't get `which cargo` status")
         .success();
@@ -163,17 +190,32 @@ async fn compile_harmony(
         Some(m) => m,
         None => {
             if cargo_exists {
-                return compile_cargo(platform, harmony_location).await;
+                CompileMethod::LocalCargo
             } else {
-                return compile_docker(platform, harmony_location).await;
+                CompileMethod::Docker
             }
         }
     };
 
-    match method {
+    let path = match method {
-        CompileMethod::LocalCargo => return compile_cargo(platform, harmony_location).await,
+        CompileMethod::LocalCargo => {
-        CompileMethod::Docker => return compile_docker(platform, harmony_location).await,
+            instrumentation::instrument(HarmonyComposerEvent::ProjectCompilationStarted {
+                details: "compiling project with cargo".to_string(),
+            })
+            .unwrap();
+            compile_cargo(platform, harmony_location).await
+        }
+        CompileMethod::Docker => {
+            instrumentation::instrument(HarmonyComposerEvent::ProjectCompilationStarted {
+                details: "compiling project with docker".to_string(),
+            })
+            .unwrap();
+            compile_docker(platform, harmony_location).await
+        }
     };
+
+    instrumentation::instrument(HarmonyComposerEvent::ProjectCompiled).unwrap();
+    path
 }
 
 // TODO: make sure this works with cargo workspaces
@@ -183,6 +225,12 @@ async fn compile_cargo(platform: String, harmony_location: String) -> PathBuf {
         .exec()
         .unwrap();
 
+    let stderr = if log_enabled!(log::Level::Debug) {
+        Stdio::inherit()
+    } else {
+        Stdio::piped()
+    };
+
     let mut cargo_build = Command::new("cargo")
         .current_dir(&harmony_location)
         .args(vec![
@@ -192,6 +240,7 @@ async fn compile_cargo(platform: String, harmony_location: String) -> PathBuf {
             "--message-format=json-render-diagnostics",
         ])
         .stdout(Stdio::piped())
+        .stderr(stderr)
         .spawn()
         .expect("run cargo command failed");
 
@@ -207,18 +256,20 @@ async fn compile_cargo(platform: String, harmony_location: String) -> PathBuf {
                         .expect("failed to get root package")
                         .manifest_path
                 {
-                    println!("{:?}", artifact);
+                    debug!("{:?}", artifact);
                     artifacts.push(artifact);
                 }
             }
             Message::BuildScriptExecuted(_script) => (),
             Message::BuildFinished(finished) => {
-                println!("{:?}", finished);
+                debug!("{:?}", finished);
             }
             _ => (), // Unknown message
         }
     }
 
+    cargo_build.wait().expect("run cargo command failed");
+
     let bin = artifacts
         .last()
         .expect("no binaries built")
@@ -234,7 +285,8 @@ async fn compile_cargo(platform: String, harmony_location: String) -> PathBuf {
         bin_out = PathBuf::from(format!("{}/harmony", harmony_location));
         let _copy_res = fs::copy(&bin, &bin_out).await;
     }
-    return bin_out;
+
+    bin_out
 }
 
 async fn compile_docker(platform: String, harmony_location: String) -> PathBuf {

k3d/src/lib.rs

@@ -2,7 +2,7 @@ mod downloadable_asset;
 use downloadable_asset::*;
 
 use kube::Client;
-use log::{debug, info, warn};
+use log::{debug, warn};
 use std::path::PathBuf;
 
 const K3D_BIN_FILE_NAME: &str = "k3d";
@@ -90,7 +90,7 @@ impl K3d {
         let latest_release = self.get_latest_release_tag().await.unwrap();
 
         let release_binary = self.get_binary_for_current_platform(latest_release).await;
-        info!("Foudn K3d binary to install : {release_binary:#?}");
+        debug!("Foudn K3d binary to install : {release_binary:#?}");
         release_binary.download_to_path(self.base_dir.clone()).await
     }
 
@@ -175,7 +175,7 @@ impl K3d {
             Err(_) => return Err("Could not get cluster_name, cannot initialize".to_string()),
         };
 
-        info!("Initializing k3d cluster '{}'", cluster_name);
+        debug!("Initializing k3d cluster '{}'", cluster_name);
 
         self.create_cluster(cluster_name)?;
         self.create_kubernetes_client().await
@@ -205,7 +205,7 @@ impl K3d {
     /// - `Err(String)` - Error message if any step failed
     pub async fn ensure_installed(&self) -> Result<Client, String> {
         if !self.is_installed() {
-            info!("K3d is not installed, downloading latest release");
+            debug!("K3d is not installed, downloading latest release");
             self.download_latest_release()
                 .await
                 .map_err(|e| format!("Failed to download k3d: {}", e))?;
@@ -216,13 +216,13 @@ impl K3d {
         }
 
         if !self.is_cluster_initialized() {
-            info!("Cluster is not initialized, initializing now");
+            debug!("Cluster is not initialized, initializing now");
             return self.initialize_cluster().await;
         }
 
         self.start_cluster().await?;
 
-        info!("K3d and cluster are already properly set up");
+        debug!("K3d and cluster are already properly set up");
         self.create_kubernetes_client().await
     }
 
@@ -325,12 +325,12 @@ impl K3d {
             return Err(format!("Failed to create cluster: {}", stderr));
         }
 
-        info!("Successfully created k3d cluster '{}'", cluster_name);
+        debug!("Successfully created k3d cluster '{}'", cluster_name);
         Ok(())
     }
 
     async fn create_kubernetes_client(&self) -> Result<Client, String> {
-        warn!("TODO this method is way too dumb, it should make sure that the client is connected to the k3d cluster actually represented by this instance, not just any default  client");
+        // TODO: Connect the client to the right k3d cluster (see https://git.nationtech.io/NationTech/harmony/issues/92)
         Client::try_default()
             .await
             .map_err(|e| format!("Failed to create Kubernetes client: {}", e))
@@ -352,7 +352,7 @@ impl K3d {
             return Err(format!("Failed to start cluster: {}", stderr));
         }
 
-        info!("Successfully started k3d cluster '{}'", cluster_name);
+        debug!("Successfully started k3d cluster '{}'", cluster_name);
         Ok(())
     }
 }

opnsense-config/Cargo.toml

@@ -22,4 +22,4 @@ tokio-util = { version = "0.7.13", features = [ "codec" ] }
 tokio-stream = "0.1.17"
 
 [dev-dependencies]
-pretty_assertions = "1.4.1"
+pretty_assertions.workspace = true