fix: added example of usage, fixed formatting

* define Ntfy ingress (naive implementation) based on current target
* use patched Ntfy Helm Chart
* create Ntfy main user only if needed
* add info logs
* better error bubbling
* instrument feature installations
* upgrade prometheus alerting charts if already installed
* harmony_composer params to control deployment `target` and `profile`

Co-authored-by: Ian Letourneau <letourneau.ian@gmail.com>
Co-authored-by: Jean-Gabriel Gill-Couture <jg@nationtech.io>
Reviewed-on: NationTech/harmony#107

.gitea/workflows/check.yml

@@ -9,7 +9,7 @@ jobs:
   check:
     runs-on: docker
     container:
-      image: hub.nationtech.io/harmony/harmony_composer:latest@sha256:eb0406fcb95c63df9b7c4b19bc50ad7914dd8232ce98e9c9abef628e07c69386
+      image: hub.nationtech.io/harmony/harmony_composer:latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.gitea/workflows/harmony_composer.yaml

@@ -7,7 +7,7 @@ on:
 jobs:
   package_harmony_composer:
     container:
-      image: hub.nationtech.io/harmony/harmony_composer:latest@sha256:eb0406fcb95c63df9b7c4b19bc50ad7914dd8232ce98e9c9abef628e07c69386
+      image: hub.nationtech.io/harmony/harmony_composer:latest
     runs-on: dind
     steps:
       - name: Checkout code
@@ -45,14 +45,14 @@ jobs:
             -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
             "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/tags/snapshot-latest" \
             | jq -r '.id // empty')
-          
+
           if [ -n "$RELEASE_ID" ]; then
             # Delete existing release
             curl -X DELETE \
               -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
               "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/$RELEASE_ID"
           fi
-          
+
           # Create new release
           RESPONSE=$(curl -X POST \
             -H "Authorization: token ${{ secrets.GITEATOKEN }}" \
@@ -65,7 +65,7 @@ jobs:
               "prerelease": true
             }' \
             "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases")
-          
+
           echo "RELEASE_ID=$(echo $RESPONSE | jq -r '.id')" >> $GITHUB_ENV
 
       - name: Upload Linux binary

examples/remove_rook_osd/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "example_remove_rook_osd"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+harmony = { version = "0.1.0", path = "../../harmony" }
+harmony_cli = { version = "0.1.0", path = "../../harmony_cli" }
+harmony_tui = { version = "0.1.0", path = "../../harmony_tui" }
+tokio.workspace = true

examples/remove_rook_osd/src/main.rs

@@ -0,0 +1,18 @@
+use harmony::{
+    inventory::Inventory, modules::storage::ceph::ceph_remove_osd_score::CephRemoveOsd,
+    topology::K8sAnywhereTopology,
+};
+
+#[tokio::main]
+async fn main() {
+    let ceph_score = CephRemoveOsd {
+        osd_deployment_name: "rook-ceph-osd-2".to_string(),
+        rook_ceph_namespace: "rook-ceph".to_string(),
+    };
+
+    let topology = K8sAnywhereTopology::from_env();
+    let inventory = Inventory::autoload();
+    harmony_cli::run(inventory, topology, vec![Box::new(ceph_score)], None)
+        .await
+        .unwrap();
+}

harmony/Cargo.toml

@@ -5,6 +5,9 @@ version.workspace = true
 readme.workspace = true
 license.workspace = true
 
+[features]
+testing = []
+
 [dependencies]
 rand = "0.9"
 hex = "0.4"

harmony/src/domain/instrumentation.rs

@@ -2,6 +2,8 @@ use log::debug;
 use once_cell::sync::Lazy;
 use tokio::sync::broadcast;
 
+use crate::modules::application::ApplicationFeatureStatus;
+
 use super::{
     interpret::{InterpretError, Outcome},
     topology::TopologyStatus,
@@ -10,13 +12,16 @@ use super::{
 #[derive(Debug, Clone)]
 pub enum HarmonyEvent {
     HarmonyStarted,
+    HarmonyFinished,
     InterpretExecutionStarted {
+        execution_id: String,
         topology: String,
         interpret: String,
         score: String,
         message: String,
     },
     InterpretExecutionFinished {
+        execution_id: String,
         topology: String,
         interpret: String,
         score: String,
@@ -27,6 +32,12 @@ pub enum HarmonyEvent {
         status: TopologyStatus,
         message: Option<String>,
     },
+    ApplicationFeatureStateChanged {
+        topology: String,
+        application: String,
+        feature: String,
+        status: ApplicationFeatureStatus,
+    },
 }
 
 static HARMONY_EVENT_BUS: Lazy<broadcast::Sender<HarmonyEvent>> = Lazy::new(|| {
@@ -36,9 +47,14 @@ static HARMONY_EVENT_BUS: Lazy<broadcast::Sender<HarmonyEvent>> = Lazy::new(|| {
 });
 
 pub fn instrument(event: HarmonyEvent) -> Result<(), &'static str> {
-    match HARMONY_EVENT_BUS.send(event) {
-        Ok(_) => Ok(()),
-        Err(_) => Err("send error: no subscribers"),
+    if cfg!(any(test, feature = "testing")) {
+        let _ = event; // Suppress the "unused variable" warning for `event`
+        Ok(())
+    } else {
+        match HARMONY_EVENT_BUS.send(event) {
+            Ok(_) => Ok(()),
+            Err(_) => Err("send error: no subscribers"),
+        }
     }
 }
 

harmony/src/domain/score.rs

@@ -5,6 +5,7 @@ use serde::Serialize;
 use serde_value::Value;
 
 use super::{
+    data::Id,
     instrumentation::{self, HarmonyEvent},
     interpret::{Interpret, InterpretError, Outcome},
     inventory::Inventory,
@@ -20,9 +21,11 @@ pub trait Score<T: Topology>:
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        let id = Id::default();
         let interpret = self.create_interpret();
 
         instrumentation::instrument(HarmonyEvent::InterpretExecutionStarted {
+            execution_id: id.clone().to_string(),
             topology: topology.name().into(),
             interpret: interpret.get_name().to_string(),
             score: self.name(),
@@ -32,6 +35,7 @@ pub trait Score<T: Topology>:
         let result = interpret.execute(inventory, topology).await;
 
         instrumentation::instrument(HarmonyEvent::InterpretExecutionFinished {
+            execution_id: id.clone().to_string(),
             topology: topology.name().into(),
             interpret: interpret.get_name().to_string(),
             score: self.name(),

harmony/src/domain/topology/k8s.rs

@@ -5,7 +5,7 @@ use k8s_openapi::{
 };
 use kube::{
     Client, Config, Error, Resource,
-    api::{Api, AttachParams, ListParams, Patch, PatchParams, ResourceExt},
+    api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
     runtime::reflector::Lookup,
@@ -17,7 +17,9 @@ use kube::{
 };
 use log::{debug, error, trace};
 use serde::{Serialize, de::DeserializeOwned};
+use serde_json::json;
 use similar::TextDiff;
+use tokio::io::AsyncReadExt;
 
 #[derive(new, Clone)]
 pub struct K8sClient {
@@ -51,6 +53,66 @@ impl K8sClient {
         })
     }
 
+    pub async fn get_deployment(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+    ) -> Result<Option<Deployment>, Error> {
+        let deps: Api<Deployment> = if let Some(ns) = namespace {
+            Api::namespaced(self.client.clone(), ns)
+        } else {
+            Api::default_namespaced(self.client.clone())
+        };
+        Ok(deps.get_opt(name).await?)
+    }
+
+    pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result<Option<Pod>, Error> {
+        let pods: Api<Pod> = if let Some(ns) = namespace {
+            Api::namespaced(self.client.clone(), ns)
+        } else {
+            Api::default_namespaced(self.client.clone())
+        };
+        Ok(pods.get_opt(name).await?)
+    }
+
+    pub async fn scale_deployment(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+        replicas: u32,
+    ) -> Result<(), Error> {
+        let deployments: Api<Deployment> = if let Some(ns) = namespace {
+            Api::namespaced(self.client.clone(), ns)
+        } else {
+            Api::default_namespaced(self.client.clone())
+        };
+
+        let patch = json!({
+            "spec": {
+                "replicas": replicas
+            }
+        });
+        let pp = PatchParams::default();
+        let scale = Patch::Apply(&patch);
+        deployments.patch_scale(name, &pp, &scale).await?;
+        Ok(())
+    }
+
+    pub async fn delete_deployment(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+    ) -> Result<(), Error> {
+        let deployments: Api<Deployment> = if let Some(ns) = namespace {
+            Api::namespaced(self.client.clone(), ns)
+        } else {
+            Api::default_namespaced(self.client.clone())
+        };
+        let delete_params = DeleteParams::default();
+        deployments.delete(name, &delete_params).await?;
+        Ok(())
+    }
+
     pub async fn wait_until_deployment_ready(
         &self,
         name: String,
@@ -76,6 +138,68 @@ impl K8sClient {
         }
     }
 
+    /// Will execute a commond in the first pod found that matches the specified label
+    /// '{label}={name}'
+    pub async fn exec_app_capture_output(
+        &self,
+        name: String,
+        label: String,
+        namespace: Option<&str>,
+        command: Vec<&str>,
+    ) -> Result<String, String> {
+        let api: Api<Pod>;
+
+        if let Some(ns) = namespace {
+            api = Api::namespaced(self.client.clone(), ns);
+        } else {
+            api = Api::default_namespaced(self.client.clone());
+        }
+        let pod_list = api
+            .list(&ListParams::default().labels(format!("{label}={name}").as_str()))
+            .await
+            .expect("couldn't get list of pods");
+
+        let res = api
+            .exec(
+                pod_list
+                    .items
+                    .first()
+                    .expect("couldn't get pod")
+                    .name()
+                    .expect("couldn't get pod name")
+                    .into_owned()
+                    .as_str(),
+                command,
+                &AttachParams::default().stdout(true).stderr(true),
+            )
+            .await;
+        match res {
+            Err(e) => Err(e.to_string()),
+            Ok(mut process) => {
+                let status = process
+                    .take_status()
+                    .expect("Couldn't get status")
+                    .await
+                    .expect("Couldn't unwrap status");
+
+                if let Some(s) = status.status {
+                    let mut stdout_buf = String::new();
+                    if let Some(mut stdout) = process.stdout().take() {
+                        stdout.read_to_string(&mut stdout_buf).await;
+                    }
+                    debug!("Status: {} - {:?}", s, status.details);
+                    if s == "Success" {
+                        Ok(stdout_buf)
+                    } else {
+                        Err(s)
+                    }
+                } else {
+                    Err("Couldn't get inner status of pod exec".to_string())
+                }
+            }
+        }
+    }
+
     /// Will execute a command in the first pod found that matches the label `app.kubernetes.io/name={name}`
     pub async fn exec_app(
         &self,
@@ -120,7 +244,7 @@ impl K8sClient {
                     .expect("Couldn't unwrap status");
 
                 if let Some(s) = status.status {
-                    debug!("Status: {}", s);
+                    debug!("Status: {} - {:?}", s, status.details);
                     if s == "Success" { Ok(()) } else { Err(s) }
                 } else {
                     Err("Couldn't get inner status of pod exec".to_string())

harmony/src/domain/topology/k8s_anywhere.rs

@@ -28,7 +28,13 @@ use super::{
     PreparationOutcome, Topology,
     k8s::K8sClient,
     oberservability::monitoring::AlertReceiver,
-    tenant::{TenantConfig, TenantManager, k8s::K8sTenantManager},
+    tenant::{
+        TenantConfig, TenantManager,
+        k8s::K8sTenantManager,
+        network_policy::{
+            K3dNetworkPolicyStrategy, NetworkPolicyStrategy, NoopNetworkPolicyStrategy,
+        },
+    },
 };
 
 #[derive(Clone, Debug)]
@@ -250,16 +256,21 @@ impl K8sAnywhereTopology {
         Ok(Some(state))
     }
 
-    async fn ensure_k8s_tenant_manager(&self) -> Result<(), String> {
+    async fn ensure_k8s_tenant_manager(&self, k8s_state: &K8sState) -> Result<(), String> {
         if self.tenant_manager.get().is_some() {
             return Ok(());
         }
 
         self.tenant_manager
             .get_or_try_init(async || -> Result<K8sTenantManager, String> {
-                // TOOD: checker si K8s ou K3d/s tenant manager (ref. issue https://git.nationtech.io/NationTech/harmony/issues/94)
                 let k8s_client = self.k8s_client().await?;
-                Ok(K8sTenantManager::new(k8s_client))
+                let network_policy_strategy: Box<dyn NetworkPolicyStrategy> = match k8s_state.source
+                {
+                    K8sSource::LocalK3d => Box::new(K3dNetworkPolicyStrategy::new()),
+                    K8sSource::Kubeconfig => Box::new(NoopNetworkPolicyStrategy::new()),
+                };
+
+                Ok(K8sTenantManager::new(k8s_client, network_policy_strategy))
             })
             .await?;
 
@@ -390,7 +401,7 @@ impl Topology for K8sAnywhereTopology {
             "no K8s client could be found or installed".to_string(),
         ))?;
 
-        self.ensure_k8s_tenant_manager()
+        self.ensure_k8s_tenant_manager(k8s_state)
             .await
             .map_err(PreparationError::new)?;
 

harmony/src/domain/topology/tenant/k8s.rs

@@ -20,24 +20,27 @@ use serde::de::DeserializeOwned;
 use serde_json::json;
 use tokio::sync::OnceCell;
 
-use super::{TenantConfig, TenantManager};
+use super::{TenantConfig, TenantManager, network_policy::NetworkPolicyStrategy};
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct K8sTenantManager {
     k8s_client: Arc<K8sClient>,
     k8s_tenant_config: Arc<OnceCell<TenantConfig>>,
+    network_policy_strategy: Box<dyn NetworkPolicyStrategy>,
 }
 
 impl K8sTenantManager {
-    pub fn new(client: Arc<K8sClient>) -> Self {
+    pub fn new(
+        client: Arc<K8sClient>,
+        network_policy_strategy: Box<dyn NetworkPolicyStrategy>,
+    ) -> Self {
         Self {
             k8s_client: client,
             k8s_tenant_config: Arc::new(OnceCell::new()),
+            network_policy_strategy,
         }
     }
-}
 
-impl K8sTenantManager {
     fn get_namespace_name(&self, config: &TenantConfig) -> String {
         config.name.clone()
     }
@@ -218,29 +221,6 @@ impl K8sTenantManager {
                   }
                 ]
               },
-              {
-                "to": [
-                  {
-                "ipBlock": {
-                    "cidr": "10.43.0.1/32",
-                    }
-                  }
-                ]
-              },
-              {
-                "to": [
-                  {
-                      //TODO this ip is from the docker network that k3d is running on
-                      //since k3d does not deploy kube-api-server as a pod it needs to ahve the ip
-                      //address opened up
-                      //need to find a way to automatically detect the ip address from the docker
-                      //network
-                "ipBlock": {
-                    "cidr": "172.18.0.0/16",
-                    }
-                  }
-                ]
-              },
               {
                 "to": [
                   {
@@ -410,12 +390,27 @@ impl K8sTenantManager {
     }
 }
 
+impl Clone for K8sTenantManager {
+    fn clone(&self) -> Self {
+        Self {
+            k8s_client: self.k8s_client.clone(),
+            k8s_tenant_config: self.k8s_tenant_config.clone(),
+            network_policy_strategy: self.network_policy_strategy.clone_box(),
+        }
+    }
+}
+
 #[async_trait]
 impl TenantManager for K8sTenantManager {
     async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> {
         let namespace = self.build_namespace(config)?;
         let resource_quota = self.build_resource_quota(config)?;
+
         let network_policy = self.build_network_policy(config)?;
+        let network_policy = self
+            .network_policy_strategy
+            .adjust_policy(network_policy, config);
+
         let resource_limit_range = self.build_limit_range(config)?;
 
         self.ensure_constraints(&namespace)?;

harmony/src/domain/topology/tenant/mod.rs

@@ -1,11 +1,11 @@
 pub mod k8s;
 mod manager;
-use std::str::FromStr;
-
-pub use manager::*;
-use serde::{Deserialize, Serialize};
+pub mod network_policy;
 
 use crate::data::Id;
+pub use manager::*;
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] // Assuming serde for Scores
 pub struct TenantConfig {

harmony/src/domain/topology/tenant/network_policy.rs

@@ -0,0 +1,120 @@
+use k8s_openapi::api::networking::v1::{
+    IPBlock, NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyPeer, NetworkPolicySpec,
+};
+
+use super::TenantConfig;
+
+pub trait NetworkPolicyStrategy: Send + Sync + std::fmt::Debug {
+    fn clone_box(&self) -> Box<dyn NetworkPolicyStrategy>;
+
+    fn adjust_policy(&self, policy: NetworkPolicy, config: &TenantConfig) -> NetworkPolicy;
+}
+
+#[derive(Clone, Debug)]
+pub struct NoopNetworkPolicyStrategy {}
+
+impl NoopNetworkPolicyStrategy {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Default for NoopNetworkPolicyStrategy {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl NetworkPolicyStrategy for NoopNetworkPolicyStrategy {
+    fn clone_box(&self) -> Box<dyn NetworkPolicyStrategy> {
+        Box::new(self.clone())
+    }
+
+    fn adjust_policy(&self, policy: NetworkPolicy, _config: &TenantConfig) -> NetworkPolicy {
+        policy
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct K3dNetworkPolicyStrategy {}
+
+impl K3dNetworkPolicyStrategy {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl Default for K3dNetworkPolicyStrategy {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl NetworkPolicyStrategy for K3dNetworkPolicyStrategy {
+    fn clone_box(&self) -> Box<dyn NetworkPolicyStrategy> {
+        Box::new(self.clone())
+    }
+
+    fn adjust_policy(&self, policy: NetworkPolicy, _config: &TenantConfig) -> NetworkPolicy {
+        let mut egress = policy
+            .spec
+            .clone()
+            .unwrap_or_default()
+            .egress
+            .clone()
+            .unwrap_or_default();
+        egress.push(NetworkPolicyEgressRule {
+            to: Some(vec![NetworkPolicyPeer {
+                ip_block: Some(IPBlock {
+                    cidr: "172.18.0.0/16".into(), // TODO: query the IP range https://git.nationtech.io/NationTech/harmony/issues/108
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }]),
+            ..Default::default()
+        });
+
+        NetworkPolicy {
+            spec: Some(NetworkPolicySpec {
+                egress: Some(egress),
+                ..policy.spec.unwrap_or_default()
+            }),
+            ..policy
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use k8s_openapi::api::networking::v1::{
+        IPBlock, NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyPeer, NetworkPolicySpec,
+    };
+
+    use super::{K3dNetworkPolicyStrategy, NetworkPolicyStrategy};
+
+    #[test]
+    pub fn should_add_ip_block_for_k3d_harmony_server() {
+        let strategy = K3dNetworkPolicyStrategy::new();
+
+        let policy =
+            strategy.adjust_policy(NetworkPolicy::default(), &super::TenantConfig::default());
+
+        let expected_policy = NetworkPolicy {
+            spec: Some(NetworkPolicySpec {
+                egress: Some(vec![NetworkPolicyEgressRule {
+                    to: Some(vec![NetworkPolicyPeer {
+                        ip_block: Some(IPBlock {
+                            cidr: "172.18.0.0/16".into(),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    }]),
+                    ..Default::default()
+                }]),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        assert_eq!(expected_policy, policy);
+    }
+}

harmony/src/modules/application/features/continuous_delivery.rs

@@ -1,7 +1,7 @@
 use std::{io::Write, process::Command, sync::Arc};
 
 use async_trait::async_trait;
-use log::{debug, error};
+use log::info;
 use serde_yaml::Value;
 use tempfile::NamedTempFile;
 
@@ -56,14 +56,11 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         chart_url: String,
         image_name: String,
     ) -> Result<(), String> {
-        error!(
-            "FIXME This works only with local k3d installations, which is fine only for current demo purposes. We assume usage of K8sAnywhereTopology"
-        );
-
-        error!("TODO hardcoded k3d bin path is wrong");
+        // TODO: This works only with local k3d installations, which is fine only for current demo purposes. We assume usage of K8sAnywhereTopology"
+        // https://git.nationtech.io/NationTech/harmony/issues/106
         let k3d_bin_path = (*HARMONY_DATA_DIR).join("k3d").join("k3d");
         // --- 1. Import the container image into the k3d cluster ---
-        debug!(
+        info!(
             "Importing image '{}' into k3d cluster 'harmony'",
             image_name
         );
@@ -80,7 +77,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         }
 
         // --- 2. Get the kubeconfig for the k3d cluster and write it to a temp file ---
-        debug!("Retrieving kubeconfig for k3d cluster 'harmony'");
+        info!("Retrieving kubeconfig for k3d cluster 'harmony'");
         let kubeconfig_output = Command::new(&k3d_bin_path)
             .args(["kubeconfig", "get", "harmony"])
             .output()
@@ -101,7 +98,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
         let kubeconfig_path = temp_kubeconfig.path().to_str().unwrap();
 
         // --- 3. Install or upgrade the Helm chart in the cluster ---
-        debug!(
+        info!(
             "Deploying Helm chart '{}' to namespace '{}'",
             chart_url, app_name
         );
@@ -131,7 +128,7 @@ impl<A: OCICompliant + HelmPackage> ContinuousDelivery<A> {
             ));
         }
 
-        debug!("Successfully deployed '{}' to local k3d cluster.", app_name);
+        info!("Successfully deployed '{}' to local k3d cluster.", app_name);
         Ok(())
     }
 }
@@ -151,14 +148,12 @@ impl<
         // Or ask for it when unknown
 
         let helm_chart = self.application.build_push_helm_package(&image).await?;
-        debug!("Pushed new helm chart {helm_chart}");
 
-        error!("TODO Make building image configurable/skippable if image already exists (prompt)");
+        // TODO: Make building image configurable/skippable if image already exists (prompt)")
+        // https://git.nationtech.io/NationTech/harmony/issues/104
         let image = self.application.build_push_oci_image().await?;
-        debug!("Pushed new docker image {image}");
 
-        debug!("Installing ContinuousDelivery feature");
-        // TODO this is a temporary hack for demo purposes, the deployment target should be driven
+        // TODO: this is a temporary hack for demo purposes, the deployment target should be driven
         // by the topology only and we should not have to know how to perform tasks like this for
         // which the topology should be responsible.
         //
@@ -171,17 +166,20 @@ impl<
         // access it. This forces every Topology to understand the concept of targets though... So
         // instead I'll create a new Capability which is MultiTargetTopology and we'll see how it
         // goes. It still does not feel right though.
+        //
+        // https://git.nationtech.io/NationTech/harmony/issues/106
         match topology.current_target() {
             DeploymentTarget::LocalDev => {
+                info!("Deploying {} locally...", self.application.name());
                 self.deploy_to_local_k3d(self.application.name(), helm_chart, image)
                     .await?;
             }
             target => {
-                debug!("Deploying to target {target:?}");
+                info!("Deploying {} to target {target:?}", self.application.name());
                 let score = ArgoHelmScore {
-                    namespace: "harmonydemo-staging".to_string(),
-                    openshift: false,
-                    domain: "argo.harmonydemo.apps.st.mcd".to_string(),
+                    namespace: "harmony-example-rust-webapp".to_string(),
+                    openshift: true,
+                    domain: "argo.harmonydemo.apps.ncd0.harmony.mcd".to_string(),
                     argo_apps: vec![ArgoApplication::from(CDApplicationConfig {
                         // helm pull oci://hub.nationtech.io/harmony/harmony-example-rust-webapp-chart --version 0.1.0
                         version: Version::from("0.1.0").unwrap(),
@@ -189,7 +187,7 @@ impl<
                         helm_chart_name: "harmony-example-rust-webapp-chart".to_string(),
                         values_overrides: None,
                         name: "harmony-demo-rust-webapp".to_string(),
-                        namespace: "harmonydemo-staging".to_string(),
+                        namespace: "harmony-example-rust-webapp".to_string(),
                     })],
                 };
                 score

harmony/src/modules/application/features/helm_argocd_score.rs

@@ -1,5 +1,4 @@
 use async_trait::async_trait;
-use log::error;
 use non_blank_string_rs::NonBlankString;
 use serde::Serialize;
 use std::str::FromStr;
@@ -50,7 +49,6 @@ impl<T: Topology + K8sclient + HelmCommand> Interpret<T> for ArgoInterpret {
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
-        error!("Uncomment below, only disabled for debugging");
         self.score.interpret(inventory, topology).await?;
 
         let k8s_client = topology.k8s_client().await?;
@@ -58,9 +56,14 @@ impl<T: Topology + K8sclient + HelmCommand> Interpret<T> for ArgoInterpret {
             .apply_yaml_many(&self.argo_apps.iter().map(|a| a.to_yaml()).collect(), None)
             .await
             .unwrap();
+
         Ok(Outcome::success(format!(
-            "Successfully installed ArgoCD and {} Applications",
-            self.argo_apps.len()
+            "ArgoCD installed with {} {}",
+            self.argo_apps.len(),
+            match self.argo_apps.len() {
+                1 => "application",
+                _ => "applications",
+            }
         )))
     }
 

harmony/src/modules/application/features/monitoring.rs

@@ -4,6 +4,7 @@ use crate::modules::application::{Application, ApplicationFeature};
 use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
 
+use crate::topology::MultiTargetTopology;
 use crate::{
     inventory::Inventory,
     modules::monitoring::{
@@ -33,6 +34,7 @@ impl<
         + 'static
         + TenantManager
         + K8sclient
+        + MultiTargetTopology
         + std::fmt::Debug
         + PrometheusApplicationMonitoring<CRDPrometheus>,
 > ApplicationFeature<T> for Monitoring
@@ -55,11 +57,11 @@ impl<
         };
         let ntfy = NtfyScore {
             namespace: namespace.clone(),
-            host: "localhost".to_string(),
+            host: "ntfy.harmonydemo.apps.ncd0.harmony.mcd".to_string(),
         };
         ntfy.interpret(&Inventory::empty(), topology)
             .await
-            .expect("couldn't create interpret for ntfy");
+            .map_err(|e| e.to_string())?;
 
         let ntfy_default_auth_username = "harmony";
         let ntfy_default_auth_password = "harmony";
@@ -96,7 +98,7 @@ impl<
         alerting_score
             .interpret(&Inventory::empty(), topology)
             .await
-            .unwrap();
+            .map_err(|e| e.to_string())?;
         Ok(())
     }
     fn name(&self) -> String {

harmony/src/modules/application/mod.rs

@@ -14,11 +14,19 @@ use serde::Serialize;
 
 use crate::{
     data::{Id, Version},
+    instrumentation::{self, HarmonyEvent},
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
     topology::Topology,
 };
 
+#[derive(Clone, Debug)]
+pub enum ApplicationFeatureStatus {
+    Installing,
+    Installed,
+    Failed { details: String },
+}
+
 pub trait Application: std::fmt::Debug + Send + Sync {
     fn name(&self) -> String;
 }
@@ -47,20 +55,41 @@ impl<A: Application, T: Topology + std::fmt::Debug> Interpret<T> for Application
                 .join(", ")
         );
         for feature in self.features.iter() {
-            debug!(
-                "Installing feature {} for application {app_name}",
-                feature.name()
-            );
+            instrumentation::instrument(HarmonyEvent::ApplicationFeatureStateChanged {
+                topology: topology.name().into(),
+                application: self.application.name(),
+                feature: feature.name(),
+                status: ApplicationFeatureStatus::Installing,
+            })
+            .unwrap();
+
             let _ = match feature.ensure_installed(topology).await {
-                Ok(()) => (),
+                Ok(()) => {
+                    instrumentation::instrument(HarmonyEvent::ApplicationFeatureStateChanged {
+                        topology: topology.name().into(),
+                        application: self.application.name(),
+                        feature: feature.name(),
+                        status: ApplicationFeatureStatus::Installed,
+                    })
+                    .unwrap();
+                }
                 Err(msg) => {
+                    instrumentation::instrument(HarmonyEvent::ApplicationFeatureStateChanged {
+                        topology: topology.name().into(),
+                        application: self.application.name(),
+                        feature: feature.name(),
+                        status: ApplicationFeatureStatus::Failed {
+                            details: msg.clone(),
+                        },
+                    })
+                    .unwrap();
                     return Err(InterpretError::new(format!(
                         "Application Interpret failed to install feature : {msg}"
                     )));
                 }
             };
         }
-        Ok(Outcome::success("successfully created app".to_string()))
+        Ok(Outcome::success("Application created".to_string()))
     }
 
     fn get_name(&self) -> InterpretName {

harmony/src/modules/application/rust.rs

@@ -10,7 +10,7 @@ use dockerfile_builder::Dockerfile;
 use dockerfile_builder::instruction::{CMD, COPY, ENV, EXPOSE, FROM, RUN, USER, WORKDIR};
 use dockerfile_builder::instruction_builder::CopyBuilder;
 use futures_util::StreamExt;
-use log::{debug, error, log_enabled};
+use log::{debug, info, log_enabled};
 use serde::Serialize;
 use tar::Archive;
 
@@ -46,7 +46,7 @@ where
     }
 
     fn name(&self) -> String {
-        format!("Application: {}", self.application.name())
+        format!("{} [ApplicationScore]", self.application.name())
     }
 }
 
@@ -73,19 +73,19 @@ impl Application for RustWebapp {
 #[async_trait]
 impl HelmPackage for RustWebapp {
     async fn build_push_helm_package(&self, image_url: &str) -> Result<String, String> {
-        debug!("Starting Helm chart build and push for '{}'", self.name);
+        info!("Starting Helm chart build and push for '{}'", self.name);
 
         // 1. Create the Helm chart files on disk.
         let chart_dir = self
             .create_helm_chart_files(image_url)
             .map_err(|e| format!("Failed to create Helm chart files: {}", e))?;
-        debug!("Successfully created Helm chart files in {:?}", chart_dir);
+        info!("Successfully created Helm chart files in {:?}", chart_dir);
 
         // 2. Package the chart into a .tgz archive.
         let packaged_chart_path = self
             .package_helm_chart(&chart_dir)
             .map_err(|e| format!("Failed to package Helm chart: {}", e))?;
-        debug!(
+        info!(
             "Successfully packaged Helm chart: {}",
             packaged_chart_path.to_string_lossy()
         );
@@ -94,7 +94,7 @@ impl HelmPackage for RustWebapp {
         let oci_chart_url = self
             .push_helm_chart(&packaged_chart_path)
             .map_err(|e| format!("Failed to push Helm chart: {}", e))?;
-        debug!("Successfully pushed Helm chart to: {}", oci_chart_url);
+        info!("Successfully pushed Helm chart to: {}", oci_chart_url);
 
         Ok(oci_chart_url)
     }
@@ -107,20 +107,20 @@ impl OCICompliant for RustWebapp {
     async fn build_push_oci_image(&self) -> Result<String, String> {
         // This function orchestrates the build and push process.
         // It's async to match the trait definition, though the underlying docker commands are blocking.
-        debug!("Starting OCI image build and push for '{}'", self.name);
+        info!("Starting OCI image build and push for '{}'", self.name);
 
         // 1. Build the image by calling the synchronous helper function.
         let image_tag = self.image_name();
         self.build_docker_image(&image_tag)
             .await
             .map_err(|e| format!("Failed to build Docker image: {}", e))?;
-        debug!("Successfully built Docker image: {}", image_tag);
+        info!("Successfully built Docker image: {}", image_tag);
 
         // 2. Push the image to the registry.
         self.push_docker_image(&image_tag)
             .await
             .map_err(|e| format!("Failed to push Docker image: {}", e))?;
-        debug!("Successfully pushed Docker image to: {}", image_tag);
+        info!("Successfully pushed Docker image to: {}", image_tag);
 
         Ok(image_tag)
     }
@@ -195,7 +195,7 @@ impl RustWebapp {
         );
 
         while let Some(msg) = image_build_stream.next().await {
-            println!("Message: {msg:?}");
+            debug!("Message: {msg:?}");
         }
 
         Ok(image_name.to_string())
@@ -219,7 +219,7 @@ impl RustWebapp {
         );
 
         while let Some(msg) = push_image_stream.next().await {
-            println!("Message: {msg:?}");
+            debug!("Message: {msg:?}");
         }
 
         Ok(image_tag.to_string())
@@ -288,9 +288,8 @@ impl RustWebapp {
                         .unwrap(),
                 );
                 // Copy the compiled binary from the builder stage.
-                error!(
-                    "FIXME Should not be using score name here, instead should use name from Cargo.toml"
-                );
+                // TODO: Should not be using score name here, instead should use name from Cargo.toml
+                // https://git.nationtech.io/NationTech/harmony/issues/105
                 let binary_path_in_builder = format!("/app/target/release/{}", self.name);
                 let binary_path_in_final = format!("/home/appuser/{}", self.name);
                 dockerfile.push(
@@ -328,9 +327,8 @@ impl RustWebapp {
                 ));
 
                 // Copy only the compiled binary from the builder stage.
-                error!(
-                    "FIXME Should not be using score name here, instead should use name from Cargo.toml"
-                );
+                // TODO: Should not be using score name here, instead should use name from Cargo.toml
+                // https://git.nationtech.io/NationTech/harmony/issues/105
                 let binary_path_in_builder = format!("/app/target/release/{}", self.name);
                 let binary_path_in_final = format!("/usr/local/bin/{}", self.name);
                 dockerfile.push(

harmony/src/modules/helm/chart.rs

@@ -55,7 +55,7 @@ impl<T: Topology + HelmCommand> Score<T> for HelmChartScore {
     }
 
     fn name(&self) -> String {
-        format!("{} {} HelmChartScore", self.release_name, self.chart_name)
+        format!("{} [HelmChartScore]", self.release_name)
     }
 }
 
@@ -225,19 +225,20 @@ impl<T: Topology + HelmCommand> Interpret<T> for HelmChartInterpret {
         match status {
             helm_wrapper_rs::HelmDeployStatus::Deployed => Ok(Outcome::new(
                 InterpretStatus::SUCCESS,
-                "Helm Chart deployed".to_string(),
+                format!("Helm Chart {} deployed", self.score.release_name),
             )),
             helm_wrapper_rs::HelmDeployStatus::PendingInstall => Ok(Outcome::new(
                 InterpretStatus::RUNNING,
-                "Helm Chart Pending install".to_string(),
+                format!("Helm Chart {} pending install...", self.score.release_name),
             )),
             helm_wrapper_rs::HelmDeployStatus::PendingUpgrade => Ok(Outcome::new(
                 InterpretStatus::RUNNING,
-                "Helm Chart pending upgrade".to_string(),
-            )),
-            helm_wrapper_rs::HelmDeployStatus::Failed => Err(InterpretError::new(
-                "Failed to install helm chart".to_string(),
+                format!("Helm Chart {} pending upgrade...", self.score.release_name),
             )),
+            helm_wrapper_rs::HelmDeployStatus::Failed => Err(InterpretError::new(format!(
+                "Helm Chart {} installation failed",
+                self.score.release_name
+            ))),
         }
     }
 

harmony/src/modules/k3d/install.rs

@@ -7,7 +7,6 @@ use serde::Serialize;
 use crate::{
     config::HARMONY_DATA_DIR,
     data::{Id, Version},
-    instrumentation::{self, HarmonyEvent},
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
     score::Score,
@@ -30,7 +29,7 @@ impl Default for K3DInstallationScore {
 }
 
 impl<T: Topology> Score<T> for K3DInstallationScore {
-    fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
         Box::new(K3dInstallationInterpret {
             score: self.clone(),
         })

harmony/src/modules/mod.rs

@@ -14,5 +14,6 @@ pub mod monitoring;
 pub mod okd;
 pub mod opnsense;
 pub mod prometheus;
+pub mod storage;
 pub mod tenant;
 pub mod tftp;

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -33,7 +33,10 @@ impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
     }
 
     fn name(&self) -> String {
-        "ApplicationMonitoringScore".to_string()
+        format!(
+            "{} monitoring [ApplicationMonitoringScore]",
+            self.application.name()
+        )
     }
 }
 
@@ -61,7 +64,9 @@ impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
 
         match result {
             Ok(outcome) => match outcome {
-                PreparationOutcome::Success { details } => Ok(Outcome::success(details)),
+                PreparationOutcome::Success { details: _ } => {
+                    Ok(Outcome::success("Prometheus installed".into()))
+                }
                 PreparationOutcome::Noop => Ok(Outcome::noop()),
             },
             Err(err) => Err(InterpretError::from(err)),

harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs

@@ -1,9 +1,25 @@
 use non_blank_string_rs::NonBlankString;
 use std::str::FromStr;
 
-use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
+use crate::{modules::helm::chart::HelmChartScore, topology::DeploymentTarget};
+
+pub fn ntfy_helm_chart_score(
+    namespace: String,
+    host: String,
+    target: DeploymentTarget,
+) -> HelmChartScore {
+    // TODO not actually the correct logic, this should be fixed by using an ingresss which is the
+    // correct k8s standard.
+    //
+    // Another option is to delegate to the topology the ingress technology it wants to use Route,
+    // Ingress or other
+    let route_enabled = match target {
+        DeploymentTarget::LocalDev => false,
+        DeploymentTarget::Staging => true,
+        DeploymentTarget::Production => true,
+    };
+    let ingress_enabled = !route_enabled;
 
-pub fn ntfy_helm_chart_score(namespace: String, host: String) -> HelmChartScore {
     let values = format!(
         r#"
 replicaCount: 1
@@ -25,23 +41,14 @@ serviceAccount:
 
 service:
   type: ClusterIP
-  port: 80
+  port: 8080
 
 ingress:
-  enabled: true
-#  annotations:
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: {host}
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
+  enabled: {ingress_enabled}
 
+route:
+  enabled: {route_enabled}
+  host: {host}
 
 autoscaling:
   enabled: false
@@ -49,7 +56,7 @@ autoscaling:
 config:
   enabled: true
   data:
-#    base-url: "https://ntfy.something.com"
+    base-url: "https://{host}"
     auth-file: "/var/cache/ntfy/user.db"
     auth-default-access: "deny-all"
     cache-file: "/var/cache/ntfy/cache.db"
@@ -59,6 +66,7 @@ config:
     enable-signup: false
     enable-login: "true"
     enable-metrics: "true"
+    listen-http: ":8080"
 
 persistence:
   enabled: true
@@ -69,16 +77,12 @@ persistence:
     HelmChartScore {
         namespace: Some(NonBlankString::from_str(&namespace).unwrap()),
         release_name: NonBlankString::from_str("ntfy").unwrap(),
-        chart_name: NonBlankString::from_str("sarab97/ntfy").unwrap(),
-        chart_version: Some(NonBlankString::from_str("0.1.7").unwrap()),
+        chart_name: NonBlankString::from_str("oci://hub.nationtech.io/harmony/ntfy").unwrap(),
+        chart_version: Some(NonBlankString::from_str("0.1.7-nationtech.1").unwrap()),
         values_overrides: None,
         values_yaml: Some(values.to_string()),
         create_namespace: true,
         install_only: false,
-        repository: Some(HelmRepository::new(
-            "sarab97".to_string(),
-            url::Url::parse("https://charts.sarabsingh.com").unwrap(),
-            true,
-        )),
+        repository: None,
     }
 }

harmony/src/modules/monitoring/ntfy/ntfy.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use log::debug;
+use log::info;
 use serde::Serialize;
 use strum::{Display, EnumString};
 
@@ -11,7 +11,7 @@ use crate::{
     inventory::Inventory,
     modules::monitoring::ntfy::helm::ntfy_helm_chart::ntfy_helm_chart_score,
     score::Score,
-    topology::{HelmCommand, K8sclient, Topology, k8s::K8sClient},
+    topology::{HelmCommand, K8sclient, MultiTargetTopology, Topology, k8s::K8sClient},
 };
 
 #[derive(Debug, Clone, Serialize)]
@@ -20,7 +20,7 @@ pub struct NtfyScore {
     pub host: String,
 }
 
-impl<T: Topology + HelmCommand + K8sclient> Score<T> for NtfyScore {
+impl<T: Topology + HelmCommand + K8sclient + MultiTargetTopology> Score<T> for NtfyScore {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
         Box::new(NtfyInterpret {
             score: self.clone(),
@@ -28,7 +28,7 @@ impl<T: Topology + HelmCommand + K8sclient> Score<T> for NtfyScore {
     }
 
     fn name(&self) -> String {
-        "NtfyScore".to_string()
+        "alert receiver [NtfyScore]".into()
     }
 }
 
@@ -77,7 +77,7 @@ impl NtfyInterpret {
                 vec![
                     "sh",
                     "-c",
-                    format!("NTFY_PASSWORD={password} ntfy user add --role={role} {username}")
+                    format!("NTFY_PASSWORD={password} ntfy user add --role={role} --ignore-exists {username}")
                         .as_str(),
                 ],
             )
@@ -89,22 +89,27 @@ impl NtfyInterpret {
 
 /// We need a ntfy interpret to wrap the HelmChartScore in order to run the score, and then bootstrap the config inside ntfy
 #[async_trait]
-impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for NtfyInterpret {
+impl<T: Topology + HelmCommand + K8sclient + MultiTargetTopology> Interpret<T> for NtfyInterpret {
     async fn execute(
         &self,
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
-        ntfy_helm_chart_score(self.score.namespace.clone(), self.score.host.clone())
-            .interpret(inventory, topology)
-            .await?;
+        ntfy_helm_chart_score(
+            self.score.namespace.clone(),
+            self.score.host.clone(),
+            topology.current_target(),
+        )
+        .interpret(inventory, topology)
+        .await?;
 
-        debug!("installed ntfy helm chart");
+        info!("installed ntfy helm chart");
         let client = topology
             .k8s_client()
             .await
             .expect("couldn't get k8s client");
 
+        info!("deploying ntfy...");
         client
             .wait_until_deployment_ready(
                 "ntfy".to_string(),
@@ -112,14 +117,14 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for NtfyInterpret {
                 None,
             )
             .await?;
-        debug!("created k8s client");
+        info!("ntfy deployed");
 
+        info!("adding user harmony");
         self.add_user(client, "harmony", "harmony", Some(NtfyRole::Admin))
             .await?;
+        info!("user added");
 
-        debug!("exec into pod done");
-
-        Ok(Outcome::success("installed ntfy".to_string()))
+        Ok(Outcome::success("Ntfy installed".to_string()))
     }
 
     fn get_name(&self) -> InterpretName {

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -61,7 +61,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> S
     }
 
     fn name(&self) -> String {
-        "CRDApplicationAlertingScore".into()
+        "prometheus alerting [CRDAlertingScore]".into()
     }
 }
 
@@ -94,7 +94,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> I
         self.install_monitors(self.service_monitors.clone(), &client)
             .await?;
         Ok(Outcome::success(
-            "deployed application monitoring composants".to_string(),
+            "K8s monitoring components installed".to_string(),
         ))
     }
 
@@ -166,7 +166,8 @@ impl K8sPrometheusCRDAlertingInterpret {
 
         let install_output = Command::new("helm")
             .args([
-                "install",
+                "upgrade",
+                "--install",
                 &chart_name,
                 tgz_path.to_str().unwrap(),
                 "--namespace",

harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs

@@ -0,0 +1,419 @@
+use std::{
+    process::Command,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use async_trait::async_trait;
+use log::{info, warn};
+use serde::{Deserialize, Serialize};
+use tokio::time::sleep;
+
+use crate::{
+    data::{Id, Version},
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    score::Score,
+    topology::{K8sclient, Topology, k8s::K8sClient},
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CephRemoveOsd {
+    pub osd_deployment_name: String,
+    pub rook_ceph_namespace: String,
+}
+
+impl<T: Topology + K8sclient> Score<T> for CephRemoveOsd {
+    fn name(&self) -> String {
+        format!("CephRemoveOsdScore")
+    }
+
+    #[doc(hidden)]
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(CephRemoveOsdInterpret {
+            score: self.clone(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct CephRemoveOsdInterpret {
+    score: CephRemoveOsd,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient> Interpret<T> for CephRemoveOsdInterpret {
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await.unwrap();
+        self.verify_ceph_toolbox_exists(client.clone()).await?;
+        self.scale_deployment(client.clone()).await?;
+        self.verify_deployment_scaled(client.clone()).await?;
+        self.delete_deployment(client.clone()).await?;
+        self.verify_deployment_deleted(client.clone()).await?;
+        let osd_id_full = self.get_ceph_osd_id().unwrap();
+        self.purge_ceph_osd(client.clone(), &osd_id_full).await?;
+        self.verify_ceph_osd_removal(client.clone(), &osd_id_full)
+            .await?;
+
+        Ok(Outcome::success(format!(
+            "Successfully removed OSD {} from rook-ceph cluster by deleting deployment {}",
+            osd_id_full, self.score.osd_deployment_name
+        )))
+    }
+    fn get_name(&self) -> InterpretName {
+        todo!()
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl CephRemoveOsdInterpret {
+    pub fn get_ceph_osd_id(&self) -> Result<String, InterpretError> {
+        let osd_id_numeric = self
+            .score
+            .osd_deployment_name
+            .split('-')
+            .nth(3)
+            .ok_or_else(|| {
+                InterpretError::new(format!(
+                    "Could not parse OSD id from deployment name {}",
+                    self.score.osd_deployment_name
+                ))
+            })?;
+        let osd_id_full = format!("osd.{}", osd_id_numeric);
+
+        info!(
+            "Targeting Ceph OSD: {} (parsed from deployment {})",
+            osd_id_full, self.score.osd_deployment_name
+        );
+
+        Ok(osd_id_full)
+    }
+
+    pub async fn verify_ceph_toolbox_exists(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let toolbox_dep = "rook-ceph-tools".to_string();
+
+        match client
+            .get_deployment(&toolbox_dep, Some(&self.score.rook_ceph_namespace))
+            .await
+        {
+            Ok(Some(deployment)) => {
+                if let Some(status) = deployment.status {
+                    let ready_count = status.ready_replicas.unwrap_or(0);
+                    if ready_count >= 1 {
+                        return Ok(Outcome::success(format!(
+                            "'{}' is ready with {} replica(s).",
+                            &toolbox_dep, ready_count
+                        )));
+                    } else {
+                        return Err(InterpretError::new(
+                            "ceph-tool-box not ready in cluster".to_string(),
+                        ));
+                    }
+                } else {
+                    Err(InterpretError::new(format!(
+                        "failed to get deployment status {}",
+                        &toolbox_dep
+                    )))
+                }
+            }
+            Ok(None) => Err(InterpretError::new(format!(
+                "Deployment '{}' not found in namespace '{}'.",
+                &toolbox_dep, self.score.rook_ceph_namespace
+            ))),
+            Err(e) => Err(InterpretError::new(format!(
+                "Failed to query for deployment '{}': {}",
+                &toolbox_dep, e
+            ))),
+        }
+    }
+
+    pub async fn scale_deployment(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        info!(
+            "Scaling down OSD deployment: {}",
+            self.score.osd_deployment_name
+        );
+        client
+            .scale_deployment(
+                &self.score.osd_deployment_name,
+                Some(&self.score.rook_ceph_namespace),
+                0,
+            )
+            .await?;
+        Ok(Outcome::success(format!(
+            "Scaled down deployment {}",
+            self.score.osd_deployment_name
+        )))
+    }
+
+    pub async fn verify_deployment_scaled(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let (timeout, interval, start) = self.build_timer();
+
+        info!("Waiting for OSD deployment to scale down to 0 replicas");
+        loop {
+            let dep = client
+                .get_deployment(
+                    &self.score.osd_deployment_name,
+                    Some(&self.score.rook_ceph_namespace),
+                )
+                .await?;
+
+            if let Some(deployment) = dep {
+                if let Some(status) = deployment.status {
+                    if status.replicas.unwrap_or(1) == 0 && status.ready_replicas.unwrap_or(1) == 0
+                    {
+                        return Ok(Outcome::success(
+                            "Deployment successfully scaled down.".to_string(),
+                        ));
+                    }
+                }
+            }
+
+            if start.elapsed() > timeout {
+                return Err(InterpretError::new(format!(
+                    "Timed out waiting for deployment {} to scale down",
+                    self.score.osd_deployment_name
+                )));
+            }
+            sleep(interval).await;
+        }
+    }
+
+    fn build_timer(&self) -> (Duration, Duration, Instant) {
+        let timeout = Duration::from_secs(120);
+        let interval = Duration::from_secs(5);
+        let start = Instant::now();
+        (timeout, interval, start)
+    }
+    pub async fn delete_deployment(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        info!(
+            "Deleting OSD deployment: {}",
+            self.score.osd_deployment_name
+        );
+        client
+            .delete_deployment(
+                &self.score.osd_deployment_name,
+                Some(&self.score.rook_ceph_namespace),
+            )
+            .await?;
+        Ok(Outcome::success(format!(
+            "deployment {} deleted",
+            self.score.osd_deployment_name
+        )))
+    }
+
+    pub async fn verify_deployment_deleted(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let (timeout, interval, start) = self.build_timer();
+
+        info!("Waiting for OSD deployment to scale down to 0 replicas");
+        loop {
+            let dep = client
+                .get_deployment(
+                    &self.score.osd_deployment_name,
+                    Some(&self.score.rook_ceph_namespace),
+                )
+                .await?;
+
+            if dep.is_none() {
+                info!(
+                    "Deployment {} successfully deleted.",
+                    self.score.osd_deployment_name
+                );
+                return Ok(Outcome::success(format!(
+                    "Deployment {} deleted.",
+                    self.score.osd_deployment_name
+                )));
+            }
+
+            if start.elapsed() > timeout {
+                return Err(InterpretError::new(format!(
+                    "Timed out waiting for deployment {} to be deleted",
+                    self.score.osd_deployment_name
+                )));
+            }
+            sleep(interval).await;
+        }
+    }
+
+    fn get_osd_tree(&self, json: serde_json::Value) -> Result<CephOsdTree, InterpretError> {
+        let nodes = json.get("nodes").ok_or_else(|| {
+            InterpretError::new("Missing 'nodes' field in ceph osd tree JSON".to_string())
+        })?;
+        let tree: CephOsdTree = CephOsdTree {
+            nodes: serde_json::from_value(nodes.clone()).map_err(|e| {
+                InterpretError::new(format!("Failed to parse ceph osd tree JSON: {}", e))
+            })?,
+        };
+        Ok(tree)
+    }
+
+    pub async fn purge_ceph_osd(
+        &self,
+        client: Arc<K8sClient>,
+        osd_id_full: &str,
+    ) -> Result<Outcome, InterpretError> {
+        info!(
+            "Purging OSD {} from Ceph cluster and removing its auth key",
+            osd_id_full
+        );
+        client
+            .exec_app_capture_output(
+                "rook-ceph-tools".to_string(),
+                "app".to_string(),
+                Some(&self.score.rook_ceph_namespace),
+                vec![
+                    format!("ceph osd purge {osd_id_full} --yes-i-really-mean-it").as_str(),
+                    format!("ceph auth del osd.{osd_id_full}").as_str(),
+                ],
+            )
+            .await?;
+        Ok(Outcome::success(format!(
+            "osd id {} removed from osd tree",
+            osd_id_full
+        )))
+    }
+
+    pub async fn verify_ceph_osd_removal(
+        &self,
+        client: Arc<K8sClient>,
+        osd_id_full: &str,
+    ) -> Result<Outcome, InterpretError> {
+        let (timeout, interval, start) = self.build_timer();
+        info!(
+            "Verifying OSD {} has been removed from the Ceph tree...",
+            osd_id_full
+        );
+        loop {
+            let output = client
+                .exec_app_capture_output(
+                    "rook-ceph-tools".to_string(),
+                    "app".to_string(),
+                    Some(&self.score.rook_ceph_namespace),
+                    vec!["ceph osd tree -f json"],
+                )
+                .await?;
+            let tree =
+                self.get_osd_tree(serde_json::from_str(&output).expect("could not extract json"));
+
+            let osd_found = tree
+                .unwrap()
+                .nodes
+                .iter()
+                .any(|node| node.name == osd_id_full);
+
+            if !osd_found {
+                return Ok(Outcome::success(format!(
+                    "Successfully verified that OSD {} is removed from the Ceph cluster.",
+                    osd_id_full,
+                )));
+            }
+
+            if start.elapsed() > timeout {
+                return Err(InterpretError::new(format!(
+                    "Timed out waiting for OSD {} to be removed from Ceph tree",
+                    osd_id_full
+                )));
+            }
+
+            warn!(
+                "OSD {} still found in Ceph tree, retrying in {:?}...",
+                osd_id_full, interval
+            );
+            sleep(interval).await;
+        }
+    }
+}
+#[derive(Debug, Deserialize, PartialEq)]
+pub struct CephOsdTree {
+    pub nodes: Vec<CephNode>,
+}
+
+#[derive(Debug, Deserialize, PartialEq)]
+pub struct CephNode {
+    pub id: i32,
+    pub name: String,
+    #[serde(rename = "type")]
+    pub node_type: String,
+    pub type_id: Option<i32>,
+    pub children: Option<Vec<i32>>,
+    pub exists: Option<i32>,
+    pub status: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use super::*;
+
+    #[test]
+    fn test_get_osd_tree() {
+        let json_data = json!({
+            "nodes": [
+                {"id": 1, "name": "osd.1", "type": "osd", "primary_affinity":"1"},
+                {"id": 2, "name": "osd.2", "type": "osd", "crush_weight": 1.22344}
+            ]
+        });
+        let interpret = CephRemoveOsdInterpret {
+            score: CephRemoveOsd {
+                osd_deployment_name: "osd-1".to_string(),
+                rook_ceph_namespace: "dummy_ns".to_string(),
+            },
+        };
+        let json = interpret.get_osd_tree(json_data).unwrap();
+
+        let expected = CephOsdTree {
+            nodes: vec![
+                CephNode {
+                    id: 1,
+                    name: "osd.1".to_string(),
+                    node_type: "osd".to_string(),
+                    type_id: None,
+                    children: None,
+                    exists: None,
+                    status: None,
+                },
+                CephNode {
+                    id: 2,
+                    name: "osd.2".to_string(),
+                    node_type: "osd".to_string(),
+                    type_id: None,
+                    children: None,
+                    exists: None,
+                    status: None,
+                },
+            ],
+        };
+
+        assert_eq!(json, expected);
+    }
+}

harmony/src/modules/storage/ceph/mod.rs

@@ -0,0 +1 @@
+pub mod ceph_remove_osd_score;

harmony/src/modules/storage/mod.rs

@@ -0,0 +1 @@
+pub mod ceph;

harmony/src/modules/tenant/mod.rs

@@ -28,7 +28,7 @@ impl<T: Topology + TenantManager> Score<T> for TenantScore {
     }
 
     fn name(&self) -> String {
-        format!("{} TenantScore", self.config.name)
+        format!("{} [TenantScore]", self.config.name)
     }
 }
 
@@ -47,8 +47,8 @@ impl<T: Topology + TenantManager> Interpret<T> for TenantInterpret {
         topology.provision_tenant(&self.tenant_config).await?;
 
         Ok(Outcome::success(format!(
-            "Successfully provisioned tenant {} with id {}",
-            self.tenant_config.name, self.tenant_config.id
+            "Tenant provisioned with id '{}'",
+            self.tenant_config.id
         )))
     }
 

harmony_cli/Cargo.toml

@@ -5,6 +5,10 @@ version.workspace = true
 readme.workspace = true
 license.workspace = true
 
+[features]
+default = ["tui"]
+tui = ["dep:harmony_tui"]
+
 [dependencies]
 assert_cmd = "2.0.17"
 clap = { version = "4.5.35", features = ["derive"] }
@@ -19,7 +23,5 @@ lazy_static = "1.5.0"
 log.workspace = true
 indicatif-log-bridge = "0.2.3"
 
-
-[features]
-default = ["tui"]
-tui = ["dep:harmony_tui"]
+[dev-dependencies]
+harmony = { path = "../harmony", features = ["testing"] }

harmony_cli/src/cli_logger.rs

@@ -1,19 +1,22 @@
 use harmony::{
     instrumentation::{self, HarmonyEvent},
+    modules::application::ApplicationFeatureStatus,
     topology::TopologyStatus,
 };
-use indicatif::{MultiProgress, ProgressBar};
+use indicatif::MultiProgress;
 use indicatif_log_bridge::LogWrapper;
+use log::error;
 use std::{
-    collections::HashMap,
     sync::{Arc, Mutex},
+    thread,
+    time::Duration,
 };
 
-use crate::progress;
+use crate::progress::{IndicatifProgressTracker, ProgressTracker};
 
 pub fn init() -> tokio::task::JoinHandle<()> {
-    configure_logger();
-    let handle = tokio::spawn(handle_events());
+    let base_progress = configure_logger();
+    let handle = tokio::spawn(handle_events(base_progress));
 
     loop {
         if instrumentation::instrument(HarmonyEvent::HarmonyStarted).is_ok() {
@@ -24,32 +27,46 @@ pub fn init() -> tokio::task::JoinHandle<()> {
     handle
 }
 
-fn configure_logger() {
+fn configure_logger() -> MultiProgress {
     let logger =
         env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).build();
     let level = logger.filter();
-    let multi = MultiProgress::new();
-    LogWrapper::new(multi.clone(), logger).try_init().unwrap();
+    let progress = MultiProgress::new();
+
+    LogWrapper::new(progress.clone(), logger)
+        .try_init()
+        .unwrap();
     log::set_max_level(level);
+
+    progress
 }
 
-async fn handle_events() {
-    instrumentation::subscribe("Harmony CLI Logger", {
-        let sections: Arc<Mutex<HashMap<String, MultiProgress>>> =
-            Arc::new(Mutex::new(HashMap::new()));
-        let progress_bars: Arc<Mutex<HashMap<String, ProgressBar>>> =
-            Arc::new(Mutex::new(HashMap::new()));
+async fn handle_events(base_progress: MultiProgress) {
+    let progress_tracker = Arc::new(IndicatifProgressTracker::new(base_progress.clone()));
+    let preparing_topology = Arc::new(Mutex::new(false));
+    let current_score: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
 
+    instrumentation::subscribe("Harmony CLI Logger", {
         move |event| {
-            let sections_clone = Arc::clone(&sections);
-            let progress_bars_clone = Arc::clone(&progress_bars);
+            let progress_tracker = Arc::clone(&progress_tracker);
+            let preparing_topology = Arc::clone(&preparing_topology);
+            let current_score = Arc::clone(&current_score);
 
             async move {
-                let mut sections = sections_clone.lock().unwrap();
-                let mut progress_bars = progress_bars_clone.lock().unwrap();
+                let mut preparing_topology = preparing_topology.lock().unwrap();
+                let mut current_score = current_score.lock().unwrap();
 
                 match event {
                     HarmonyEvent::HarmonyStarted => {}
+                    HarmonyEvent::HarmonyFinished => {
+                        progress_tracker.add_section(
+                            "harmony-summary",
+                            &format!("\n{} Harmony completed\n\n", crate::theme::EMOJI_HARMONY),
+                        );
+                        progress_tracker.add_section("harmony-finished", "\n\n");
+                        thread::sleep(Duration::from_millis(200));
+                        return false;
+                    }
                     HarmonyEvent::TopologyStateChanged {
                         topology,
                         status,
@@ -60,110 +77,124 @@ async fn handle_events() {
                         match status {
                             TopologyStatus::Queued => {}
                             TopologyStatus::Preparing => {
-                                let section = progress::new_section(format!(
-                                    "{} Preparing environment: {topology}...",
-                                    crate::theme::EMOJI_TOPOLOGY,
-                                ));
-                                (*sections).insert(section_key, section);
+                                progress_tracker.add_section(
+                                    &section_key,
+                                    &format!(
+                                        "\n{} Preparing environment: {topology}...",
+                                        crate::theme::EMOJI_TOPOLOGY
+                                    ),
+                                );
+                                (*preparing_topology) = true;
                             }
                             TopologyStatus::Success => {
-                                let section = (*sections).get(&section_key).unwrap();
-                                let progress = progress::add_spinner(section, "".into());
-
-                                progress::success(
-                                    section,
-                                    Some(progress),
-                                    message.unwrap_or("".into()),
-                                );
-
-                                (*sections).remove(&section_key);
+                                (*preparing_topology) = false;
+                                progress_tracker.add_task(&section_key, "topology-success", "");
+                                progress_tracker
+                                    .finish_task("topology-success", &message.unwrap_or("".into()));
                             }
                             TopologyStatus::Noop => {
-                                let section = (*sections).get(&section_key).unwrap();
-                                let progress = progress::add_spinner(section, "".into());
-
-                                progress::skip(
-                                    section,
-                                    Some(progress),
-                                    message.unwrap_or("".into()),
-                                );
-
-                                (*sections).remove(&section_key);
+                                (*preparing_topology) = false;
+                                progress_tracker.add_task(&section_key, "topology-skip", "");
+                                progress_tracker
+                                    .skip_task("topology-skip", &message.unwrap_or("".into()));
                             }
                             TopologyStatus::Error => {
-                                let section = (*sections).get(&section_key).unwrap();
-                                let progress = progress::add_spinner(section, "".into());
-
-                                progress::error(
-                                    section,
-                                    Some(progress),
-                                    message.unwrap_or("".into()),
-                                );
-
-                                (*sections).remove(&section_key);
+                                progress_tracker.add_task(&section_key, "topology-error", "");
+                                (*preparing_topology) = false;
+                                progress_tracker
+                                    .fail_task("topology-error", &message.unwrap_or("".into()));
                             }
                         }
                     }
                     HarmonyEvent::InterpretExecutionStarted {
+                        execution_id: task_key,
                         topology,
-                        interpret,
+                        interpret: _,
                         score,
                         message,
                     } => {
-                        let section_key = if (*sections).contains_key(&topology_key(&topology)) {
+                        let is_key_topology = (*preparing_topology)
+                            && progress_tracker.contains_section(&topology_key(&topology));
+                        let is_key_current_score = current_score.is_some()
+                            && progress_tracker
+                                .contains_section(&score_key(&current_score.clone().unwrap()));
+                        let is_key_score = progress_tracker.contains_section(&score_key(&score));
+
+                        let section_key = if is_key_topology {
                             topology_key(&topology)
-                        } else if (*sections).contains_key(&score_key(&score)) {
-                            score_key(&interpret)
+                        } else if is_key_current_score {
+                            score_key(&current_score.clone().unwrap())
+                        } else if is_key_score {
+                            score_key(&score)
                         } else {
+                            (*current_score) = Some(score.clone());
                             let key = score_key(&score);
-                            let section = progress::new_section(format!(
-                                "\n{} Interpreting score: {score}...",
-                                crate::theme::EMOJI_SCORE,
-                            ));
-                            (*sections).insert(key.clone(), section);
+                            progress_tracker.add_section(
+                                &key,
+                                &format!(
+                                    "{} Interpreting score: {score}...",
+                                    crate::theme::EMOJI_SCORE
+                                ),
+                            );
                             key
                         };
-                        let section = (*sections).get(&section_key).unwrap();
-                        let progress_bar = progress::add_spinner(section, message);
 
-                        (*progress_bars).insert(interpret_key(&interpret), progress_bar);
+                        progress_tracker.add_task(&section_key, &task_key, &message);
                     }
                     HarmonyEvent::InterpretExecutionFinished {
-                        topology,
-                        interpret,
+                        execution_id: task_key,
+                        topology: _,
+                        interpret: _,
                         score,
                         outcome,
                     } => {
-                        let has_topology = (*sections).contains_key(&topology_key(&topology));
-                        let section_key = if has_topology {
-                            topology_key(&topology)
-                        } else {
-                            score_key(&score)
-                        };
-
-                        let section = (*sections).get(&section_key).unwrap();
-                        let progress_bar =
-                            (*progress_bars).get(&interpret_key(&interpret)).cloned();
-
-                        let _ = section.clear();
+                        if current_score.is_some() && current_score.clone().unwrap() == score {
+                            (*current_score) = None;
+                        }
 
                         match outcome {
                             Ok(outcome) => match outcome.status {
                                 harmony::interpret::InterpretStatus::SUCCESS => {
-                                    progress::success(section, progress_bar, outcome.message)
+                                    progress_tracker.finish_task(&task_key, &outcome.message);
                                 }
                                 harmony::interpret::InterpretStatus::NOOP => {
-                                    progress::skip(section, progress_bar, outcome.message)
+                                    progress_tracker.skip_task(&task_key, &outcome.message);
                                 }
-                                _ => progress::error(section, progress_bar, outcome.message),
+                                _ => progress_tracker.fail_task(&task_key, &outcome.message),
                             },
                             Err(err) => {
-                                progress::error(section, progress_bar, err.to_string());
+                                error!("Interpret error: {err}");
+                                progress_tracker.fail_task(&task_key, &err.to_string());
                             }
                         }
+                    }
+                    HarmonyEvent::ApplicationFeatureStateChanged {
+                        topology: _,
+                        application,
+                        feature,
+                        status,
+                    } => {
+                        if let Some(score) = &(*current_score) {
+                            let section_key = score_key(score);
+                            let task_key = app_feature_key(&application, &feature);
 
-                        if !has_topology {
-                            (*progress_bars).remove(&section_key);
+                            match status {
+                                ApplicationFeatureStatus::Installing => {
+                                    let message = format!("Feature '{}' installing...", feature);
+                                    progress_tracker.add_task(&section_key, &task_key, &message);
+                                }
+                                ApplicationFeatureStatus::Installed => {
+                                    let message = format!("Feature '{}' installed", feature);
+                                    progress_tracker.finish_task(&task_key, &message);
+                                }
+                                ApplicationFeatureStatus::Failed { details } => {
+                                    let message = format!(
+                                        "Feature '{}' installation failed: {}",
+                                        feature, details
+                                    );
+                                    progress_tracker.fail_task(&task_key, &message);
+                                }
+                            }
                         }
                     }
                 }
@@ -182,6 +213,6 @@ fn score_key(score: &str) -> String {
     format!("score-{score}")
 }
 
-fn interpret_key(interpret: &str) -> String {
-    format!("interpret-{interpret}")
+fn app_feature_key(application: &str, feature: &str) -> String {
+    format!("app-{application}-{feature}")
 }

harmony_cli/src/lib.rs

@@ -1,5 +1,6 @@
 use clap::Parser;
 use clap::builder::ArgPredicate;
+use harmony::instrumentation;
 use harmony::inventory::Inventory;
 use harmony::maestro::Maestro;
 use harmony::{score::Score, topology::Topology};
@@ -97,6 +98,7 @@ pub async fn run<T: Topology + Send + Sync + 'static>(
 
     let result = init(maestro, args_struct).await;
 
+    instrumentation::instrument(instrumentation::HarmonyEvent::HarmonyFinished).unwrap();
     let _ = tokio::try_join!(cli_logger_handle);
     result
 }
@@ -130,8 +132,9 @@ async fn init<T: Topology + Send + Sync + 'static>(
 
     // if list option is specified, print filtered list and exit
     if args.list {
-        println!("Available scores:");
-        println!("{}", list_scores_with_index(&scores_vec));
+        let num_scores = scores_vec.len();
+        println!("Available scores {num_scores}:");
+        println!("{}\n\n", list_scores_with_index(&scores_vec));
         return Ok(());
     }
 
@@ -163,7 +166,7 @@ async fn init<T: Topology + Send + Sync + 'static>(
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use harmony::{
         inventory::Inventory,
         maestro::Maestro,

harmony_cli/src/progress.rs

@@ -1,50 +1,147 @@
+use indicatif::{MultiProgress, ProgressBar};
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use indicatif::{MultiProgress, ProgressBar};
-
-pub fn new_section(title: String) -> MultiProgress {
-    let multi_progress = MultiProgress::new();
-    let _ = multi_progress.println(title);
-
-    multi_progress
+pub trait ProgressTracker: Send + Sync {
+    fn contains_section(&self, id: &str) -> bool;
+    fn add_section(&self, id: &str, message: &str);
+    fn add_task(&self, section_id: &str, task_id: &str, message: &str);
+    fn finish_task(&self, id: &str, message: &str);
+    fn fail_task(&self, id: &str, message: &str);
+    fn skip_task(&self, id: &str, message: &str);
+    fn clear(&self);
 }
 
-pub fn add_spinner(multi_progress: &MultiProgress, message: String) -> ProgressBar {
-    let progress = multi_progress.add(ProgressBar::new_spinner());
-
-    progress.set_style(crate::theme::SPINNER_STYLE.clone());
-    progress.set_message(message);
-    progress.enable_steady_tick(Duration::from_millis(100));
-
-    progress
+struct Section {
+    header_index: usize,
+    task_count: usize,
+    pb: ProgressBar,
 }
 
-pub fn success(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
-    if let Some(progress) = progress {
-        multi_progress.remove(&progress)
+struct IndicatifProgressTrackerState {
+    sections: HashMap<String, Section>,
+    tasks: HashMap<String, ProgressBar>,
+    pb_count: usize,
+}
+
+#[derive(Clone)]
+pub struct IndicatifProgressTracker {
+    mp: MultiProgress,
+    state: Arc<Mutex<IndicatifProgressTrackerState>>,
+}
+
+impl IndicatifProgressTracker {
+    pub fn new(base: MultiProgress) -> Self {
+        let sections = HashMap::new();
+        let tasks = HashMap::new();
+
+        let state = Arc::new(Mutex::new(IndicatifProgressTrackerState {
+            sections,
+            tasks,
+            pb_count: 0,
+        }));
+
+        Self { mp: base, state }
+    }
+}
+
+impl ProgressTracker for IndicatifProgressTracker {
+    fn add_section(&self, id: &str, message: &str) {
+        let mut state = self.state.lock().unwrap();
+
+        let header_pb = self
+            .mp
+            .add(ProgressBar::new(1).with_style(crate::theme::SECTION_STYLE.clone()));
+        header_pb.finish_with_message(message.to_string());
+
+        let header_index = state.pb_count;
+        state.pb_count += 1;
+
+        state.sections.insert(
+            id.to_string(),
+            Section {
+                header_index,
+                task_count: 0,
+                pb: header_pb,
+            },
+        );
     }
 
-    let progress = multi_progress.add(ProgressBar::new_spinner());
-    progress.set_style(crate::theme::SUCCESS_SPINNER_STYLE.clone());
-    progress.finish_with_message(message);
-}
+    fn add_task(&self, section_id: &str, task_id: &str, message: &str) {
+        let mut state = self.state.lock().unwrap();
 
-pub fn error(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
-    if let Some(progress) = progress {
-        multi_progress.remove(&progress)
+        let insertion_index = {
+            let current_section = state
+                .sections
+                .get(section_id)
+                .expect("Section ID not found");
+            current_section.header_index + current_section.task_count + 1 // +1 to insert after header
+        };
+
+        let pb = self.mp.insert(insertion_index, ProgressBar::new_spinner());
+        pb.set_style(crate::theme::SPINNER_STYLE.clone());
+        pb.set_prefix("  ");
+        pb.set_message(message.to_string());
+        pb.enable_steady_tick(Duration::from_millis(80));
+
+        state.pb_count += 1;
+
+        let section = state
+            .sections
+            .get_mut(section_id)
+            .expect("Section ID not found");
+        section.task_count += 1;
+
+        // We inserted a new progress bar, so we must update the header_index
+        // for all subsequent sections.
+        for (id, s) in state.sections.iter_mut() {
+            if id != section_id && s.header_index >= insertion_index {
+                s.header_index += 1;
+            }
+        }
+
+        state.tasks.insert(task_id.to_string(), pb);
     }
 
-    let progress = multi_progress.add(ProgressBar::new_spinner());
-    progress.set_style(crate::theme::ERROR_SPINNER_STYLE.clone());
-    progress.finish_with_message(message);
-}
-
-pub fn skip(multi_progress: &MultiProgress, progress: Option<ProgressBar>, message: String) {
-    if let Some(progress) = progress {
-        multi_progress.remove(&progress)
+    fn finish_task(&self, id: &str, message: &str) {
+        let state = self.state.lock().unwrap();
+        if let Some(pb) = state.tasks.get(id) {
+            pb.set_style(crate::theme::SUCCESS_SPINNER_STYLE.clone());
+            pb.finish_with_message(message.to_string());
+        }
     }
 
-    let progress = multi_progress.add(ProgressBar::new_spinner());
-    progress.set_style(crate::theme::SKIP_SPINNER_STYLE.clone());
-    progress.finish_with_message(message);
+    fn fail_task(&self, id: &str, message: &str) {
+        let state = self.state.lock().unwrap();
+        if let Some(pb) = state.tasks.get(id) {
+            pb.set_style(crate::theme::ERROR_SPINNER_STYLE.clone());
+            pb.finish_with_message(message.to_string());
+        }
+    }
+
+    fn skip_task(&self, id: &str, message: &str) {
+        let state = self.state.lock().unwrap();
+        if let Some(pb) = state.tasks.get(id) {
+            pb.set_style(crate::theme::SKIP_SPINNER_STYLE.clone());
+            pb.finish_with_message(message.to_string());
+        }
+    }
+
+    fn contains_section(&self, id: &str) -> bool {
+        let state = self.state.lock().unwrap();
+        state.sections.contains_key(id)
+    }
+
+    fn clear(&self) {
+        let mut state = self.state.lock().unwrap();
+
+        state.tasks.values().for_each(|p| self.mp.remove(p));
+        state.tasks.clear();
+        state.sections.values().for_each(|s| self.mp.remove(&s.pb));
+        state.sections.clear();
+        state.pb_count = 0;
+
+        let _ = self.mp.clear();
+    }
 }

harmony_cli/src/theme.rs

@@ -11,17 +11,24 @@ pub static EMOJI_TOPOLOGY: Emoji<'_, '_> = Emoji("📦", "");
 pub static EMOJI_SCORE: Emoji<'_, '_> = Emoji("🎶", "");
 
 lazy_static! {
+    pub static ref SECTION_STYLE: ProgressStyle = ProgressStyle::default_spinner()
+        .template("{wide_msg:.bold}")
+        .unwrap();
     pub static ref SPINNER_STYLE: ProgressStyle = ProgressStyle::default_spinner()
-        .template("    {spinner:.green} {msg}")
+        .template("    {spinner:.green} {wide_msg}")
         .unwrap()
         .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]);
     pub static ref SUCCESS_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
         .clone()
         .tick_strings(&[format!("{}", EMOJI_SUCCESS).as_str()]);
-    pub static ref SKIP_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
+    pub static ref SKIP_SPINNER_STYLE: ProgressStyle = ProgressStyle::default_spinner()
+        .template("    {spinner:.orange} {wide_msg}")
+        .unwrap()
         .clone()
         .tick_strings(&[format!("{}", EMOJI_SKIP).as_str()]);
-    pub static ref ERROR_SPINNER_STYLE: ProgressStyle = SPINNER_STYLE
+    pub static ref ERROR_SPINNER_STYLE: ProgressStyle = ProgressStyle::default_spinner()
+        .template("    {spinner:.red} {wide_msg}")
+        .unwrap()
         .clone()
         .tick_strings(&[format!("{}", EMOJI_ERROR).as_str()]);
 }

harmony_composer/src/harmony_composer_logger.rs

@@ -1,10 +1,6 @@
-use indicatif::{MultiProgress, ProgressBar};
-use indicatif_log_bridge::LogWrapper;
-use log::error;
-use std::{
-    collections::HashMap,
-    sync::{Arc, Mutex},
-};
+use harmony_cli::progress::{IndicatifProgressTracker, ProgressTracker};
+use indicatif::MultiProgress;
+use std::sync::Arc;
 
 use crate::instrumentation::{self, HarmonyComposerEvent};
 
@@ -22,85 +18,59 @@ pub fn init() -> tokio::task::JoinHandle<()> {
 }
 
 fn configure_logger() {
-    let logger =
-        env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).build();
-    let level = logger.filter();
-    let multi = MultiProgress::new();
-    LogWrapper::new(multi.clone(), logger).try_init().unwrap();
-    log::set_max_level(level);
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).build();
 }
 
 pub async fn handle_events() {
-    const PROGRESS_SETUP: &str = "project-initialization";
+    let progress_tracker = Arc::new(IndicatifProgressTracker::new(MultiProgress::new()));
+
+    const SETUP_SECTION: &str = "project-initialization";
+    const COMPILTATION_TASK: &str = "compilation";
     const PROGRESS_DEPLOYMENT: &str = "deployment";
 
     instrumentation::subscribe("Harmony Composer Logger", {
-        let progresses: Arc<Mutex<HashMap<String, MultiProgress>>> =
-            Arc::new(Mutex::new(HashMap::new()));
-        let compilation_progress = Arc::new(Mutex::new(None::<ProgressBar>));
-
         move |event| {
-            let progresses_clone = Arc::clone(&progresses);
-            let compilation_progress_clone = Arc::clone(&compilation_progress);
+            let progress_tracker = Arc::clone(&progress_tracker);
 
             async move {
-                let mut progresses_guard = progresses_clone.lock().unwrap();
-                let mut compilation_progress_guard = compilation_progress_clone.lock().unwrap();
-
                 match event {
                     HarmonyComposerEvent::HarmonyComposerStarted => {}
                     HarmonyComposerEvent::ProjectInitializationStarted => {
-                        let multi_progress = harmony_cli::progress::new_section(format!(
-                            "{} Initializing Harmony project...",
-                            harmony_cli::theme::EMOJI_HARMONY,
-                        ));
-                        (*progresses_guard).insert(PROGRESS_SETUP.to_string(), multi_progress);
+                        progress_tracker.add_section(
+                            SETUP_SECTION,
+                            &format!(
+                                "{} Initializing Harmony project...",
+                                harmony_cli::theme::EMOJI_HARMONY,
+                            ),
+                        );
                     }
-                    HarmonyComposerEvent::ProjectInitialized => println!("\n"),
+                    HarmonyComposerEvent::ProjectInitialized => {}
                     HarmonyComposerEvent::ProjectCompilationStarted { details } => {
-                        let initialization_progress =
-                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
-                        let _ = initialization_progress.clear();
-
-                        let progress =
-                            harmony_cli::progress::add_spinner(initialization_progress, details);
-                        *compilation_progress_guard = Some(progress);
+                        progress_tracker.add_task(SETUP_SECTION, COMPILTATION_TASK, &details);
                     }
                     HarmonyComposerEvent::ProjectCompiled => {
-                        let initialization_progress =
-                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
-
-                        harmony_cli::progress::success(
-                            initialization_progress,
-                            (*compilation_progress_guard).take(),
-                            "project compiled".to_string(),
-                        );
+                        progress_tracker.finish_task(COMPILTATION_TASK, "project compiled");
                     }
                     HarmonyComposerEvent::ProjectCompilationFailed { details } => {
-                        let initialization_progress =
-                            (*progresses_guard).get(PROGRESS_SETUP).unwrap();
-
-                        harmony_cli::progress::error(
-                            initialization_progress,
-                            (*compilation_progress_guard).take(),
-                            "failed to compile project".to_string(),
+                        progress_tracker.fail_task(COMPILTATION_TASK, &format!("failed to compile project:\n{details}"));
+                    }
+                    HarmonyComposerEvent::DeploymentStarted { target, profile } => {
+                        progress_tracker.add_section(
+                            PROGRESS_DEPLOYMENT,
+                            &format!(
+                                "\n{} Deploying project on target '{target}' with profile '{profile}'...\n",
+                                harmony_cli::theme::EMOJI_DEPLOY,
+                            ),
                         );
-
-                        error!("{details}");
                     }
-                    HarmonyComposerEvent::DeploymentStarted { target } => {
-                        let multi_progress = harmony_cli::progress::new_section(format!(
-                            "{} Starting deployment to {target}...\n\n",
-                            harmony_cli::theme::EMOJI_DEPLOY
-                        ));
-                        (*progresses_guard).insert(PROGRESS_DEPLOYMENT.to_string(), multi_progress);
+                    HarmonyComposerEvent::DeploymentCompleted => {
+                        progress_tracker.clear();
                     }
-                    HarmonyComposerEvent::DeploymentCompleted => println!("\n"),
+                    HarmonyComposerEvent::DeploymentFailed { details } => {
+                        progress_tracker.add_task(PROGRESS_DEPLOYMENT, "deployment-failed", "");
+                        progress_tracker.fail_task("deployment-failed", &details);
+                    },
                     HarmonyComposerEvent::Shutdown => {
-                        for (_, progresses) in (*progresses_guard).iter() {
-                            progresses.clear().unwrap();
-                        }
-
                         return false;
                     }
                 }

harmony_composer/src/instrumentation.rs

@@ -2,16 +2,28 @@ use log::debug;
 use once_cell::sync::Lazy;
 use tokio::sync::broadcast;
 
+use crate::{HarmonyProfile, HarmonyTarget};
+
 #[derive(Debug, Clone)]
 pub enum HarmonyComposerEvent {
     HarmonyComposerStarted,
     ProjectInitializationStarted,
     ProjectInitialized,
-    ProjectCompilationStarted { details: String },
+    ProjectCompilationStarted {
+        details: String,
+    },
     ProjectCompiled,
-    ProjectCompilationFailed { details: String },
-    DeploymentStarted { target: String },
+    ProjectCompilationFailed {
+        details: String,
+    },
+    DeploymentStarted {
+        target: HarmonyTarget,
+        profile: HarmonyProfile,
+    },
     DeploymentCompleted,
+    DeploymentFailed {
+        details: String,
+    },
     Shutdown,
 }
 
@@ -23,9 +35,18 @@ static HARMONY_COMPOSER_EVENT_BUS: Lazy<broadcast::Sender<HarmonyComposerEvent>>
     });
 
 pub fn instrument(event: HarmonyComposerEvent) -> Result<(), &'static str> {
-    match HARMONY_COMPOSER_EVENT_BUS.send(event) {
-        Ok(_) => Ok(()),
-        Err(_) => Err("send error: no subscribers"),
+    #[cfg(not(test))]
+    {
+        match HARMONY_COMPOSER_EVENT_BUS.send(event) {
+            Ok(_) => Ok(()),
+            Err(_) => Err("send error: no subscribers"),
+        }
+    }
+
+    #[cfg(test)]
+    {
+        let _ = event; // Suppress the "unused variable" warning for `event`
+        Ok(())
     }
 }
 

harmony_composer/src/main.rs

@@ -49,14 +49,11 @@ struct CheckArgs {
 
 #[derive(Args, Clone, Debug)]
 struct DeployArgs {
-    #[arg(long, default_value_t = false)]
-    staging: bool,
+    #[arg(long = "target", short = 't', default_value = "local")]
+    harmony_target: HarmonyTarget,
 
-    #[arg(long, default_value_t = false)]
-    prod: bool,
-
-    #[arg(long, default_value_t = false)]
-    smoke_test: bool,
+    #[arg(long = "profile", short = 'p', default_value = "dev")]
+    harmony_profile: HarmonyProfile,
 }
 
 #[derive(Args, Clone, Debug)]
@@ -68,6 +65,38 @@ struct AllArgs {
     deploy: DeployArgs,
 }
 
+#[derive(Clone, Debug, clap::ValueEnum)]
+enum HarmonyTarget {
+    Local,
+    Remote,
+}
+
+impl std::fmt::Display for HarmonyTarget {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            HarmonyTarget::Local => f.write_str("local"),
+            HarmonyTarget::Remote => f.write_str("remote"),
+        }
+    }
+}
+
+#[derive(Clone, Debug, clap::ValueEnum)]
+enum HarmonyProfile {
+    Dev,
+    Staging,
+    Production,
+}
+
+impl std::fmt::Display for HarmonyProfile {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            HarmonyProfile::Dev => f.write_str("dev"),
+            HarmonyProfile::Staging => f.write_str("staging"),
+            HarmonyProfile::Production => f.write_str("production"),
+        }
+    }
+}
+
 #[tokio::main]
 async fn main() {
     let hc_logger_handle = harmony_composer_logger::init();
@@ -122,26 +151,39 @@ async fn main() {
                 );
             }
             Commands::Deploy(args) => {
-                let deploy = if args.staging {
-                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
-                        target: "staging".to_string(),
-                    })
-                    .unwrap();
-                    todo!("implement staging deployment")
-                } else if args.prod {
-                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
-                        target: "prod".to_string(),
-                    })
-                    .unwrap();
-                    todo!("implement prod deployment")
-                } else {
-                    instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
-                        target: "dev".to_string(),
-                    })
-                    .unwrap();
-                    Command::new(harmony_bin_path).arg("-y").arg("-a").spawn()
+                instrumentation::instrument(HarmonyComposerEvent::DeploymentStarted {
+                    target: args.harmony_target.clone(),
+                    profile: args.harmony_profile.clone(),
+                })
+                .unwrap();
+
+                if matches!(args.harmony_profile, HarmonyProfile::Dev)
+                    && !matches!(args.harmony_target, HarmonyTarget::Local)
+                {
+                    instrumentation::instrument(HarmonyComposerEvent::DeploymentFailed {
+                        details: format!(
+                            "Cannot run profile '{}' on target '{}'. Profile '{}' can run locally only.",
+                            args.harmony_profile, args.harmony_target, args.harmony_profile
+                        ),
+                    }).unwrap();
+                    return;
                 }
-                .expect("failed to run harmony deploy");
+
+                let use_local_k3d = match args.harmony_target {
+                    HarmonyTarget::Local => true,
+                    HarmonyTarget::Remote => false,
+                };
+
+                let mut command = Command::new(harmony_bin_path);
+                command
+                    .env("HARMONY_USE_LOCAL_K3D", format!("{use_local_k3d}"))
+                    .env("HARMONY_PROFILE", format!("{}", args.harmony_profile))
+                    .arg("-y")
+                    .arg("-a");
+
+                info!("{:?}", command);
+
+                let deploy = command.spawn().expect("failed to run harmony deploy");
 
                 let deploy_output = deploy.wait_with_output().unwrap();
                 debug!("{}", String::from_utf8(deploy_output.stdout).unwrap());