From 624e4330bb540fba62c5fc22d5083f51e63babed Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 13:36:30 -0400 Subject: [PATCH 1/5] boilerplate --- harmony/src/domain/topology/k8s_anywhere.rs | 86 ++++++++++++++++++- harmony/src/domain/topology/tenant/manager.rs | 2 +- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index bcd95bc..9153b73 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -6,6 +6,7 @@ use log::{info, warn}; use tokio::sync::OnceCell; use crate::{ + executors::ExecutorError, interpret::{InterpretError, Outcome}, inventory::Inventory, maestro::Maestro, @@ -13,7 +14,11 @@ use crate::{ topology::LocalhostTopology, }; -use super::{HelmCommand, K8sclient, Topology, k8s::K8sClient}; +use super::{ + HelmCommand, K8sclient, Topology, + k8s::K8sClient, + tenant::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, +}; struct K8sState { client: Arc, @@ -21,6 +26,7 @@ struct K8sState { message: String, } +#[derive(Debug)] enum K8sSource { LocalK3d, Kubeconfig, @@ -209,3 +215,81 @@ impl Topology for K8sAnywhereTopology { } impl HelmCommand for K8sAnywhereTopology {} + +impl TenantManager for K8sAnywhereTopology { + fn provision_tenant<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 TenantConfig, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn update_tenant_resource_limits<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + new_limits: &'life2 ResourceLimits, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn update_tenant_network_policy<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + new_policy: &'life2 TenantNetworkPolicy, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn deprovision_tenant<'life0, 'life1, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + todo!() + } +} diff --git a/harmony/src/domain/topology/tenant/manager.rs b/harmony/src/domain/topology/tenant/manager.rs index b1b7eb3..4166261 100644 --- a/harmony/src/domain/topology/tenant/manager.rs +++ b/harmony/src/domain/topology/tenant/manager.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use crate::executors::ExecutorError; #[async_trait] -pub trait TenantManager: Send + Sync + std::fmt::Debug { +pub trait TenantManager { /// Provisions a new tenant based on the provided configuration. /// This operation should be idempotent; if a tenant with the same `config.name` /// already exists and matches the config, it will succeed without changes. -- 2.39.5 From 97fba07f4e717955f61ded06ad0d1607cc26ea53 Mon Sep 17 00:00:00 2001 From: Willem Date: Thu, 29 May 2025 14:35:58 -0400 Subject: [PATCH 2/5] feat: adding kubernetes implentation of tenant manager --- harmony/src/domain/topology/k8s_anywhere.rs | 91 +++++---------------- harmony/src/domain/topology/tenant/k8s.rs | 88 ++++++++++++++++++++ harmony/src/domain/topology/tenant/mod.rs | 11 +-- 3 files changed, 114 insertions(+), 76 deletions(-) create mode 100644 harmony/src/domain/topology/tenant/k8s.rs diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 9153b73..7743cae 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -15,9 +15,7 @@ use crate::{ }; use super::{ - HelmCommand, K8sclient, Topology, - k8s::K8sClient, - tenant::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, + k8s::K8sClient, tenant::{k8s::K8sTenantManager, ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, HelmCommand, K8sclient, Topology }; struct K8sState { @@ -34,8 +32,10 @@ enum K8sSource { pub struct K8sAnywhereTopology { k8s_state: OnceCell>, + tenant_manager: K8sTenantManager, } + #[async_trait] impl K8sclient for K8sAnywhereTopology { async fn k8s_client(&self) -> Result, String> { @@ -216,80 +216,29 @@ impl Topology for K8sAnywhereTopology { impl HelmCommand for K8sAnywhereTopology {} +#[async_trait] impl TenantManager for K8sAnywhereTopology { - fn provision_tenant<'life0, 'life1, 'async_trait>( - &'life0 self, - config: &'life1 TenantConfig, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { + self.tenant_manager.provision_tenant(config).await } - fn update_tenant_resource_limits<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - new_limits: &'life2 ResourceLimits, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn update_tenant_resource_limits( + &self, + tenant_name: &str, + new_limits: &ResourceLimits, + ) -> Result<(), ExecutorError> { + self.tenant_manager.update_tenant_resource_limits(tenant_name, new_limits).await } - fn update_tenant_network_policy<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - new_policy: &'life2 TenantNetworkPolicy, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn update_tenant_network_policy( + &self, + tenant_name: &str, + new_policy: &TenantNetworkPolicy, + ) -> Result<(), ExecutorError> { + self.tenant_manager.update_tenant_network_policy(tenant_name, new_policy).await } - fn deprovision_tenant<'life0, 'life1, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + self.tenant_manager.deprovision_tenant(tenant_name).await } } diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs new file mode 100644 index 0000000..24837a0 --- /dev/null +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use crate::{executors::ExecutorError, topology::k8s::K8sClient}; +use async_trait::async_trait; +use k8s_openapi::api::core::v1::Namespace; +use serde_json::json; + +use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; + +pub struct K8sTenantManager { + k8s_client: Arc, +} + +#[async_trait] +impl TenantManager for K8sTenantManager { + async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { + let namespace = json!( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "labels": { + "harmony.nationtech.io/tenant.id": config.id, + "name": config.name, + + }, + "name": config.name, + }, + } + ); + todo!("Validate that when tenant already exists (by id) that name has not changed"); + + let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + + + let resource_quota = json!( + { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": config.name + }, + "spec": { + "hard": { + "cpu": config.resource_limits.cpu_limit_cores, + "memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), + }, + "scopeSelector": { + "matchExpressions": [ + { + "operator": "In", + "scopeName": "PriorityClass", + "values": ["high"] + } + ] + } + } + } + ] + } + + ); + } + + async fn update_tenant_resource_limits( + &self, + tenant_name: &str, + new_limits: &ResourceLimits, + ) -> Result<(), ExecutorError> { + todo!() + } + + async fn update_tenant_network_policy( + &self, + tenant_name: &str, + new_policy: &TenantNetworkPolicy, + ) -> Result<(), ExecutorError> { + todo!() + } + + async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + todo!() + } +} diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index 0704a34..dc016e7 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -1,4 +1,5 @@ mod manager; +pub mod k8s; pub use manager::*; use serde::{Deserialize, Serialize}; @@ -29,17 +30,17 @@ pub struct TenantConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct ResourceLimits { /// Requested/guaranteed CPU cores (e.g., 2.0). - pub cpu_request_cores: Option, + pub cpu_request_cores: f32, /// Maximum CPU cores the tenant can burst to (e.g., 4.0). - pub cpu_limit_cores: Option, + pub cpu_limit_cores: f32, /// Requested/guaranteed memory in Gigabytes (e.g., 8.0). - pub memory_request_gb: Option, + pub memory_request_gb: f32, /// Maximum memory in Gigabytes tenant can burst to (e.g., 16.0). - pub memory_limit_gb: Option, + pub memory_limit_gb: f32, /// Total persistent storage allocation in Gigabytes across all volumes. - pub storage_total_gb: Option, + pub storage_total_gb: f32, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -- 2.39.5 From 5e51f7490cb8282d100ae23d380efd5694447c2e Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 15:41:57 -0400 Subject: [PATCH 3/5] Update request quota --- harmony/src/domain/topology/tenant/k8s.rs | 67 +++++++++++------------ 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 24837a0..08ea7a4 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -21,49 +21,46 @@ impl TenantManager for K8sTenantManager { "metadata": { "labels": { "harmony.nationtech.io/tenant.id": config.id, - "name": config.name, - + "harmony.nationtech.io/tenant.name": config.name, }, "name": config.name, }, } ); - todo!("Validate that when tenant already exists (by id) that name has not changed"); + todo!("Validate that when tenant already exists (by id) that name has not changed"); - let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + let resource_quota = json!( + { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": config.name, + "labels": { + "harmony.nationtech.io/tenant.id": config.id, + "harmony.nationtech.io/tenant.name": config.name, + }, + "namespace": config.name, + }, + "spec": { + "hard": { + "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), + "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), + "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), + "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), + "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb) + } + } + } + ] + } - let resource_quota = json!( - { - "apiVersion": "v1", - "kind": "List", - "items": [ - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "metadata": { - "name": config.name - }, - "spec": { - "hard": { - "cpu": config.resource_limits.cpu_limit_cores, - "memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), - }, - "scopeSelector": { - "matchExpressions": [ - { - "operator": "In", - "scopeName": "PriorityClass", - "values": ["high"] - } - ] - } - } - } - ] - } - - ); + ); } async fn update_tenant_resource_limits( -- 2.39.5 From 6490e5e82a840abae553d78ceb46d125683dd47b Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 15:49:46 -0400 Subject: [PATCH 4/5] Hardcode some limits to protect the overall cluster --- harmony/src/domain/topology/tenant/k8s.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 08ea7a4..bf817c5 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -53,7 +53,15 @@ impl TenantManager for K8sTenantManager { "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), - "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb) + "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb), + "pods": "20", + "services": "10", + "configmaps": "30", + "secrets": "30", + "persistentvolumeclaims": "15", + "services.loadbalancers": "2", + "services.nodeports": "5", + } } } -- 2.39.5 From 7c809bf18a8b8a65c42567c7042b46e43b301d35 Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 16:03:58 -0400 Subject: [PATCH 5/5] Make k8stenantmanager a oncecell --- harmony/src/domain/topology/k8s_anywhere.rs | 37 ++++++++++++++++----- harmony/src/domain/topology/tenant/k8s.rs | 2 ++ harmony/src/domain/topology/tenant/mod.rs | 2 +- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 7743cae..369f030 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -1,4 +1,4 @@ -use std::{process::Command, sync::Arc}; +use std::{io::Error, process::Command, sync::Arc}; use async_trait::async_trait; use inquire::Confirm; @@ -15,7 +15,11 @@ use crate::{ }; use super::{ - k8s::K8sClient, tenant::{k8s::K8sTenantManager, ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, HelmCommand, K8sclient, Topology + HelmCommand, K8sclient, Topology, + k8s::K8sClient, + tenant::{ + ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy, k8s::K8sTenantManager, + }, }; struct K8sState { @@ -32,10 +36,9 @@ enum K8sSource { pub struct K8sAnywhereTopology { k8s_state: OnceCell>, - tenant_manager: K8sTenantManager, + tenant_manager: OnceCell, } - #[async_trait] impl K8sclient for K8sAnywhereTopology { async fn k8s_client(&self) -> Result, String> { @@ -57,6 +60,7 @@ impl K8sAnywhereTopology { pub fn new() -> Self { Self { k8s_state: OnceCell::new(), + tenant_manager: OnceCell::new(), } } @@ -165,6 +169,15 @@ impl K8sAnywhereTopology { Ok(Some(state)) } + + fn get_k8s_tenant_manager(&self) -> Result<&K8sTenantManager, ExecutorError> { + match self.tenant_manager.get() { + Some(t) => Ok(t), + None => Err(ExecutorError::UnexpectedError( + "K8sTenantManager not available".to_string(), + )), + } + } } struct K8sAnywhereConfig { @@ -219,7 +232,9 @@ impl HelmCommand for K8sAnywhereTopology {} #[async_trait] impl TenantManager for K8sAnywhereTopology { async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { - self.tenant_manager.provision_tenant(config).await + self.get_k8s_tenant_manager()? + .provision_tenant(config) + .await } async fn update_tenant_resource_limits( @@ -227,7 +242,9 @@ impl TenantManager for K8sAnywhereTopology { tenant_name: &str, new_limits: &ResourceLimits, ) -> Result<(), ExecutorError> { - self.tenant_manager.update_tenant_resource_limits(tenant_name, new_limits).await + self.get_k8s_tenant_manager()? + .update_tenant_resource_limits(tenant_name, new_limits) + .await } async fn update_tenant_network_policy( @@ -235,10 +252,14 @@ impl TenantManager for K8sAnywhereTopology { tenant_name: &str, new_policy: &TenantNetworkPolicy, ) -> Result<(), ExecutorError> { - self.tenant_manager.update_tenant_network_policy(tenant_name, new_policy).await + self.get_k8s_tenant_manager()? + .update_tenant_network_policy(tenant_name, new_policy) + .await } async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { - self.tenant_manager.deprovision_tenant(tenant_name).await + self.get_k8s_tenant_manager()? + .deprovision_tenant(tenant_name) + .await } } diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index bf817c5..88cf712 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -2,11 +2,13 @@ use std::sync::Arc; use crate::{executors::ExecutorError, topology::k8s::K8sClient}; use async_trait::async_trait; +use derive_new::new; use k8s_openapi::api::core::v1::Namespace; use serde_json::json; use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; +#[derive(new)] pub struct K8sTenantManager { k8s_client: Arc, } diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index dc016e7..e1e93a2 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -1,5 +1,5 @@ -mod manager; pub mod k8s; +mod manager; pub use manager::*; use serde::{Deserialize, Serialize}; -- 2.39.5