From 624e4330bb540fba62c5fc22d5083f51e63babed Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 13:36:30 -0400 Subject: [PATCH 01/16] boilerplate --- harmony/src/domain/topology/k8s_anywhere.rs | 86 ++++++++++++++++++- harmony/src/domain/topology/tenant/manager.rs | 2 +- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index bcd95bc..9153b73 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -6,6 +6,7 @@ use log::{info, warn}; use tokio::sync::OnceCell; use crate::{ + executors::ExecutorError, interpret::{InterpretError, Outcome}, inventory::Inventory, maestro::Maestro, @@ -13,7 +14,11 @@ use crate::{ topology::LocalhostTopology, }; -use super::{HelmCommand, K8sclient, Topology, k8s::K8sClient}; +use super::{ + HelmCommand, K8sclient, Topology, + k8s::K8sClient, + tenant::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, +}; struct K8sState { client: Arc, @@ -21,6 +26,7 @@ struct K8sState { message: String, } +#[derive(Debug)] enum K8sSource { LocalK3d, Kubeconfig, @@ -209,3 +215,81 @@ impl Topology for K8sAnywhereTopology { } impl HelmCommand for K8sAnywhereTopology {} + +impl TenantManager for K8sAnywhereTopology { + fn provision_tenant<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 TenantConfig, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn update_tenant_resource_limits<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + new_limits: &'life2 ResourceLimits, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn update_tenant_network_policy<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + new_policy: &'life2 TenantNetworkPolicy, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + todo!() + } + + fn deprovision_tenant<'life0, 'life1, 'async_trait>( + &'life0 self, + tenant_name: &'life1 str, + ) -> ::core::pin::Pin< + Box< + dyn ::core::future::Future> + + ::core::marker::Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + todo!() + } +} diff --git a/harmony/src/domain/topology/tenant/manager.rs b/harmony/src/domain/topology/tenant/manager.rs index b1b7eb3..4166261 100644 --- a/harmony/src/domain/topology/tenant/manager.rs +++ b/harmony/src/domain/topology/tenant/manager.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use crate::executors::ExecutorError; #[async_trait] -pub trait TenantManager: Send + Sync + std::fmt::Debug { +pub trait TenantManager { /// Provisions a new tenant based on the provided configuration. /// This operation should be idempotent; if a tenant with the same `config.name` /// already exists and matches the config, it will succeed without changes. From 97fba07f4e717955f61ded06ad0d1607cc26ea53 Mon Sep 17 00:00:00 2001 From: Willem Date: Thu, 29 May 2025 14:35:58 -0400 Subject: [PATCH 02/16] feat: adding kubernetes implentation of tenant manager --- harmony/src/domain/topology/k8s_anywhere.rs | 91 +++++---------------- harmony/src/domain/topology/tenant/k8s.rs | 88 ++++++++++++++++++++ harmony/src/domain/topology/tenant/mod.rs | 11 +-- 3 files changed, 114 insertions(+), 76 deletions(-) create mode 100644 harmony/src/domain/topology/tenant/k8s.rs diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 9153b73..7743cae 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -15,9 +15,7 @@ use crate::{ }; use super::{ - HelmCommand, K8sclient, Topology, - k8s::K8sClient, - tenant::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, + k8s::K8sClient, tenant::{k8s::K8sTenantManager, ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, HelmCommand, K8sclient, Topology }; struct K8sState { @@ -34,8 +32,10 @@ enum K8sSource { pub struct K8sAnywhereTopology { k8s_state: OnceCell>, + tenant_manager: K8sTenantManager, } + #[async_trait] impl K8sclient for K8sAnywhereTopology { async fn k8s_client(&self) -> Result, String> { @@ -216,80 +216,29 @@ impl Topology for K8sAnywhereTopology { impl HelmCommand for K8sAnywhereTopology {} +#[async_trait] impl TenantManager for K8sAnywhereTopology { - fn provision_tenant<'life0, 'life1, 'async_trait>( - &'life0 self, - config: &'life1 TenantConfig, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { + self.tenant_manager.provision_tenant(config).await } - fn update_tenant_resource_limits<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - new_limits: &'life2 ResourceLimits, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn update_tenant_resource_limits( + &self, + tenant_name: &str, + new_limits: &ResourceLimits, + ) -> Result<(), ExecutorError> { + self.tenant_manager.update_tenant_resource_limits(tenant_name, new_limits).await } - fn update_tenant_network_policy<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - new_policy: &'life2 TenantNetworkPolicy, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn update_tenant_network_policy( + &self, + tenant_name: &str, + new_policy: &TenantNetworkPolicy, + ) -> Result<(), ExecutorError> { + self.tenant_manager.update_tenant_network_policy(tenant_name, new_policy).await } - fn deprovision_tenant<'life0, 'life1, 'async_trait>( - &'life0 self, - tenant_name: &'life1 str, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() + async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + self.tenant_manager.deprovision_tenant(tenant_name).await } } diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs new file mode 100644 index 0000000..24837a0 --- /dev/null +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use crate::{executors::ExecutorError, topology::k8s::K8sClient}; +use async_trait::async_trait; +use k8s_openapi::api::core::v1::Namespace; +use serde_json::json; + +use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; + +pub struct K8sTenantManager { + k8s_client: Arc, +} + +#[async_trait] +impl TenantManager for K8sTenantManager { + async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { + let namespace = json!( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "labels": { + "harmony.nationtech.io/tenant.id": config.id, + "name": config.name, + + }, + "name": config.name, + }, + } + ); + todo!("Validate that when tenant already exists (by id) that name has not changed"); + + let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + + + let resource_quota = json!( + { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": config.name + }, + "spec": { + "hard": { + "cpu": config.resource_limits.cpu_limit_cores, + "memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), + }, + "scopeSelector": { + "matchExpressions": [ + { + "operator": "In", + "scopeName": "PriorityClass", + "values": ["high"] + } + ] + } + } + } + ] + } + + ); + } + + async fn update_tenant_resource_limits( + &self, + tenant_name: &str, + new_limits: &ResourceLimits, + ) -> Result<(), ExecutorError> { + todo!() + } + + async fn update_tenant_network_policy( + &self, + tenant_name: &str, + new_policy: &TenantNetworkPolicy, + ) -> Result<(), ExecutorError> { + todo!() + } + + async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + todo!() + } +} diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index 0704a34..dc016e7 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -1,4 +1,5 @@ mod manager; +pub mod k8s; pub use manager::*; use serde::{Deserialize, Serialize}; @@ -29,17 +30,17 @@ pub struct TenantConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct ResourceLimits { /// Requested/guaranteed CPU cores (e.g., 2.0). - pub cpu_request_cores: Option, + pub cpu_request_cores: f32, /// Maximum CPU cores the tenant can burst to (e.g., 4.0). - pub cpu_limit_cores: Option, + pub cpu_limit_cores: f32, /// Requested/guaranteed memory in Gigabytes (e.g., 8.0). - pub memory_request_gb: Option, + pub memory_request_gb: f32, /// Maximum memory in Gigabytes tenant can burst to (e.g., 16.0). - pub memory_limit_gb: Option, + pub memory_limit_gb: f32, /// Total persistent storage allocation in Gigabytes across all volumes. - pub storage_total_gb: Option, + pub storage_total_gb: f32, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] From 5e51f7490cb8282d100ae23d380efd5694447c2e Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 15:41:57 -0400 Subject: [PATCH 03/16] Update request quota --- harmony/src/domain/topology/tenant/k8s.rs | 67 +++++++++++------------ 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 24837a0..08ea7a4 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -21,49 +21,46 @@ impl TenantManager for K8sTenantManager { "metadata": { "labels": { "harmony.nationtech.io/tenant.id": config.id, - "name": config.name, - + "harmony.nationtech.io/tenant.name": config.name, }, "name": config.name, }, } ); - todo!("Validate that when tenant already exists (by id) that name has not changed"); + todo!("Validate that when tenant already exists (by id) that name has not changed"); - let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + let resource_quota = json!( + { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": config.name, + "labels": { + "harmony.nationtech.io/tenant.id": config.id, + "harmony.nationtech.io/tenant.name": config.name, + }, + "namespace": config.name, + }, + "spec": { + "hard": { + "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), + "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), + "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), + "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), + "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb) + } + } + } + ] + } - let resource_quota = json!( - { - "apiVersion": "v1", - "kind": "List", - "items": [ - { - "apiVersion": "v1", - "kind": "ResourceQuota", - "metadata": { - "name": config.name - }, - "spec": { - "hard": { - "cpu": config.resource_limits.cpu_limit_cores, - "memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), - }, - "scopeSelector": { - "matchExpressions": [ - { - "operator": "In", - "scopeName": "PriorityClass", - "values": ["high"] - } - ] - } - } - } - ] - } - - ); + ); } async fn update_tenant_resource_limits( From 6490e5e82a840abae553d78ceb46d125683dd47b Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 15:49:46 -0400 Subject: [PATCH 04/16] Hardcode some limits to protect the overall cluster --- harmony/src/domain/topology/tenant/k8s.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 08ea7a4..bf817c5 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -53,7 +53,15 @@ impl TenantManager for K8sTenantManager { "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), - "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb) + "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb), + "pods": "20", + "services": "10", + "configmaps": "30", + "secrets": "30", + "persistentvolumeclaims": "15", + "services.loadbalancers": "2", + "services.nodeports": "5", + } } } From 7c809bf18a8b8a65c42567c7042b46e43b301d35 Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 16:03:58 -0400 Subject: [PATCH 05/16] Make k8stenantmanager a oncecell --- harmony/src/domain/topology/k8s_anywhere.rs | 37 ++++++++++++++++----- harmony/src/domain/topology/tenant/k8s.rs | 2 ++ harmony/src/domain/topology/tenant/mod.rs | 2 +- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 7743cae..369f030 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -1,4 +1,4 @@ -use std::{process::Command, sync::Arc}; +use std::{io::Error, process::Command, sync::Arc}; use async_trait::async_trait; use inquire::Confirm; @@ -15,7 +15,11 @@ use crate::{ }; use super::{ - k8s::K8sClient, tenant::{k8s::K8sTenantManager, ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}, HelmCommand, K8sclient, Topology + HelmCommand, K8sclient, Topology, + k8s::K8sClient, + tenant::{ + ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy, k8s::K8sTenantManager, + }, }; struct K8sState { @@ -32,10 +36,9 @@ enum K8sSource { pub struct K8sAnywhereTopology { k8s_state: OnceCell>, - tenant_manager: K8sTenantManager, + tenant_manager: OnceCell, } - #[async_trait] impl K8sclient for K8sAnywhereTopology { async fn k8s_client(&self) -> Result, String> { @@ -57,6 +60,7 @@ impl K8sAnywhereTopology { pub fn new() -> Self { Self { k8s_state: OnceCell::new(), + tenant_manager: OnceCell::new(), } } @@ -165,6 +169,15 @@ impl K8sAnywhereTopology { Ok(Some(state)) } + + fn get_k8s_tenant_manager(&self) -> Result<&K8sTenantManager, ExecutorError> { + match self.tenant_manager.get() { + Some(t) => Ok(t), + None => Err(ExecutorError::UnexpectedError( + "K8sTenantManager not available".to_string(), + )), + } + } } struct K8sAnywhereConfig { @@ -219,7 +232,9 @@ impl HelmCommand for K8sAnywhereTopology {} #[async_trait] impl TenantManager for K8sAnywhereTopology { async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { - self.tenant_manager.provision_tenant(config).await + self.get_k8s_tenant_manager()? + .provision_tenant(config) + .await } async fn update_tenant_resource_limits( @@ -227,7 +242,9 @@ impl TenantManager for K8sAnywhereTopology { tenant_name: &str, new_limits: &ResourceLimits, ) -> Result<(), ExecutorError> { - self.tenant_manager.update_tenant_resource_limits(tenant_name, new_limits).await + self.get_k8s_tenant_manager()? + .update_tenant_resource_limits(tenant_name, new_limits) + .await } async fn update_tenant_network_policy( @@ -235,10 +252,14 @@ impl TenantManager for K8sAnywhereTopology { tenant_name: &str, new_policy: &TenantNetworkPolicy, ) -> Result<(), ExecutorError> { - self.tenant_manager.update_tenant_network_policy(tenant_name, new_policy).await + self.get_k8s_tenant_manager()? + .update_tenant_network_policy(tenant_name, new_policy) + .await } async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { - self.tenant_manager.deprovision_tenant(tenant_name).await + self.get_k8s_tenant_manager()? + .deprovision_tenant(tenant_name) + .await } } diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index bf817c5..88cf712 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -2,11 +2,13 @@ use std::sync::Arc; use crate::{executors::ExecutorError, topology::k8s::K8sClient}; use async_trait::async_trait; +use derive_new::new; use k8s_openapi::api::core::v1::Namespace; use serde_json::json; use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; +#[derive(new)] pub struct K8sTenantManager { k8s_client: Arc, } diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index dc016e7..e1e93a2 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -1,5 +1,5 @@ -mod manager; pub mod k8s; +mod manager; pub use manager::*; use serde::{Deserialize, Serialize}; From 045954f8d363d42757bcd3409ca2eedf513ede45 Mon Sep 17 00:00:00 2001 From: Taha Hawa Date: Thu, 29 May 2025 18:06:16 -0400 Subject: [PATCH 06/16] start network policy --- harmony/src/domain/topology/tenant/k8s.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 88cf712..5feac85 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -71,6 +71,21 @@ impl TenantManager for K8sTenantManager { } ); + + let network_policy = json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": format!("{}-network-policy", config.name), + }, + "spec": { + "podSelector": {}, + "egress": [], + "ingress": [], + "policyTypes": [ + ] + } + }); } async fn update_tenant_resource_limits( From 2ff70db0b1d0bbdc833b0f917759daf8d1677d5d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Jun 2025 13:52:40 -0400 Subject: [PATCH 07/16] wip: Tenant example project --- Cargo.lock | 15 ++++ adr/tenant/NetworkPolicy.yaml | 41 ++++++++++ adr/tenant/TestDeployment.yaml | 95 +++++++++++++++++++++++ harmony/src/domain/data/id.rs | 6 ++ harmony/src/domain/topology/tenant/mod.rs | 22 ++++++ harmony/src/modules/tenant/mod.rs | 2 +- 6 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 adr/tenant/NetworkPolicy.yaml create mode 100644 adr/tenant/TestDeployment.yaml diff --git a/Cargo.lock b/Cargo.lock index 7c721b1..ded4c85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,21 @@ dependencies = [ "url", ] +[[package]] +name = "example-tenant" +version = "0.1.0" +dependencies = [ + "cidr", + "env_logger", + "harmony", + "harmony_cli", + "harmony_macros", + "harmony_types", + "log", + "tokio", + "url", +] + [[package]] name = "example-tui" version = "0.1.0" diff --git a/adr/tenant/NetworkPolicy.yaml b/adr/tenant/NetworkPolicy.yaml new file mode 100644 index 0000000..5bb1c71 --- /dev/null +++ b/adr/tenant/NetworkPolicy.yaml @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: tenant-isolation-policy + namespace: testtenant +spec: + podSelector: {} # Selects all pods in the namespace + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: {} # Allow from all pods in the same namespace + egress: + - to: + - podSelector: {} # Allow to all pods in the same namespace + - to: + - podSelector: {} + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns # Target the openshift-dns namespace + # Note, only opening port 53 is not enough, will have to dig deeper into this one eventually + # ports: + # - protocol: UDP + # port: 53 + # - protocol: TCP + # port: 53 + # Allow egress to public internet only + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 # RFC1918 + - 172.16.0.0/12 # RFC1918 + - 192.168.0.0/16 # RFC1918 + - 169.254.0.0/16 # Link-local + - 127.0.0.0/8 # Loopback + - 224.0.0.0/4 # Multicast + - 240.0.0.0/4 # Reserved + - 100.64.0.0/10 # Carrier-grade NAT + - 0.0.0.0/8 # Reserved diff --git a/adr/tenant/TestDeployment.yaml b/adr/tenant/TestDeployment.yaml new file mode 100644 index 0000000..a075ba8 --- /dev/null +++ b/adr/tenant/TestDeployment.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: testtenant +--- +apiVersion: v1 +kind: Namespace +metadata: + name: testtenant2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-web + namespace: testtenant +spec: + replicas: 1 + selector: + matchLabels: + app: test-web + template: + metadata: + labels: + app: test-web + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-web + namespace: testtenant +spec: + selector: + app: test-web + ports: + - port: 80 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-client + namespace: testtenant +spec: + replicas: 1 + selector: + matchLabels: + app: test-client + template: + metadata: + labels: + app: test-client + spec: + containers: + - name: curl + image: curlimages/curl:latest + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-web + namespace: testtenant2 +spec: + replicas: 1 + selector: + matchLabels: + app: test-web + template: + metadata: + labels: + app: test-web + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-web + namespace: testtenant2 +spec: + selector: + app: test-web + ports: + - port: 80 + targetPort: 8080 diff --git a/harmony/src/domain/data/id.rs b/harmony/src/domain/data/id.rs index e215eb4..b9f9e7a 100644 --- a/harmony/src/domain/data/id.rs +++ b/harmony/src/domain/data/id.rs @@ -16,3 +16,9 @@ impl std::fmt::Display for Id { f.write_str(&self.value) } } + +impl Default for Id { + fn default() -> Self { + todo!() + } +} diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index e1e93a2..4bbefef 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -27,6 +27,28 @@ pub struct TenantConfig { pub labels_or_tags: HashMap, } +impl Default for TenantConfig { + fn default() -> Self { + let id = Id::default(); + Self { + name: format!("tenant_{id}"), + id, + resource_limits: ResourceLimits { + cpu_request_cores: 4.0, + cpu_limit_cores: 4.0, + memory_request_gb: 4.0, + memory_limit_gb: 4.0, + storage_total_gb: 20.0, + }, + network_policy: TenantNetworkPolicy { + default_inter_tenant_ingress: InterTenantIngressPolicy::DenyAll, + default_internet_egress: InternetEgressPolicy::AllowAll, + }, + labels_or_tags: HashMap::new(), + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct ResourceLimits { /// Requested/guaranteed CPU cores (e.g., 2.0). diff --git a/harmony/src/modules/tenant/mod.rs b/harmony/src/modules/tenant/mod.rs index 5ee212c..72412ec 100644 --- a/harmony/src/modules/tenant/mod.rs +++ b/harmony/src/modules/tenant/mod.rs @@ -14,7 +14,7 @@ use crate::{ #[derive(Debug, Serialize, Clone)] pub struct TenantScore { - config: TenantConfig, + pub config: TenantConfig, } impl Score for TenantScore { From 5127f44ab3104c0a62ea26576746cf3a58747249 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Jun 2025 13:56:40 -0400 Subject: [PATCH 08/16] docs: Add note about pod privilege escalation in ADR 011 Tenant --- adr/011-multi-tenant-cluster.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adr/011-multi-tenant-cluster.md b/adr/011-multi-tenant-cluster.md index 73cd824..88fb0ea 100644 --- a/adr/011-multi-tenant-cluster.md +++ b/adr/011-multi-tenant-cluster.md @@ -137,8 +137,9 @@ Our approach addresses both customer and team multi-tenancy requirements: ### Implementation Roadmap 1. **Phase 1**: Implement VPN access and manual tenant provisioning 2. **Phase 2**: Deploy TenantScore automation for namespace, RBAC, and NetworkPolicy management -3. **Phase 3**: Integrate Keycloak for centralized identity management -4. **Phase 4**: Add advanced monitoring and per-tenant observability +4. **Phase 3**: Work on privilege escalation from pods, audit for weaknesses, enforce security policies on pod runtimes +3. **Phase 4**: Integrate Keycloak for centralized identity management +4. **Phase 5**: Add advanced monitoring and per-tenant observability ### TenantScore Structure Preview ```rust From ec17ccc2461aeea308d38b27dc3c732128c125d8 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Jun 2025 13:59:48 -0400 Subject: [PATCH 09/16] feat: Add example-tenant (WIP) --- examples/tenant/Cargo.toml | 18 ++++++++++++++++ examples/tenant/src/main.rs | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 examples/tenant/Cargo.toml create mode 100644 examples/tenant/src/main.rs diff --git a/examples/tenant/Cargo.toml b/examples/tenant/Cargo.toml new file mode 100644 index 0000000..94267da --- /dev/null +++ b/examples/tenant/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "example-tenant" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } +cidr = { workspace = true } +tokio = { workspace = true } +harmony_macros = { path = "../../harmony_macros" } +log = { workspace = true } +env_logger = { workspace = true } +url = { workspace = true } diff --git a/examples/tenant/src/main.rs b/examples/tenant/src/main.rs new file mode 100644 index 0000000..a389ac6 --- /dev/null +++ b/examples/tenant/src/main.rs @@ -0,0 +1,41 @@ +use harmony::{ + data::Id, + inventory::Inventory, + maestro::Maestro, + modules::tenant::TenantScore, + topology::{K8sAnywhereTopology, tenant::TenantConfig}, +}; + +#[tokio::main] +async fn main() { + let tenant = TenantScore { + config: TenantConfig { + id: Id::default(), + name: "TestTenant".to_string(), + ..Default::default() + }, + }; + + let mut maestro = Maestro::::initialize( + Inventory::autoload(), + K8sAnywhereTopology::new(), + ) + .await + .unwrap(); + + maestro.register_all(vec![Box::new(tenant)]); + harmony_cli::init(maestro, None).await.unwrap(); +} + +// TODO write tests +// - Create Tenant with default config mostly, make sure namespace is created +// - deploy sample client/server app with nginx unprivileged and a service +// - exec in the client pod and validate the following +// - can reach internet +// - can reach server pod +// - can resolve dns queries to internet +// - can resolve dns queries to services +// - cannot reach services and pods in other namespaces +// - Create Tenant with specific cpu/ram/storage requests / limits and make sure they are enforced by trying to +// deploy a pod with lower requests/limits (accepted) and higher requests/limits (rejected) +// - Create TenantCredentials and make sure they give only access to the correct tenant From 8e472e4c65dbbb6e4b3851fb60422e25f5e04413 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 8 Jun 2025 21:23:29 -0400 Subject: [PATCH 10/16] feat: Add Default implementation for Harmony Id along with documentation. This Id implementation is optimized for ease of use. Ids are prefixed with the unix epoch and suffixed with 7 alphanumeric characters. But Ids can also contain any String the user wants to pass it --- Cargo.lock | 8 ++++++++ harmony/Cargo.toml | 2 ++ harmony/src/domain/data/id.rs | 31 ++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ded4c85..a2d8c40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1431,6 +1431,7 @@ dependencies = [ "harmony_macros", "harmony_types", "helm-wrapper-rs", + "hex", "http 1.3.1", "inquire", "k3d-rs", @@ -1442,6 +1443,7 @@ dependencies = [ "non-blank-string-rs", "opnsense-config", "opnsense-config-xml", + "rand 0.9.1", "reqwest 0.11.27", "russh", "rust-ipmi", @@ -1566,6 +1568,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hex-literal" version = "0.4.1" diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index fcf69cf..b98ec49 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -6,6 +6,8 @@ readme.workspace = true license.workspace = true [dependencies] +rand = "0.9" +hex = "0.4" libredfish = "0.1.1" reqwest = { version = "0.11", features = ["blocking", "json"] } russh = "0.45.0" diff --git a/harmony/src/domain/data/id.rs b/harmony/src/domain/data/id.rs index b9f9e7a..1f64054 100644 --- a/harmony/src/domain/data/id.rs +++ b/harmony/src/domain/data/id.rs @@ -1,5 +1,23 @@ +use std::time::SystemTime; +use std::time::UNIX_EPOCH; +use rand::distr::Alphanumeric; +use rand::distr::SampleString; + use serde::{Deserialize, Serialize}; +/// A unique identifier designed for ease of use. +/// +/// You can pass it any String to use and Id, or you can use the default format with `Id::default()` +/// +/// The default format looks like this +/// +/// `462d4c_g2COgai` +/// +/// The first part is the unix timesamp in hexadecimal which makes Id easily sorted by creation time. +/// Second part is a serie of 7 random characters. +/// +/// **It is not meant to be very secure or unique**, it is suitable to generate up to 10 000 items per +/// second with a reasonable collision rate of 0,000014 % as calculated by this calculator : https://kevingal.com/apps/collision.html #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Id { value: String, @@ -19,6 +37,17 @@ impl std::fmt::Display for Id { impl Default for Id { fn default() -> Self { - todo!() + let start = SystemTime::now(); + let since_the_epoch = start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + let timestamp = since_the_epoch.as_secs(); + + let hex_timestamp = format!("{:x}", timestamp & 0xffffff); + + let random_part: String = Alphanumeric.sample_string(&mut rand::rng(), 7); + + let value = format!("{}_{}", hex_timestamp, random_part); + Self { value } } } From 14fc4345c1f15959f5cff280634d10d4537cbfcb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 8 Jun 2025 23:24:41 -0400 Subject: [PATCH 11/16] feat: Initialize k8s tenant properly --- harmony/src/domain/data/id.rs | 4 ++-- harmony/src/domain/topology/k8s_anywhere.rs | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/harmony/src/domain/data/id.rs b/harmony/src/domain/data/id.rs index 1f64054..2950324 100644 --- a/harmony/src/domain/data/id.rs +++ b/harmony/src/domain/data/id.rs @@ -1,7 +1,7 @@ -use std::time::SystemTime; -use std::time::UNIX_EPOCH; use rand::distr::Alphanumeric; use rand::distr::SampleString; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; use serde::{Deserialize, Serialize}; diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 369f030..bd2f261 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -170,6 +170,22 @@ impl K8sAnywhereTopology { Ok(Some(state)) } + async fn ensure_k8s_tenant_manager(&self) -> Result<(), String> { + if let Some(_) = self.tenant_manager.get() { + return Ok(()); + } + + self.tenant_manager + .get_or_try_init(async || -> Result { + let k8s_client = self.k8s_client().await?; + Ok(K8sTenantManager::new(k8s_client)) + }) + .await + .unwrap(); + + Ok(()) + } + fn get_k8s_tenant_manager(&self) -> Result<&K8sTenantManager, ExecutorError> { match self.tenant_manager.get() { Some(t) => Ok(t), @@ -217,6 +233,10 @@ impl Topology for K8sAnywhereTopology { "No K8s client could be found or installed".to_string(), ))?; + self.ensure_k8s_tenant_manager() + .await + .map_err(|e| InterpretError::new(e))?; + match self.is_helm_available() { Ok(()) => Ok(Outcome::success(format!( "{} + helm available", From 24e466fadd600a6774561f5a73cbeeab6ad9476e Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 8 Jun 2025 23:51:11 -0400 Subject: [PATCH 12/16] fix: formatting --- harmony/src/domain/data/id.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harmony/src/domain/data/id.rs b/harmony/src/domain/data/id.rs index 1f64054..2950324 100644 --- a/harmony/src/domain/data/id.rs +++ b/harmony/src/domain/data/id.rs @@ -1,7 +1,7 @@ -use std::time::SystemTime; -use std::time::UNIX_EPOCH; use rand::distr::Alphanumeric; use rand::distr::SampleString; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; use serde::{Deserialize, Serialize}; From 00e71b97f67ba1fc5e2af2ebf646f7af85c040fb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 9 Jun 2025 13:39:38 -0400 Subject: [PATCH 13/16] chore: Move ADR helper files into folders with their corresponding ADR number --- .../main_context_prompt.md | 0 adr/{core-abstractions => 003-abstractions}/topology/Cargo.toml | 0 adr/{core-abstractions => 003-abstractions}/topology/src/main.rs | 0 .../topology/src/main_claude37_2.rs | 0 .../topology/src/main_claudev1.rs | 0 .../topology/src/main_gemini25pro.rs | 0 .../topology/src/main_geminifail.rs | 0 .../topology/src/main_right.rs | 0 .../topology/src/main_v1.rs | 0 adr/{core-abstractions => 003-abstractions}/topology2/Cargo.toml | 0 adr/{core-abstractions => 003-abstractions}/topology2/src/main.rs | 0 .../topology2/src/main_capabilities.rs | 0 .../topology2/src/main_v1.rs | 0 .../topology2/src/main_v2.rs | 0 .../topology2/src/main_v4.rs | 0 adr/{tenant => 011-tenant}/NetworkPolicy.yaml | 0 adr/{tenant => 011-tenant}/TestDeployment.yaml | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename adr/{core-abstractions => 003-abstractions}/main_context_prompt.md (100%) rename adr/{core-abstractions => 003-abstractions}/topology/Cargo.toml (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_claude37_2.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_claudev1.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_gemini25pro.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_geminifail.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_right.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology/src/main_v1.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/Cargo.toml (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/src/main.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/src/main_capabilities.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/src/main_v1.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/src/main_v2.rs (100%) rename adr/{core-abstractions => 003-abstractions}/topology2/src/main_v4.rs (100%) rename adr/{tenant => 011-tenant}/NetworkPolicy.yaml (100%) rename adr/{tenant => 011-tenant}/TestDeployment.yaml (100%) diff --git a/adr/core-abstractions/main_context_prompt.md b/adr/003-abstractions/main_context_prompt.md similarity index 100% rename from adr/core-abstractions/main_context_prompt.md rename to adr/003-abstractions/main_context_prompt.md diff --git a/adr/core-abstractions/topology/Cargo.toml b/adr/003-abstractions/topology/Cargo.toml similarity index 100% rename from adr/core-abstractions/topology/Cargo.toml rename to adr/003-abstractions/topology/Cargo.toml diff --git a/adr/core-abstractions/topology/src/main.rs b/adr/003-abstractions/topology/src/main.rs similarity index 100% rename from adr/core-abstractions/topology/src/main.rs rename to adr/003-abstractions/topology/src/main.rs diff --git a/adr/core-abstractions/topology/src/main_claude37_2.rs b/adr/003-abstractions/topology/src/main_claude37_2.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_claude37_2.rs rename to adr/003-abstractions/topology/src/main_claude37_2.rs diff --git a/adr/core-abstractions/topology/src/main_claudev1.rs b/adr/003-abstractions/topology/src/main_claudev1.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_claudev1.rs rename to adr/003-abstractions/topology/src/main_claudev1.rs diff --git a/adr/core-abstractions/topology/src/main_gemini25pro.rs b/adr/003-abstractions/topology/src/main_gemini25pro.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_gemini25pro.rs rename to adr/003-abstractions/topology/src/main_gemini25pro.rs diff --git a/adr/core-abstractions/topology/src/main_geminifail.rs b/adr/003-abstractions/topology/src/main_geminifail.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_geminifail.rs rename to adr/003-abstractions/topology/src/main_geminifail.rs diff --git a/adr/core-abstractions/topology/src/main_right.rs b/adr/003-abstractions/topology/src/main_right.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_right.rs rename to adr/003-abstractions/topology/src/main_right.rs diff --git a/adr/core-abstractions/topology/src/main_v1.rs b/adr/003-abstractions/topology/src/main_v1.rs similarity index 100% rename from adr/core-abstractions/topology/src/main_v1.rs rename to adr/003-abstractions/topology/src/main_v1.rs diff --git a/adr/core-abstractions/topology2/Cargo.toml b/adr/003-abstractions/topology2/Cargo.toml similarity index 100% rename from adr/core-abstractions/topology2/Cargo.toml rename to adr/003-abstractions/topology2/Cargo.toml diff --git a/adr/core-abstractions/topology2/src/main.rs b/adr/003-abstractions/topology2/src/main.rs similarity index 100% rename from adr/core-abstractions/topology2/src/main.rs rename to adr/003-abstractions/topology2/src/main.rs diff --git a/adr/core-abstractions/topology2/src/main_capabilities.rs b/adr/003-abstractions/topology2/src/main_capabilities.rs similarity index 100% rename from adr/core-abstractions/topology2/src/main_capabilities.rs rename to adr/003-abstractions/topology2/src/main_capabilities.rs diff --git a/adr/core-abstractions/topology2/src/main_v1.rs b/adr/003-abstractions/topology2/src/main_v1.rs similarity index 100% rename from adr/core-abstractions/topology2/src/main_v1.rs rename to adr/003-abstractions/topology2/src/main_v1.rs diff --git a/adr/core-abstractions/topology2/src/main_v2.rs b/adr/003-abstractions/topology2/src/main_v2.rs similarity index 100% rename from adr/core-abstractions/topology2/src/main_v2.rs rename to adr/003-abstractions/topology2/src/main_v2.rs diff --git a/adr/core-abstractions/topology2/src/main_v4.rs b/adr/003-abstractions/topology2/src/main_v4.rs similarity index 100% rename from adr/core-abstractions/topology2/src/main_v4.rs rename to adr/003-abstractions/topology2/src/main_v4.rs diff --git a/adr/tenant/NetworkPolicy.yaml b/adr/011-tenant/NetworkPolicy.yaml similarity index 100% rename from adr/tenant/NetworkPolicy.yaml rename to adr/011-tenant/NetworkPolicy.yaml diff --git a/adr/tenant/TestDeployment.yaml b/adr/011-tenant/TestDeployment.yaml similarity index 100% rename from adr/tenant/TestDeployment.yaml rename to adr/011-tenant/TestDeployment.yaml From 8c65aef127398489ff334efa40ab62fd81d3b2e4 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 9 Jun 2025 13:58:40 -0400 Subject: [PATCH 14/16] feat: Can now apply any k8s resource type, both namespaced or cluster scoped --- harmony/src/domain/topology/k8s.rs | 87 ++++++++++++--------- harmony/src/domain/topology/k8s_anywhere.rs | 13 +-- harmony/src/modules/k8s/resource.rs | 4 +- 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index 08868e8..b969cad 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -1,11 +1,11 @@ use derive_new::new; -use k8s_openapi::NamespaceResourceScope; +use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope}; use kube::{ Api, Client, Config, Error, Resource, api::PostParams, config::{KubeConfigOptions, Kubeconfig}, }; -use log::error; +use log::{debug, error, trace}; use serde::de::DeserializeOwned; #[derive(new)] @@ -20,52 +20,31 @@ impl K8sClient { }) } - pub async fn apply_all< - K: Resource - + std::fmt::Debug - + Sync - + DeserializeOwned - + Default - + serde::Serialize - + Clone, - >( - &self, - resource: &Vec, - ) -> Result, kube::Error> + pub async fn apply(&self, resource: &K, ns: Option<&str>) -> Result where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, + ::Scope: ApplyStrategy, ::DynamicType: Default, { - let mut result = vec![]; - for r in resource.iter() { - let api: Api = Api::all(self.client.clone()); - result.push(api.create(&PostParams::default(), &r).await?); - } - Ok(result) + debug!("Applying resource {:?} with ns {:?}", resource.meta().name, ns); + trace!("{:#?}", serde_json::to_string(resource)); + + let api: Api = <::Scope as ApplyStrategy>::get_api(&self.client, ns); + api.create(&PostParams::default(), &resource).await } - pub async fn apply_namespaced( - &self, - resource: &Vec, - ns: Option<&str>, - ) -> Result, Error> + pub async fn apply_many(&self, resource: &Vec, ns: Option<&str>) -> Result, Error> where - K: Resource - + Clone - + std::fmt::Debug - + DeserializeOwned - + serde::Serialize - + Default, + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, + ::Scope: ApplyStrategy, ::DynamicType: Default, { - let mut resources = Vec::new(); + let mut result = Vec::new(); for r in resource.iter() { - let api: Api = match ns { - Some(ns) => Api::namespaced(self.client.clone(), ns), - None => Api::default_namespaced(self.client.clone()), - }; - resources.push(api.create(&PostParams::default(), &r).await?); + result.push(self.apply(r, ns).await?); } - Ok(resources) + + Ok(result) } pub(crate) async fn from_kubeconfig(path: &str) -> Option { @@ -86,3 +65,35 @@ impl K8sClient { )) } } + +pub trait ApplyStrategy { + fn get_api(client: &Client, ns: Option<&str>) -> Api; +} + +/// Implementation for all resources that are cluster-scoped. +/// It will always use `Api::all` and ignore the namespace parameter. +impl ApplyStrategy for ClusterResourceScope +where + K: Resource, + ::DynamicType: Default, +{ + fn get_api(client: &Client, _ns: Option<&str>) -> Api { + Api::all(client.clone()) + } +} + +/// Implementation for all resources that are namespace-scoped. +/// It will use `Api::namespaced` if a namespace is provided, otherwise +/// it falls back to the default namespace configured in your kubeconfig. +impl ApplyStrategy for NamespaceResourceScope +where + K: Resource, + ::DynamicType: Default, +{ + fn get_api(client: &Client, ns: Option<&str>) -> Api { + match ns { + Some(ns) => Api::namespaced(client.clone(), ns), + None => Api::default_namespaced(client.clone()), + } + } +} diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index bd2f261..fd0685d 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -6,6 +6,7 @@ use log::{info, warn}; use tokio::sync::OnceCell; use crate::{ + data::Id, executors::ExecutorError, interpret::{InterpretError, Outcome}, inventory::Inventory, @@ -259,27 +260,27 @@ impl TenantManager for K8sAnywhereTopology { async fn update_tenant_resource_limits( &self, - tenant_name: &str, + tenant_id: &Id, new_limits: &ResourceLimits, ) -> Result<(), ExecutorError> { self.get_k8s_tenant_manager()? - .update_tenant_resource_limits(tenant_name, new_limits) + .update_tenant_resource_limits(tenant_id, new_limits) .await } async fn update_tenant_network_policy( &self, - tenant_name: &str, + tenant_id: &Id, new_policy: &TenantNetworkPolicy, ) -> Result<(), ExecutorError> { self.get_k8s_tenant_manager()? - .update_tenant_network_policy(tenant_name, new_policy) + .update_tenant_network_policy(tenant_id, new_policy) .await } - async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { self.get_k8s_tenant_manager()? - .deprovision_tenant(tenant_name) + .deprovision_tenant(tenant_id) .await } } diff --git a/harmony/src/modules/k8s/resource.rs b/harmony/src/modules/k8s/resource.rs index 6880292..3c0b2bf 100644 --- a/harmony/src/modules/k8s/resource.rs +++ b/harmony/src/modules/k8s/resource.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use k8s_openapi::NamespaceResourceScope; use kube::Resource; +use log::info; use serde::{Serialize, de::DeserializeOwned}; use crate::{ @@ -75,11 +76,12 @@ where _inventory: &Inventory, topology: &T, ) -> Result { + info!("Applying {} resources", self.score.resource.len()); topology .k8s_client() .await .expect("Environment should provide enough information to instanciate a client") - .apply_namespaced(&self.score.resource, self.score.namespace.as_deref()) + .apply_many(&self.score.resource, self.score.namespace.as_deref()) .await?; Ok(Outcome::success( From 6cf61ae67ce70f86e092841725e68fa6758532b9 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 9 Jun 2025 13:59:49 -0400 Subject: [PATCH 15/16] feat: Tenant manager k8s implementation progress : ResourceQuota, NetworkPolicy and Namespace look good. Still WIP --- harmony/src/domain/topology/tenant/k8s.rs | 141 ++++++++++++++++-- harmony/src/domain/topology/tenant/manager.rs | 17 +-- 2 files changed, 129 insertions(+), 29 deletions(-) diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 5feac85..ed51d96 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -1,9 +1,17 @@ use std::sync::Arc; -use crate::{executors::ExecutorError, topology::k8s::K8sClient}; +use crate::{data::Id, executors::ExecutorError, topology::k8s::K8sClient}; use async_trait::async_trait; use derive_new::new; -use k8s_openapi::api::core::v1::Namespace; +use k8s_openapi::{ + NamespaceResourceScope, + api::{ + core::v1::{Namespace, ResourceQuota}, + networking::v1::NetworkPolicy, + }, +}; +use kube::Resource; +use serde::de::DeserializeOwned; use serde_json::json; use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; @@ -13,9 +21,29 @@ pub struct K8sTenantManager { k8s_client: Arc, } -#[async_trait] -impl TenantManager for K8sTenantManager { - async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { +impl K8sTenantManager { + fn get_namespace_name(&self, config: &TenantConfig) -> String { + config.name.clone() + } + + fn ensure_constraints(&self, namespace: &Namespace) -> Result<(), ExecutorError> { + todo!("Validate that when tenant already exists (by id) that name has not changed"); + todo!("Make sure other Tenant constraints are respected by this k8s implementation"); + } + + async fn apply_resource< + K: Resource + std::fmt::Debug + Sync + DeserializeOwned + Default + serde::Serialize + Clone, + >( + &self, + resource: K, + ) -> Result + where + ::DynamicType: Default, + { + todo!("Apply tenant labels on resource and apply resource with k8s client properly") + } + + fn build_namespace(&self, config: &TenantConfig) -> Result { let namespace = json!( { "apiVersion": "v1", @@ -25,14 +53,19 @@ impl TenantManager for K8sTenantManager { "harmony.nationtech.io/tenant.id": config.id, "harmony.nationtech.io/tenant.name": config.name, }, - "name": config.name, + "name": self.get_namespace_name(config), }, } ); - todo!("Validate that when tenant already exists (by id) that name has not changed"); - - let namespace: Namespace = serde_json::from_value(namespace).unwrap(); + serde_json::from_value(namespace).map_err(|e| { + ExecutorError::ConfigurationError(format!( + "Could not build TenantManager Namespace. {}", + e + )) + }) + } + fn build_resource_quota(&self, config: &TenantConfig) -> Result { let resource_quota = json!( { "apiVersion": "v1", @@ -47,7 +80,7 @@ impl TenantManager for K8sTenantManager { "harmony.nationtech.io/tenant.id": config.id, "harmony.nationtech.io/tenant.name": config.name, }, - "namespace": config.name, + "namespace": self.get_namespace_name(config), }, "spec": { "hard": { @@ -71,7 +104,15 @@ impl TenantManager for K8sTenantManager { } ); + serde_json::from_value(resource_quota).map_err(|e| { + ExecutorError::ConfigurationError(format!( + "Could not build TenantManager ResourceQuota. {}", + e + )) + }) + } + fn build_network_policy(&self, config: &TenantConfig) -> Result { let network_policy = json!({ "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", @@ -80,17 +121,87 @@ impl TenantManager for K8sTenantManager { }, "spec": { "podSelector": {}, - "egress": [], - "ingress": [], + "egress": [ + { "to": [ {"podSelector": {}}]}, + { "to": + [ + { + "podSelector": {}, + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name":"openshift-dns" + } + } + }, + ] + }, + { "to": [ + { + "ipBlock": { + + "cidr": "0.0.0.0/0", + // See https://en.wikipedia.org/wiki/Reserved_IP_addresses + "except": [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "192.18.0.0/15", + "198.51.100.0/24", + "169.254.0.0/16", + "203.0.113.0/24", + "127.0.0.0/8", + + // Not sure we should block this one as it is + // used for multicast. But better block more than less. + "224.0.0.0/4", + "240.0.0.0/4", + "100.64.0.0/10", + "233.252.0.0/24", + "0.0.0.0/8", + ], + } + } + ] + }, + ], + "ingress": [ + { "from": [ {"podSelector": {}}]} + ], "policyTypes": [ + "Ingress", "Egress", ] } }); + + serde_json::from_value(network_policy).map_err(|e| { + ExecutorError::ConfigurationError(format!( + "Could not build TenantManager NetworkPolicy. {}", + e + )) + }) + } +} + +#[async_trait] +impl TenantManager for K8sTenantManager { + async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError> { + let namespace = self.build_namespace(config)?; + let resource_quota = self.build_resource_quota(config)?; + let network_policy = self.build_network_policy(config)?; + + self.ensure_constraints(&namespace)?; + self.apply_resource(namespace).await?; + self.apply_resource(resource_quota).await?; + self.apply_resource(network_policy).await?; + todo!(); } async fn update_tenant_resource_limits( &self, - tenant_name: &str, + tenant_id: &Id, new_limits: &ResourceLimits, ) -> Result<(), ExecutorError> { todo!() @@ -98,13 +209,13 @@ impl TenantManager for K8sTenantManager { async fn update_tenant_network_policy( &self, - tenant_name: &str, + tenant_id: &Id, new_policy: &TenantNetworkPolicy, ) -> Result<(), ExecutorError> { todo!() } - async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError> { + async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { todo!() } } diff --git a/harmony/src/domain/topology/tenant/manager.rs b/harmony/src/domain/topology/tenant/manager.rs index 4166261..df042c8 100644 --- a/harmony/src/domain/topology/tenant/manager.rs +++ b/harmony/src/domain/topology/tenant/manager.rs @@ -16,31 +16,20 @@ pub trait TenantManager { async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError>; /// Updates the resource limits for an existing tenant. - /// - /// # Arguments - /// * `tenant_name`: The logical name of the tenant to update. - /// * `new_limits`: The new set of resource limits to apply. async fn update_tenant_resource_limits( &self, - tenant_name: &str, + tenant_id: &Id, new_limits: &ResourceLimits, ) -> Result<(), ExecutorError>; /// Updates the high-level network isolation policy for an existing tenant. - /// - /// # Arguments - /// * `tenant_name`: The logical name of the tenant to update. - /// * `new_policy`: The new network policy to apply. async fn update_tenant_network_policy( &self, - tenant_name: &str, + tenant_id: &Id, new_policy: &TenantNetworkPolicy, ) -> Result<(), ExecutorError>; /// Decommissions an existing tenant, removing its isolated context and associated resources. /// This operation should be idempotent. - /// - /// # Arguments - /// * `tenant_name`: The logical name of the tenant to deprovision. - async fn deprovision_tenant(&self, tenant_name: &str) -> Result<(), ExecutorError>; + async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError>; } From 8d8120bbfd7db9d2e21917c049d24d1d59e2a2c6 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 9 Jun 2025 14:00:46 -0400 Subject: [PATCH 16/16] fix: K8s ingress module was completely broken, fixed resource definition structure and types --- harmony/src/domain/topology/k8s.rs | 6 +++++- harmony/src/modules/k8s/ingress.rs | 26 ++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index b969cad..cfaae1f 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -26,7 +26,11 @@ impl K8sClient { ::Scope: ApplyStrategy, ::DynamicType: Default, { - debug!("Applying resource {:?} with ns {:?}", resource.meta().name, ns); + debug!( + "Applying resource {:?} with ns {:?}", + resource.meta().name, + ns + ); trace!("{:#?}", serde_json::to_string(resource)); let api: Api = <::Scope as ApplyStrategy>::get_api(&self.client, ns); diff --git a/harmony/src/modules/k8s/ingress.rs b/harmony/src/modules/k8s/ingress.rs index 883d721..d07d82f 100644 --- a/harmony/src/modules/k8s/ingress.rs +++ b/harmony/src/modules/k8s/ingress.rs @@ -1,5 +1,6 @@ use harmony_macros::ingress_path; use k8s_openapi::api::networking::v1::Ingress; +use log::{debug, trace}; use serde::Serialize; use serde_json::json; @@ -56,22 +57,24 @@ impl Score for K8sIngressScore { let ingress = json!( { "metadata": { - "name": self.name + "name": self.name.to_string(), }, "spec": { "rules": [ - { "host": self.host, + { "host": self.host.to_string(), "http": { "paths": [ { "path": path, "pathType": path_type.as_str(), - "backend": [ - { - "service": self.backend_service, - "port": self.port + "backend": { + "service": { + "name": self.backend_service.to_string(), + "port": { + "number": self.port, + } } - ] + } } ] } @@ -81,13 +84,16 @@ impl Score for K8sIngressScore { } ); + trace!("Building ingresss object from Value {ingress:#}"); let ingress: Ingress = serde_json::from_value(ingress).unwrap(); + debug!( + "Successfully built Ingress for host {:?}", + ingress.metadata.name + ); Box::new(K8sResourceInterpret { score: K8sResourceScore::single( ingress.clone(), - self.namespace - .clone() - .map(|f| f.as_c_str().to_str().unwrap().to_string()), + self.namespace.clone().map(|f| f.to_string()), ), }) }