feat: K8s Tenant looks good, basic isolation working now #56
| @ -10,8 +10,8 @@ use harmony::{ | ||||
| async fn main() { | ||||
|     let tenant = TenantScore { | ||||
|         config: TenantConfig { | ||||
|             id: Id::default(), | ||||
|             name: "TestTenant".to_string(), | ||||
|             id: Id::from_str("test-tenant-id"), | ||||
|             name: "testtenant".to_string(), | ||||
|             ..Default::default() | ||||
|         }, | ||||
|     }; | ||||
|  | ||||
| @ -27,6 +27,10 @@ impl Id { | ||||
|     pub fn from_string(value: String) -> Self { | ||||
|         Self { value } | ||||
|     } | ||||
| 
 | ||||
|     pub fn from_str(value: &str) -> Self { | ||||
|         Self::from_string(value.to_string()) | ||||
|     } | ||||
| } | ||||
| 
 | ||||
| impl std::fmt::Display for Id { | ||||
|  | ||||
| @ -31,7 +31,10 @@ impl K8sClient { | ||||
|             resource.meta().name, | ||||
|             ns | ||||
|         ); | ||||
|         trace!("{:#?}", serde_json::to_string(resource)); | ||||
|         trace!( | ||||
|             "{:#}", | ||||
|             serde_json::to_value(resource).unwrap_or(serde_json::Value::Null) | ||||
|         ); | ||||
| 
 | ||||
|         let api: Api<K> = <<K as Resource>::Scope as ApplyStrategy<K>>::get_api(&self.client, ns); | ||||
|         api.create(&PostParams::default(), &resource).await | ||||
|  | ||||
| @ -257,30 +257,4 @@ impl TenantManager for K8sAnywhereTopology { | ||||
|             .provision_tenant(config) | ||||
|             .await | ||||
|     } | ||||
| 
 | ||||
|     async fn update_tenant_resource_limits( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_limits: &ResourceLimits, | ||||
|     ) -> Result<(), ExecutorError> { | ||||
|         self.get_k8s_tenant_manager()? | ||||
|             .update_tenant_resource_limits(tenant_id, new_limits) | ||||
|             .await | ||||
|     } | ||||
| 
 | ||||
|     async fn update_tenant_network_policy( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_policy: &TenantNetworkPolicy, | ||||
|     ) -> Result<(), ExecutorError> { | ||||
|         self.get_k8s_tenant_manager()? | ||||
|             .update_tenant_network_policy(tenant_id, new_policy) | ||||
|             .await | ||||
|     } | ||||
| 
 | ||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { | ||||
|         self.get_k8s_tenant_manager()? | ||||
|             .deprovision_tenant(tenant_id) | ||||
|             .await | ||||
|     } | ||||
| } | ||||
|  | ||||
| @ -1,20 +1,21 @@ | ||||
| use std::sync::Arc; | ||||
| 
 | ||||
| use crate::{data::Id, executors::ExecutorError, topology::k8s::K8sClient}; | ||||
| use crate::{ | ||||
|     executors::ExecutorError, | ||||
|     topology::k8s::{ApplyStrategy, K8sClient}, | ||||
| }; | ||||
| use async_trait::async_trait; | ||||
| use derive_new::new; | ||||
| use k8s_openapi::{ | ||||
|     NamespaceResourceScope, | ||||
|     api::{ | ||||
|         core::v1::{Namespace, ResourceQuota}, | ||||
|         networking::v1::NetworkPolicy, | ||||
|     }, | ||||
| use k8s_openapi::api::{ | ||||
|     core::v1::{Namespace, ResourceQuota}, | ||||
|     networking::v1::NetworkPolicy, | ||||
| }; | ||||
| use kube::Resource; | ||||
| use log::{debug, info, warn}; | ||||
| use serde::de::DeserializeOwned; | ||||
| use serde_json::json; | ||||
| 
 | ||||
| use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; | ||||
| use super::{TenantConfig, TenantManager}; | ||||
| 
 | ||||
| #[derive(new)] | ||||
| pub struct K8sTenantManager { | ||||
| @ -26,21 +27,40 @@ impl K8sTenantManager { | ||||
|         config.name.clone() | ||||
|     } | ||||
| 
 | ||||
|     fn ensure_constraints(&self, namespace: &Namespace) -> Result<(), ExecutorError> { | ||||
|         todo!("Validate that when tenant already exists (by id) that name has not changed"); | ||||
|         todo!("Make sure other Tenant constraints are respected by this k8s implementation"); | ||||
|     fn ensure_constraints(&self, _namespace: &Namespace) -> Result<(), ExecutorError> { | ||||
|         warn!("Validate that when tenant already exists (by id) that name has not changed"); | ||||
|         warn!("Make sure other Tenant constraints are respected by this k8s implementation"); | ||||
|         Ok(()) | ||||
|     } | ||||
| 
 | ||||
|     async fn apply_resource< | ||||
|         K: Resource + std::fmt::Debug + Sync + DeserializeOwned + Default + serde::Serialize + Clone, | ||||
|     >( | ||||
|         &self, | ||||
|         resource: K, | ||||
|         mut resource: K, | ||||
|         config: &TenantConfig, | ||||
|     ) -> Result<K, ExecutorError> | ||||
|     where | ||||
|         <K as kube::Resource>::DynamicType: Default, | ||||
|         <K as kube::Resource>::Scope: ApplyStrategy<K>, | ||||
|     { | ||||
|         todo!("Apply tenant labels on resource and apply resource with k8s client properly") | ||||
|         self.apply_labels(&mut resource, config); | ||||
|         self.k8s_client | ||||
|             .apply(&resource, Some(&self.get_namespace_name(config))) | ||||
|             .await | ||||
|             .map_err(|e| { | ||||
|                 ExecutorError::UnexpectedError(format!("Could not create Tenant resource : {e}")) | ||||
|             }) | ||||
|     } | ||||
| 
 | ||||
|     fn apply_labels<K: Resource>(&self, resource: &mut K, config: &TenantConfig) { | ||||
|         let labels = resource.meta_mut().labels.get_or_insert_default(); | ||||
|         labels.insert( | ||||
|             "app.kubernetes.io/managed-by".to_string(), | ||||
|             "harmony".to_string(), | ||||
|         ); | ||||
|         labels.insert("harmony/tenant-id".to_string(), config.id.to_string()); | ||||
|         labels.insert("harmony/tenant-name".to_string(), config.name.clone()); | ||||
|     } | ||||
| 
 | ||||
|     fn build_namespace(&self, config: &TenantConfig) -> Result<Namespace, ExecutorError> { | ||||
| @ -50,7 +70,7 @@ impl K8sTenantManager { | ||||
|                 "kind": "Namespace", | ||||
|                 "metadata": { | ||||
|                     "labels": { | ||||
|                         "harmony.nationtech.io/tenant.id": config.id, | ||||
|                         "harmony.nationtech.io/tenant.id": config.id.to_string(), | ||||
|                         "harmony.nationtech.io/tenant.name": config.name, | ||||
|                     }, | ||||
|                 "name": self.get_namespace_name(config), | ||||
| @ -68,40 +88,35 @@ impl K8sTenantManager { | ||||
|     fn build_resource_quota(&self, config: &TenantConfig) -> Result<ResourceQuota, ExecutorError> { | ||||
|         let resource_quota = json!( | ||||
|          { | ||||
|            "apiVersion": "v1", | ||||
|            "kind": "List", | ||||
|            "items": [ | ||||
|              { | ||||
|                "apiVersion": "v1", | ||||
|                "kind": "ResourceQuota", | ||||
|                "metadata": { | ||||
|                  "name": config.name, | ||||
|                  "labels": { | ||||
|                   "harmony.nationtech.io/tenant.id": config.id, | ||||
|                   "harmony.nationtech.io/tenant.name": config.name, | ||||
|                  }, | ||||
|                  "namespace": self.get_namespace_name(config), | ||||
|                }, | ||||
|                "spec": { | ||||
|                  "hard": { | ||||
|                    "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), | ||||
|                    "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), | ||||
|                    "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), | ||||
|                    "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), | ||||
|                    "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb), | ||||
|                    "pods": "20", | ||||
|                    "services": "10", | ||||
|                    "configmaps": "30", | ||||
|                    "secrets": "30", | ||||
|                    "persistentvolumeclaims": "15", | ||||
|                    "services.loadbalancers": "2", | ||||
|                    "services.nodeports": "5", | ||||
|           "apiVersion": "v1", | ||||
|           "kind": "ResourceQuota", | ||||
|           "metadata": { | ||||
|             "name": format!("{}-quota", config.name), | ||||
|             "labels": { | ||||
|              "harmony.nationtech.io/tenant.id": config.id.to_string(), | ||||
|              "harmony.nationtech.io/tenant.name": config.name, | ||||
|             }, | ||||
|             "namespace": self.get_namespace_name(config), | ||||
|           }, | ||||
|           "spec": { | ||||
|             "hard": { | ||||
|               "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), | ||||
|               "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), | ||||
|               "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), | ||||
|               "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), | ||||
|               "requests.storage": format!("{:.3}Gi", config.resource_limits.storage_total_gb), | ||||
|               "pods": "20", | ||||
|               "services": "10", | ||||
|               "configmaps": "30", | ||||
|               "secrets": "30", | ||||
|               "persistentvolumeclaims": "15", | ||||
|               "services.loadbalancers": "2", | ||||
|               "services.nodeports": "5", | ||||
|               "limits.ephemeral-storage": "10Gi", | ||||
| 
 | ||||
|                  } | ||||
|                } | ||||
|              } | ||||
|            ] | ||||
|          } | ||||
|             } | ||||
|           } | ||||
|         } | ||||
| 
 | ||||
|         ); | ||||
|         serde_json::from_value(resource_quota).map_err(|e| { | ||||
| @ -193,29 +208,20 @@ impl TenantManager for K8sTenantManager { | ||||
|         let network_policy = self.build_network_policy(config)?; | ||||
| 
 | ||||
|         self.ensure_constraints(&namespace)?; | ||||
|         self.apply_resource(namespace).await?; | ||||
|         self.apply_resource(resource_quota).await?; | ||||
|         self.apply_resource(network_policy).await?; | ||||
|         todo!(); | ||||
|     } | ||||
| 
 | ||||
|     async fn update_tenant_resource_limits( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_limits: &ResourceLimits, | ||||
|     ) -> Result<(), ExecutorError> { | ||||
|         todo!() | ||||
|     } | ||||
|         debug!("Creating namespace for tenant {}", config.name); | ||||
|         self.apply_resource(namespace, config).await?; | ||||
| 
 | ||||
|     async fn update_tenant_network_policy( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_policy: &TenantNetworkPolicy, | ||||
|     ) -> Result<(), ExecutorError> { | ||||
|         todo!() | ||||
|     } | ||||
|         debug!("Creating resource_quota for tenant {}", config.name); | ||||
|         self.apply_resource(resource_quota, config).await?; | ||||
| 
 | ||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { | ||||
|         todo!() | ||||
|         debug!("Creating network_policy for tenant {}", config.name); | ||||
|         self.apply_resource(network_policy, config).await?; | ||||
| 
 | ||||
|         info!( | ||||
|             "Success provisionning K8s tenant id {} name {}", | ||||
|             config.id, config.name | ||||
|         ); | ||||
|         Ok(()) | ||||
|     } | ||||
| } | ||||
|  | ||||
| @ -5,31 +5,14 @@ use crate::executors::ExecutorError; | ||||
| 
 | ||||
| #[async_trait] | ||||
| pub trait TenantManager { | ||||
|     /// Provisions a new tenant based on the provided configuration.
 | ||||
|     /// This operation should be idempotent; if a tenant with the same `config.name`
 | ||||
|     /// Creates or update tenant based on the provided configuration.
 | ||||
|     /// This operation should be idempotent; if a tenant with the same `config.id`
 | ||||
|     /// already exists and matches the config, it will succeed without changes.
 | ||||
|     ///
 | ||||
|     /// If it exists but differs, it will be updated, or return an error if the update
 | ||||
|     /// action is not supported
 | ||||
|     ///
 | ||||
|     /// # Arguments
 | ||||
|     /// * `config`: The desired configuration for the new tenant.
 | ||||
|     async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError>; | ||||
| 
 | ||||
|     /// Updates the resource limits for an existing tenant.
 | ||||
|     async fn update_tenant_resource_limits( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_limits: &ResourceLimits, | ||||
|     ) -> Result<(), ExecutorError>; | ||||
| 
 | ||||
|     /// Updates the high-level network isolation policy for an existing tenant.
 | ||||
|     async fn update_tenant_network_policy( | ||||
|         &self, | ||||
|         tenant_id: &Id, | ||||
|         new_policy: &TenantNetworkPolicy, | ||||
|     ) -> Result<(), ExecutorError>; | ||||
| 
 | ||||
|     /// Decommissions an existing tenant, removing its isolated context and associated resources.
 | ||||
|     /// This operation should be idempotent.
 | ||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError>; | ||||
| } | ||||
|  | ||||
| @ -3,8 +3,6 @@ mod manager; | ||||
| pub use manager::*; | ||||
| use serde::{Deserialize, Serialize}; | ||||
| 
 | ||||
| use std::collections::HashMap; | ||||
| 
 | ||||
| use crate::data::Id; | ||||
| 
 | ||||
| #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] // Assuming serde for Scores
 | ||||
| @ -21,10 +19,6 @@ pub struct TenantConfig { | ||||
| 
 | ||||
|     /// High-level network isolation policies for the tenant.
 | ||||
|     pub network_policy: TenantNetworkPolicy, | ||||
| 
 | ||||
|     /// Key-value pairs for provider-specific tagging, labeling, or metadata.
 | ||||
|     /// Useful for billing, organization, or filtering within the provider's console.
 | ||||
|     pub labels_or_tags: HashMap<String, String>, | ||||
| } | ||||
| 
 | ||||
| impl Default for TenantConfig { | ||||
| @ -44,7 +38,6 @@ impl Default for TenantConfig { | ||||
|                 default_inter_tenant_ingress: InterTenantIngressPolicy::DenyAll, | ||||
|                 default_internet_egress: InternetEgressPolicy::AllowAll, | ||||
|             }, | ||||
|             labels_or_tags: HashMap::new(), | ||||
|         } | ||||
|     } | ||||
| } | ||||
|  | ||||
		Loading…
	
		Reference in New Issue
	
	Block a user