feat: K8s Tenant looks good, basic isolation working now #56
| @ -10,8 +10,8 @@ use harmony::{ | |||||||
| async fn main() { | async fn main() { | ||||||
|     let tenant = TenantScore { |     let tenant = TenantScore { | ||||||
|         config: TenantConfig { |         config: TenantConfig { | ||||||
|             id: Id::default(), |             id: Id::from_str("test-tenant-id"), | ||||||
|             name: "TestTenant".to_string(), |             name: "testtenant".to_string(), | ||||||
|             ..Default::default() |             ..Default::default() | ||||||
|         }, |         }, | ||||||
|     }; |     }; | ||||||
|  | |||||||
| @ -27,6 +27,10 @@ impl Id { | |||||||
|     pub fn from_string(value: String) -> Self { |     pub fn from_string(value: String) -> Self { | ||||||
|         Self { value } |         Self { value } | ||||||
|     } |     } | ||||||
|  | 
 | ||||||
|  |     pub fn from_str(value: &str) -> Self { | ||||||
|  |         Self::from_string(value.to_string()) | ||||||
|  |     } | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| impl std::fmt::Display for Id { | impl std::fmt::Display for Id { | ||||||
|  | |||||||
| @ -31,7 +31,10 @@ impl K8sClient { | |||||||
|             resource.meta().name, |             resource.meta().name, | ||||||
|             ns |             ns | ||||||
|         ); |         ); | ||||||
|         trace!("{:#?}", serde_json::to_string(resource)); |         trace!( | ||||||
|  |             "{:#}", | ||||||
|  |             serde_json::to_value(resource).unwrap_or(serde_json::Value::Null) | ||||||
|  |         ); | ||||||
| 
 | 
 | ||||||
|         let api: Api<K> = <<K as Resource>::Scope as ApplyStrategy<K>>::get_api(&self.client, ns); |         let api: Api<K> = <<K as Resource>::Scope as ApplyStrategy<K>>::get_api(&self.client, ns); | ||||||
|         api.create(&PostParams::default(), &resource).await |         api.create(&PostParams::default(), &resource).await | ||||||
|  | |||||||
| @ -257,30 +257,4 @@ impl TenantManager for K8sAnywhereTopology { | |||||||
|             .provision_tenant(config) |             .provision_tenant(config) | ||||||
|             .await |             .await | ||||||
|     } |     } | ||||||
| 
 |  | ||||||
|     async fn update_tenant_resource_limits( |  | ||||||
|         &self, |  | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_limits: &ResourceLimits, |  | ||||||
|     ) -> Result<(), ExecutorError> { |  | ||||||
|         self.get_k8s_tenant_manager()? |  | ||||||
|             .update_tenant_resource_limits(tenant_id, new_limits) |  | ||||||
|             .await |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     async fn update_tenant_network_policy( |  | ||||||
|         &self, |  | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_policy: &TenantNetworkPolicy, |  | ||||||
|     ) -> Result<(), ExecutorError> { |  | ||||||
|         self.get_k8s_tenant_manager()? |  | ||||||
|             .update_tenant_network_policy(tenant_id, new_policy) |  | ||||||
|             .await |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { |  | ||||||
|         self.get_k8s_tenant_manager()? |  | ||||||
|             .deprovision_tenant(tenant_id) |  | ||||||
|             .await |  | ||||||
|     } |  | ||||||
| } | } | ||||||
|  | |||||||
| @ -1,20 +1,21 @@ | |||||||
| use std::sync::Arc; | use std::sync::Arc; | ||||||
| 
 | 
 | ||||||
| use crate::{data::Id, executors::ExecutorError, topology::k8s::K8sClient}; | use crate::{ | ||||||
|  |     executors::ExecutorError, | ||||||
|  |     topology::k8s::{ApplyStrategy, K8sClient}, | ||||||
|  | }; | ||||||
| use async_trait::async_trait; | use async_trait::async_trait; | ||||||
| use derive_new::new; | use derive_new::new; | ||||||
| use k8s_openapi::{ | use k8s_openapi::api::{ | ||||||
|     NamespaceResourceScope, |     core::v1::{Namespace, ResourceQuota}, | ||||||
|     api::{ |     networking::v1::NetworkPolicy, | ||||||
|         core::v1::{Namespace, ResourceQuota}, |  | ||||||
|         networking::v1::NetworkPolicy, |  | ||||||
|     }, |  | ||||||
| }; | }; | ||||||
| use kube::Resource; | use kube::Resource; | ||||||
|  | use log::{debug, info, warn}; | ||||||
| use serde::de::DeserializeOwned; | use serde::de::DeserializeOwned; | ||||||
| use serde_json::json; | use serde_json::json; | ||||||
| 
 | 
 | ||||||
| use super::{ResourceLimits, TenantConfig, TenantManager, TenantNetworkPolicy}; | use super::{TenantConfig, TenantManager}; | ||||||
| 
 | 
 | ||||||
| #[derive(new)] | #[derive(new)] | ||||||
| pub struct K8sTenantManager { | pub struct K8sTenantManager { | ||||||
| @ -26,21 +27,40 @@ impl K8sTenantManager { | |||||||
|         config.name.clone() |         config.name.clone() | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     fn ensure_constraints(&self, namespace: &Namespace) -> Result<(), ExecutorError> { |     fn ensure_constraints(&self, _namespace: &Namespace) -> Result<(), ExecutorError> { | ||||||
|         todo!("Validate that when tenant already exists (by id) that name has not changed"); |         warn!("Validate that when tenant already exists (by id) that name has not changed"); | ||||||
|         todo!("Make sure other Tenant constraints are respected by this k8s implementation"); |         warn!("Make sure other Tenant constraints are respected by this k8s implementation"); | ||||||
|  |         Ok(()) | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     async fn apply_resource< |     async fn apply_resource< | ||||||
|         K: Resource + std::fmt::Debug + Sync + DeserializeOwned + Default + serde::Serialize + Clone, |         K: Resource + std::fmt::Debug + Sync + DeserializeOwned + Default + serde::Serialize + Clone, | ||||||
|     >( |     >( | ||||||
|         &self, |         &self, | ||||||
|         resource: K, |         mut resource: K, | ||||||
|  |         config: &TenantConfig, | ||||||
|     ) -> Result<K, ExecutorError> |     ) -> Result<K, ExecutorError> | ||||||
|     where |     where | ||||||
|         <K as kube::Resource>::DynamicType: Default, |         <K as kube::Resource>::DynamicType: Default, | ||||||
|  |         <K as kube::Resource>::Scope: ApplyStrategy<K>, | ||||||
|     { |     { | ||||||
|         todo!("Apply tenant labels on resource and apply resource with k8s client properly") |         self.apply_labels(&mut resource, config); | ||||||
|  |         self.k8s_client | ||||||
|  |             .apply(&resource, Some(&self.get_namespace_name(config))) | ||||||
|  |             .await | ||||||
|  |             .map_err(|e| { | ||||||
|  |                 ExecutorError::UnexpectedError(format!("Could not create Tenant resource : {e}")) | ||||||
|  |             }) | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     fn apply_labels<K: Resource>(&self, resource: &mut K, config: &TenantConfig) { | ||||||
|  |         let labels = resource.meta_mut().labels.get_or_insert_default(); | ||||||
|  |         labels.insert( | ||||||
|  |             "app.kubernetes.io/managed-by".to_string(), | ||||||
|  |             "harmony".to_string(), | ||||||
|  |         ); | ||||||
|  |         labels.insert("harmony/tenant-id".to_string(), config.id.to_string()); | ||||||
|  |         labels.insert("harmony/tenant-name".to_string(), config.name.clone()); | ||||||
|     } |     } | ||||||
| 
 | 
 | ||||||
|     fn build_namespace(&self, config: &TenantConfig) -> Result<Namespace, ExecutorError> { |     fn build_namespace(&self, config: &TenantConfig) -> Result<Namespace, ExecutorError> { | ||||||
| @ -50,7 +70,7 @@ impl K8sTenantManager { | |||||||
|                 "kind": "Namespace", |                 "kind": "Namespace", | ||||||
|                 "metadata": { |                 "metadata": { | ||||||
|                     "labels": { |                     "labels": { | ||||||
|                         "harmony.nationtech.io/tenant.id": config.id, |                         "harmony.nationtech.io/tenant.id": config.id.to_string(), | ||||||
|                         "harmony.nationtech.io/tenant.name": config.name, |                         "harmony.nationtech.io/tenant.name": config.name, | ||||||
|                     }, |                     }, | ||||||
|                 "name": self.get_namespace_name(config), |                 "name": self.get_namespace_name(config), | ||||||
| @ -68,40 +88,35 @@ impl K8sTenantManager { | |||||||
|     fn build_resource_quota(&self, config: &TenantConfig) -> Result<ResourceQuota, ExecutorError> { |     fn build_resource_quota(&self, config: &TenantConfig) -> Result<ResourceQuota, ExecutorError> { | ||||||
|         let resource_quota = json!( |         let resource_quota = json!( | ||||||
|          { |          { | ||||||
|            "apiVersion": "v1", |           "apiVersion": "v1", | ||||||
|            "kind": "List", |           "kind": "ResourceQuota", | ||||||
|            "items": [ |           "metadata": { | ||||||
|              { |             "name": format!("{}-quota", config.name), | ||||||
|                "apiVersion": "v1", |             "labels": { | ||||||
|                "kind": "ResourceQuota", |              "harmony.nationtech.io/tenant.id": config.id.to_string(), | ||||||
|                "metadata": { |              "harmony.nationtech.io/tenant.name": config.name, | ||||||
|                  "name": config.name, |             }, | ||||||
|                  "labels": { |             "namespace": self.get_namespace_name(config), | ||||||
|                   "harmony.nationtech.io/tenant.id": config.id, |           }, | ||||||
|                   "harmony.nationtech.io/tenant.name": config.name, |           "spec": { | ||||||
|                  }, |             "hard": { | ||||||
|                  "namespace": self.get_namespace_name(config), |               "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), | ||||||
|                }, |               "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), | ||||||
|                "spec": { |               "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), | ||||||
|                  "hard": { |               "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), | ||||||
|                    "limits.cpu": format!("{:.0}",config.resource_limits.cpu_limit_cores), |               "requests.storage": format!("{:.3}Gi", config.resource_limits.storage_total_gb), | ||||||
|                    "limits.memory": format!("{:.3}Gi", config.resource_limits.memory_limit_gb), |               "pods": "20", | ||||||
|                    "requests.cpu": format!("{:.0}",config.resource_limits.cpu_request_cores), |               "services": "10", | ||||||
|                    "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), |               "configmaps": "30", | ||||||
|                    "requests.storage": format!("{:.3}", config.resource_limits.storage_total_gb), |               "secrets": "30", | ||||||
|                    "pods": "20", |               "persistentvolumeclaims": "15", | ||||||
|                    "services": "10", |               "services.loadbalancers": "2", | ||||||
|                    "configmaps": "30", |               "services.nodeports": "5", | ||||||
|                    "secrets": "30", |               "limits.ephemeral-storage": "10Gi", | ||||||
|                    "persistentvolumeclaims": "15", |  | ||||||
|                    "services.loadbalancers": "2", |  | ||||||
|                    "services.nodeports": "5", |  | ||||||
| 
 | 
 | ||||||
|                  } |             } | ||||||
|                } |           } | ||||||
|              } |         } | ||||||
|            ] |  | ||||||
|          } |  | ||||||
| 
 | 
 | ||||||
|         ); |         ); | ||||||
|         serde_json::from_value(resource_quota).map_err(|e| { |         serde_json::from_value(resource_quota).map_err(|e| { | ||||||
| @ -193,29 +208,20 @@ impl TenantManager for K8sTenantManager { | |||||||
|         let network_policy = self.build_network_policy(config)?; |         let network_policy = self.build_network_policy(config)?; | ||||||
| 
 | 
 | ||||||
|         self.ensure_constraints(&namespace)?; |         self.ensure_constraints(&namespace)?; | ||||||
|         self.apply_resource(namespace).await?; |  | ||||||
|         self.apply_resource(resource_quota).await?; |  | ||||||
|         self.apply_resource(network_policy).await?; |  | ||||||
|         todo!(); |  | ||||||
|     } |  | ||||||
| 
 | 
 | ||||||
|     async fn update_tenant_resource_limits( |         debug!("Creating namespace for tenant {}", config.name); | ||||||
|         &self, |         self.apply_resource(namespace, config).await?; | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_limits: &ResourceLimits, |  | ||||||
|     ) -> Result<(), ExecutorError> { |  | ||||||
|         todo!() |  | ||||||
|     } |  | ||||||
| 
 | 
 | ||||||
|     async fn update_tenant_network_policy( |         debug!("Creating resource_quota for tenant {}", config.name); | ||||||
|         &self, |         self.apply_resource(resource_quota, config).await?; | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_policy: &TenantNetworkPolicy, |  | ||||||
|     ) -> Result<(), ExecutorError> { |  | ||||||
|         todo!() |  | ||||||
|     } |  | ||||||
| 
 | 
 | ||||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError> { |         debug!("Creating network_policy for tenant {}", config.name); | ||||||
|         todo!() |         self.apply_resource(network_policy, config).await?; | ||||||
|  | 
 | ||||||
|  |         info!( | ||||||
|  |             "Success provisionning K8s tenant id {} name {}", | ||||||
|  |             config.id, config.name | ||||||
|  |         ); | ||||||
|  |         Ok(()) | ||||||
|     } |     } | ||||||
| } | } | ||||||
|  | |||||||
| @ -5,31 +5,14 @@ use crate::executors::ExecutorError; | |||||||
| 
 | 
 | ||||||
| #[async_trait] | #[async_trait] | ||||||
| pub trait TenantManager { | pub trait TenantManager { | ||||||
|     /// Provisions a new tenant based on the provided configuration.
 |     /// Creates or update tenant based on the provided configuration.
 | ||||||
|     /// This operation should be idempotent; if a tenant with the same `config.name`
 |     /// This operation should be idempotent; if a tenant with the same `config.id`
 | ||||||
|     /// already exists and matches the config, it will succeed without changes.
 |     /// already exists and matches the config, it will succeed without changes.
 | ||||||
|  |     ///
 | ||||||
|     /// If it exists but differs, it will be updated, or return an error if the update
 |     /// If it exists but differs, it will be updated, or return an error if the update
 | ||||||
|     /// action is not supported
 |     /// action is not supported
 | ||||||
|     ///
 |     ///
 | ||||||
|     /// # Arguments
 |     /// # Arguments
 | ||||||
|     /// * `config`: The desired configuration for the new tenant.
 |     /// * `config`: The desired configuration for the new tenant.
 | ||||||
|     async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError>; |     async fn provision_tenant(&self, config: &TenantConfig) -> Result<(), ExecutorError>; | ||||||
| 
 |  | ||||||
|     /// Updates the resource limits for an existing tenant.
 |  | ||||||
|     async fn update_tenant_resource_limits( |  | ||||||
|         &self, |  | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_limits: &ResourceLimits, |  | ||||||
|     ) -> Result<(), ExecutorError>; |  | ||||||
| 
 |  | ||||||
|     /// Updates the high-level network isolation policy for an existing tenant.
 |  | ||||||
|     async fn update_tenant_network_policy( |  | ||||||
|         &self, |  | ||||||
|         tenant_id: &Id, |  | ||||||
|         new_policy: &TenantNetworkPolicy, |  | ||||||
|     ) -> Result<(), ExecutorError>; |  | ||||||
| 
 |  | ||||||
|     /// Decommissions an existing tenant, removing its isolated context and associated resources.
 |  | ||||||
|     /// This operation should be idempotent.
 |  | ||||||
|     async fn deprovision_tenant(&self, tenant_id: &Id) -> Result<(), ExecutorError>; |  | ||||||
| } | } | ||||||
|  | |||||||
| @ -3,8 +3,6 @@ mod manager; | |||||||
| pub use manager::*; | pub use manager::*; | ||||||
| use serde::{Deserialize, Serialize}; | use serde::{Deserialize, Serialize}; | ||||||
| 
 | 
 | ||||||
| use std::collections::HashMap; |  | ||||||
| 
 |  | ||||||
| use crate::data::Id; | use crate::data::Id; | ||||||
| 
 | 
 | ||||||
| #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] // Assuming serde for Scores
 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] // Assuming serde for Scores
 | ||||||
| @ -21,10 +19,6 @@ pub struct TenantConfig { | |||||||
| 
 | 
 | ||||||
|     /// High-level network isolation policies for the tenant.
 |     /// High-level network isolation policies for the tenant.
 | ||||||
|     pub network_policy: TenantNetworkPolicy, |     pub network_policy: TenantNetworkPolicy, | ||||||
| 
 |  | ||||||
|     /// Key-value pairs for provider-specific tagging, labeling, or metadata.
 |  | ||||||
|     /// Useful for billing, organization, or filtering within the provider's console.
 |  | ||||||
|     pub labels_or_tags: HashMap<String, String>, |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| impl Default for TenantConfig { | impl Default for TenantConfig { | ||||||
| @ -44,7 +38,6 @@ impl Default for TenantConfig { | |||||||
|                 default_inter_tenant_ingress: InterTenantIngressPolicy::DenyAll, |                 default_inter_tenant_ingress: InterTenantIngressPolicy::DenyAll, | ||||||
|                 default_internet_egress: InternetEgressPolicy::AllowAll, |                 default_internet_egress: InternetEgressPolicy::AllowAll, | ||||||
|             }, |             }, | ||||||
|             labels_or_tags: HashMap::new(), |  | ||||||
|         } |         } | ||||||
|     } |     } | ||||||
| } | } | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user