From a0c0905c3bcabc9d46658384175cfd77106eedc3 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Mar 2026 10:56:48 -0500 Subject: [PATCH 1/6] wip: zitadel deployment --- Cargo.lock | 12 + harmony/src/domain/topology/k8s/mod.rs | 16 + .../domain/topology/k8s_anywhere/postgres.rs | 1 - harmony/src/modules/k8s/resource.rs | 4 +- harmony/src/modules/postgresql/cnpg/crd.rs | 8 + harmony/src/modules/postgresql/operator.rs | 6 +- harmony/src/modules/postgresql/score_k8s.rs | 7 + harmony/src/modules/zitadel/mod.rs | 388 +++++++++++++++++- 8 files changed, 426 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31f96f6..7653931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2293,6 +2293,18 @@ dependencies = [ "url", ] +[[package]] +name = "example-zitadel" +version = "0.1.0" +dependencies = [ + "harmony", + "harmony_cli", + "harmony_macros", + "harmony_types", + "tokio", + "url", +] + [[package]] name = "example_validate_ceph_cluster_health" version = "0.1.0" diff --git a/harmony/src/domain/topology/k8s/mod.rs b/harmony/src/domain/topology/k8s/mod.rs index f5171ad..a5d45ce 100644 --- a/harmony/src/domain/topology/k8s/mod.rs +++ b/harmony/src/domain/topology/k8s/mod.rs @@ -103,6 +103,12 @@ pub struct DrainOptions { pub timeout: Duration, } +pub enum WriteMode { + CreateOrUpdate, + Create, + Update, +} + impl Default for DrainOptions { fn default() -> Self { Self { @@ -834,6 +840,16 @@ impl K8sClient { K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, ::DynamicType: Default, { + self.apply_with_strategy(resource, namespace, WriteMode::CreateOrUpdate).await + } + + pub async fn apply_with_strategy(&self, resource: &K, namespace: Option<&str>, apply_strategy: WriteMode) -> Result + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, + ::DynamicType: Default, + { + todo!("Refactoring in progress: Handle the apply_strategy parameter and add utility functions like apply that set it for ease of use (create, update)"); + debug!( "Applying resource {:?} with ns {:?}", resource.meta().name, diff --git a/harmony/src/domain/topology/k8s_anywhere/postgres.rs b/harmony/src/domain/topology/k8s_anywhere/postgres.rs index 2bf800b..76356c6 100644 --- a/harmony/src/domain/topology/k8s_anywhere/postgres.rs +++ b/harmony/src/domain/topology/k8s_anywhere/postgres.rs @@ -1,7 +1,6 @@ use async_trait::async_trait; use crate::{ - interpret::Outcome, inventory::Inventory, modules::postgresql::{ K8sPostgreSQLScore, diff --git a/harmony/src/modules/k8s/resource.rs b/harmony/src/modules/k8s/resource.rs index 83e10a7..2e22f60 100644 --- a/harmony/src/modules/k8s/resource.rs +++ b/harmony/src/modules/k8s/resource.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use k8s_openapi::{NamespaceResourceScope, ResourceScope}; +use k8s_openapi::ResourceScope; use kube::Resource; use log::info; use serde::{Serialize, de::DeserializeOwned}; @@ -109,7 +109,7 @@ where topology .k8s_client() .await - .expect("Environment should provide enough information to instanciate a client") + .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}"))) .apply_many(&self.score.resource, self.score.namespace.as_deref()) .await?; diff --git a/harmony/src/modules/postgresql/cnpg/crd.rs b/harmony/src/modules/postgresql/cnpg/crd.rs index c8f6126..76bbeae 100644 --- a/harmony/src/modules/postgresql/cnpg/crd.rs +++ b/harmony/src/modules/postgresql/cnpg/crd.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use kube::{CustomResource, api::ObjectMeta}; use serde::{Deserialize, Serialize}; @@ -16,6 +18,10 @@ pub struct ClusterSpec { pub image_name: Option, pub storage: Storage, pub bootstrap: Bootstrap, + /// This must be set to None if you want cnpg to generate a superuser secret + #[serde(skip_serializing_if = "Option::is_none")] + pub superuser_secret: Option>, + pub enable_superuser_access: bool, } impl Default for Cluster { @@ -34,6 +40,8 @@ impl Default for ClusterSpec { image_name: None, storage: Storage::default(), bootstrap: Bootstrap::default(), + superuser_secret: None, + enable_superuser_access: false, } } } diff --git a/harmony/src/modules/postgresql/operator.rs b/harmony/src/modules/postgresql/operator.rs index d908361..6298ddb 100644 --- a/harmony/src/modules/postgresql/operator.rs +++ b/harmony/src/modules/postgresql/operator.rs @@ -52,8 +52,8 @@ pub struct CloudNativePgOperatorScore { pub source_namespace: String, } -impl Default for CloudNativePgOperatorScore { - fn default() -> Self { +impl CloudNativePgOperatorScore { + fn default_openshift() -> Self { Self { namespace: "openshift-operators".to_string(), channel: "stable-v1".to_string(), @@ -68,7 +68,7 @@ impl CloudNativePgOperatorScore { pub fn new(namespace: &str) -> Self { Self { namespace: namespace.to_string(), - ..Default::default() + ..Self::default_openshift() } } } diff --git a/harmony/src/modules/postgresql/score_k8s.rs b/harmony/src/modules/postgresql/score_k8s.rs index 5e3cb08..5c4b834 100644 --- a/harmony/src/modules/postgresql/score_k8s.rs +++ b/harmony/src/modules/postgresql/score_k8s.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use serde::Serialize; use crate::interpret::Interpret; @@ -66,6 +68,11 @@ impl Score for K8sPostgreSQLScore { owner: "app".to_string(), }, }, + // superuser_secret: Some(BTreeMap::from([( + // "name".to_string(), + // format!("{}-superuser", self.config.cluster_name.clone()), + // )])), + enable_superuser_access: true, ..ClusterSpec::default() }; diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 1bd8a05..f4ebcc0 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -1,43 +1,387 @@ +use base64::{Engine, prelude::BASE64_STANDARD}; +use rand::{thread_rng, Rng}; +use rand::distributions::Alphanumeric; +use k8s_openapi::api::core::v1::Namespace; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use k8s_openapi::{ByteString, api::core::v1::Secret}; +use std::collections::BTreeMap; use std::str::FromStr; +use async_trait::async_trait; use harmony_macros::hurl; +use harmony_types::id::Id; +use harmony_types::storage::StorageSize; +use log::{debug, error, info, trace, warn}; use non_blank_string_rs::NonBlankString; use serde::Serialize; use crate::{ - interpret::Interpret, + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, modules::helm::chart::{HelmChartScore, HelmRepository}, + modules::k8s::resource::K8sResourceScore, + modules::postgresql::capability::{PostgreSQL, PostgreSQLClusterRole, PostgreSQLConfig}, score::Score, topology::{HelmCommand, K8sclient, Topology}, }; +const NAMESPACE: &str = "zitadel"; +const PG_CLUSTER_NAME: &str = "zitadel-pg"; +const MASTERKEY_SECRET_NAME: &str = "zitadel-masterkey"; + +/// Opinionated Zitadel deployment score. +/// +/// Deploys a PostgreSQL cluster (via the [`PostgreSQL`] trait) and the Zitadel +/// Helm chart into the same namespace. Intended as a central multi-tenant IdP +/// with SSO for OKD/OpenShift, OpenBao, Harbor, Grafana, Nextcloud, Ente +/// Photos, and others. +/// +/// # Ingress annotations +/// No controller-specific ingress annotations are set. The Zitadel service +/// already carries the Traefik h2c annotation for k3s/k3d by default. +/// Add annotations via `values_overrides` depending on your distribution: +/// - NGINX: `nginx.ingress.kubernetes.io/backend-protocol: GRPC` +/// - OpenShift HAProxy: `haproxy.router.openshift.io/*` or use OpenShift Routes +/// - AWS ALB: set `ingress.controller: aws` +/// +/// # Database credentials +/// CNPG creates a `-superuser` secret with key `password`. Because +/// `envVarsSecret` injects secret keys verbatim as env var names and the CNPG +/// key (`password`) does not match ZITADEL's expected name +/// (`ZITADEL_DATABASE_POSTGRES_USER_PASSWORD`), individual `env` entries with +/// `valueFrom.secretKeyRef` are used instead. For environments with an +/// External Secrets Operator or similar, create a dedicated secret with the +/// correct ZITADEL env var names and switch to `envVarsSecret`. #[derive(Debug, Serialize, Clone)] pub struct ZitadelScore { - /// Host used for external access (ingress) + /// External domain (e.g. `"auth.example.com"`). pub host: String, } -impl Score for ZitadelScore { +impl Score for ZitadelScore { fn name(&self) -> String { "ZitadelScore".to_string() } #[doc(hidden)] fn create_interpret(&self) -> Box> { - // TODO exec pod commands to initialize secret store if not already done + Box::new(ZitadelInterpret { + host: self.host.clone(), + }) + } +} + +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +struct ZitadelInterpret { + host: String, +} + +#[async_trait] +impl Interpret for ZitadelInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + info!( + "[Zitadel] Starting full deployment — namespace: '{NAMESPACE}', host: '{}'", + self.host + ); + + info!("Creating namespace {NAMESPACE} if it does not exist"); + K8sResourceScore::single( + Namespace { + metadata: ObjectMeta { + name: Some(NAMESPACE.to_string()), + ..Default::default() + }, + ..Default::default() + }, + None, + ) + .interpret(inventory, topology) + .await?; + + // --- Step 1: PostgreSQL ------------------------------------------- + + let pg_config = PostgreSQLConfig { + cluster_name: PG_CLUSTER_NAME.to_string(), + instances: 2, + storage_size: StorageSize::gi(10), + role: PostgreSQLClusterRole::Primary, + namespace: NAMESPACE.to_string(), + }; + + debug!( + "[Zitadel] Deploying PostgreSQL cluster '{}' — instances: {}, storage: 10Gi, namespace: '{}'", + pg_config.cluster_name, pg_config.instances, pg_config.namespace + ); + + topology.deploy(&pg_config).await.map_err(|e| { + let msg = format!( + "[Zitadel] PostgreSQL deployment failed for '{}': {e}", + pg_config.cluster_name + ); + error!("{msg}"); + InterpretError::new(msg) + })?; + + info!( + "[Zitadel] PostgreSQL cluster '{}' deployed", + pg_config.cluster_name + ); + + // --- Step 2: Resolve internal DB endpoint ------------------------- + + debug!( + "[Zitadel] Resolving internal endpoint for cluster '{}'", + pg_config.cluster_name + ); + + let endpoint = topology.get_endpoint(&pg_config).await.map_err(|e| { + let msg = format!( + "[Zitadel] Failed to resolve endpoint for cluster '{}': {e}", + pg_config.cluster_name + ); + error!("{msg}"); + InterpretError::new(msg) + })?; + + info!( + "[Zitadel] DB endpoint resolved — host: '{}', port: {}", + endpoint.host, endpoint.port + ); + + // The CNPG-managed superuser secret contains 'password', 'username', + // 'host', 'port', 'dbname', 'uri'. We reference 'password' directly + // via env.valueFrom.secretKeyRef because CNPG's key names do not + // match ZITADEL's required env var names. + let pg_user_secret = format!("{PG_CLUSTER_NAME}-app"); + let pg_superuser_secret = format!("{PG_CLUSTER_NAME}-superuser"); + let db_host = &endpoint.host; + let db_port = endpoint.port; let host = &self.host; - let values_yaml = Some(format!(r#""#)); + debug!( + "[Zitadel] DB credentials source — secret: '{pg_user_secret}', key: 'password'" + ); + debug!( + "[Zitadel] DB credentials source — superuser secret: '{pg_superuser_secret}', key: 'password'" + ); - todo!("This is not complete yet"); + // --- Step 3: Create masterkey secret ------------------------------------ - HelmChartScore { - namespace: Some(NonBlankString::from_str("zitadel").unwrap()), + debug!( + "[Zitadel] Creating masterkey secret '{}' in namespace '{}'", + MASTERKEY_SECRET_NAME, NAMESPACE + ); + + + // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes. + let masterkey: String = thread_rng() + .sample_iter(&Alphanumeric) + .take(32) + .map(char::from) + .collect(); + let masterkey_bytes = BASE64_STANDARD.encode(&masterkey); + + let mut masterkey_data: BTreeMap = BTreeMap::new(); + masterkey_data.insert("masterkey".to_string(), ByteString(masterkey_bytes.into())); + + let masterkey_secret = Secret { + metadata: ObjectMeta { + name: Some(MASTERKEY_SECRET_NAME.to_string()), + namespace: Some(NAMESPACE.to_string()), + ..ObjectMeta::default() + }, + data: Some(masterkey_data), + ..Secret::default() + }; + + topology + .k8s_client() + .await + .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}"))) + .create(masterkey_secret) + .await?; + + K8sResourceScore::single(masterkey_secret, Some(NAMESPACE.to_string())) + .interpret(inventory, topology) + .await + .map_err(|e| { + let msg = format!("[Zitadel] Failed to create masterkey secret: {e}"); + error!("{msg}"); + InterpretError::new(msg) + })?; + + info!( + "[Zitadel] Masterkey secret '{}' created", + MASTERKEY_SECRET_NAME + ); + + // --- Step 4: Build Helm values ------------------------------------ + + warn!( + "[Zitadel] No ingress controller annotations are set. \ + Add controller-specific annotations for your distribution: \ + NGINX → 'nginx.ingress.kubernetes.io/backend-protocol: GRPC'; \ + OpenShift HAProxy → 'haproxy.router.openshift.io/*' or use Routes; \ + AWS ALB → set ingress.controller=aws." + ); + + let values_yaml = format!( + r#"zitadel: + masterkeySecretName: "{MASTERKEY_SECRET_NAME}" + configmapConfig: + ExternalDomain: "{host}" + ExternalSecure: true + TLS: + Enabled: false + Database: + Postgres: + Host: "{db_host}" + Port: {db_port} + Database: zitadel + MaxOpenConns: 20 + MaxIdleConns: 10 + User: + Username: postgres + SSL: + Mode: require + Admin: + Username: postgres + SSL: + Mode: require +# Directly import credentials from the postgres secret +# TODO : use a less privileged postgres user +env: + - name: ZITADEL_DATABASE_POSTGRES_USER_USERNAME + valueFrom: + secretKeyRef: + name: "{pg_superuser_secret}" + key: user + - name: ZITADEL_DATABASE_POSTGRES_USER_PASSWORD + valueFrom: + secretKeyRef: + name: "{pg_superuser_secret}" + key: password + - name: ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: "{pg_superuser_secret}" + key: user + - name: ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: "{pg_superuser_secret}" + key: password +# Security context for OpenShift restricted PSA compliance +podSecurityContext: + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault +# Init job security context (runs before main deployment) +initJob: + podSecurityContext: + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault +# Setup job security context +setupJob: + podSecurityContext: + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault +ingress: + enabled: true + annotations: {{}} + hosts: + - host: "{host}" + paths: + - path: / + pathType: Prefix +login: + enabled: true + podSecurityContext: + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: null + fsGroup: null + seccompProfile: + type: RuntimeDefault + ingress: + enabled: true + annotations: {{}} + hosts: + - host: "{host}" + paths: + - path: /ui/v2/login + pathType: Prefix"# + ); + + trace!("[Zitadel] Helm values YAML:\n{values_yaml}"); + + // --- Step 5: Deploy Helm chart ------------------------------------ + + info!( + "[Zitadel] Deploying Helm chart 'zitadel/zitadel' as release 'zitadel' in namespace '{NAMESPACE}'" + ); + + let result = HelmChartScore { + namespace: Some(NonBlankString::from_str(NAMESPACE).unwrap()), release_name: NonBlankString::from_str("zitadel").unwrap(), chart_name: NonBlankString::from_str("zitadel/zitadel").unwrap(), chart_version: None, values_overrides: None, - values_yaml, + values_yaml: Some(values_yaml), create_namespace: true, install_only: false, repository: Some(HelmRepository::new( @@ -46,6 +390,30 @@ impl Score for ZitadelScore { true, )), } - .create_interpret() + .interpret(inventory, topology) + .await; + + match &result { + Ok(_) => info!("[Zitadel] Helm chart deployed successfully"), + Err(e) => error!("[Zitadel] Helm chart deployment failed: {e}"), + } + + result + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("Zitadel") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + vec![] } } -- 2.39.5 From 2e1f1b84473d0a8c5620eb43eff3dc7e73d526fc Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Mar 2026 14:21:15 -0500 Subject: [PATCH 2/6] feat: Refactor K8sClient into separate, publishable crate, and add zitadel example --- Cargo.lock | 23 + Cargo.toml | 4 +- examples/k8s_drain_node/Cargo.toml | 3 +- examples/k8s_drain_node/src/main.rs | 2 +- examples/k8s_write_file_on_node/Cargo.toml | 3 +- examples/k8s_write_file_on_node/src/main.rs | 2 +- examples/operatorhub_catalog/src/main.rs | 2 +- examples/zitadel/Cargo.toml | 14 + examples/zitadel/src/main.rs | 19 + examples/zitadel/zitadel-9.24.0.tgz | Bin 0 -> 55437 bytes harmony-k8s/Cargo.toml | 23 + harmony-k8s/src/apply.rs | 552 ++++ .../k8s => harmony-k8s/src}/bundle.rs | 2 +- harmony-k8s/src/client.rs | 105 + .../k8s => harmony-k8s/src}/config.rs | 0 harmony-k8s/src/discovery.rs | 79 + .../k8s => harmony-k8s/src}/helper.rs | 2 +- harmony-k8s/src/lib.rs | 13 + harmony-k8s/src/main.rs | 3 + harmony-k8s/src/node.rs | 673 +++++ harmony-k8s/src/pod.rs | 187 ++ harmony-k8s/src/resources.rs | 301 ++ harmony-k8s/src/types.rs | 100 + harmony/Cargo.toml | 6 +- harmony/src/domain/topology/ha_cluster.rs | 3 +- harmony/src/domain/topology/k8s/mod.rs | 2631 ----------------- .../topology/k8s_anywhere/k8s_anywhere.rs | 9 +- harmony/src/domain/topology/mod.rs | 1 - harmony/src/domain/topology/network.rs | 3 +- harmony/src/domain/topology/tenant/k8s.rs | 9 +- harmony/src/infra/network_manager.rs | 2 +- .../application/features/helm_argocd_score.rs | 3 +- harmony/src/modules/argocd/mod.rs | 3 +- .../modules/cert_manager/cluster_issuer.rs | 3 +- harmony/src/modules/k8s/failover.rs | 3 +- harmony/src/modules/k8s/resource.rs | 2 +- .../crd/crd_alertmanager_config.rs | 4 +- .../crd/rhob_alertmanager_config.rs | 6 +- harmony/src/modules/monitoring/ntfy/ntfy.rs | 3 +- harmony/src/modules/monitoring/okd/config.rs | 6 +- harmony/src/modules/nats/score_nats_k8s.rs | 3 +- harmony/src/modules/postgresql/operator.rs | 2 +- .../k8s_prometheus_alerting_score.rs | 6 +- .../modules/prometheus/rhob_alerting_score.rs | 3 +- .../storage/ceph/ceph_remove_osd_score.rs | 3 +- .../ceph/ceph_validate_health_score.rs | 3 +- harmony/src/modules/zitadel/mod.rs | 15 +- 47 files changed, 2152 insertions(+), 2692 deletions(-) create mode 100644 examples/zitadel/Cargo.toml create mode 100644 examples/zitadel/src/main.rs create mode 100644 examples/zitadel/zitadel-9.24.0.tgz create mode 100644 harmony-k8s/Cargo.toml create mode 100644 harmony-k8s/src/apply.rs rename {harmony/src/domain/topology/k8s => harmony-k8s/src}/bundle.rs (99%) create mode 100644 harmony-k8s/src/client.rs rename {harmony/src/domain/topology/k8s => harmony-k8s/src}/config.rs (100%) create mode 100644 harmony-k8s/src/discovery.rs rename {harmony/src/domain/topology/k8s => harmony-k8s/src}/helper.rs (99%) create mode 100644 harmony-k8s/src/lib.rs create mode 100644 harmony-k8s/src/main.rs create mode 100644 harmony-k8s/src/node.rs create mode 100644 harmony-k8s/src/pod.rs create mode 100644 harmony-k8s/src/resources.rs create mode 100644 harmony-k8s/src/types.rs delete mode 100644 harmony/src/domain/topology/k8s/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 7653931..637f625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1902,6 +1902,7 @@ dependencies = [ "cidr", "env_logger", "harmony", + "harmony-k8s", "harmony_cli", "harmony_macros", "harmony_types", @@ -1919,6 +1920,7 @@ dependencies = [ "cidr", "env_logger", "harmony", + "harmony-k8s", "harmony_cli", "harmony_macros", "harmony_types", @@ -2738,6 +2740,7 @@ dependencies = [ "env_logger", "fqdn", "futures-util", + "harmony-k8s", "harmony_execution", "harmony_inventory_agent", "harmony_macros", @@ -2760,6 +2763,7 @@ dependencies = [ "opnsense-config-xml", "option-ext", "pretty_assertions", + "rand 0.9.2", "reqwest 0.11.27", "russh", "rust-ipmi", @@ -2786,6 +2790,25 @@ dependencies = [ "walkdir", ] +[[package]] +name = "harmony-k8s" +version = "0.1.0" +dependencies = [ + "inquire 0.7.5", + "k8s-openapi", + "kube", + "log", + "pretty_assertions", + "reqwest 0.12.23", + "serde", + "serde_json", + "serde_yaml", + "similar", + "tokio", + "tokio-retry", + "url", +] + [[package]] name = "harmony-node-readiness-endpoint" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 357b6d2..d3f7076 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ "adr/agent_discovery/mdns", "brocade", "harmony_agent", - "harmony_agent/deploy", "harmony_node_readiness", + "harmony_agent/deploy", "harmony_node_readiness", "harmony-k8s", ] [workspace.package] @@ -38,6 +38,8 @@ tokio = { version = "1.40", features = [ "macros", "rt-multi-thread", ] } +tokio-retry = "0.3.0" +tokio-util = "0.7.15" cidr = { features = ["serde"], version = "0.2" } russh = "0.45" russh-keys = "0.45" diff --git a/examples/k8s_drain_node/Cargo.toml b/examples/k8s_drain_node/Cargo.toml index d8ded7f..c804a8e 100644 --- a/examples/k8s_drain_node/Cargo.toml +++ b/examples/k8s_drain_node/Cargo.toml @@ -10,9 +10,10 @@ publish = false harmony = { path = "../../harmony" } harmony_cli = { path = "../../harmony_cli" } harmony_types = { path = "../../harmony_types" } +harmony_macros = { path = "../../harmony_macros" } +harmony-k8s = { path = "../../harmony-k8s" } cidr.workspace = true tokio.workspace = true -harmony_macros = { path = "../../harmony_macros" } log.workspace = true env_logger.workspace = true url.workspace = true diff --git a/examples/k8s_drain_node/src/main.rs b/examples/k8s_drain_node/src/main.rs index 71cf4b3..5720fe1 100644 --- a/examples/k8s_drain_node/src/main.rs +++ b/examples/k8s_drain_node/src/main.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use harmony::topology::k8s::{DrainOptions, K8sClient}; +use harmony_k8s::{DrainOptions, K8sClient}; use log::{info, trace}; #[tokio::main] diff --git a/examples/k8s_write_file_on_node/Cargo.toml b/examples/k8s_write_file_on_node/Cargo.toml index b735441..96bd344 100644 --- a/examples/k8s_write_file_on_node/Cargo.toml +++ b/examples/k8s_write_file_on_node/Cargo.toml @@ -10,9 +10,10 @@ publish = false harmony = { path = "../../harmony" } harmony_cli = { path = "../../harmony_cli" } harmony_types = { path = "../../harmony_types" } +harmony_macros = { path = "../../harmony_macros" } +harmony-k8s = { path = "../../harmony-k8s" } cidr.workspace = true tokio.workspace = true -harmony_macros = { path = "../../harmony_macros" } log.workspace = true env_logger.workspace = true url.workspace = true diff --git a/examples/k8s_write_file_on_node/src/main.rs b/examples/k8s_write_file_on_node/src/main.rs index f37e171..eb88b83 100644 --- a/examples/k8s_write_file_on_node/src/main.rs +++ b/examples/k8s_write_file_on_node/src/main.rs @@ -1,4 +1,4 @@ -use harmony::topology::k8s::{DrainOptions, K8sClient, NodeFile}; +use harmony_k8s::{K8sClient, NodeFile}; use log::{info, trace}; #[tokio::main] diff --git a/examples/operatorhub_catalog/src/main.rs b/examples/operatorhub_catalog/src/main.rs index 8e35024..09ef182 100644 --- a/examples/operatorhub_catalog/src/main.rs +++ b/examples/operatorhub_catalog/src/main.rs @@ -9,7 +9,7 @@ use harmony::{ #[tokio::main] async fn main() { let operatorhub_catalog = OperatorHubCatalogSourceScore::default(); - let cnpg_operator = CloudNativePgOperatorScore::default(); + let cnpg_operator = CloudNativePgOperatorScore::default_openshift(); harmony_cli::run( Inventory::autoload(), diff --git a/examples/zitadel/Cargo.toml b/examples/zitadel/Cargo.toml new file mode 100644 index 0000000..9e7a7e8 --- /dev/null +++ b/examples/zitadel/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "example-zitadel" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_macros = { path = "../../harmony_macros" } +harmony_types = { path = "../../harmony_types" } +tokio.workspace = true +url.workspace = true diff --git a/examples/zitadel/src/main.rs b/examples/zitadel/src/main.rs new file mode 100644 index 0000000..78bef1d --- /dev/null +++ b/examples/zitadel/src/main.rs @@ -0,0 +1,19 @@ +use harmony::{ + inventory::Inventory, modules::zitadel::ZitadelScore, topology::K8sAnywhereTopology, +}; + +#[tokio::main] +async fn main() { + let zitadel = ZitadelScore { + host: "sso.sto1.nationtech.io".to_string(), + }; + + harmony_cli::run( + Inventory::autoload(), + K8sAnywhereTopology::from_env(), + vec![Box::new(zitadel)], + None, + ) + .await + .unwrap(); +} diff --git a/examples/zitadel/zitadel-9.24.0.tgz b/examples/zitadel/zitadel-9.24.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bf608921af6d405f67c026bca483528e5f0f2360 GIT binary patch literal 55437 zcmV)1K+V4&iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYecN;g7FFb$iQ{Yhd9$R-sO7hF>tadWzDzaj0BikBLPVR1; zQm&aBq*;su!urmW=F!4d1xmyNMd9Zp`zmp1vM! zU5V+9%42=F)nzgksgY*7)n!V?k^P4kX1vv9^pEC~#J1xT`-fKWOty7Z41}M^lfQk# z4tme`d;8t}CtDLPQ^RE{a(IK7aG4x3U1XWc&HoJt&RELrr0BCP#)>>?o@2G4Rz)5O z=-xFRNi!}6;jG=iTV|SxL&mc#krB64-nw=p|Bv3Y7rp(hdt84vm}{OCLf?-7Sh)WW z{{7$o{l)Xz{(rvz?=QdZ|A+Y5X0ODM7l~oW0kEgdSh#&1Z1>*QHoF{4&4w}&EK;zG zG~-EP{}{?qk@G1Zz&S)!(URvhH1FhXx_ z(F2EDjBT@SmpxU3Uqxhg{EYJ=HF6@#N!j7Dn_LW1$6f zOSI&B~lxv zhKyfxnE=tNgH<4{HP90$x?B$J4H7ljv(j-dn#6lCvIUBSEk-Yx@C+wxd$s-kI2L9s zawcw#$WsoikctRi&k8Nro6F1devkEqfvM+(U^iotdICt8tW0C-kh9CPe%FY6B2#YH zkY%~Lo!YgR=_nUkJ3=KQ?;%6-;ZQ_Ivkw<%Y$Q^FRGcM?Jia#=y41Q5xt*%s_48sME>86>agn-` zZaBKf-WJ+WhZAn1F>oIZ)-jEUnM~PK6(x;T&Jx9AHsA?QBav5W@Cu$kWStImeVWol zFVFflUJR6kCmPs+X^~&s#rx4Rf=Dd`$GcvWz#gFuw(pm_N2b~c9$VheMKZO^q*BI- zN9?QY06h1o6vuse)<=f5J)gh}sb`f2gOhF+Zh#l!w?gJZ(+7Q>9OtGlqFflJa^IsB zIej@wMSLuB;PZh9iF+f>*vbSMvq9yoRhd48&>^tg4 zq}MW6sXYMLHP59blBXGCZU1nECIiR8IHTMgvX?KOKO_3Q<hn6 zNeyFK7%PBu$il#Zc1KulVVp%tp^fXD`f-#xt#lXC?}Nr;E>m$yIs2AL<47zc;!zl* zCjaUhK&5BJlt+m4zP z3~3}!{Bp7=7u<-L4W^azSqoES>{m5_M|BXnTnVJsqI-0lG^TmrHnQ?&!DTJ z%&p(rDal`d2awrvak&?fLtQ%cFOfhcKd{N)mNr zchpddBu1rU!c#sHvAraz2{$t0Niua)3pF~|X&+hLey`#20-lhXrOLR=k?kC?E=6%Nb-mH%`d&=Wxfvg_v`7;B zj{mEpv6yfM*QlTDr@PwZG9CTw1Q<6fK5?c-GCfYDNX_aGzt2bERu7iT5!tRPGEJSn zJ=k3pR_H-%I`?`($E1pjBxd4PYJ*}5P8npftjIH^g=Q)(spxoW!E^>nC4#3v2k&|X zoNiZ(@pvLrX}KHrwcGoall>lo^Um_d>CxLy?|*!Ea=~(yh?&vDLlPI;i8jA89mmji zslC^5eWio`ayc3s#)n4aH72(&Kpqt#{eow|oifsPcNu5@_2Z>K5WW|%2OPU1|7)X2 zV+t<O-o=avPZgXEY1;Tl&>;wefmk$I z;0Df~bTe+AFkK9X@|GwNbX3v;+6DBeovL#_V$8kqFr;gx#XJ1=s3MC_k>B8`IX!yI zj_S{qf~WevliN(9D)x#S+wZgeXaCXNKj`isT<-55!hio$9l#}+dJclmXo;n^Vv6Ba z)Glt83oTM$$n&F1Br`1Dz#mn;pqq1U)*4*H9>>#>6yeJDCOvcJj0ADXKAbl1Z>zWz zDQdi>WPtmJ>)`186chxW#))7FbT>@YjShP}1-kH1HQ4n7?G{>et(>zrEmfojk60A1 zp?C>8#BGq&B}<`2j~xNk7j6A!6|A!LpTJDI5ga5>%DDu`%PLG=>1}ns##K6=c^pq^ z;rYd02ZT2RxwUi_$hbL9++;$7trIeQ6umC%lht zJ01(}q}!FfQvJkoBQDz^>rAFy4Yr%ETT6R)tiIG61t&cFiS#=BXNMi7sY2UjX(bJx zCf>qTq{*TlVlODToFdhh_$FNE^6DXmDxXoR?F#3@wNWW6vQf@syb6c!55|WZa4eEZ zPmlNDeq>LSF5GsKWY_*q-ORM?#v&0$bTgI6XnIJa>cUqr3TIlYJai+h0(Qtc`<=Nv zxzv+^#(2*kTBQ;cSgRV(?o_ff+b??7*V zEiEM8er;cQC8lt+Qg1WX6yi0IL*8&79N}A~4aN=3j6RwTO?KI%_Kb=g3$z{Yu8s*arZQ4Voz3jJQnI-ZH?c!N3 zukE2_g);0HV~-{>ep+cOZaJ6_pFd-RsS(-=c{DA{)PObcsDFHViewn`oJU6FV0_I3 z!^=~iWMiHd6Oqekr^j9?_U`>9Sh*)fVr19$%|4_uQn9G@wQHBv&{8EL9htH1ej>Pb z_E#9$E*uc$bI9JWNeh8RK4d0m-8f=L-T&d;-}k%!!T!1zi|f52=V`1a?Ehn9!DH5q z7+AQfcTj^OMru5q7>dMX+GA%*YfEKR&)j-U1Wz?f@H;%-lEkfxh+!pq>_ozOr-8|Q z-zrDqxXwzBwA1n*lNE10p->$yH_UK-xi4_Oxd6EYE)ta_10G#@v%wbGg>1Wh`+t@(uY|BV$F$zr7Gt@% zmRhRxjkNTep2>+chwNa#dK~<(TnrD{|GN5%-una>SP2JK?^mtT?Y-cWJ*j$Ef6+aj z$zG&#(YrqAovRqUFVakpRccixe_qdiwZq;7D%3y2S%Zs!d1ZnJz{eZfeio4tmXm5)#_lHcqqqN(F zn|@x1U2=Khc`$!o-V;x>E@f54Lx`q<%GJ*;DOzH3+@{C&Ch-f(FYG`x*~wcdzd!}F zbi)kJkH~O=-SR*WMcxiH6?@g56ch5vSg~ZcA`Ox5`xBXxXwir4dF^?AyZm{6TRwlV zPaQNUq%t*PB=VoDuP{6x3B&lc$gSpheExwIMkda7pNTvYso^7{A~ruh|A0o-JH8EB zwZ#kHLc_*)N+Tv0Ysip*YkSwvdT(Km^fM6!tnt&&<;OehD8A-tBx1kgz_nxdU3H&X z4N+A^hvQ+#-Pl6g>uMVFJZ7(zkHm1hm0c=EZU-&GoxV$*S;hbDhyEoS^J~Gle?55v zNxwhO)kK)FDD)1RmVG-O^UMh6Hl^m7Z{DTvckIYO`*3daiW{&dQGyF{DO|c*j`ZGa z*T}#)oJ01LV-|N)W_>Fg@Rnouo@1MGJB(DW3ImqB3>=^_6St8Nv1SJ^_j^Ia!Y?EB zsciiVy%mjYBc2W(7D*)z|@ESFGrj36E6uGfewa)$8o-I$4a}}3o$+r|v6l~-rEqCe; zQ@_KhLFJ$=&=3`?o754qFT^Gf)$K6EtV{3h4#Ii4KT4fp1w&4$e$y2LwwSZJ408wv z-JVsIGUVi9+&;#u%RRnK>4gJP5bg3nNTO*cMirbBK+x!p>@YL8oYW%^9zZj@k{T z5YmH}`r{*{aoA4knkjy};gSfhWip@2Gzh)2BE{Ym_u@fU>!cg;Zj_t7i1#A&iEq2h z8HTH%9#wfih9{0iEc3jUnj@Bj3El#y5)D3>swp?EgzrD z!2Uk#f-8e1o@29Yuve0=nXb4%O0Y0j=kT~}k?YzO{MM*$BD8{lrGa&jYeEx45Th2X zia)Zed`t!g4A&^lkxus*Ord=jTja1~Ql`ijxie#cKqE64AwtWdJP> zpvbQ}b5^Ygj(p3B2JSR){XNkPy;U>BgQ%gx6cjM{baHvt-+_>Rks22VMAr)+oi<}^ zp%+7sVEXHrM8j&Zr@0vV_)meh|9Bnk3#Gy)Yasu#Lq^_mdoNz*j1IFPJcWo?Fsb6D)eCN9~Nok zmmVxlK4utbg@>H3F1Map?#heNJRiK+YYUo5uzBtY6YZHq?>R{ts1GoZEmjTE9u^6( z&u@i{u2?K43c{$3&?by!#UKb(gsb8FFFvdtgB{+zzpOYQYR2mV1-BbW^=NT{IT&{A z)P!HE5!}D*vR@Jv@no#D`K9z;^K=T~niy;`l&EO^!sA$?)$moS?I!yLMjG+-377_1 zJWcsTMqv&_BCk+?I9D+{J>O+FV@Y?jEEm^O6&l3uu{S!}o0*DPEXg1k##RT`Gz#M` z;kG8Qo z1o zNPUMWa(QGoKSnOv>jxi+LuN0MkxB6Rt~Bhgu_YT^7VV1t&2q-xK&E?o?6yT}JNLD|a5iQsnA*t>j|9O zK-RTfI`4-UA{C3J|8xg}qDub;`oM~`8hK%<8n{4!r;!L1ITf1c9&G%58Hfh zz>dEhcPh}M)yOmtO>;fVH=l>>Rz;LiBgwN|-P*;Vt8bL3BJS$nk}lYxcjnR9PJwcy z-04!e+H5s5K5j67ZvQ73vu~sx+Yaz*0_i=KL3lb6*@Ex^e~^&zxb~#C1WTPnW zNwrtDqA93c@6{eXeAYX7(L3-x&@mZWpS!I7kpGhBBVBDenc615ipU&JrrlhOq)P2? zE)}NIxZBNCZoUbUrQGMdFsciI)`c0TwJ*nlCuZD*y;J$B)9sFp$-0yv_RW3|{?`fY z{kf?P*fZ@+;udq!jpw-%*SYD(@`khKIlroj9Sol)#*s4wT%kN|&^i&%$nP~Y(oal&Syi8R$ zva}tFXc{FIx@Jn7zFpEPWvz->{iNJF?59qB!*q68=Y~tu`FU%r%oZzS3Yr4mcf*v4 zcjDIce$^FAjTKp2+s=%Prnf?i+@%=TtTLeY4tXT%O20HDYV_82?WC^+(vx%rv5e95 z8pG*#nce3YfPG@PH35Y}Se$1;U^N95b~l^Lfz`APrt}2Zc;b zSEdx@?`Is^FwojeWc6AbViMtuBd&q?{Cq;i*uCPNu9#%Upd@!mm|Qq{d#)Vn1qs z`lTusEY7)1DX2VG3Ax&0d;i<4S&4Vi%mBGSsGR1&p_Xyv#~5nTm@YoM;p=!E zm0&4UNUMM-Xs`0w{3M-F&O(9{1RKLaj;SFZF3xtn?S$fBa`9Usw1L1}07d~@f)AIK zLkL~*>MT4Yf3M0~+_ujBJ`hut#&p@E#EQ_12Z_Kg8-hgfr~b*s52wc`pFZ?YE&G88@BUlSk*LpFUiiebcd8%;DbNU&(XyH)rcQ{Hs-0{^p)L z{?NaC|Mt_x$=^Sm^e;cXIXQZDa?$^$16lijty&@BA)NE^EXqh57C6tvP~Q5*#JHV5!3_OWz2x!0}?6W4}I> z032sCQ>x9goMI)^JA7vu9LbKDmgy)F?jr^nl`IFe5yOnDUbRvACX(Eli00bI=dxQPv>;_PFcyWt=LCEgh}`$4$;2poPuJ)C+Z_I zeAIIYZ`$c%5SRZ1ZmNWlXcZ6uqHyj$96qP+=;MeHFrn>sNhS@sAaa3CBnkuN2Q)|BZNxQvN!=QK);cCUx_Zh%nKWM?(IvkU0B#UjLPn^4yYcvU z;D{-~X;>u5ya6n8m01lf_K6B!c9o{>$_->Tabd-mRPVBZT*PZN1+buywEMpdL}xn6 zc?QStsWwu1L$geOMer{WpFhCG1O#jO4W$(wtuXphTS@ppBy;Bw-ZETZK4#;HWJ1qY z=1C|=upjL6z*MB=L35@>f@W?vY(JJW=+2M5%yD00!kLQ`7P%e{S9h+?xi9+5p-bsz z)uirn(09x0-mJs5Z4~8#I7$W)LCzb=E#=P@j?!ACfk2P%WB z`(eWIm1CnWi8FaFyc2{KsgcQSzf2bqimx`8E`oMQu@8#v7%OC}%+jxln72jIQkF1y5&moBg5m?=pR49)e}u}p4q zIVkK>(#g$?awALh@O%tKs{B$bo;-i{iooNd0Rvgam zyhMSmnC~g)v`45|+QpIJg+Xe#ThWdJU;Oj^{Xj6E=gO!^B_6;Ovz`#u30{eJEax#B z82Zb=p~u2IgWiZFb0l|N z*<(2JD2tfvIAXDgN+UM2t8(tgS54-tj&`*RncKvR-9QUFKdawg=J%U~_X{Us0ggHv z3&P*UxD|hBr*SDpN*p^6D~%5xLN6GaUM@#Xq5-QpSy_)GaLYv?ixYcE##pAYU^ilb zh`BQ^a}-z-!cnbGtP11pbS@#8;`--q*~320Vmq=a9aKn;9ga_Y(8 zZ~rs5DmUSU$KPdOECDo8?Dja}Vc!OG!toVShofaJz#Sr$Mj|tM&xyEhD$I?7^KXx) zySGihbR+w25~N(05!~f=CDaRVGSd!OSGmw(yNnTL7s(+QpiH=b-N4~Y<2SlvA9WTL zcJyPP9i4q!_8;sPlJLH(Lwrm|tL@LwZR;1TzNCw`D%tb`*?q`(o^uzJCXJW`Azs)3 z5hThmuv}##H&Rq$!2lb$DeXG!9)D^t2^-g5g+I+NWd60YpStdlj*Sli-sh?h+6_@m z4%ts9w>&bt z!*)s8?>ca3I?K83#+xg;;c|r^9kRVb?p;6ID~;$gM59^PLMPU>5|>-LcYV;KnO&as zgN^ai&*jHEbIKT0$bxksDn!7e6%(`uQ?DnZk=dmnwaG%W2hWvo=JKR80@KNML?EZL zL!1ex(Um6x!R;z{7)7F+aTbdh22d&x4o-DOD<^0u>JKEh6U9}-cC{a$yMX*eACu&Q zVbpKhD_iVpubj1)S_AckpcG*!JS3+FCck8bPx(SVvuXi2lqe6MI816(7ThHPyQ;wK7G6to zhJR+nBujj%Ra&08pBYvxQ!!p3XK}KspjtDVa#M(6A%MVkCB~u*A@XDMa4-9%ZTfNt@VX$G7m>95J%K{-@Ky(rybB|iT2aei9x4a%9-=jL zy$`jdpxy&Fg)0zP>dbC*rM4Jl#UVS`-=FwTCItC)@a!*dC4Gt$sXg_w>8Y0o&)})K zhw3{O3+E36$2QpiOGDE+I`onREja@L_usfkD4SZPp-l%Mn7|T_Uc%MrNj=MS)D_nf z>=S!97EJY2G2rakfBWQ6oMF7Xh7???q^u<7l*IDG9ajFX(JW0tx~f8(d33Q1VKzY* z=%~xnxNH1?EOE-eW}}Jjv?-5<@b%P_#eJ~D+;72M0uqD$uu-&uP19k=*#=p{BiO(g zJ8!HmFmjoBEK=P>MN|#Wk{Bukq5XZtc$6!x+3zY9_rYe{oaO{1=BbfgUJ}6ldJo)} zsYH#YeVFI5N-a@jB`R}X6U&+S4N(bf{rH_X6|%A8R~LC$+H+$^Q{(!fa&k_Q1zKa0 zg5GJ4{o4d%)7!f^i*7IrNG!%%@)^q^M21S8 zJ~2VAf*I$k7>%1FrZF|tI=1BftCMtadFY=@)Z^LD=IGg z!=d8N1-e}VaZ8W41-e}TaqI3S4-9ns#{@&IT;~J*3!tKssTUaVT8(O%Ozkm{sT2*X z1Y{eHc{&oUy<7KQXe ztNA7fUP0#H2KBX`i?Y==5uxHktz<3xw1lNe3G{f?b`6QtucdiY3|KXtjH^7KWC6tQb zhIzxKd2zezFtm1hTh;7DsCGkrVFkhrVdpHxe0L?YNW9-{VO@G|+*H8KaeQ&web25B zde2|{8>P^Sbf~OO3x@qE`>o&!LCZnEm?5w=V;YZxh3mwyXL?d~Xg4{|2MXPDI>|jI zlQgH2_(at8Q%bg=z8%C*%^VL`B1Z{kNn$DQ>#=#DnlKS%%i*#3+mxG%yG}Lct~6w> zkW5)Ccg1I}+_=(OkiRl7eZ>m8dAYvZL543pUWvyjCA;oz%_N|#mH)z?Qr0wJ)vp26(1#_gW+ss$v~Y*$_mx}z?lG2`^;M1p%V9l0Ah=H^jh=*Fy@ ztuQ68>OAxz69cn@Y>{-$(+X{!|TQ#I4&ajihMq`a2iw z1Szz=pgW(Q3r29?^|t1vbK29l65KKyYY|vad4hH2*i)M2ZUDIUs~YH?t!*~&HT!$) zG>zr8j0+6X&QyxA{5-8_Wm-UCS2GqeXX+;1+HMN7@sSlIdcersn{5dC+NOgk?k}FH zA^OJ2D2kqFLVnPUvs9(sTq&?Vli!ci3jSDD5k z!8jBk+Y{AHu(IC13$oxfyp~2%-B1haHP*IOu*Sg#vUl#awopCxXKQ=T{?T8~=Be+T z2IR}iwwXymowIZ9%;*a^@+~S8;Zyc1W8XdWpqqA3;6_(zGNp~|_9${Bw{lz+y4+fL zHN45COJM+ABj^&VT9v|pFv_$kg+bM}S8||B*j@k+>ae}QzFQ><3@mU?cp(!5>U@uR zG|{*ktf$Fw0=YA8UqJYKED)I(G%)9!m3TiYyLW!U4H}y$ykPC%f*Dy>jUUXo)?4-R z-isp(sxbu+ZW9!mCnW&#Y^{3ZYa&Au5-423g{w5hfO_YzwP2bdLy?sSZWihj2;2NaCeVrS*xd+D@qoWsZm?o^R~lLlGvGbu`GGUWnNC5JaK;k~)MJl}f;h8}c0fNNX> z;w`&4>0g3`A6=Xum#(BB!FmvF_5&OiDTybqX@mgPe49 zo2`+TKAV-k`JD7HBi)L%a+z1iPq#wGc#5$h4Dj?k*k|x4LCCo}0$}|H(PTS!a4=vV ztEIZPN_4lZ;TC0C);r!Ts6&dF^m*^>DXG^xZ8P9^=@KR2wdu&igrX7u^c?V;GiA_e zluQ8x1svR|RgrM*{rDVC_JZ{I)so{e8+tb1yiA&>tCmZguOfi|W&bbx!L0)~k@4G= z>Z)zIRD0XJQ~_Hh+Sgu49X}mEi_kG^1pvCstq`#Cp)LrR3V>c+@_#im|Ct-qrV{hF zS*Mm!|19b-nWAyoPXk;*Q6hK_7rUpfh}h*>A5EVNQuLoi>zRjV^5{eRFbOvIEHffW zwB27~D6gE+kp_=8#WI0D9UeBj)tcz(CC-g;)avu-~FOrk2z*C_DtQsG0!rQLd9FG zafvzYm6HNdPatzO=*jz^bqE{)E9VDvdDe$C(+~<_t2Gir6W#!+SCCO*PkOIDmmnkd z;z&j0!a!!fol8`(0RNzZeqa=ok-^DWr3LHI&2C!ItlqsoefR%%9dpBRyzNr)ds_i0 z!s?I+^~rBFM4)aESzsMp0rD>fA<4}c1DgSDxG%_tI$*xHn1!Vv3st(e23UZw9%x`C6_}^|B@zg4c_V;Um;qMs zunJoZG%1l3?iEL2O%#Pd)Pq+XL#Ldh*~xL}zXGn%46A_kG4S56mmw=ST!l?pjvI4T zH=U_EYutc{2|Njcx{ZO^O;AajEElni@Sa5YaXh7BLEaD?L=9=f^J~&GF8q*&L#ykT z>01BkMo8g(EaKU$d0|7jv-QkbZ=q^nGu|ACi83Ngx2>nmHYd%NiMGD|v;Elzp1qa} zT)KBju?-Szfz8p3oe;X9uhScNe7yyjvMK!84sp<#!T=d^)Qsh5+$ByV;5K}jOv?u$ z@2{2jX0=S-#ysBA_((KBqIfF6OlPYi`|5xwH0ZA`o&;sPO&ainOyT>>6W(T*n83+% zU(AQ)A`|2-2WvBQ@SkDoFTGLR+U3At#(T*(Zpt?HD@YU&G{G`*stjcy3o_WBJHCM{ zC*zFdhpcn*Tfvh~&7|xiO6g+(QtFU(-YLgCosG$PpZ>i0pLtel)cXlbCnl?w7%N3PF_ixU|nhF5wiG5UMEm)sw*@qPNonyUSihBqLMaFi_KpN#dEfT7n;)1VXTURZy_n8a7 zJQLn;9IS81z+M~*~jw&ViqCP17huh z=+eC)x)j1^DN!xsE_mC$iDG6;U!f~0bZWf$sPX{rSE$>cV+xz)K%-eIBB3?Ur(t+= zZ_7^Qztmt!Jx^EQxf-rasO7GKgyP*W!4ejw7R`+6`#ef|TV4VgXpnH_+hUvs#-xLi zBW^Ph1y$?Aq^Q~QpDV=Vm$PjyoDX{QuWEo-g#e%2n=EU=_mDaj#NUH;l7{cC8w;dkV8O{UH+qcNeLC&4Y4YbFhO;OvU_|9(b+K~NG{Gd{A%hN=~B4u?1%PxN}dXe(~?7kVgz)2 z9Cj>toXAwb>CN*v5n5YXgd>7Y78!XhUQz3y#`XS2Y+oCzBL@ORR+?1;w5fU=bZ-yl zG7M}%F*y_=6`;a(EyKLuTpoMWR9LO!K^+qA0PWS3`4%v4N&AMKRaT3RA zE~vGj2IGBTrSY7xj(ehWh%qFTlM;$A*VJ~opc%Z$a+*uar zfwHnlP$m|28qJ>SE18v3g}FhAV0Sq0cn_R6srmD#md`0u(H&VWUc}X_MU}$4v|d4C zXJzI9Cm)6zdT**Tsh~fP>f&RR)^Q?uT4ZyyWGKf3VYd_LLS73`_BhPEO}Dl|6JbS4 z+KtQBGew4{Ix&aKmd0=vN>KN8QPU^(8fkJKUEwsW#`Z+BL=HtXjS{izBVo)~OsGDJ zmfE6~G;BoS zM?%s8NQJC?Zn#V8oKoNokp)$piXvXqvMETY?22x0xgbK{T}KXvj;l69Ao44eK|erD zmDbiAw>YLS!ow7Lg~Y(y4X7SD?^!U<6^w2og5m)bV&M8PC2vOGcpd%$hRFFTElT#%07i8gsla?lPH1 zNddkcVn0xq#I!7!hs1-NC^XnvO>jjs#`mr75BOh7U>4F)p zTlTEKLF@+RYf{&P*>s)1@aB4xOMBcq?ZJs@c7Ak8mnPMu7NABH;Jd-`y@*jM?li`< z2Y-9^P&Mw72oWMxe5ycMu{5EvUWw_CcJR3q@uv&n4&5_2~HRf z06GwDbRoE%%f%vu!5ueThyTa!4v@M;=h?j0Gs7J8mO#kX*nRf8R(fdC7>r}yz2nuG zAT+(M>1-{jTRsc7IlqQEz)1yF%kXC-e{NFT!D2oosH$l9>B5$rA5{OMX^#M z+P#WL5F%HS-f1q4#kucRslA~x_Xex9H&y1|T3n)ib{i}z*KY5?0qN_-+9|d_nYOFi zzO`Mc+rCTw3kZf6NoF?}SiWH=fHJNu16b|b6wwu=rHhB&1T&T8SE?(&))r`8Q+^Xn zJ6liwZallFT=*P*T~J7VmR;u-l?Pn0rn2phDF_(;zRZcj9c5};l%5w>lcDMgKHyV3jy+>?rphx1T|*&H5LeuuyPAm zX$|2Jh(^GTaqRg#zz+OB)oZ-HEK&xi2UW1!0ZzRs8%zT_&&dy^)aAFhb`7f=pej04 zkRCg#KPO!P0O0|a`I>f&nBm>f6NKkl?gLCjPcBe%S(!)NbD&vhc5mtkb0Wxsj0yfG zsAdkm&8s;P)R=I-?WWJlnz%G;cga0=z^*l-R8n@E1|6VSmNjT9^P{Y7gRX1xM?baF zyBmy98y;QlxY@f48JHTqvz}cl%IGK=B!r&e6QMI63BAj-@a293_A7twXCmrVN#Wvb z-^t{I7TON>2SBU&CM<^Ia;|>l@Cu-VU?7*ize*Y8hPpXq!7B&*&Mm#YqmqHQ z#>pjUyocsJ9YU>~U9=rJvx+^m0I0lVr~pt_4#SDu5UL^%^2_`~zG5%@S*gpjhADD( zN@bp#+bwE#&D}gPP~2Qv1+Odl5^+dS>MV3fRQStK5Rc7 z#}5cDQe%Y6v%Vv6bEdi1*Dz~5Kn3yjoV-LU(-3X<;gG|@f~x6*^o(NHBDWL|Qp~ll z+K(?;Rd853gCB`~xHzlP_ydN51tNuvTFN1iRj1(q9Xa-hBZFT*>S+6D-z9z&hTH39ED6tQzdIM`vBfbMJ5`BFhI_y2cir?-{G%QAAB*Q8$lp!qg1 zzsz31Z|FX6Dm+LFGAT3!P1*&YS%QG@sFfeXmB!M8a+60`F@chqL)wC;DWJaJ*ei{^ zspMgWm6|wb#bF17ME6)YjbPZT`)xx|ndYS~tE9{pkKAD5X6ZY>gXg>=6au?2+}^gf z!{RTQifcsx$hD3j{(84k&jsHPml;spwzdg54URi+F2@0u!IhrdH`X{#opTXNZwqRGzrX{^L+k3H5002%#7VRmz(B5 znV6X3SbLD-*U|aPMp)FXY(t=ZD%X3p2M?e14qo(JQm2m$|EW`N*m0ORoc!l5F}gFE zI{*10Rf$rm{mqqI>V%EE-Av`?o0l)1KXacqVs3-ZHH)CV!L9HEETG>9_-ADoyCt4i z+yeQ@wIVU(kZJ(KV=%?w59}O(yER@~uFwJ;!ZE96$QOhkn~6HmJ=Vu?6SM^cu?2)u zZ>6d}bnyVY%vUvnlagZQMQU(x#|f7cy^C({S2EuP45D*x#=A_Ih&USCHn1~-lKbk< z$43<$9>hj2Yd;5k%)7RsE_opr0Sl0h_*TW@kiB?Trru-2ce8!eMpRR4a2y_$x;hjX zdKaJY8=_JPiiOpstL-<=nK1qud%|Z8Jc{)NbhFwv2Pk|({X71JL7@>YTx!5tuEVi= z@OGLE16^5A_HM=gh(r3@-w??WsX!i zQ-v9*B8}aBBBcd*-pS7U}s z4q^SZ)bQINOtp7`cBEa_L#3IrL8wX*8y0EgQE4h#K}|?atdN>5M?-~vDB8|XV~Z+& z1osq?0(c5B%3DXhSuXKrV2=zQ0F_qwlemIsk}8NEvTJv4UP*gs_Dk^wS8#6Gw=OvT z@ejWoF59f<>N#~-kNxwX-O048!>&-ld{yZTkaJ2fc87iUKI6a9N3&CWhx9~HV6-^c zuA`E26bjKW)c_05iYux1YLl;Yes>zs`zU1G5h4>&fKsU9CUBLxz$$G)Rq&OUaC!zE zbRCRcSza)_EssT#iJSo13##)iYeBhoH~gvwUU$G!pIh4{%F|#PmSpjneq5UFP~}}3 z>u&jpL-qHGS6G&X%2Eee9S(XBio0Tg1zo8k)l}x#DhgD90d|I~0lFK6)$KCFN4sH_ zU>pm4pk4X{5aw_}TqPE)r9l8KF#XW^@1e@Ji$k7LR0IZb*AuL@zqIolZKf%T)MOeI zn?;0XZ=}@1u));(T_KYbA;>XI5Um+f*Q<#!8u{Uz@W(nt>P!XbSs25R6u>IJKlCMe zzah+sO807tKB9WFPkr&+0EQcoaLUf^R>+r1_%=jY;`sOQpH7_^KC-fk3tOLdE$T8a zEAaRZVT~|oIdn+4qRrwqvj>Msrq$re=eIpe7nD)witw`2M)NnZ+%Z)YKI zddo;&3n(z{1{s#Q{sIF*9o1?(vUHW|5U$djMmVLDhvKrVcu1XEA>s_d;Ud1k3v?~Z!`sK+LwCQb!4n-uIM?1=&!XO9ZtNR87%IW( zU*gHYlT4;!?*Q9CB){q}y8W?GoAjv(4(;GHrzil}%M~hk_vDONuEpRtMBpzuA?ib` z9udKt&;wW=5mwCjLjn6l-Nwo%m0J0N6^O@35~-Q1sitx)mNMtK_|u{v5|DdXl&FVi z3Y;AaA6w~Xe}8}f&o5rUzx(_9)qkJ;`NfMrzx>m|%Y&Eu&-S0~AH4k2{=tjqFAo01 z_BS_#`9Fm=Jpa@Fo!?cT+!ylm58#>3|H6gu9FqU`aBt7@K$rMI<)ghg=R>n+m#BO2 zY>(RG(c7_CP5UgGEZnCh!h7eC{ljCEcO152nZ9(rR^Yobe-YTUBeqG!?MMLoq1Fc9f6Q@vB|-T~gcvTX^W5(8C2r zQ{7vK044U!?ED*dr2lr#&MZmWl)&8Bw29d3BAnbE-J~|p8ETL(xii!1pR*4xZWo3gk~qNzr;Dn-Zxj zBO`5cBHonJdJcC0x4H|k)x1eR7VWReytwv4Z8iT@QJ9?Xe&hPmJ*o;$8TBCwm6~IX z;7>u7ZK&PnVVfrb^atu3-IV;`+KYt?)}VqPLAC+;nX9<|3;% z7gxQ(sOlv#)qgZm<+-Dkw5r6hwPC)^2NZ5HoN$xDgc}ScoEu2EnEB=}Ss0_*X4UQ~ z&UbTBz8i}1T^8Z{N89Ydy0+f#v^MS8W|P?qo623-aOT3wc?*9`Mry-DE~3`K94_Uf zV{*+TQy23nY5~!tpA@ zS(eLdEU8WLQkKFp&((|udM64PK3b6t0E_fhkTmbLv?mZ?M6voWqw)}~nt2?xAzUL*-! z8{3?eXDZ^!d)UAi?y%co{#Q*k>Ct6WUhIkcJX3_Y|3I$-6VDy@cxV`i@Tbswk=jSg z=rt2W079`IsR=w9uF+};Cn1}O6uE)}XDCPZ#ORfo9(8&g`Ya|&%(O$N+>$@qx!AO> zZB9v`nG*_28R&JQ20RJzn!S{8Cno6TN>NatzfEHV^RDyW>=1GoAP%6{-}NN53*1D| ziL$d&>`5}8-Z4-Ud&O|gdh}ul&+)P2V@5xA);m!#cjilQ#ncn_DeWf+3JOn|as-Sn zpzmIvzWaX|>XPSnt*30v(>M``W7qaL-HbaW9m({zN7S#SKMir%J3w8{V`X#7c7*#B_Yqd&L9%1@-%VfAcqU_!_{nCZuZ&Fe4 zhicxMAV;K$`xBp%5%Bm1oi9-%8DSY@D)ns2<2%<*&*V2&$u!E}0oc)xeRg#AZP|C| zl1J~N!{^>46;=%k@nCMQ^bb#GHu8)EiWYdY5gqnA+Fm#vu|#S(`t5T;C~Uykq)k8r zfKBGtqO|E}+>DpgzUi3vjpiTsGyN=-QfviDg>CKFmo{rXkrgM^*#FWARwvMYveoCf z8L#vNTSXk#e-?P78%K7D?|G=vwQ9M%)#4WOzrD-hWmXB-`srud(3fZZd7x&MZQo#v z2k;cUNjShWR09zNZ7}sFJ!g-HtHo?;5xhPCNU);(N=)xvkbLg@Ly*{heb6fxq?upW z3!I>;{9jhw_6oJU*RJFJ#YAvBVk|4c7ediD71e`n1(&@b1WVY{oD3#kqpTj{{r?)# z+I|O@q1v^Yb`md4VJ(8OVTNDX3qm>)1koL-Nf30h*sQy)*S=*P$^}bLWhhrGL0Q{YSc~%O z>aVc+51D!5Po^(A?d1Muv3kzQI_3KO_$&4KLe1Y}%;&C@hf7Tf5L~p?{mb-}{@>4n zB7P%#!88>u<7=Wy?Rr=#F1G1Q;$~E#T1~T!>KygrMQbR*Yn39~s2bZ)gst^FHd6_1 zqtmIBscJouwQ9|+KkhbLOO0f@al6YrM2mTJc?e{SJix99VJkix>Vzl8V-f3f&?n|G z#b>iQRnaNVEC;g>r;YFw8=XpFl#aMv{^ghP%~+w@P;Xr-1~peK8Q+{AF&y+fqRpG* zTGpn481rkX=DTd?P0N=mp|l8G5a#-$3L%_|Tp%c>x=E`c-!61=tZvwEg^aF}sV9Pc zSZEX)GPSIf*~1e|6m0jtu7sZ`Wb6~_A?RLt1M|1UY>A`z6VO%aAmKs*e3-4{|_bIu}fd_ z-dBNwPIkf0`SwHq5+e|JSveD{-I2+ztY5SP@~UjdW1g9^VNe{AzS;3LrX_@j=e{=N z+`5p#zSnZ3_hvh7yKrqH3s#6Kcx)1^Yv|cglYc=ot%$VB7Q?-clx{Fu-;l|a{ zTfhE*);<63vX?>cI<7*Mw72%7*0vZ{WzNphqN<>~jYg;sm@yCq6*Gkg4IsSzE0IE! z=`H|if4Dfq%xs?y4ju-M-yMjl^00@PkWi5i z;2b~nh${V)iyuypPd~L=nL0Ble5rJ47xcni0m#N&~7au$9_(T8l{o79$Cx8EN(!c!l=H%$r z$wmL0kDWJ48+yQ@U%XNiE>rjD5(Ccsb8FdowUGH|kEidq z_N38lP{_n|WlEP$sLeX$kqE1RBV!V}fPYJ1iP&&tL<9KIJW8wsg5+ngDHNn80rM~8 z5pRM4@~hi!y&p6ZX3px@qG3F=3kAh|4Sv~N_``Cvjs+{e8fwR~t&WHtbey(B(?BKw zREAk3kAE3Pmb1&tsaGfNNfO|Jlq#WxxY(6_jOfWIYB9RFgAdYAQF3WPCl zO;-iPXajnIh!t8aci68I@Y)PmG1KK{%mOWdEap-4VE`@cbFHuVcL^$Q0b?B%F>eC*2uiUI0LA9$bO%YZ6p&&bF&_??;tqfm|1w9;ax94z*{2G$n3z(2Se zgu%@FR)R9{Pp=7SP(Hs7w1IzmT@V5HvjWtCf9Kke2fl|pk9wsi+PD7f&TAy8%4@wib83cmo4H=Ct1R9nvub84chU)4XI#yj%Kc9z{=rci0VM2+j%aA^0u|y$L6&tcZ?Eohkwc^Gg#GVlWZ)e z0%#Cmv6ErBVPE~iD8{dqjKl6(q>@BxEN2h`gEo)I6@syk#_VI~9~sQzpJ9y4EKp~C zVLoeg$yyGX_n{Z&Bd-0&4g=4(N)ZmF>X4zc7}hL|c0wF+&=wOJ8LVAV8a(d@#4uH{ z=!--|WtfEZK zV>-%t23dWl+DPTg_F&;csZt2M_yIA`M^+d0-)x#bV55^}daP0-Zrg+6z0ZhEtp1UR zYgj|<3hrd$*7!Ic-Ea)m5%7H|L-)q&oUpsL43~VryEcms^)T>s zO=0q_trHbDAe9L2`f-=Xqki3SBM6R9(iTOmad3wGwUA~jIBKBhU z^xQ5okiwGX*mYax8?rND70)<)^$ z5ZEuVF58-kE%Y>HwUBXv$KUdVr;*j?J(2tjKX&Z7XL30xtjr=k~U{`!{28UG?~g<6ppAnnN@C_VLAj< zK)5a4W9YyJA(M*8;0&b(&*lT&0@szEPV6!Xc_%|Da;JpA@SUsGSDLkF`p1s@P2;bl zJc~Ydb{>Q;K{;66!SLqt@|@wbcpDQuF=Ky$Ph-R>Kvi%uDbgi41RI@`X%nwHYo$Oo zv=6)J{6PGAqcO~hN0HDVk>06TSdOL^RuYlQ>)JqyURB~?xza=xu{|EIWh`=*C?2x` zHQC^4cn6yR?aZSnT5J{ga+I@q=?8YP>>R6M{IjKvps_8mAyp%(T5&_=?x`2*&kj@tPKC z$-v7iZ@wSL=pMG$Yv_S-7K<37!sGf*YjEvU3JX!TSvB#p+5>gqix1Nleh>#hx`{ly z`h;-KVD*6fV1cM@L`Jt2!5tA(8roAS3uSbdDxhb+Q*v~{W zATJZfCDI#q%V1DM*{gV@@O*FpPpn9dOlEszx`?1+;2+X~<)-S3ues{TpD`x-AzVFf z-jB%9V^kvYRRb=5`uWm7w!_NbsRAoS7pYdkm3EEIU4tv36DYe-&SzHc{NJ$cM_ivOxyPTISe98vXApUH#V2z|@zE!DQ7;3-aTYTH7 z$9YS#YV>1Z1f{9gfaL7?69)Oc&#Vwt`4i2mByw2^Ob$!z*Z6UToNovYAyxUB%j3}= z>@AIorMM)FTumwoKm<~n3ur*<-lTC+B9&f?bTzPxyVKxOPOK_A$?iahEEiDffvH@M zq-9Z%A_8iPW9M@zgSWRZ02PZy*he2IAplnJ0$0c`(OUWE^gQCL?y!q;!4Whgs6!;v zSYFGx;E9vEv|Sa;1>9fh<6Rc#Dua^0skV0v|2aeh#d0{U^52Sy=WAd8A%Si6dy`Ln zyCI0fy4VUJsJv0}bjZ>oNp=wx7=y?FkGV{xBl`)#N(a1jWA~UPeu*zuKJ@Tm`}dbI z0Y#8M?;Esy=$LKH zb?J_}JnK7KR3Izj0Ey6f8vjsCi3jYdrNT8|W7~TZPI&U7@TIfc@l$wk*QIg!@DuHg z(e%8ev9YIR`QYdKy0s7I3t9%NL#bl*Z~%I)Wc@M_wiTrfKHp<~e{3dJDw{D+S(QWn zP*gXGUQ#H-H?32=5gEwec@g|aq;aOuPLwMr9!sp& zK9mU=Ibr3$lZhzHSjrTO|A^QZg>g_{wil=b1B^Se@7ZutXOSN1cPhP5s(tbsB_%f1 zx6+KWRHfZqDFc-QsbIL&2&;or>o0kWb7~1T!sl^NuZMlG9v`$05qmgngY`D^Di{X$gk1opTzkzF zd1yZ&Pl7}PFpX3GF5GK)WUrDyB}|CgEMeM3z}+jz!KU`=0`P+Q>=`IR0cOFEV|yy` zR>%b_p%z#Qp*Ux;x=CT#n=uP^tAGVqQkGH>W+R>Eo$@mzt&(gWY{4er3T8w%>;T|o zo>|$LOKwD9(Zb0=wXj%5ShIA{BVXt5gAsRU;fjLPNYL+ zAB%XlO8nv4@ClDbGi(8y&`KL#1`F_LCVa1Yuvf*}ABpF*@2jd2c$M-Z1t@))mO5U{ zCMp&^c3yrED3xgYMBg(d$Uj8a4+W3kr^(c+#-OlIwGor{ z_7lu>2S6VYvf&z@saX&d)O0a1^M)LG}nYqE|g;UF|1~cGGRcC>|nTsSSk2 z{Xl1Y>{N*~einV~K)4*ZP2EC6E!8LyJO}8Tr*@T9Vc6wae{OWaS~%UyAu}EXDuWHC z?E8{1s{m(|swW%5&`!?@9LT$3sctxp2;J7)s+YEF=L+8_Syw~aSQTNzGM39sUjc4| z(k?vY275~4XQw^*hL3~=NO?l)M|J)59HHKn!J?eWltmn4UyIb9?wADcN7e>TbrqT~ z!Qj{&_68W$YyJe!T=qKXa-rPj_g)372G;!w8-|o!0NJrNR0je+%nGF*&d=Em-u+M+&pm4oB>oX(riA?cq9P=y_sb(DS0j>gQ1ptF! zGawep_2UEQb&!^Uf+q|@+9=r;LW$f3LIkch?iS~w196@x5qa9Mqjnk~M`D@VEq33? zBg@IxUOAtKDDvoa^cCSknv(nK& zC}L#fwRk0XoXAwH>YH-mZ{S-gy;I&VfyE_%RRdsdHOZ`yKq+oNv@|D0#VU70#CwX+ zOJ`Rkrg8PyX=QT>!O?I!sN0!kfW5Z9_eSZQc&EQoZq>kai1a=8zQw5m5#Ja>TrV3 zJF$aM(L{|#E(YM%l7n<`P}95mx1OW99YY>o& z);_3YT^`Geidtv6=zcA1z2>SYiBR6|u$q65=;EuW{{ens&r{)@cPtS*kV1P(B-vneA5jC zm6dV;LOsGTONtRGLk-YJWp!n@fOLJKdO&cco5)m7c(S~Ril4<}k*h1z!SQ3?cs^~T zcj@8EU6#7$qPn@Cy)Nbal*oIZ#>14RDko6cj^P)i32BUj1?MKD=UW!vWipMDBDQK0 z8OZFfS7NG5a~cN7fn9;c-0-<%`T{rT@}+HKj$4GU`zUf1J~w{uqfUZY5g(WR2+Eo)cWY62W3JireQz4IB4|;4#mARg&x*CmT7$-#fWjT=2p%;#%Hu;o2ja76d z@}5*d3>ZdBMQW09BdPKwh{xlL)9!m__xOu{V|bUQoE{C1)G7O|;0Z#(AmGlIYKyq^ zw6}~Fpu;sWwh163OBRPaMr8>+WEQ&jf*nw^LOePf#2v1rH(Z(*w_j!Il4n}VRHWt{ zk?PB{RxDgKR`UY3q4$x!27r<6Ur&ryDf?B5`abVVidG3;GwS9++LhHq$x;)>;*hLl z;xo5n{V}$V}skaUyrl>osfrqvu`*C-u@-E@ksZQ47oDjbnB(EePxPy5KoajS%t24*R?6vnb(O`$Ve&RGfx=Ri-7oO41b3 z4gSQ>Qn=kO;bXaFJqp8htNUHs;wX=@D9SFJ4Jw^q9pys`@J@UP%Y}!x7L@@}s4*IC zM>$W6gkmP_Fd+dgQ?+!R2dldB2e3D^sjkw@oh0Hm9(!3Br6X?F++(rNpxS_y z!f(!x7!LYDN_h;#m|x2U1y|-x%fs#B2}BFRTz>@C%Ly+84z5_;r1oSJ3%4GGp6umC z+T~@Y>fTu8@^^cXBmhh5ctGrJ`_pHTh%UXc>!;n19=H zHV<0cWnP7mP%kOx1K_0Pj7kU=QiFm(Pjr$4h+6elp~A?AUqUpUUofoUfr^mu+Y5r> zSsm0G^sEN((-PU|L8=PQD`>TB{7--S`7J!!&qTDtj^b;cMj|e^KR_j_)S*#2WO_`+ z1TZ{W@t$9LO4BT1FJD8b8wAJuqAQm^g2mS#fH&?!^2XZ_{YwHIDXU{)aT-!PN`OcV zG%nlmm}jPJSi(1U>6@Jp{A8+%+}WXOY6Qv+5Eg-$&~l{rW;-t>>K&g5 zrs3BHt78Dv7Sx0I@1wV8)jXe%G`#MVPm+(F3QniE%|v7`sF2jN*|hcgOxnj^ko7gp zI1H@N*rTI`{j!k~0X#Aoe8PN2G{?-Lpo8~FrtbK=mYki?r}X-=2H`I23!aGBeE~@W zucdjP=^YnL1p_8W4~-iZs`h3Xpa|fCFzyoOCp^R3#XW+G-yN5|HrCcl?pDI7I90mb zu z%QR$()CRcHqr+DRsW4t+m+9~>7wA89^Y{M_5;fSHaGCB!lX%a$U3zwxK1^EID{i>e zO+UBK>zoX{?R^C+jc46)`RCp!y*&GC?ikxT3fxZ%$BKW)`65J^r6X%~U{;3Ks^#)l z_u&(9^7$25Rv#uy(7h8ND_7V?ce-hRCv$C1sZ`MC-nwUMBvV8ZI_LJk@yeTcxd!N; zXsHSM_N#2Y`HG?YY2)#Hi+K*;i|JQdyw6wg{ywZk57>5n@ww7a?>c1vJaO1s53@TD z-}iFIY50B?UJP3IvhG=1|5@Seof*RiZO?*7HJi1bT>YsvsMa~>tM5Gj@Z)0PliN(< z9pKd>sGipSeR0db(i81luc8RAD;2*A^RL3ZX$)V5`3gUQ*84}{zH4ut^h3A@Gj3ar_@3>Wrvw&`fp(A8jP*uovzC_nnXf%3EyL;NcCa$5 z)_)$usqmv1v;tI|dGPDv-B0Vs?B2RO>wj*Uxaz_`;DOTkO}o&Yc+l6$Q9P09ac(e# zWA#9l#o;MtEEunV- zVuA-!nMbXF=p>hTClYZDD9pYn-W@6v%v-W!e!NOGl#{nzkwz*OF*`n*q5QST<#6iO zDwt+|4GU(MBzUGFv?ym3#nAuz+1{~=u5!hr@vCoL#4S`6;Y^EU*wu2BinuZel`~%X zfZ7bEj5D`7$48W08nly|X`^ziXVS!nknKmcxMf-|u+I(Kiw9k;lWxSjQEv7kZdsnr_ZOTqSmkre>+M)e?1%3X5fj-xF&izGFR>WUTP;vvYsvkQj zB0hEkF-Tk4zAM-pSlDe=Vqy4Y+z@Fwm_pkww#Y#)rgYG#d}sE=gK1R)Cn4=U<5m(? z<79J@2!5B+3iDRxgBJPyDi1+*!Y@ySfD~%C2xUqWRZw&|gLd?($|r;p2n<*%7@J)* zF)02JS(5%ZvJJ5Q zJXSi2X2!HhM2mHfi1i0_5BJD*GZ~wKAp^#PqMq10@@1}yK_c{6DFcPv-E&YZyRr2C z!(=M9EkiJm66=*CKuVQB691~HZY|N}-OqYV-m=z!Z-2*{1GTk9ExkQAe_i60zGHGD>qHHB!u0GW=W&ei1UyN+a&tok`!sv{|Fie)-EG`f;{BVS0?TB7wptGL@*`#I z>?*PntC1yDq?7Gtb8K)1BvB0q69AOV*!u3jgNw&tUgS`s6g%OZo<<&GfQyTZi;Mew z>ExV_)1h=Sai^nW|1JB}lCt5=Tb^7ELA?{qv*K+6LGlkLBP}UmqBS!L$P73rkqkw& zyb7zs3rLKvgvtr_;hbOXJ9h5wwWvQJ_C0WJ`8lhb6uo}AC5Z5ZirJ=Z7 zA)?e!$s={iHs9*?olkXaKO#xU=cx`^8zDgwK_S1jT4=rG-c?Q~*OKg#B%+I6s|(_g zAo3nVOx^w-8qT4l`x~K4<_#^ zohjf5v9>)VISQf>9R&Z2gI{-ozv{AU;dVr>wr3nC5u2iqXhLv=f>2HL`1SeO#o*v{ z&_jL6Y*{y5I7C4FPdx9U6DEXB9GES@l;A|5*!-q8g;@l(x7yyNmtNri)Nort)*F|fLUR8Zr*f_iDe}I#&hjiZ^-yfqp=^IE0YjK> zVX?@2sq@}v>1}kGl-ks@!L_Io^bTECQ_XH=P09FgjSsV{>c!|;(*ZGJ{mFOMyf9i<6BqT*RnHM`+ zYefpZ=~hfB(07teb(#mWx9f5Xf*6XP9Pz|l<=|H zjRqfTOkIzoi~5q!LOFwa^>NIuMUzYvs80xpGbZ=JUDF<*qxX24#-xY*74IRNsGw(! z?4qX@fcMb3L3mH`|9PMVvA{0F|F!QgSH_}ra?NsO_;b8Z$;%(dko5{yXix0Qsl0M# z98NOP3>kw$<9b-1w-mOqQ)gp{^@|mrR&^c1h4@`a7WXJt2ojI_;CpmP7Wr4L*+}Xt z?Y&GyqbS0)Y@|CH!)(brtx3r{cNIOx)@Rzf)BfaFg!e^!uigrUsl!&z)KW_DgFITV z27~jyagV!4Hw4d3&&oYU(1M{27%okeC+~3*&rS6pH6=;-lqPz)l{l0jnkZ4kW&e(~ zze>z-goZfANvN}7f>UH!cp9AayT+x~R2GxGK3nVZn_jE>U;3|`)=leAzn{m-l9Ei* z7)v6yPtOKN{jPj3*EYt^&d$!WCr{wNJ3BkY|L#2B+x^q-)7_^#dpmnOyHEeLv-@O! z_t~G&&e|YY@XVCp{->Qgk5!!9XYvdtIJtz3eu}vmrHk6DA3mV&p*rrnz_4y^x~zddSA7QzDUl<^a`NKQ2h;_1^twm+KM-rs%_H;?k|95Pf`XSVy)q<;ZdT?^ z<^BJ5LgJKg(Uobu9vWM`|M#ElKQHe8r+d54H~aq~o(~_k|J*uypMr6!P_GH3{ch{e z+c!5rU|Pg?g%`RIFo%NdSGWWFT*K17YUv8ge^keF5~64O@F$)2XQL5)kJ>>SnZ8I8 zsXto)9i*5-&>xh)MgrQ`!xgI!a?qIyIVIS;4w$FU(alW&UzQ;_gey71ZjQwQQ8-Nj zbKTaDM0aT1e4p(D76lrT5YGfbf=zRUzyRMPN@7#$1gh`Yat&0V$Hw*ZP}onTdhBbcU0;`_d`=S6$^v`cmr)M3RRo@^3su8P5Yx;QPZUM_a-YI z3)jz=0DJF6bCSwnbXd@xSz=36Ugxdcpg8vllLSK5&<^*2-XG5X`V$GAD z-WjO4in=nBrtSrcu758seR=O>Epq#(Vy;Uzlt-zG|JYh$SU8PDR!k4J{sQ2kRYtDG zW%4OV|NVm*?IUDdR{}!&FKyI*+g_PL9kS_^C8jq20`$V~8bqsj0;qea3d?3fT~xCQ zA&lB5F$pE(C#LhR+e8wNEB4Sqn%armouq5OspdB~Eu5y^eAN`QDgfkTk;YV>P$5zK zzuS6@k7%SMm@V`X{W4>cRQF;66@uZsmlMzWM((R;{3kVN)zrEsr0ULohbYaa7_w=~ z5|YT={Mu&j)s$^KHqD1iDV!!$=2oO?3F16j8vv-ft%k1;4qD9t1c$DBfDrnvVGc3D z$x1+m`ipx2)AU?}OR3oG3b=#_x_iK-ge_VH;!EQm+!RD`*t-W1b=XTKh*pJe@3YG( z656c_WN9C00GHENC5H{3}1B<|^AOUBeeJ`K1$B#GqFN-@hNMXA*~w zBt>o_<$2W6RP`KOYYj&e9xa-Z0x)Y)fy$MIjkU7y#_(%i{!kNGz~PIGv-u2wgFe^{ zn)B7hGtdn7p=Pjv?P@Nv!VEMISO$3mvG8sJmOU=4%+{EZ+jCVuWj>3yHN*=@9RaZ>05eJ zoLRDLgt4l|jqz5z?0eX&?00n>F7KuWlNUtJcv9~R31(13j4mdk+d7BL8Nz8l%I=SU zLn9o=0)Zo*iPc%?FS$V9P~rPowMO;L?bIdv7oNsVJ_st%Z@zx)7CNG#%$Bm;%U0IS za`Z3o8ELo(Vx<)jjnQ9pkXeNLqN~bc#1~uN0_)-qD6ge>eZ*CuTJN*_q4v7I7x1IT zlwRBXlcVz`s9!gYSN6K_1Za-#-;TOPJ|&7agJubSa=w*}Yk=IZ1*7Q#iFBx&f}K?3 z3?YOv%XP_a?GL=wwGOgdxlb#2g!{&Qf<7Y2{vlYB`xNF;Gjz#h-fZ-u1-StL{tLC5 zHBFN!x0pd`39sU;2^b}z-2*0K zk|b9PFe6O4P2+307%w7=K5L2;(|;XSEEuKm#l?VB#ltNU_NwB)%mkV=^^e1b^LE{| z7V}WYru+SqVn(n=s3;~cTZ)TPp1>MJl-k__3%T5fOEBRJ1kQ`qR6SQsQ$`bs-dvpc z9>n<-?W zo0}Z?If^%ET66DJJEFWLvizrLaCn|=2(=jul^6DcaiuJX*b^pnP!YfbvUGKQ=hnAC zVLc>6HR->r4=qKtTqK$PF&YL|&iN(oJ$s6#nDdk9dtdR;iMJf1PyH^=l3>KPWx)NyZ2T1>0d7Q z#cq46V*=UijX~gqO8P5_Iyx&}E8n-hHMCzJ{NwDR`}+9w?4o;qFi>KrsilrAuMhbw zq&QaZTcGGv-(TUFM&1=3oG*~XV-pQ}&5}nFjWCT7OT5{(BB~%5&UBC;5JSR}KuDZK zm^*(Y@VNuFE4r=LvDDZ8Q2S_Tj;-K-C&=`Q@ONl5OG29=GR0hIoiKRanI2mftLlh~ zV>%{@jOR#@6mvZ%edp!GtwREF#zOC6NXgXF&R#a0{+;%C@`s<01aUH)(Ks?es2brq zi7-oIIMZ5F{BOoPXi5|HAK({QhLg;iB06Y1!yG4)5OBiQ>!^LU0QvfaS#UeMoDGR_ zlF69Bmv(Z!7f2%ybq{fhhcu>=64CuW`^vd%l^ndHF8r_Uy6e7H6Y~D4CYJs4945D1 zFiEkTxZpXct-W32!atS}ws<7K*Hp!)%`NAra`YrIJoI!WNtw5L;TWw$zijH%u+sdN#+p zK)99LdVsKwmU{S9Ggw1bxtzKwXQAqx3gqp)md?`2%npq7pTEqAkc%`@b=yo!g&MM7 zZ-ayCtwr5liipHCCQ-}mB{fypP1I6Twv`el^+{moJDf>0!2;n}u$GvG3LkpEnXROq z(-plqLDOUnvK!LG1DdV407JF=s!nijh@d4KDMMh(aT^@R?7CNjBz+O zJ~RJd!)|dL;~&dO$^?~+&)ZOafk?OjtE2DniSFYyi`2+_U37HPKZ0Muz)r4FTQU}l zt^rba+165=NdkC5#!AAQBQZ7C@yql1%&NCqdp!HwU$6DbmY%bXBRW| z1?;IDax5k$-Zh~y0^X#Aw?A)ZvXAcIf}~7|KkXWq2#vt%yHPEA>uyI&)}wIq@F9HB zv=47hSG=~^x%dupr+fLj7Eh&K~6Zrn|$ zq&>3hJt=asZFkEXDGNgA{ikT>rUpO+MLnCF>|PgHTGcbn5qmY2=O(%RXZ4vV1&!#H zhRpuJ1`aQdgEMrs+ueWi7vOH$zRn}f;nKq#bmqqY==p-M@KWKAwewwQVz-}HY+#>$ zy#8>U(u8bZekIxq7P6|pbu8YtT|b8dJ(c_#F41;YuwSFgQDAplcU-LshNinO^0PZF z1wxiNt39T*)&^IBC(Qado6^@ z4)>L|&8}8@y5cfr7keA7->T5k`dz-pY1b(U%N-vB4UnXAy-{?Ww` z$A?F6-}H|z-VV;bKRSK;?eWRci?+Ut2fF&rc8XnABm3pS`SIH~7bhWR>%pyxwT){2cO+1GntgA7@Sh|t( zQNJ~Oq0;oPzg#2R5h)4PCw4dW&~6L4;56OY6w7dO;!SGdw2JXw{`k-I_;xFs?!7fp zPiZph(I(5;G&}rrjQ%m)vC)^^f$6&Y4Hlq(M(rATp}hs5i3#3&`b^BGzNdp5O?wRH zUB{XatiOJn`+DWEEmJqniz|;__W$4YluU(&-u?nt0E3U>8p0@U}axcQlNmm>zg<4{Da4h+%Ih6G$+pPep z{rtZf)lp&i=v63!+LhbD#O}2+vOxhjP4#IW^JtEYMkXV~Xz^jZ3h^HH7=527rg1r3dUv`-G-;nOo{o%4n23Zy4+O(47Jqb}jCHnHi ztF!*#^x*Z;4GIDTK$p5cG*)?n%-Lyj<|@qFs2Q8USFucy^MbZZe$g7~eTv;Lb1R(t*+p8jmp^qagEt|CY1qQF7(Fb@;bd zbAj+ovMH7{1nW2y+K=$ueeoJ!5(K_0b2ft-PSpabWSXA?zHHfD?F^$c;z5>T)d%JT zMJyq1h=W+L4$`^i^OaYH71LngrZjUaf-%Eqm#oT@vrh(&=kToHGPv#e7RQ`35QlK($qnqF6_PHeU zUPfXI=kV?EMSpO7+8-R89v;1YeQ@~d`1I)eqko+r3|>`DtB2a#nPGQZU>)XFVyIt6 z!=vO1^*liITFAoL>-yHOh-hTLC zXK-_)mA(D`$=i#wv%%rf#h@B7+d0vBTNk4*&7yFMp#?{4sf7C-!CwygOb`yho3!S4 z8Leht)C^Xn(+@5KZ}DwTEF&;zVVoKnYB#>j?R*J-hZTQKpCb3@5XD6ADGOL?ceXF&s$1*-w0M@dN zW#Cq}YYwp9X>OA>*@k%r*1h-bv|L8V`@4zLTduuy58oj%2Z^P~<6#yq9+vtz?sb$fwHw zXA9D;%K>bO{qO0MCnfv;?*7ik{`U~iD*j)2yI)o^zoC~vRq~3vTm6=}X4Sfo-Gh~{ z5|>rgWpxi#zx2vZVD%m3_=p)iVX`>UnPeiwF-^vKccRQFSurO3bO^Z$%VVu(z^qQS zxgNu1&ElpkIw-DhbN*a;8(5EtG~=kZG!oX6duakWyeN#g(tvsm?u?et!3tTQP_wBG zvd~pkF1#9Gmzg1#n#~M1UddKf{mrU2s{swdYZzho0B)9;8Z zYC%)b(v;_25#L*3-bgLhZZ&&kGEXe%!Q?I~3~XXjUy>Nl`y^yZRCUhh@m29nvD=hz z3JkDOhq>0pu11)~GfoB*PQ--8(bC?EA2iTD*GOc3)NwVH3v*QLyLc8i5vW^pocFS(?NK8U_% zULCQ)h{j~Qxl-Q5$=snxD&&tmgut9F^jO{KZr54=lBQc*HI@`V{HJ#QwT)J{qg2T+ zN=W^|TdqbolHHvKM8Jt)GaeGL)N{)657WLzD6MwwL+A^Wd&B=3K_)gRwdk_T%xwx1 zm)}QDh{-Gc<`|_c^16%|(TO==E1+2-Y3$`HF^rv4kS1NQuG_Y4+qP}n)3$BfwvB0P z+SatDZQJf|_kT{rzTOe*qHd}ps-hM$^L?L;=Yc#K(`LXEEQy&#FGYP_V~~~Ro^KMP zrqmQG!Y7Rv9rrDAh&=Tck^9c%9J7=eLj{!`yFfv)7M6j_FqW*~3m)d-NkWq@9h3}) zI*F|jAUm`0^Xz@&=Gi{`x8s3?4X^!Ms}|YlB)|DI>Gr;`Om<0c9)81W%Z)J#YmO@|lqCo!E=r_qHUtYskb>T$LboKa z6Hp0c19?i+;Kjnj^v~o~qFAC0yg7nKFPCYNw&mtJ^N`i#4C^r(?r^hXFs%6;q9i2S zuImVY%P5v46fjf%H&&ubF~^Nec2_N+CgyN*Etk;qT}P|Fw_hsA(4#hHwWnSd*PLnH z>>3SG>AxTYPw8WPrauUx@O#SOjPqWAya^Ti) z#O!2eMB7g`m~Tjyp!0?61*9&c>C_QhzMG13CPA~PM7+h^KUy1y>9?Qd5^IpY80!J6 zXh40X-+phf33xy zevKVxzXYP$!IxQGcMi}$bKu6ZI5zU+MI|+AP1V_QF@M1MHADXB}wyDCERPNQpy1+QT&3&}TrzeXTV@7M8{~8iJt5l`TRk5e@ zbTeAt@@MV1LkO(2WpHh2!l|-06_UWM*eG4#Ir~4~U)_0KSA;EOlFIB?QlPXXPE>6I z39N+UmkcRrt6_b;AThriqt0|j9qTEw{SsBP((Tr{EU)~jh5!@7<_@@4cuV6Ji#>HQ zp>K+^wu%$3A&zMTlWRFdQ_Ni2iiP?`eGse0cRJ;?zp1M zbOHBcr*hPgVbwX6kiSG9P@eT7*46B#YRF7Pm#kY8ZI^b}Va|%qOV4X)uLRdRpL*x& z#etjsPzO4^Kg8&-)+)6ws9ay8d#cnenLFz0pfHh#pt#|f z$53bEVxixSdlACbxPnQ!emNlcNKm`OY;%v*=U})e@1A`m~O*e0OykbRfYCo|Da|@aLNUN=tkq} zB!eeT#!~j8XK0Y+deUnLqh?d#+2AXfDxZ!TY>?zxo#3>0ZqY@6ih@GS6hV@18ZTPR zo4rdGE!}6?#iVdw4`nePB?3CqB$WBfC_B3pxvLlpnG^$SBu*&7iiB(+sScr5IOH*3 zxBQ%=%c6zL`AfKW+@@M$m{k~Q*4o=8Sq!Wg_IJNusVMxm{8H(%5Lf(;c7l^qN`>hK zb(}Y2HC@>V$hA48dOu=9wnl7h<1&+eW{76T$^|Nd?}9!GL!e!r6h5WVR1nPlR4f_s z-TRaYEuXcstPJ=MniL%aGUB8@mr|DQ=^qwM!<>rxT5T~^B5|N;3{Qoeb#c^*_SSeU zMZ>BcLN%?N>4*o9l!67w0<{wMQc#>^H^4*8h$#@=o2m?dHST=6k*Lt)y##jFF7&YS@``-Lk~_ zO;~5R=J<>Px);pRmI;H7`~?C&y_0r?l2_XT=TF4T`gYaad1U0#q7Qphx*9--ZZyE_ z`E#QD`zC~sP=F{tg?WN?b=Cc8c^f9U@M#Y;*(#P`1HLn?PYj9t7ssvxf831AR7DSf zlG;G`$l;4_4e-JHG4h_ESYUr1AUORn68TdEMUg;FvR=mE@g_teK1F|X5-CGi#&zW3 zcc1&cuT4-qREvL-RAcgwiLHyu?ixsr(J3fmpc!{E zvP;7!BWlc>deHTF`?T-#BlG$^u9W$Uy+K+M$`CnzZ1}?oI$gHVpQ?h{fk`uDW<%Zs zhEVV__5ny!&U->&R(Y&hUKgwIF@f9^ege!;*~g<5pp&F)Tdy8aSfTT+tGW;QK$ix1 z&PTfm;QdhKxZe@wdtn5yPir{NIdA)sS9Hi{-OXuE+R*K&%eOX#YQEh$V>g_)G^c9r zR1=-3i!7-9)rFg)q_|)Qac}zFj#g8U-?NH#XpXX@!lPaDLmyk0IN#uF_nW*YlpoE7 zYLdX;MR{{@xH(%a{(Lced6W3|iLAI^v6uQ2^($x((Xje~u!pTW7kr9_=q0ZMW)2&Y za7d)t0r#`WA0EcUNaT?HAR~H2-yfXXN6)$06yCV?HY4bbsH;aNDlQ)Btqh`$SeERU zlN^`zfz4hs{I2Kex*iX-?u^e5&iRUk0C`(9A-GqP_LRgnfn3_9%w+;5qr80x3syO$ zYf*l&O}nzMv94^!lI)3wO`rp5za`VN67ctR-eMRc)jVJ}r~6(=E_1wg30MWNgsK+c z#ceW;BwrFBb5=s3;(cvNot~1bn-59xso~1tL9BJ_vQVbniyNqCrdt_+uA9ZTHG)Ld zLW~)jg6XdL9dug%S2r|y&9%}})6XGmfCO?V&gfo9@$0Q$0Dqq4Yi`@V>HFY^tvSbW z0ubd3djPmo1}w4ySh=onPu*~GUu<~KJP6PB;h|G@$MF`G&Q+Iq4lI@j&w1U9*^57Z zW{Gcl=;@k2g3CZ=g)yR^oz}e~;d&xN^_ERd3r(ibAKUK5+2UVM%B$CYx7$HCHVyDU zXcoAo&c*BBPsVH4UrW!yJZ;N?R|O9C{`O>3e1&L>YW`IzdIWOrZ?&cAwkf`At{J}0 z`v|tlJ}F)TooDu?-8%r)binsaxWiE9u35T3?AOi2cT4hT`-52u;7d3^`12RWH=(W8 zuN+(x>>pjh<pP9Ry6k+6wI)Ys1cElq^rn<}aWmKkvL zd#!ycYX(|OXv1hvHM#7ax?*uwBw_t1dx-$HY3}d*#$c(~RjhTh?1z*_d#%!_4Pwmf zQ@69;?@TlUun`;!LRc0&XiB(NM05@-76~MphA7TMFjBBLST=K9lYt#z2nPLNu--Bvkm#MhQssq(vp@Swz ziLSX!3c;ZhM#C1;kHv}PrGeGj)fzo>!? zi}gXl^l?D<+;e{VzTk6#-5~(dOXl5>hEU{H^PpDw9@1N1{IdPQS1M~K-U~NqXTYc7 zuRlEwX3RTA2W0ZY*B*z+_X*|Us*gQ}$_N@}4?~KaTXU1nYG*4&X;s;5YeKTm9rqHx z^REdRq9wmpYy9Hqi(-D)OP3puk-2J=q(KE}(6my!>pXGTfe6Hn0EuL%s>nZWMIib= zRgBZQjk4qV!o34%jLH~-F*T{Bwy4?lO8#T5t@sUAVb6`b?q?0^)w**yn7{3S5sO4{ zq7TET`zUsDRlRR-P%d*JXH}C)dv2BOrX6sKK)4yegkw+7-N!Opl)>2S@6-%zps-Qed-Kmj=T zUOf5eF8fSSId%Q<(@M;QYXh*44eJf$>nyuG+1{?i|YyFHN6hbLG58N53J$eH1 z>wb}b_-0}I9rh$qpJqjow;IDUj`J>m;)araPu$Kt;U9;W=MMtce zmi*fwc8EQI)_xuErSb!VJfdxb51_$HzmjJKOszdqb<(hRq+rIL3rQtM>-qzrJr|Y~ zj1N*>aQU_(E)@A)1{Ey1cMnvy?!`Z=ZjUCzU+2c~t4^`+-U|h}x+V*oEc5bC-zf7?MdkHzmDeBZTH$jH>`8X+2NIQAgJO- zRBe*^-yoqNeC7Jga#T;r^aZ_3zeG9CP^NTAl=h;1QD>yHP`lDXY`UwCYn(&YNBPBR~3ng_0doF*|LKUro z$)oJobx%+Fiz3q$T&3vmPH|MFQ{(!HR7;GluA=%h7%4_smp{+ zpZ;sC5cjgUjwHw}zvGKY-&Y`5=x1MsrN(F{m`SX7ZNd;0(hZp!Q4y|4stm!1GE7Sr z%0pnh07xp3@8v5s1`9X{Gd_7pAeovd%l!7@Oe%|lKDD1L2bVwzi6|@_T^r}XT|0iE z*(lZWS8$G7&z3dML7T!H;kGYDda*4{j6*Xih(mL}QIU$r?(_^+Y~?l z7rNjuc)H3~A&>P%i6sHh4$oWVhM?;xOSYmpVWhq(LbO~%uX)C3y_K`(8>F!OR)x!U zzyj#T0~AX3aBKnqR$gBfTNOv2H;Ntw_WS*W`)^9>fQ_kdgEt?XWsAsFk5anBhg@;S zs@)c~g?r8xOF1()>^X>|;h^c8?raaM01MT=i1E4T`B>W?YI%fura1KC8voIo9kPGy zj-pG41LSH7P?DVr*phDs>*P|hV|MVq^GgA$%Gf>cCE+3J(;R?$`#(5roy6kTubDsl zx`K0113H*lKX(;T=`Sin2Q#8#}o_@50vn!=M({vM)X@B|o7)D{A?n z_tihOCHuWM+(4Z1v6IN?v_q zk`aIh$D^WbJz+G;0b2S2r5%75j{=A1pJS$BY-N`+Wh?~=dC@xlDSA|eL-JIpK4rF0 zj7H$4UV@|_QIhDO`GW1Oc_y1Qsn??cO8)LEd5%f8XQ<)q;~sT<+a?tVrAQp;$iY&k z-EDO?6Y&N*Ml8<1mQ+4QRAWEz=5Iry+;Z!=vw-b3Rz=cJbE%FcY58RTYnMzh5mVNc z@0JSQsmxouNBf}mAyDiXU8z557CAgW@Tuq?#4MbZLA(3VKl6dP3`7T{I08N*$w~=! z_dcF+eUuM+Idi)B54(!=fc9@L2Fkxzib3X|0XJU4fR^@$$F=QnfZ*`K5tx@=8qq9$ zz!zf{QhV74kjMA>v97HDOV@@-{EW@=bJB|0Pd7E6$3BvNX>*!J7S(z#%4YwVi8xG| zL06tU>AE@y!Wj!ZZ{^WuUYODP3q=(M_jz?5U2_{Hshcm|oBTO|_9P%q=a#`E(EJ5( z()}U+-QV8601zBs9KP`+A6-?0@3#Lw7X6fN?x6W)rv!EykI6V732SE@$;&!a{e?KW z&i)R>&;s~Twy6FF@UN|n2O!>8&UF6yYOvS<4dV+XG%VxIESd+`&G|3pxkdjkw8A$Y z*TzGQUw_r<_$=fi=;C<$0Zi{|DGWkGGlrlIx_o-!B#3y6qmT@9{}OeMKN^1c%lHbk z>WP)DofaoF)uA4yIO2TVx`gOVo$FMuG6X}U!-(x?o9g>)3phW)Lr%>jvs+}!rv|=z z#p7q7_o?rGF0xzWu!7sFfF+vfYIriLiipE{S_%A5G(n-_ft^bUC%LnWMM=Deu{4>F z5vb4a>;*%8z@bGS7(jQb`gQ45Why{%zv-_+#DF37W8A)OE z@C#hp6fZf1OX3Rf(N75YTOTs^7`iU3d-2;-g9)VKtP{7&UB;k>#-Rv3Pq4t(f~WHR zVP}%oG(Y^s8nEHh)vmb&5cKK#@7t>i&|~JM*dRf|$XJT&$CU;B)iw;U_>pl-CsQ@T zk#saW26#C6ciac{!*>hy-xyrE8x9z5L{v{C&kobUMIi}~laP3yF3ge|%29g;FXi`M zBqM0_ZXPOrlqU~f%; zfP8N%e15h(ej0{A7_Gha18P0L1%3MZTeyEpw%Q+2{u?!4)fw*}U9-HcBlrxDh(2jM zc*{8@-Y#mK0@VRF@gG+FttWyjv+Ea3pmaaK$Oip6pg=I7>ki&pDimeD4L^8szji|SmQ`MR zf2_kgP%D&;-5G+d9|4mnG}wj_pnsCT0A?rB)PSLuUit3`NU!QoK)=VTIV!lLaR^s} z{!gZ4m+8MGYN{CI$F|anji@%eJw*7eCChbqENXfE>*5v36mwp!w0IMi(&|6P20sUB z8ky`QoXTdRkd)2k-tiKR%d$1|0y5=EOOpfHt+2;yk0FZ`OyV(WZUwSEO|G^;m4_RO z7@aNAFpCOmCQu%7+vED3&Wssgnn!0^PcGp5XshJ3GLD<28Ut~aK6IYX$mq2Xd+<_! zKV@L+>x>Oxy@fZG#<(IzlA$%vjmK^jWI3sHV>Lj{TJy)w7EBro#v5GI^~ibV)O)qm zq9^UC%Ix;sj$`}CWq%G)#j71~Ahc|4afP&l!Gj*6^&E6mh1G*Wv!bgaz(VxYbNQgc za8?c!GJ5&3RnS++P$QrH>d4Cu!#XP9#LJFDv1M!$u=R7#f6?BGz-aRaVAfyxduL~D z7$7hmGL%@F7;;iaCSu<{oT|UxA7Hb~G1n{T4lu;S@A36p%9D6u8Xf%V|8RYIYcct5 z=-v7?$MK&+yWsnv0+itlDgvc98$Q6FFNp=c_sWh3-(w^+nG2+(v-kFEHFk{A7T2~f zcRGCigaIeTPCtu83#YB+#gyo+W-YcfE<%m+3#kvl_KLb1 z*IC$xxItE7vSX9H87aZON|z<6Na2T39`Vz-qj6PeEsgt;M5O21v|vqdHGV2}$1EvBgcyCA3F<%i7*RRK2kmmixJ9+q&uQu+*Q4HM)v~ zscnRbo#SB52&G{;2A6bFx3I8k-|#|U45I}9gW8=aYzcYMsRR02GDJ8ufG~p=TOI+& zMpj5p6r-qYYR>oy>~`mfZAEtmTlkGno8-V==l;oX!!R5_Aysds93a8J$C?B)Iw{~b?fgBW-}-OQd(F1x;F$_NJk2>o z3N6d*ItaIKNr7!m4W+!+XdAI}6aAg%$GNmeMRyB$qhF9g^I?+MmkrGjbgFv}lB-NT za%qzo;}(K_>o!#I_R-}YKigmv(an;Ftqgs4qb>#UsyB>9B}K@qEr=*0Hfj)mwE(;4 znkw#cJwp_b`MMP5Szfuo+#TGEH=nz?Cc@a=rf*GV9*NjL%ZJo|!KCXdquO%w-3KA7 z4vMpVKfP6aJe_@Ac|2F`@MT?Rs^0bBi+GTz$n`2+pif1h$kxb}_2gJW`GB@^;Eack z(9;fn1e!3AAP2AuJL@kH)uUZy`0O$lQLCFIhHi|MTIbLFkM+e;MDHK2`Ck_o*?$8& zy#q$p{i`9zC%~@LgUo_*6NyY#rKcUmNS<(X0>0{(ahTXzPZw+~^bePa5f)2^GF?;g zf>DOrxN@r8Rs>w#4Zzg8sFvBGckJd)pO^FNYn~B~pr+4acIvtp_U z*oUG2UjDfl;s_tuITBUCze)6!*lyJ^EcscxZg*#PUoRg|R#w9a^BQkazG7xNi0+{B zW=f0ko*c|t4%_-9b}XAgsVKOxhYHrH^AA4oUFB;Zhor~LRg=YTtIXW7oASCT#DV(a za|sFv;o1TZDz#~Y`9AI*PeGu5Z@lB+Q&Op!gc+%lP+BX(aSvpWh-M3@RtlX!f@*O% zi+3!k&MzvXAgfHN^Nb`K)fZ&O3BxVfydGoiF+-&D8bL*<+$gyOn$vZv>ii~ro{uyl zKcZ2V?QO$tssY=i-Eys@cskzK(iBrES+BnWhKQBc2dm^E#rKg=yQ&2_;wEwVqhwrk z>iOs@f1u?=z~!{8pwW^2@q!A#E0oShYWDiW>Q&uZyS5PA$Sc`v?_1ujvi+5+cx zbIHfzw8YDudstjv2S$Qog?hGZ?-9`7eeu10A>bBp;|@6g-afm&J`tn(Gra6F!|u=5 z!No7|b^R-KOLIE%l-A=iE=sJ0EjVPWR!VuPnQ+gn{>D)Rbw4$pQsqWhESYvvyLcF> zC;?jhC4#Q!5H2DmW2aI+puPbm_%tW~YIOtQU8BT*_?jW6o~c6U+clNh8Oa}t5u_?y zc`>hyIH$Vrk|Y7=*FnBM=AH%G2a~Bwg1W?%f}Gkxl#zaTf(qkNYiJlLb1E9 z00Nb0!O%i&ixrP%e##*>H?YjUZ)a<&bSbvJd;=MFq%E_I`oI}iQd5mdU}vjn?vRfrAM7QkDG1yrp(`cjm6po{4nI*TU(TMQC&gn z$R)-cmTa5P-x5J_|HQhxw|cKp(ZF)2!JzA1WhTK7D(l>0jGw!Bw<_Wyyo$?Ra*TgA z->t28V?1P#iK|1&%vn%Txk|^KJBe4BEw^aU9-o)`WtJRt2K9K>Mx^@hwMT_pC0FAy zZLM9Ch*!D=$T}t+{Blo$k)}CRHdUSaTh+z&`hoQIj08mPs7f%1=N)PV$H{#naE$Mr z3}$Q;f#DYF^D?|SA6;~~hb*2gg|*i5qbUAqNoFs_A0hW&DhY}6eAAYtq3!UAL2cm* z7!YBV=_azf%{(fJbig)7@6~Ul=v@nx$;(scG8nyGphxNx}^No_ecak`H;S>y8P z5yS`4J0lMUHDU9Tt{9R~wr`ei56BEB^17l?p9uprbJlgBSU|ls^ew({tpL`r^X{;^ z?%$g|3qU9dJLXh6JVNzeBjvu;QgbI5j>9stZqG!su-_6Aca~K|5gnpPXmj%^$)G_B zp^*!zRUu78#Qno4A!ddh=9VuKy^>4lEG71>#H252-rOMi<>^mwBi$cKSutP8rR#%e zH0n9u5>)cCTjtIDyP@^E_JQVXJCWlV>`bodvQGX3nS6)$d5{y`&Ipel;8NAR3+L+| zZh2;cn(oChE1p-;@Ce1!@}E0ylmp!((}((gdAlC=He|=s?e|R;{T`U!NO!-^%ahl@ zdFl#&Z_6d6uM3z?M|jpsamPzju}`fAhv1il91ruRFDK|TFYH&EQNxe4txWcX5x4a# z0igAs(t%u1$mAeYnefF2*wa#o%yLnaSg8J>XvJM<7_^Kc#TOlBt%Hm<8V63!Qt%pP zk^%j-I=g2%KdD;m_6oM}_s_RnQo((?r!@|Oyuy_iVm+;Q*9aQXmJ9C+j$QeEG-UW4 zP6E-Vron0v_}=_407)x?#%+dL46laXW+prDeh#Bz=na+~pfQY64Bp1rCxrRl2Hf7L zLECNhJm1}5NYue}4aVD5H?gw)9_gxa85De3(tS|VFE&oCb}34zOZWx<*<>LZVC&}^ zmarh{n)vm0Jv74$fM3D|VaRHR;v*`_D!yXyHjS0%>KZgMRfhi~iRHSv{>`qz$5EaU z9V@I4FTV8L`bD9pMhL~Jsu@4_-obUX{`R5KmPc%32{VjB`h3Mr)X zeLR|_+98Nw2&vu)Q$JA|aV;;!kmsKuv8+h4OjF(rH<2b9Lb4ql7(S)&WU?Lj8T&on4j)U2!`(S?RS@ck~JN%AXE?Yk|qpfSgv8ZkL1S@SPA!c=kWkS#TB0Li z4P60LrmGSCXSO9`PNq#3S`e=~G#8LXDI6#1fKS|R<5O*FbZ^LGAqG_isY&@*=z(_~ znRai0XCy{Mijq>Z^^-sKB-!Y7Z`@$0vjt~(+?vKE$fO~O%5*Y$iTf3XU@L@RIL7L37cuk<{(G%DJ`Dt6p(pI zGtr<=Q)=9)Z|j@J=*`WoHf!B(G%)=~e&RyYVpZ^}BGYAv^Gpz7OoY8e`3f#4prM6t zoQn31nOrdUjLQDFQS-IgmEZV(9E5CA$y+~sD*SzP)BR2pqp-y?r;rc|Bpz%F3}1^x_385Iof z=Uew+z2F{{Ng%+m%2anFc$u3&XI-|vSA&Orh&$4+Um0>=bb(zvGXbqy()uXG9Q1c0b5kF)VmT19>gmEk> z%$+rnUgUxdla#QyvI=HGId`@NJ|zP2=2FVjSCleYQdO8{c2r>f#jVKG%t3s@;F^wN z_ZzZ$!0A}}kxBO3z}eUGj?t*4E5PETRgo0MB)<-=_6Im%njQeblIDXOOSHiu_0_cZJ|u$9|al- z5zO3YDo1}<^bY+5@M;FXRH1db>=_2b70CRN0181Tt;QN!G;ywbK~T<4+BINl(&Y!m zi<@=aN^KN>><#@-+ zMqS%leCh+eC|KEbU)ZFiD3fMdI?xYLqV#k6)nIb0SDJzr#V%Qb3@=!w|Cy0MTiF_y z$&2NTi5KO;%Ur5uqY4ay|D_6K?*Pp}LM^?!ePa9psg<^Ru2GMC-CH@_Wgu~-c387m zL&Y0_Ge8*+_d5>A_L6{z#57>-8*3r+lvWw@KRdEjK?D-4WkUv{cZ5xa!PrnpE=QB9 zkm;vhkLK#)j&Q&27>b><`Vs@hl=5p1Ag=^fO3P+&(E8dhe zBr>*j&l*y13+-U%di0U%!~CY;0k=t8S~lvdP>|lyCAW^`$3M&AKYlP@;tt}k5&N32 z$xv>KP|{!s@zsilA-fA?c#5+M^+v4<((kMRM%l5vQqzM``-ZoBse|}}*f!8ml->{6 zvxOE7tuJSE@$w+eo0kca6>|ACyviiNs7AO#)t#Nf)=C?YckKBwZ2|>uZ=_^*|M0Tx z>~bAI=}&lS5W7ya^FjUipuIAhi#@2O_sL-ym$M*J z?s$OOE#)h`sPUE>*KP34e_eoMs}jHU*lb!qMlU3_x)-G`(vEO-T2+<{mgTZu&k5~2 zfzMdU$?{$6(C53H1s?g3`8L4tfVeesFshg+SlAkd3Yc8ot@o{82zae;SvgoO=ebNh z*CR5OcX~P&=V+1A;1#b3&ps&3mAebXn9`C{$GZ-S;1vv5oo&{$ejgeP6kZb)NS9fx z;c*9w?nZ(fL&u7`J4Ph&qw^v`*dsO zeUEvtEt`xSB5J@~&;4#5F3%Npjh<s+l7(DZW(!&%BBAfvn(wG#V;f zAD19qBU{co+o(teBE9P1A5tyEe9W8J#qzbnCAP;d|G9%!ID&zQ#v zKtMAg`R3*`52ja)aH?EPki`cI!BSRwRdFt6)1z!l;hH=keMI~YdZa<;zZ;S}L$L5a z|4Bkp?wIj^xE8AuOI%Nr_S8DE&3+T2yU7TU=1rRlP%j8cuNi+uyM$2fi7L}B)u`Lv zi1Lm&ldSVUXm+n|+k5QJ4e*g)8GPXBWo~c^?pPu#1WZY+Pv@=a^NBF|BH&O>!0WMY zs$)Y;hmf1j+1n}rmU{fGm38H`s-PxB9Ajde`?F`N2b>cBO}e_#n(%wtY$-zI;Pe-vu9|YjMHHpEs-krg)Jp9yIvnq+ZIwAFSH7b} zJlqj?3pj^6(^lX<7bb@4Z)?kC6%PIf_%HSL=Rvwurg|~yxvde;6H^5wuL5hz5K9PE zx=3I5=nHizV2Jxww;E_Rn2K4Hk)*cw^O^*wKg%fj0=xI~H2?<4Z!XBxkRGXfSR+jn zY_UlSs;Ob~N5W|4^T5c|TjT>(`sM200$r}C5p&?rR-KZ~(TP~mZ!Vh_QF^S3PVn>F zPx#w3>a6N+JyJwj&c=3$kWFJI#kiQVhp)EqYB(0UC8tE}n)e^N7)EvOPVJVOcXTJ; z11&4WM{Dtyv3WxD@(dTY92YVwhGgsUdZO@SJ&$&|gn(Hk(lTOS#7 zenz}8SLl`QMBw)okv$YAwWBZUES}vjNNgYzA}9>G7s5u<}^0~#Kr=NYHgoM_G^ITmD-Kl^pzVTsiH zqkeAwfxqkf>{E%ec4;TMl;A7+X$E8Fv|WwXk-p_u2Ji80%9-84w$PpRgTgm^a;*fP zBlRi0dRvyyJ2Zfb`XA@m{Q0_)opr)BaY?wa+L5^}1ko&(!Uq$86 zAccrSt+`WBjktlZMiGgskR#;imAT@^HNq{N=898^x)Fn{Pc;SlO(pp9GUpIA|-fXD4cuC>SsrmeSWVGg8wmK9F_k<64QAK%&@ft$H zn=vXfH#Q|~#7}&oX1)I8D29aBV=6XgEJwwEx_y@qD0J}u_f)KEr-OZ#E80kr6-A#i zKG-&dtYE&v08zc#FO#W3NH~E19Q>mYk0)YWP6M`7eIYO0raoaO}Tk-`u(NjgcQz zf=7nWo$G`#qWg;RKvbzo6v7jR-sOf0E57i3E*0hrO{_f|EXGPbvH??De3!z2f}5j% z><*MxaB+Tvg_i=ih`%HCL zqg^%b6)+SMa@G~cFP;SFVe8Tsu z-2}RDZmwP@=Z;+U8EXjYcbxNxCqM4mIn=oc>ZaffHt$$r0JEszV7B#+mp~nq_iRMa(Xl*xA?|{_@Ir;{@AasRk!vz@pd%kNDbLD$D64xJXfG7VYCz4ydYsrB{G}!NHb0THL&J15Rv3#s> z6|5Qw94B5cHMEcVkt#lnGgk{NM7#Rrcb_XRp$pAXOx$z~TvtBdQ~`vH<_v@qzP(br z`$CZ+F@uMczINlgcbF@FN{;SZUJ~1%C2p!x_fNBgk&B^bO~dU4X6O6oEIO98|KgB* z8Lr#3V4z9u5og%t8=R31Owv+?^NysHNt{x!#4lVdK#sT>H^>A=e@% zJCjAnecWL3G<(@nxMbn4m%|e#(1|44!-f>u+b|uKTb*G1Gp|Xo^k^~kfD_^pu#z+N zbExXZtp{Kr7%D#C$ zGzgkwke@b>4RoCs9?E)b#qPX}lprsxpZFL*=G8P*JoPkLYj#iO$ko>8ZKda8@feV* zXluO4E7Q}2`!Ij19^vQ`bc>FtR=27|?-wk4XkB**v=QU8Kh^yD4uF5;21$Y>|2FZwjBXhE_`Dx^mvWDBp?& zU9B55(a;0CB!0c6VRQD0w{;`1xn*HK5}^fG;*`tI-w00a+>rEFv98wHI6T^QOQWM) zGP7a)71y@b^_jR@kK(FAI@+N?LLv{Z!DQNu?YJEo`h2Ypu|0o%=QT2Nru&{R7T=XH zhi(GI0e({S!PJs=S~|uv$IL}*ZuQ-p1NRU62ZM1%@3#<4CkxW5-+x8pSeJ1o1j(mX zb=qqE*9?sVTjqCeaEXIs?REWT$Qr`GaJbl9gkOm-98*%U2Ud{JRyGlRWt-h`l>Z9qbqCR*qR8-* zpgShiw`?;C7q2x;G);78W z=wTft@*%8=C3DPnaHgdh)`}b;9}94A7fepkB1pmKg}VV8DG*AY)9xXcN793gMwB)- z74%LFnWGE`ctM3n_6@}@!ld!k0cxV6{r!}OSI)M~o+j#@kh4Ew#oCuEBKd5r2dNU+c7qshah*wX@9YWGnDw z*=uye9@#q%ruHo46I!NA;bi!TQ50lUhItOSIsAJ1$j9FlZ}=zFRu=t)r^KGoxOJSt z1S@q0L!m9#aUd@E#peikmg6_Tjl^KhrdyFBtUlJ5tZ1$lK;r>EIfmQ4+GJC~Xu0b_ z++K;GLrxy*uuxgEf?@CZ1ftqeQLq|=P6ZmDNC4mM#x2FdDebO`Q6POf^5Ij2K3B-%p{9}BtaY>@PFE|bk6#jtGKkvP z2xAQYilWNRohPalcUp>VRD5~pqm=}$Qv5iVz9#`=dPqI0AroQ~({;b{hnz4fZty@F zcfbc16@O&TL77^Hd4Uq;W(Yg5=TQdba=&OM7uwg2Win5@DNKqZJLtATCyr>LsuMNf zeH-$ePJe)pzDiH%ar|oznrf&3S$?KWq3G|vO|ZB-_m4%b?(j%iieUg$4>2RKii2%Q zqGxbf_G%b2fYk|M@pI#&@pAi zrkvph6jxRMO>y5wVv~YpIBM$HhKP_k9bO!Q!aH{F`C`Mm{8--gI_j9jdyG8>$`Qv0 z%Q;Zeiib`=iCz=iPJqUo5qSb|&f^J!0O)F<)&;lH30EP9S>A{Eu{$u~N$z})PM(D~ z;)NT~eQG=^3%FQbNQFbRTXQV5&cr`9&?%yryu#|?zkjfZW3S~A%NQ`8SraW*^9*rs zUv#-SYS+pJG7|Lx1;w;Ol$XPe8ZTj+!J8>BBJzL5A}QNOaXw@@M!L0S?lqgb(C*ru zohu(5sY-*U+w==V2wnJ?Gc;%gHS^SH1Jsuw4G>$GmkZ(H;k5Ke8nUcQfSv5%P+;AA zucds)eq46+DLQNQhJ^iKU|Fw7?s-TI6l!D7@s_rwCYgWQ)h0YRsNyWr?ClQ2o-$Q? zGi{I53Y%~$bRh`}$RYPr$uVOmqNkFWku=4(sIYmBrT;8vD8a6p{~N(hf{#k1I^s$# z1k?OrJ354^g`%#|WTP+*j)ZEO;0!Js-Ye6BV?rBcFh<)Z=>Ls$++tmx8>Up(6r<(? zZ&A5u(`D78YzWH8yc#*_nv%a%dQvCUiwK#xGx~JE>GH0`3G8-^dOhWJ%pv~tHBd%R z?2e_+i&uVlBItwFpCmy*N?`hkC*hx4c@>A-8CQGAQfz}*Ne{6G4CJzmK$_Rn9*F?Q zo}LVoSD1h6=bfZhx^$I81eaT$Tb6rHmWctS+dy(eR$PRT&Jt5DYfPy;>VO9oyU-ggrRgYyPmvY!nI!Zl zuw`>BnmOoROP_msVPC^JI_-4Og2eQ~`+7BZJrTbV&TH$8&$v}mRbaP16SyR;>b<0aTiR=>ZKVy29>F2q{ZGEbp9Gq7Wy9xd`fr=*Il?jx6dBVmNN|xfB}=V{96t6p~~OVxqo0XIw&2gNiB6)$)nC<4Pt zs57F02uEkC$Bb(YSSw2`@G?qww{3Mwx#!ljXqE@(yY)%U@@Cv`Nd>P9^={&$WV*_x z&&jREn6izK>5iB;khR9B}ZLLmbLc5%Tdhk8BzV*|Ij zYx&)Lu&>iz9ojl1;;FPIx)r=DCl`@vzbI(Ejn%zC{C-H$FDHpdbmB45 zqgx}47mLs1&d)l$Q*vR3yt?#vok_r^?)A;eapU3GIr`H5yydnG6rd0r8q@VI?a%vg zxz(?MxVjE)7J{vy<4ZWDx!A20p1|*;Ch-_4+G-hMX`K(xNBpV};K+g%KM zK3UbmN_E?0pU39IhB0qJ+01sCot^cm((=BpX6t$(Keo%%>&dju_f&qfi2mLxZ;Lf& zj!^s#rqs8V%wtxtkuC(&Uwu>xF2o*QjtayHgwYPRha^?YU7)`49YgeaW48)vOPAmE zcgtLJER4HH-;#(sTCSr}C!a~?r%Y1z%V-wEJZ{%GFdOA=hI-wUvYA@ybu9hRFSb1u zX|_kC=q(Di) z`<_rJeMis5=-wLk8)zlwD^EcfmuCRv>V_*y5!nF4#1~+2g?KV2Fu%QLT^6Sq!h$1f$>Eik*{d^16TE`1da&(sRqbN* z^UX2+S~oZ%O1ajO{#b$g)1hJNZd0M;;dsDR2EEqzDWkUCC6o0s32p=PfcLLsOC;Y~ z)ne*>B0JVKLBEMs@vXHr-dT5axekGF*<|34y!KcA6u72=(%Bo>FIiYhN&Dkj7iB}c zo-{@%FoMRe;5Gem#TO+}KZXt-$^x`Pm;VX57Dnl`Ske1fad9mo)c{>baJtd9{?Bb` zEDN)>h_t|oy%g?kRj&jd@XR;`3MMybCC3&C5G^8?~R{(f_Qj7&Zf4{LA zqrG|}*7*>}aj~_huQA;z;Cw*f z16Y+Vb?pE$DpW1;jyXO+M@0^PRy1$zwh!LRpUHp0gpZ*4_s}dYRc=c zw_$2bth3@q72s>Axl!5eeOBGDs4>>g%>DN}P*Q=a+68j`E=zP&qWsQ9I;whIcbSg* zG1gtE!;Yjkv`+N-Jt@e3`p007r?HcKK>tMLggW;i_dO(Xt}T@5`QUj`I{!L&T_|Lo z8#U)J>uURfNkG<5q?A$G3mYt^!}bK|ekm_+CCpW&&kJP?tLN$!F*Lj4ieh9-MIf6< z+{WH^P@iDMHv6`aJmcs)D$i1}Wz|Xmu({S4xB;bwk^N+<8hHp?qj{ltLLQ%f{Skez zX~W7(Qzow6`_Mi&@|ybl6%(@EYqvXXuy6OCKH1;9x#_g?^)Pe``%b4%Zuj6EXn#w& zkjEw-Np*RW<)b~vP#M47gP^naYpoIep3MEXXM9}vO#RceRevffX!qI&xkKXE`Ts^> z($y(0%=ex48=*jj?{bM^Z#wOB31I02rL~tA0_t-EKmA=;gP>HK$UlJ26ApX9A@% zIi@0xlA=>A)M=|7HnQL==O{=M%yUKObuAVn9(B>TMtZT9V3*ZA3oFV4$+YsUgM=@~ zc_(6_6vOr~X2Wf@#kRv~wC!r;C|;V|-O;cwVxXo&Zqhtg=qen)T+Aialz;@`Qu|0) zUo=0Pxv5k!GclEqr-PIJmJ-w^iWZDxq`_xagVJMi5QSw~9H_+GG8WyC%QkxKjilue z`0bL++vu?^mT*btTfn+a2#aS{1uq<1O;+pbS?W}oaFp4rJBi5^SnHh&bU{1-%pT%H zUWL%}qt^jRLKcw-9Uhbrq$5f#Nw7z-2dmv_8|)|Im6-P-K{H3s<0Ue7(22udj-;A+SV5N+j#Zuoq>RVqT&Ag1mfIYKZGvFlmthc#Iha(>P$M}m7$g4@1HaRig#Y&MLEm@pA8}pYK82xNH5GrvhO_%FW`bwcgkKhQHEr+8^GKav zwiQ4+pPI}|BnkOEwTVUqNdyI#3aIP{zBilGk)BJVNWe)jj!a4vOz z678Ty6K&o;=pP;*gNYA^QY{Jla2jrP(MyI-&jvb^*DRJa zQ2nAe300D^*H6cumug-yNyc&lXEniMW@Jx*r~@v>*>_#^oh_MlP35GABu7CMqJ!Xn zaq#O-@K^Nr?TB1$&p1vZHbozyY$*ytHOJ%E=Vup#gVR9|^(C`jn%bibLhFFRo-iS- zK{Uhtl;A|5*!-qe*5F@}4LI)@`PLOi)%`NAkzqT^)LJ#Pa7Q8aHbQ1L%UT(;1ZGY^ z@T%o9sGykGL(K+Ny12=jS=VU$?jTlXld^)Od($rL{`%s9rb3Zqyxc8KtU>`|Nb96U zbGB$JxxtE2KAtj3dX{d=Oo3Ir^zbw_o7mN*w2g0H%cGnwK9>%}W|g1i4HauG=#ihw zy=L&wCGb~l=8}+9rwb``YYz*A)9=r%jXu@Kjc zM?zO{R{Xw$`mYD)e%(oZ9%}Dt0|cB>M(1qO-DG$FG**~{IOD=zx3~Yikd_tC21j-# zC!lGY5WSQD=$mq+jr9(9EwcXq*}K~2 zwr%8}*Z35u-I>}m0wwvOP5h7DI(Cx3u8xzjo#c9+ObLmw#F`>_1SMC~_PgI;7Yl-v z{E%F3=5EG%6GsB^#KmHF@!KDQN&M`qGnQ3dVDoFcleHU=}A@?6;g27FZ$`m&Zp#B0UrWxBTv&XG#Vvko(M zW|yx!%7aaZ_5<|a?z~AAmY!Ach<2lO_c8(A;-gTm83RyLfUU&?oSeszg7P?s7&+Yl zfRLg-8|mDnra}_W1hXR+K;(qBF0{(C3P+5+v8JO-vBMvrIkB>m}kWq)eCx^yvSzbH2<_gmSH%`NSEX8 zI({Rtpsg3g_rv*-PE=%HmsP=FCGPEJj|qN~=D<+`Sk#S^SDQ@mBMM(|I5D(C z-gtlUN<6hc4svUt)AC$oif@2qqG2-C#!()1cqNDUnQrgZYc6Rk}C-4cd79bNNT;SgnnukG*xiMDB~-8qD3-fFya zoLuNMG?=Kl%5(wiEQldTux{4vk*WgIvhSMZwPb}+YrhQQuZvT zP~f}c7s{-Db?f{H)AN5$^J995U8gl;0OewM^&WNY`{*KVpS6p+wz#u{K*_UqgE#^9 zF&zI#xDB9{Rsj}3i%!ArwMD6s&1!j<8mkH)6GCR0@W*}j3m}rWU3ptlt$?sZxV&xO z!ARUvQ3=DZMb4u>GC(*Y0or7R|Zwf6O_9Wc(H zr_(Vv<*2U96^Tkcx;9H3YJV;4;ac~AJ@3i`_QZea;j@>A$8TOn z(-E0h*@1t1y!heebVc1@9ge8;k~cf ziQ7IiYfe4s{o8G|2lqenx%}qx(0XBVRPq0x4eNb5osJ_jNmZ5rl%3eUew>xrK9!N~ zNh{;2?ha)#R^590C5hauYi&JYHN6}w(=|$#qe(-aH|+{`t_Wj){NZ&ZUjqq`TprI0 z+v*yR_+NjmC{!&9b$1C_6d#sqg2jEvUm#Tm5zT;dK14R6H-Jrm4f9-QGc+ZA!8E@r zrJ)#FhnsX-xRfZj=SGa3C|Ty{AjzRs=;U|}7r14B*E&OOdg;@Oymg~<{yME1Z|!!4 zvk61`IFZ$N{^oz!0U3MYbG|xgK{aCHhD*pwvDT)?(5*-gK+B9$=UK zR2P72UsY+dZwkDV83sm5;0!Wcm!pX?RY+tLM7QcA56UOWlR8`llQIY_CO0K^C`y{t z?qxR;+-UENL*;a>2?RuFjK;pX+Q)*|Hpnr~=);o~)cmy3Rrr_b_I*Z{NL~3_BX0(d zw9&;RHI?Hks?SFR`m@cu1UjxST<1QMus0m*@9)a;xs-QWtF%X9NuMaOzdH zYzq${O9V!VOsgV?V@0aWa}bgh73~N|*C1v_m8~iU;q4MD9^Mt6VywyrO^`&hXAFc+F4!nuIG(Hm4?DLJux?_ejdY|9^> z+NwW1L(!Z+bC$1<=l&x|^8=@t(i}|y*t?`s^PPvNjII-M)2-yVJKu?VQE$_kmB-pT zCg(RPm~#jV|dYV?jbs4srT-`YD9-DedBcYm* zP(u){roe%lm#r3?UCoh|;7YTPH*`j)&CJvA$M9LJ>>%OPF0@7A$wXW_q|OwN2ZwKd zJ$d_jaQf@f$>8PbMI5m~g;XQPXRI1tsM6^AR#8L;Af&Tl=o>l|PDgM^4}rY~0<<1A zM8+}P=#Oge93%Oj&KH1v0jqLIHekQC(pbU~L8oohyWUMC(IL=r!Bplmm^gQ~fXdiW z|IVN7G1iaBH(5QWY%AgrK=E0=GWQp(AWk}k4MCT8PCwfY8d^Q;Bs;bhwi6Vm$+y+R zaUGQ3<1u*;qIMYg^Xa1g-vQ03s{(@k>4=9}4_pYVaKu%bsr^D$`?iKmk7OnaSxrQx zMKw_((KmUfWug-MHIzQEPNwElnjU;-0mc%Cmku-4icRX7eb0`=(v3@T+Z-<-s>b8) zEbY#>x=?4#8v;h;j}$9)ba>0L^*X%m;B9psR2Uj}I)ey}z3zZ$He(0;@|&Z>_ZNe= zr=Qq?(?=(Ibj^R!v(l*SioeCc)-MoXbJD>M_zM&}=*2|lV;6#38TIb)8%5TShu2V0 zYgEeulHY+SLg#N2r;T@q7Z{WUM}SB7vD;y(|)?@Vf_7o9+!=vEEb zaqUW0a6M#*&JFef%8{&1RvHIvLvP`L{)K%V_Pre8TwCMm*bRZWuYs%h_2BgH4f>?U zebKYLMkzUZ+2F?V@DRr5mjmi9Vxn=3i?Q$+G{Z!jDn=aGRL?^YdOadoOUm#V3tw6P zE`r6OWP(!{{$!T2fouDst}I~lh;PYli<;SKF^!`(M}wWiTP}9k)*w>J9^+g9=frl@ z_kV9iyZ-Nf^vwU=i~8FQ6AETx{sU@_kB4WcgVWca0+I}(gm~(=y|Jq3ss)dvY?)g0 zkjxk8!#PIDDewovx@}(h0NaF>`ygw?F8X)qO3=~JGCxId#8!K&d@jorA;4F)A_wC~ zmF-U2X6$fh#Yh9y9X0X{7YWAv-LkfWULIz^#Ju5u+un43-))nV8lLMLez4EyThZ=r z#4cR~)SagKso=*ioSh&mEWRCW)mCaPS5r)@cSjjrnaoq8%SBiAd%fPXO;Qa=rsOz+ zb(RF2GWx2~tdE=%&bz(`r8Z3{_nWa(UAfzEQgTAAI&;sCsAAOE0fy(d*|gx(G}pc( zCUW)|f?UGTECul!(_^QkAN6EsE;*y%ijrFcxru}b^{Rfb&D08 zDz6#z2weYqcqek`fQQrH&90*F%d{ot2m!}nEoV%5sfT@RcKA%VI*k~)QRxw$m`9zgz zRUb4P4>D?Y$8KP&&Vok^pNHfjbT0OS^&zs&n(7^Z)=13YC{$Uc%3bI(R4R8@|0co6 zL!Gu*j_B02boh0eCt$PAQ+0zL4D4o(k_jVj$!{~$7^)nNbZPZnw%B8g^FR69Ea4x( z$hiOWCphO1EzSS54T#VAd|qh*4e^=0Zc;Ek>?TmbjU{DXx2z5-6p z8rh0xT31J96$d?umoq}+R!>$}g5lcuwo2-sS5|`Xo1k2~Kr;#6FB=ixtxV$EvomAA zG;+7+lYJb-@4Z70CO+Qe;m-v+>=NTQ4{ z%0-qZU8Ey}s0H&YG|^oLxNc2RCQBu9O=+!(xJj#tfP;NgXh5b2inwkzBWs$=%xJd+ z+`)!Fyr#X0##=)eIt=l}2qU1lTVQB5#}}AJ%r4aHr-nL@7Y5}-yo;_K1+Yq;sbPww zoDgYq@ep?x!2j_?Gy6&@GUht+(YZo{ zZN-sa2)a2(Wktt+tIiogHMGh-LOUO!osZDYM`-6GwDS?#`3UWNgmykcJ0GE)|7Fn5 b$Is*E@$>k(^YgC&00960a(zM`02&1Vinqpu literal 0 HcmV?d00001 diff --git a/harmony-k8s/Cargo.toml b/harmony-k8s/Cargo.toml new file mode 100644 index 0000000..f989c9c --- /dev/null +++ b/harmony-k8s/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "harmony-k8s" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +kube.workspace = true +k8s-openapi.workspace = true +tokio.workspace = true +tokio-retry.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +log.workspace = true +similar.workspace = true +reqwest.workspace = true +url.workspace = true +inquire.workspace = true + +[dev-dependencies] +pretty_assertions.workspace = true diff --git a/harmony-k8s/src/apply.rs b/harmony-k8s/src/apply.rs new file mode 100644 index 0000000..e624766 --- /dev/null +++ b/harmony-k8s/src/apply.rs @@ -0,0 +1,552 @@ +use kube::{ + Client, Error, Resource, + api::{ + Api, ApiResource, DynamicObject, GroupVersionKind, Patch, PatchParams, + PostParams, ResourceExt, + }, + core::ErrorResponse, + discovery::Scope, + error::DiscoveryError, +}; +use log::{debug, error, trace, warn}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use similar::TextDiff; +use url::Url; + +use crate::client::K8sClient; +use crate::helper; +use crate::types::WriteMode; + +/// The field-manager token sent with every server-side apply request. +pub const FIELD_MANAGER: &str = "harmony-k8s"; + +// ── Private helpers ────────────────────────────────────────────────────────── + +/// Serialise any `Serialize` payload to a [`DynamicObject`] via JSON. +fn to_dynamic(payload: &T) -> Result { + serde_json::from_value(serde_json::to_value(payload).map_err(Error::SerdeError)?) + .map_err(Error::SerdeError) +} + +/// Fetch the current resource, display a unified diff against `payload`, and +/// return `()`. All output goes to stdout (same behaviour as before). +/// +/// A 404 is treated as "resource would be created" — not an error. +async fn show_dry_run( + api: &Api, + name: &str, + payload: &T, +) -> Result<(), Error> { + let new_yaml = serde_yaml::to_string(payload) + .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); + + match api.get(name).await { + Ok(current) => { + println!("\nDry-run for resource: '{name}'"); + let mut current_val = + serde_yaml::to_value(¤t).unwrap_or(serde_yaml::Value::Null); + if let Some(map) = current_val.as_mapping_mut() { + map.remove(&serde_yaml::Value::String("status".to_string())); + } + let current_yaml = serde_yaml::to_string(¤t_val) + .unwrap_or_else(|_| "Failed to serialize current resource".to_string()); + + if current_yaml == new_yaml { + println!("No changes detected."); + } else { + println!("Changes detected:"); + let diff = TextDiff::from_lines(¤t_yaml, &new_yaml); + for change in diff.iter_all_changes() { + let sign = match change.tag() { + similar::ChangeTag::Delete => "-", + similar::ChangeTag::Insert => "+", + similar::ChangeTag::Equal => " ", + }; + print!("{sign}{change}"); + } + } + Ok(()) + } + Err(Error::Api(ErrorResponse { code: 404, .. })) => { + println!("\nDry-run for new resource: '{name}'"); + println!("Resource does not exist. Would be created:"); + for line in new_yaml.lines() { + println!("+{line}"); + } + Ok(()) + } + Err(e) => { + error!("Failed to fetch resource '{name}' for dry-run: {e}"); + Err(e) + } + } +} + +/// Execute the real (non-dry-run) apply, respecting [`WriteMode`]. +async fn do_apply( + api: &Api, + name: &str, + payload: &T, + patch_params: &PatchParams, + write_mode: &WriteMode, +) -> Result { + match write_mode { + WriteMode::CreateOrUpdate => { + // TODO refactor this arm to perform self.update and if fail with 404 self.create + // This will avoid the repetition of the api.patch and api.create calls within this + // function body. This makes the code more maintainable + match api.patch(name, patch_params, &Patch::Apply(payload)).await { + Ok(obj) => Ok(obj), + Err(Error::Api(ErrorResponse { code: 404, .. })) => { + debug!("Resource '{name}' not found via SSA, falling back to POST"); + let dyn_obj = to_dynamic(payload)?; + api.create(&PostParams::default(), &dyn_obj).await.map_err(|e| { + error!("Failed to create '{name}': {e}"); + e + }) + } + Err(e) => { + error!("Failed to apply '{name}': {e}"); + Err(e) + } + } + } + WriteMode::Create => { + let dyn_obj = to_dynamic(payload)?; + api.create(&PostParams::default(), &dyn_obj).await.map_err(|e| { + error!("Failed to create '{name}': {e}"); + e + }) + } + WriteMode::Update => { + match api.patch(name, patch_params, &Patch::Apply(payload)).await { + Ok(obj) => Ok(obj), + Err(Error::Api(ErrorResponse { code: 404, .. })) => Err(Error::Api(ErrorResponse { + code: 404, + message: format!( + "Resource '{name}' not found and WriteMode is UpdateOnly" + ), + reason: "NotFound".to_string(), + status: "Failure".to_string(), + })), + Err(e) => { + error!("Failed to update '{name}': {e}"); + Err(e) + } + } + } + } +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +impl K8sClient { + /// Server-side apply: create if absent, update if present. + /// Equivalent to `kubectl apply`. + pub async fn apply(&self, resource: &K, namespace: Option<&str>) -> Result + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + Serialize, + ::DynamicType: Default, + { + self.apply_with_strategy(resource, namespace, WriteMode::CreateOrUpdate) + .await + } + + /// POST only — returns an error if the resource already exists. + pub async fn create(&self, resource: &K, namespace: Option<&str>) -> Result + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + Serialize, + ::DynamicType: Default, + { + self.apply_with_strategy(resource, namespace, WriteMode::Create) + .await + } + + /// Server-side apply only — returns an error if the resource does not exist. + pub async fn update(&self, resource: &K, namespace: Option<&str>) -> Result + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + Serialize, + ::DynamicType: Default, + { + self.apply_with_strategy(resource, namespace, WriteMode::Update) + .await + } + + pub async fn apply_with_strategy( + &self, + resource: &K, + namespace: Option<&str>, + write_mode: WriteMode, + ) -> Result + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + Serialize, + ::DynamicType: Default, + { + debug!( + "apply_with_strategy: {:?} ns={:?}", + resource.meta().name, + namespace + ); + trace!("{:#}", serde_json::to_value(resource).unwrap_or_default()); + + let dyntype = K::DynamicType::default(); + let gvk = GroupVersionKind { + group: K::group(&dyntype).to_string(), + version: K::version(&dyntype).to_string(), + kind: K::kind(&dyntype).to_string(), + }; + + let discovery = self.discovery().await?; + let (ar, caps) = discovery.resolve_gvk(&gvk).ok_or_else(|| { + Error::Discovery(DiscoveryError::MissingResource(format!( + "Cannot resolve GVK: {gvk:?}" + ))) + })?; + + let effective_ns = if caps.scope == Scope::Cluster { + None + } else { + namespace.or_else(|| resource.meta().namespace.as_deref()) + }; + + let api: Api = + get_dynamic_api(ar, caps, self.client.clone(), effective_ns, false); + + let name = resource + .meta() + .name + .as_deref() + .expect("Kubernetes resource must have a name"); + + if self.dry_run { + show_dry_run(&api, name, resource).await?; + return Ok(resource.clone()); + } + + let patch_params = PatchParams::apply(FIELD_MANAGER); + do_apply(&api, name, resource, &patch_params, &write_mode) + .await + .and_then(helper::dyn_to_typed) + } + + /// Applies resources in order, one at a time + pub async fn apply_many(&self, resources: &[K], ns: Option<&str>) -> Result, Error> + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned + Serialize, + ::DynamicType: Default, + { + let mut result = Vec::new(); + for r in resources.iter() { + let res = self.apply(r, ns).await; + if res.is_err() { + // NOTE: this may log sensitive data; downgrade to debug if needed. + warn!( + "Failed to apply k8s resource: {}", + serde_json::to_string_pretty(r).map_err(Error::SerdeError)? + ); + } + result.push(res?); + } + Ok(result) + } + + /// Apply a [`DynamicObject`] resource using server-side apply. + pub async fn apply_dynamic( + &self, + resource: &DynamicObject, + namespace: Option<&str>, + force_conflicts: bool, + ) -> Result { + trace!("apply_dynamic {resource:#?} ns={namespace:?} force={force_conflicts}"); + + let discovery = self.discovery().await?; + let type_meta = resource.types.as_ref().ok_or_else(|| { + Error::BuildRequest(kube::core::request::Error::Validation( + "DynamicObject must have types (apiVersion and kind)".to_string(), + )) + })?; + + let gvk = GroupVersionKind::try_from(type_meta).map_err(|_| { + Error::BuildRequest(kube::core::request::Error::Validation(format!( + "Invalid GVK in DynamicObject: {type_meta:?}" + ))) + })?; + + let (ar, caps) = discovery.resolve_gvk(&gvk).ok_or_else(|| { + Error::Discovery(DiscoveryError::MissingResource(format!( + "Cannot resolve GVK: {gvk:?}" + ))) + })?; + + let effective_ns = if caps.scope == Scope::Cluster { + None + } else { + namespace.or_else(|| resource.metadata.namespace.as_deref()) + }; + + let api = get_dynamic_api(ar, caps, self.client.clone(), effective_ns, false); + let name = resource + .metadata + .name + .as_deref() + .ok_or_else(|| { + Error::BuildRequest(kube::core::request::Error::Validation( + "DynamicObject must have metadata.name".to_string(), + )) + })?; + + debug!( + "apply_dynamic kind={:?} name='{name}' ns={effective_ns:?}", + resource.types.as_ref().map(|t| &t.kind), + ); + + // NOTE would be nice to improve cohesion between the dynamic and typed apis and avoid copy + // pasting the dry_run and some more logic + if self.dry_run { + show_dry_run(&api, name, resource).await?; + return Ok(resource.clone()); + } + + let mut patch_params = PatchParams::apply(FIELD_MANAGER); + patch_params.force = force_conflicts; + + do_apply(&api, name, resource, &patch_params, &WriteMode::CreateOrUpdate).await + } + + pub async fn apply_dynamic_many( + &self, + resources: &[DynamicObject], + namespace: Option<&str>, + force_conflicts: bool, + ) -> Result, Error> { + let mut result = Vec::new(); + for r in resources.iter() { + result.push(self.apply_dynamic(r, namespace, force_conflicts).await?); + } + Ok(result) + } + + pub async fn apply_yaml_many( + &self, + #[allow(clippy::ptr_arg)] yaml: &Vec, + ns: Option<&str>, + ) -> Result<(), Error> { + for y in yaml.iter() { + self.apply_yaml(y, ns).await?; + } + Ok(()) + } + + pub async fn apply_yaml( + &self, + yaml: &serde_yaml::Value, + ns: Option<&str>, + ) -> Result<(), Error> { + // NOTE wouldn't it be possible to parse this into a DynamicObject and simply call + // apply_dynamic instead of reimplementing api interactions? + let obj: DynamicObject = + serde_yaml::from_value(yaml.clone()).expect("YAML must deserialise to DynamicObject"); + let name = obj.metadata.name.as_ref().expect("YAML must have a name"); + + let api_version = yaml["apiVersion"].as_str().expect("missing apiVersion"); + let kind = yaml["kind"].as_str().expect("missing kind"); + + let mut it = api_version.splitn(2, '/'); + let first = it.next().unwrap(); + let (g, v) = match it.next() { + Some(second) => (first, second), + None => ("", first), + }; + + let api_resource = ApiResource::from_gvk(&GroupVersionKind::gvk(g, v, kind)); + let namespace = ns.unwrap_or_else(|| { + obj.metadata + .namespace + .as_deref() + .expect("YAML must have a namespace when ns is not provided") + }); + + let api: Api = + Api::namespaced_with(self.client.clone(), namespace, &api_resource); + + println!("Applying '{name}' in namespace '{namespace}'..."); + let patch_params = PatchParams::apply(FIELD_MANAGER); + let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?; + println!("Successfully applied '{}'.", result.name_any()); + Ok(()) + } + + /// Equivalent to `kubectl apply -f `. + pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> { + let patch_params = PatchParams::apply(FIELD_MANAGER); + let discovery = self.discovery().await?; + + let yaml = reqwest::get(url) + .await + .expect("Could not fetch URL") + .text() + .await + .expect("Could not read response body"); + + for doc in multidoc_deserialize(&yaml).expect("Failed to parse YAML from URL") { + let obj: DynamicObject = + serde_yaml::from_value(doc).expect("YAML document is not a valid object"); + let namespace = obj.metadata.namespace.as_deref().or(ns); + let type_meta = obj.types.as_ref().expect("Object is missing TypeMeta"); + let gvk = GroupVersionKind::try_from(type_meta) + .expect("Object has invalid GroupVersionKind"); + let name = obj.name_any(); + + if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) { + let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false); + trace!( + "Applying {}:\n{}", + gvk.kind, + serde_yaml::to_string(&obj).unwrap_or_default() + ); + let data: Value = serde_json::to_value(&obj).expect("serialisation failed"); + let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?; + debug!("Applied {} '{name}'", gvk.kind); + } else { + warn!("Skipping document with unknown GVK: {gvk:?}"); + } + } + Ok(()) + } + + /// Build a dynamic API client from a [`DynamicObject`]'s type metadata. + pub(crate) fn get_api_for_dynamic_object( + &self, + object: &DynamicObject, + ns: Option<&str>, + ) -> Result, Error> { + let ar = object + .types + .as_ref() + .and_then(|t| { + let parts: Vec<&str> = t.api_version.split('/').collect(); + match parts.as_slice() { + [version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk( + "", version, &t.kind, + ))), + [group, version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk( + group, version, &t.kind, + ))), + _ => None, + } + }) + .ok_or_else(|| { + Error::BuildRequest(kube::core::request::Error::Validation( + format!("Invalid apiVersion in DynamicObject: {object:#?}"), + )) + })?; + + Ok(match ns { + Some(ns) => Api::namespaced_with(self.client.clone(), ns, &ar), + None => Api::default_namespaced_with(self.client.clone(), &ar), + }) + } +} + +// ── Free functions ─────────────────────────────────────────────────────────── + +pub(crate) fn get_dynamic_api( + resource: kube::api::ApiResource, + capabilities: kube::discovery::ApiCapabilities, + client: Client, + ns: Option<&str>, + all: bool, +) -> Api { + if capabilities.scope == Scope::Cluster || all { + Api::all_with(client, &resource) + } else if let Some(namespace) = ns { + Api::namespaced_with(client, namespace, &resource) + } else { + Api::default_namespaced_with(client, &resource) + } +} + +pub(crate) fn multidoc_deserialize( + data: &str, +) -> Result, serde_yaml::Error> { + use serde::Deserialize; + let mut docs = vec![]; + for de in serde_yaml::Deserializer::from_str(data) { + docs.push(serde_yaml::Value::deserialize(de)?); + } + Ok(docs) +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod apply_tests { + use std::collections::BTreeMap; + use std::time::{SystemTime, UNIX_EPOCH}; + + use k8s_openapi::api::core::v1::ConfigMap; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use kube::api::{DeleteParams, TypeMeta}; + + use super::*; + + #[tokio::test] + #[ignore = "requires kubernetes cluster"] + async fn apply_creates_new_configmap() { + let client = K8sClient::try_default().await.unwrap(); + let ns = "default"; + let name = format!("test-cm-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + + let cm = ConfigMap { + metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + data: Some(BTreeMap::from([("key1".to_string(), "value1".to_string())])), + ..Default::default() + }; + + assert!(client.apply(&cm, Some(ns)).await.is_ok()); + + let api: Api = Api::namespaced(client.client.clone(), ns); + let _ = api.delete(&name, &DeleteParams::default()).await; + } + + #[tokio::test] + #[ignore = "requires kubernetes cluster"] + async fn apply_is_idempotent() { + let client = K8sClient::try_default().await.unwrap(); + let ns = "default"; + let name = format!("test-idem-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + + let cm = ConfigMap { + metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + data: Some(BTreeMap::from([("key".to_string(), "value".to_string())])), + ..Default::default() + }; + + assert!(client.apply(&cm, Some(ns)).await.is_ok(), "first apply failed"); + assert!(client.apply(&cm, Some(ns)).await.is_ok(), "second apply failed (not idempotent)"); + + let api: Api = Api::namespaced(client.client.clone(), ns); + let _ = api.delete(&name, &DeleteParams::default()).await; + } + + #[tokio::test] + #[ignore = "requires kubernetes cluster"] + async fn apply_dynamic_creates_new_resource() { + let client = K8sClient::try_default().await.unwrap(); + let ns = "default"; + let name = format!("test-dyn-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + + let obj = DynamicObject { + types: Some(TypeMeta { api_version: "v1".to_string(), kind: "ConfigMap".to_string() }), + metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + data: serde_json::json!({}), + }; + + let result = client.apply_dynamic(&obj, Some(ns), false).await; + assert!(result.is_ok(), "apply_dynamic failed: {:?}", result.err()); + + let api: Api = Api::namespaced(client.client.clone(), ns); + let _ = api.delete(&name, &DeleteParams::default()).await; + } +} diff --git a/harmony/src/domain/topology/k8s/bundle.rs b/harmony-k8s/src/bundle.rs similarity index 99% rename from harmony/src/domain/topology/k8s/bundle.rs rename to harmony-k8s/src/bundle.rs index d826201..ee77cc7 100644 --- a/harmony/src/domain/topology/k8s/bundle.rs +++ b/harmony-k8s/src/bundle.rs @@ -56,7 +56,7 @@ use kube::{Error, Resource, ResourceExt, api::DynamicObject}; use serde::Serialize; use serde_json; -use crate::domain::topology::k8s::K8sClient; +use crate::K8sClient; /// A ResourceBundle represents a logical unit of work consisting of multiple /// Kubernetes resources that should be applied or deleted together. diff --git a/harmony-k8s/src/client.rs b/harmony-k8s/src/client.rs new file mode 100644 index 0000000..1e828ac --- /dev/null +++ b/harmony-k8s/src/client.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; + +use kube::{Client, Config, Discovery, Error}; +use kube::config::{KubeConfigOptions, Kubeconfig}; +use log::error; +use serde::Serialize; +use tokio::sync::OnceCell; + +use crate::types::KubernetesDistribution; + +// TODO not cool, should use a proper configuration mechanism +// cli arg, env var, config file +fn read_dry_run_from_env() -> bool { + std::env::var("DRY_RUN") + .map(|v| v == "true" || v == "1") + .unwrap_or(false) +} + +#[derive(Clone)] +pub struct K8sClient { + pub(crate) client: Client, + /// When `true` no mutation is sent to the API server; diffs are printed + /// to stdout instead. Initialised from the `DRY_RUN` environment variable. + pub(crate) dry_run: bool, + pub(crate) k8s_distribution: Arc>, + pub(crate) discovery: Arc>, +} + +impl Serialize for K8sClient { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!("K8sClient serialization is not meaningful; remove this impl if unused") + } +} + +impl std::fmt::Debug for K8sClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "K8sClient {{ namespace: {}, dry_run: {} }}", + self.client.default_namespace(), + self.dry_run, + )) + } +} + +impl K8sClient { + /// Create a client, reading `DRY_RUN` from the environment. + pub fn new(client: Client) -> Self { + Self { + dry_run: read_dry_run_from_env(), + client, + k8s_distribution: Arc::new(OnceCell::new()), + discovery: Arc::new(OnceCell::new()), + } + } + + /// Create a client that always operates in dry-run mode, regardless of + /// the environment variable. + pub fn new_dry_run(client: Client) -> Self { + Self { dry_run: true, ..Self::new(client) } + } + + /// Returns `true` if this client is operating in dry-run mode. + pub fn is_dry_run(&self) -> bool { + self.dry_run + } + + pub async fn try_default() -> Result { + Ok(Self::new(Client::try_default().await?)) + } + + pub async fn from_kubeconfig(path: &str) -> Option { + Self::from_kubeconfig_with_opts(path, &KubeConfigOptions::default()).await + } + + pub async fn from_kubeconfig_with_context( + path: &str, + context: Option, + ) -> Option { + let mut opts = KubeConfigOptions::default(); + opts.context = context; + Self::from_kubeconfig_with_opts(path, &opts).await + } + + pub async fn from_kubeconfig_with_opts( + path: &str, + opts: &KubeConfigOptions, + ) -> Option { + let k = match Kubeconfig::read_from(path) { + Ok(k) => k, + Err(e) => { + error!("Failed to load kubeconfig from {path}: {e}"); + return None; + } + }; + Some(Self::new( + Client::try_from( + Config::from_custom_kubeconfig(k, opts).await.unwrap(), + ) + .unwrap(), + )) + } +} diff --git a/harmony/src/domain/topology/k8s/config.rs b/harmony-k8s/src/config.rs similarity index 100% rename from harmony/src/domain/topology/k8s/config.rs rename to harmony-k8s/src/config.rs diff --git a/harmony-k8s/src/discovery.rs b/harmony-k8s/src/discovery.rs new file mode 100644 index 0000000..e65f85a --- /dev/null +++ b/harmony-k8s/src/discovery.rs @@ -0,0 +1,79 @@ +use std::time::Duration; + +use kube::{Discovery, Error}; +use log::{debug, error, info, trace, warn}; +use tokio::sync::Mutex; +use tokio_retry::{Retry, strategy::ExponentialBackoff}; + +use crate::client::K8sClient; +use crate::types::KubernetesDistribution; + +impl K8sClient { + pub async fn get_apiserver_version( + &self, + ) -> Result { + self.client.clone().apiserver_version().await + } + + /// Runs (and caches) Kubernetes API discovery with exponential-backoff retries. + pub async fn discovery(&self) -> Result<&Discovery, Error> { + let retry_strategy = ExponentialBackoff::from_millis(1000) + .max_delay(Duration::from_secs(32)) + .take(6); + + let attempt = Mutex::new(0u32); + Retry::spawn(retry_strategy, || async { + let mut n = attempt.lock().await; + *n += 1; + match self + .discovery + .get_or_try_init(async || { + debug!("Running Kubernetes API discovery (attempt {})", *n); + let d = Discovery::new(self.client.clone()).run().await?; + debug!("Kubernetes API discovery completed"); + Ok(d) + }) + .await + { + Ok(d) => Ok(d), + Err(e) => { + warn!("Kubernetes API discovery failed (attempt {}): {}", *n, e); + Err(e) + } + } + }) + .await + .map_err(|e| { + error!("Kubernetes API discovery failed after all retries: {}", e); + e + }) + } + + /// Detect which Kubernetes distribution is running. Result is cached for + /// the lifetime of the client. + pub async fn get_k8s_distribution(&self) -> Result { + self.k8s_distribution + .get_or_try_init(async || { + debug!("Detecting Kubernetes distribution"); + let api_groups = self.client.list_api_groups().await?; + trace!("list_api_groups: {:?}", api_groups); + + let version = self.get_apiserver_version().await?; + + if api_groups.groups.iter().any(|g| g.name == "project.openshift.io") { + info!("Detected distribution: OpenshiftFamily"); + return Ok(KubernetesDistribution::OpenshiftFamily); + } + + if version.git_version.contains("k3s") { + info!("Detected distribution: K3sFamily"); + return Ok(KubernetesDistribution::K3sFamily); + } + + info!("Distribution not identified, using Default"); + Ok(KubernetesDistribution::Default) + }) + .await + .cloned() + } +} diff --git a/harmony/src/domain/topology/k8s/helper.rs b/harmony-k8s/src/helper.rs similarity index 99% rename from harmony/src/domain/topology/k8s/helper.rs rename to harmony-k8s/src/helper.rs index b0917f8..808ea12 100644 --- a/harmony/src/domain/topology/k8s/helper.rs +++ b/harmony-k8s/src/helper.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::time::Duration; -use crate::topology::KubernetesDistribution; +use crate::KubernetesDistribution; use super::bundle::ResourceBundle; use super::config::PRIVILEGED_POD_IMAGE; diff --git a/harmony-k8s/src/lib.rs b/harmony-k8s/src/lib.rs new file mode 100644 index 0000000..ec9556b --- /dev/null +++ b/harmony-k8s/src/lib.rs @@ -0,0 +1,13 @@ +pub mod apply; +pub mod bundle; +pub mod client; +pub mod config; +pub mod discovery; +pub mod helper; +pub mod node; +pub mod pod; +pub mod resources; +pub mod types; + +pub use client::K8sClient; +pub use types::{DrainOptions, KubernetesDistribution, NodeFile, ScopeResolver, WriteMode}; diff --git a/harmony-k8s/src/main.rs b/harmony-k8s/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/harmony-k8s/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/harmony-k8s/src/node.rs b/harmony-k8s/src/node.rs new file mode 100644 index 0000000..c7a7c85 --- /dev/null +++ b/harmony-k8s/src/node.rs @@ -0,0 +1,673 @@ +use std::collections::BTreeMap; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use k8s_openapi::api::core::v1::{ + ConfigMap, ConfigMapVolumeSource, Node, Pod, Volume, VolumeMount, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::{ + Error, + api::{Api, DeleteParams, EvictParams, ListParams, PostParams}, + core::ErrorResponse, + error::DiscoveryError, +}; +use log::{debug, error, info, warn}; +use tokio::time::sleep; + +use crate::client::K8sClient; +use crate::helper::{self, PrivilegedPodConfig}; +use crate::types::{DrainOptions, NodeFile}; + +impl K8sClient { + pub async fn cordon_node(&self, node_name: &str) -> Result<(), Error> { + Api::::all(self.client.clone()).cordon(node_name).await?; + Ok(()) + } + + pub async fn uncordon_node(&self, node_name: &str) -> Result<(), Error> { + Api::::all(self.client.clone()).uncordon(node_name).await?; + Ok(()) + } + + pub async fn wait_for_node_ready(&self, node_name: &str) -> Result<(), Error> { + self.wait_for_node_ready_with_timeout(node_name, Duration::from_secs(600)) + .await + } + + async fn wait_for_node_ready_with_timeout( + &self, + node_name: &str, + timeout: Duration, + ) -> Result<(), Error> { + let api: Api = Api::all(self.client.clone()); + let start = tokio::time::Instant::now(); + let poll = Duration::from_secs(5); + loop { + if start.elapsed() > timeout { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Node '{node_name}' did not become Ready within {timeout:?}" + )))); + } + match api.get(node_name).await { + Ok(node) => { + if node + .status + .as_ref() + .and_then(|s| s.conditions.as_ref()) + .map(|conds| conds.iter().any(|c| c.type_ == "Ready" && c.status == "True")) + .unwrap_or(false) + { + debug!("Node '{node_name}' is Ready"); + return Ok(()); + } + } + Err(e) => debug!("Error polling node '{node_name}': {e}"), + } + sleep(poll).await; + } + } + + async fn wait_for_node_not_ready( + &self, + node_name: &str, + timeout: Duration, + ) -> Result<(), Error> { + let api: Api = Api::all(self.client.clone()); + let start = tokio::time::Instant::now(); + let poll = Duration::from_secs(5); + loop { + if start.elapsed() > timeout { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Node '{node_name}' did not become NotReady within {timeout:?}" + )))); + } + match api.get(node_name).await { + Ok(node) => { + let is_ready = node + .status + .as_ref() + .and_then(|s| s.conditions.as_ref()) + .map(|conds| conds.iter().any(|c| c.type_ == "Ready" && c.status == "True")) + .unwrap_or(false); + if !is_ready { + debug!("Node '{node_name}' is NotReady"); + return Ok(()); + } + } + Err(e) => debug!("Error polling node '{node_name}': {e}"), + } + sleep(poll).await; + } + } + + async fn list_pods_on_node(&self, node_name: &str) -> Result, Error> { + let api: Api = Api::all(self.client.clone()); + Ok(api + .list(&ListParams::default().fields(&format!("spec.nodeName={node_name}"))) + .await? + .items) + } + + fn is_mirror_pod(pod: &Pod) -> bool { + pod.metadata + .annotations + .as_ref() + .map(|a| a.contains_key("kubernetes.io/config.mirror")) + .unwrap_or(false) + } + + fn is_daemonset_pod(pod: &Pod) -> bool { + pod.metadata + .owner_references + .as_ref() + .map(|refs| refs.iter().any(|r| r.kind == "DaemonSet")) + .unwrap_or(false) + } + + fn has_emptydir_volume(pod: &Pod) -> bool { + pod.spec + .as_ref() + .and_then(|s| s.volumes.as_ref()) + .map(|vols| vols.iter().any(|v| v.empty_dir.is_some())) + .unwrap_or(false) + } + + fn is_completed_pod(pod: &Pod) -> bool { + pod.status + .as_ref() + .and_then(|s| s.phase.as_deref()) + .map(|phase| phase == "Succeeded" || phase == "Failed") + .unwrap_or(false) + } + + fn classify_pods_for_drain( + pods: &[Pod], + options: &DrainOptions, + ) -> Result<(Vec, Vec), String> { + let mut evictable = Vec::new(); + let mut skipped = Vec::new(); + let mut blocking = Vec::new(); + + for pod in pods { + let name = pod.metadata.name.as_deref().unwrap_or(""); + let ns = pod.metadata.namespace.as_deref().unwrap_or(""); + let qualified = format!("{ns}/{name}"); + + if Self::is_mirror_pod(pod) { + skipped.push(format!("{qualified} (mirror pod)")); + continue; + } + if Self::is_completed_pod(pod) { + skipped.push(format!("{qualified} (completed)")); + continue; + } + if Self::is_daemonset_pod(pod) { + if options.ignore_daemonsets { + skipped.push(format!("{qualified} (DaemonSet-managed)")); + } else { + blocking.push(format!( + "{qualified} is managed by a DaemonSet (set ignore_daemonsets to skip)" + )); + } + continue; + } + if Self::has_emptydir_volume(pod) && !options.delete_emptydir_data { + blocking.push(format!( + "{qualified} uses emptyDir volumes (set delete_emptydir_data to allow eviction)" + )); + continue; + } + evictable.push(pod.clone()); + } + + if !blocking.is_empty() { + return Err(format!( + "Cannot drain node — the following pods block eviction:\n - {}", + blocking.join("\n - ") + )); + } + Ok((evictable, skipped)) + } + + async fn evict_pod(&self, pod: &Pod) -> Result<(), Error> { + let name = pod.metadata.name.as_deref().unwrap_or_default(); + let ns = pod.metadata.namespace.as_deref().unwrap_or_default(); + debug!("Evicting pod {ns}/{name}"); + Api::::namespaced(self.client.clone(), ns) + .evict(name, &EvictParams::default()) + .await + .map(|_| ()) + } + + /// Drains a node: cordon → classify → evict & wait. + pub async fn drain_node( + &self, + node_name: &str, + options: &DrainOptions, + ) -> Result<(), Error> { + debug!("Cordoning '{node_name}'"); + self.cordon_node(node_name).await?; + + let pods = self.list_pods_on_node(node_name).await?; + debug!("Found {} pod(s) on '{node_name}'", pods.len()); + + let (evictable, skipped) = + Self::classify_pods_for_drain(&pods, options).map_err(|msg| { + error!("{msg}"); + Error::Discovery(DiscoveryError::MissingResource(msg)) + })?; + + for s in &skipped { + info!("Skipping pod: {s}"); + } + if evictable.is_empty() { + info!("No pods to evict on '{node_name}'"); + return Ok(()); + } + info!("Evicting {} pod(s) from '{node_name}'", evictable.len()); + + let mut start = tokio::time::Instant::now(); + let poll = Duration::from_secs(5); + let mut pending = evictable; + + loop { + for pod in &pending { + match self.evict_pod(pod).await { + Ok(()) => {} + Err(Error::Api(ErrorResponse { code: 404, .. })) => {} + Err(Error::Api(ErrorResponse { code: 429, .. })) => { + warn!( + "PDB blocked eviction of {}/{}; will retry", + pod.metadata.namespace.as_deref().unwrap_or(""), + pod.metadata.name.as_deref().unwrap_or("") + ); + } + Err(e) => { + error!( + "Failed to evict {}/{}: {e}", + pod.metadata.namespace.as_deref().unwrap_or(""), + pod.metadata.name.as_deref().unwrap_or("") + ); + return Err(e); + } + } + } + + sleep(poll).await; + + let mut still_present = Vec::new(); + for pod in pending { + let ns = pod.metadata.namespace.as_deref().unwrap_or_default(); + let name = pod.metadata.name.as_deref().unwrap_or_default(); + match self.get_pod(name, Some(ns)).await? { + Some(_) => still_present.push(pod), + None => debug!("Pod {ns}/{name} evicted"), + } + } + pending = still_present; + + if pending.is_empty() { + break; + } + + if start.elapsed() > options.timeout { + match helper::prompt_drain_timeout_action( + node_name, + pending.len(), + options.timeout, + )? { + helper::DrainTimeoutAction::Accept => break, + helper::DrainTimeoutAction::Retry => { + start = tokio::time::Instant::now(); + continue; + } + helper::DrainTimeoutAction::Abort => { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Drain aborted. {} pod(s) remaining on '{node_name}'", + pending.len() + )))); + } + } + } + debug!("Waiting for {} pod(s) on '{node_name}'", pending.len()); + } + + debug!("'{node_name}' drained successfully"); + Ok(()) + } + + /// Safely reboots a node: drain → reboot → wait for Ready → uncordon. + pub async fn reboot_node( + &self, + node_name: &str, + drain_options: &DrainOptions, + timeout: Duration, + ) -> Result<(), Error> { + info!("Starting reboot for '{node_name}'"); + let node_api: Api = Api::all(self.client.clone()); + + let boot_id_before = node_api + .get(node_name) + .await? + .status + .as_ref() + .and_then(|s| s.node_info.as_ref()) + .map(|ni| ni.boot_id.clone()) + .ok_or_else(|| { + Error::Discovery(DiscoveryError::MissingResource(format!( + "Node '{node_name}' has no boot_id in status" + ))) + })?; + + info!("Draining '{node_name}'"); + self.drain_node(node_name, drain_options).await?; + + let start = tokio::time::Instant::now(); + + info!("Scheduling reboot for '{node_name}'"); + let reboot_cmd = + "echo rebooting ; nohup bash -c 'sleep 5 && nsenter -t 1 -m -- systemctl reboot'"; + match self.run_privileged_command_on_node(node_name, reboot_cmd).await { + Ok(_) => debug!("Reboot command dispatched"), + Err(e) => debug!("Reboot command error (expected if node began shutdown): {e}"), + } + + info!("Waiting for '{node_name}' to begin shutdown"); + self.wait_for_node_not_ready(node_name, timeout.saturating_sub(start.elapsed())) + .await?; + + if start.elapsed() > timeout { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Timeout during reboot of '{node_name}' (shutdown phase)" + )))); + } + + info!("Waiting for '{node_name}' to come back online"); + self.wait_for_node_ready_with_timeout( + node_name, + timeout.saturating_sub(start.elapsed()), + ) + .await?; + + if start.elapsed() > timeout { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Timeout during reboot of '{node_name}' (ready phase)" + )))); + } + + let boot_id_after = node_api + .get(node_name) + .await? + .status + .as_ref() + .and_then(|s| s.node_info.as_ref()) + .map(|ni| ni.boot_id.clone()) + .ok_or_else(|| { + Error::Discovery(DiscoveryError::MissingResource(format!( + "Node '{node_name}' has no boot_id after reboot" + ))) + })?; + + if boot_id_before == boot_id_after { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Node '{node_name}' did not actually reboot (boot_id unchanged: {boot_id_before})" + )))); + } + + info!("'{node_name}' rebooted ({boot_id_before} → {boot_id_after})"); + self.uncordon_node(node_name).await?; + info!("'{node_name}' reboot complete ({:?})", start.elapsed()); + Ok(()) + } + + /// Write a set of files to a node's filesystem via a privileged ephemeral pod. + pub async fn write_files_to_node( + &self, + node_name: &str, + files: &[NodeFile], + ) -> Result { + let ns = self.client.default_namespace(); + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + let name = format!("harmony-k8s-writer-{suffix}"); + + debug!("Writing {} file(s) to '{node_name}'", files.len()); + + let mut data = BTreeMap::new(); + let mut script = String::from("set -e\n"); + for (i, file) in files.iter().enumerate() { + let key = format!("f{i}"); + data.insert(key.clone(), file.content.clone()); + script.push_str(&format!("mkdir -p \"$(dirname \"/host{}\")\"\n", file.path)); + script.push_str(&format!("cp \"/payload/{key}\" \"/host{}\"\n", file.path)); + script.push_str(&format!("chmod {:o} \"/host{}\"\n", file.mode, file.path)); + } + + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(ns.to_string()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + + let cm_api: Api = Api::namespaced(self.client.clone(), ns); + cm_api.create(&PostParams::default(), &cm).await?; + debug!("Created ConfigMap '{name}'"); + + let (host_vol, host_mount) = helper::host_root_volume(); + let payload_vol = Volume { + name: "payload".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: name.clone(), + ..Default::default() + }), + ..Default::default() + }; + let payload_mount = VolumeMount { + name: "payload".to_string(), + mount_path: "/payload".to_string(), + ..Default::default() + }; + + let bundle = helper::build_privileged_bundle( + PrivilegedPodConfig { + name: name.clone(), + namespace: ns.to_string(), + node_name: node_name.to_string(), + container_name: "writer".to_string(), + command: vec!["/bin/bash".to_string(), "-c".to_string(), script], + volumes: vec![payload_vol, host_vol], + volume_mounts: vec![payload_mount, host_mount], + host_pid: false, + host_network: false, + }, + &self.get_k8s_distribution().await?, + ); + + bundle.apply(self).await?; + debug!("Created privileged pod bundle '{name}'"); + + let result = self.wait_for_pod_completion(&name, ns).await; + + debug!("Cleaning up '{name}'"); + let _ = bundle.delete(self).await; + let _ = cm_api.delete(&name, &DeleteParams::default()).await; + + result + } + + /// Run a privileged command on a node via an ephemeral pod. + pub async fn run_privileged_command_on_node( + &self, + node_name: &str, + command: &str, + ) -> Result { + let namespace = self.client.default_namespace(); + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + let name = format!("harmony-k8s-cmd-{suffix}"); + + debug!("Running privileged command on '{node_name}': {command}"); + + let (host_vol, host_mount) = helper::host_root_volume(); + let bundle = helper::build_privileged_bundle( + PrivilegedPodConfig { + name: name.clone(), + namespace: namespace.to_string(), + node_name: node_name.to_string(), + container_name: "runner".to_string(), + command: vec!["/bin/bash".to_string(), "-c".to_string(), command.to_string()], + volumes: vec![host_vol], + volume_mounts: vec![host_mount], + host_pid: true, + host_network: true, + }, + &self.get_k8s_distribution().await?, + ); + + bundle.apply(self).await?; + debug!("Privileged pod '{name}' created"); + + let result = self.wait_for_pod_completion(&name, namespace).await; + + debug!("Cleaning up '{name}'"); + let _ = bundle.delete(self).await; + + result + } +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use k8s_openapi::api::core::v1::{EmptyDirVolumeSource, PodSpec, PodStatus, Volume}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; + + use super::*; + + fn base_pod(name: &str, ns: &str) -> Pod { + Pod { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(ns.to_string()), + ..Default::default() + }, + spec: Some(PodSpec::default()), + status: Some(PodStatus { phase: Some("Running".to_string()), ..Default::default() }), + } + } + + fn mirror_pod(name: &str, ns: &str) -> Pod { + let mut pod = base_pod(name, ns); + pod.metadata.annotations = Some(std::collections::BTreeMap::from([( + "kubernetes.io/config.mirror".to_string(), + "abc123".to_string(), + )])); + pod + } + + fn daemonset_pod(name: &str, ns: &str) -> Pod { + let mut pod = base_pod(name, ns); + pod.metadata.owner_references = Some(vec![OwnerReference { + api_version: "apps/v1".to_string(), + kind: "DaemonSet".to_string(), + name: "some-ds".to_string(), + uid: "uid-ds".to_string(), + ..Default::default() + }]); + pod + } + + fn emptydir_pod(name: &str, ns: &str) -> Pod { + let mut pod = base_pod(name, ns); + pod.spec = Some(PodSpec { + volumes: Some(vec![Volume { + name: "scratch".to_string(), + empty_dir: Some(EmptyDirVolumeSource::default()), + ..Default::default() + }]), + ..Default::default() + }); + pod + } + + fn completed_pod(name: &str, ns: &str, phase: &str) -> Pod { + let mut pod = base_pod(name, ns); + pod.status = Some(PodStatus { phase: Some(phase.to_string()), ..Default::default() }); + pod + } + + fn default_opts() -> DrainOptions { + DrainOptions::default() + } + + // All test bodies are identical to the original — only the module path changed. + + #[test] + fn empty_pod_list_returns_empty_vecs() { + let (e, s) = K8sClient::classify_pods_for_drain(&[], &default_opts()).unwrap(); + assert!(e.is_empty()); + assert!(s.is_empty()); + } + + #[test] + fn normal_pod_is_evictable() { + let pods = vec![base_pod("web", "default")]; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); + assert_eq!(e.len(), 1); + assert!(s.is_empty()); + } + + #[test] + fn mirror_pod_is_skipped() { + let pods = vec![mirror_pod("kube-apiserver", "kube-system")]; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); + assert!(e.is_empty()); + assert!(s[0].contains("mirror pod")); + } + + #[test] + fn completed_pods_are_skipped() { + for phase in ["Succeeded", "Failed"] { + let pods = vec![completed_pod("job", "batch", phase)]; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); + assert!(e.is_empty()); + assert!(s[0].contains("completed")); + } + } + + #[test] + fn daemonset_skipped_when_ignored() { + let pods = vec![daemonset_pod("fluentd", "logging")]; + let opts = DrainOptions { ignore_daemonsets: true, ..default_opts() }; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); + assert!(e.is_empty()); + assert!(s[0].contains("DaemonSet-managed")); + } + + #[test] + fn daemonset_blocks_when_not_ignored() { + let pods = vec![daemonset_pod("fluentd", "logging")]; + let opts = DrainOptions { ignore_daemonsets: false, ..default_opts() }; + let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); + assert!(err.contains("DaemonSet") && err.contains("logging/fluentd")); + } + + #[test] + fn emptydir_blocks_without_flag() { + let pods = vec![emptydir_pod("cache", "default")]; + let opts = DrainOptions { delete_emptydir_data: false, ..default_opts() }; + let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); + assert!(err.contains("emptyDir") && err.contains("default/cache")); + } + + #[test] + fn emptydir_evictable_with_flag() { + let pods = vec![emptydir_pod("cache", "default")]; + let opts = DrainOptions { delete_emptydir_data: true, ..default_opts() }; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); + assert_eq!(e.len(), 1); + assert!(s.is_empty()); + } + + #[test] + fn multiple_blocking_all_reported() { + let pods = vec![daemonset_pod("ds", "ns1"), emptydir_pod("ed", "ns2")]; + let opts = DrainOptions { ignore_daemonsets: false, delete_emptydir_data: false, ..default_opts() }; + let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); + assert!(err.contains("ns1/ds") && err.contains("ns2/ed")); + } + + #[test] + fn mixed_pods_classified_correctly() { + let pods = vec![ + base_pod("web", "default"), + mirror_pod("kube-apiserver", "kube-system"), + daemonset_pod("fluentd", "logging"), + completed_pod("job", "batch", "Succeeded"), + base_pod("api", "default"), + ]; + let (e, s) = K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); + let names: Vec<&str> = e.iter().map(|p| p.metadata.name.as_deref().unwrap()).collect(); + assert_eq!(names, vec!["web", "api"]); + assert_eq!(s.len(), 3); + } + + #[test] + fn mirror_checked_before_completed() { + let mut pod = mirror_pod("static-etcd", "kube-system"); + pod.status = Some(PodStatus { phase: Some("Succeeded".to_string()), ..Default::default() }); + let (_, s) = K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); + assert!(s[0].contains("mirror pod"), "got: {}", s[0]); + } + + #[test] + fn completed_checked_before_daemonset() { + let mut pod = daemonset_pod("collector", "monitoring"); + pod.status = Some(PodStatus { phase: Some("Failed".to_string()), ..Default::default() }); + let (_, s) = K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); + assert!(s[0].contains("completed"), "got: {}", s[0]); + } +} diff --git a/harmony-k8s/src/pod.rs b/harmony-k8s/src/pod.rs new file mode 100644 index 0000000..353fa5b --- /dev/null +++ b/harmony-k8s/src/pod.rs @@ -0,0 +1,187 @@ +use std::time::Duration; + +use k8s_openapi::api::core::v1::Pod; +use kube::{ + Error, + api::{Api, AttachParams, ListParams}, + error::DiscoveryError, + runtime::reflector::Lookup, +}; +use log::debug; +use tokio::io::AsyncReadExt; +use tokio::time::sleep; + +use crate::client::K8sClient; + +impl K8sClient { + pub async fn get_pod( + &self, + name: &str, + namespace: Option<&str>, + ) -> Result, Error> { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + api.get_opt(name).await + } + + pub async fn wait_for_pod_ready( + &self, + pod_name: &str, + namespace: Option<&str>, + ) -> Result<(), Error> { + let mut elapsed = 0u64; + let interval = 5u64; + let timeout_secs = 120u64; + loop { + if let Some(p) = self.get_pod(pod_name, namespace).await? { + if let Some(phase) = p.status.and_then(|s| s.phase) { + if phase.to_lowercase() == "running" { + return Ok(()); + } + } + } + if elapsed >= timeout_secs { + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Pod '{}' in '{}' did not become ready within {timeout_secs}s", + pod_name, + namespace.unwrap_or(""), + )))); + } + sleep(Duration::from_secs(interval)).await; + elapsed += interval; + } + } + + /// Polls a pod until it reaches `Succeeded` or `Failed`, then returns its + /// logs. Used internally by node operations. + pub(crate) async fn wait_for_pod_completion( + &self, + name: &str, + namespace: &str, + ) -> Result { + let api: Api = Api::namespaced(self.client.clone(), namespace); + let poll_interval = Duration::from_secs(2); + for _ in 0..60 { + sleep(poll_interval).await; + let p = api.get(name).await?; + match p.status.and_then(|s| s.phase).as_deref() { + Some("Succeeded") => { + let logs = api.logs(name, &Default::default()).await.unwrap_or_default(); + debug!("Pod {namespace}/{name} succeeded. Logs: {logs}"); + return Ok(logs); + } + Some("Failed") => { + let logs = api.logs(name, &Default::default()).await.unwrap_or_default(); + debug!("Pod {namespace}/{name} failed. Logs: {logs}"); + return Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Pod '{name}' failed.\n{logs}" + )))); + } + _ => {} + } + } + Err(Error::Discovery(DiscoveryError::MissingResource(format!( + "Timed out waiting for pod '{name}'" + )))) + } + + /// Execute a command in the first pod matching `{label}={name}`. + pub async fn exec_app_capture_output( + &self, + name: String, + label: String, + namespace: Option<&str>, + command: Vec<&str>, + ) -> Result { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + let pod_list = api + .list(&ListParams::default().labels(&format!("{label}={name}"))) + .await + .expect("Failed to list pods"); + + let pod_name = pod_list + .items + .first() + .expect("No matching pod") + .name() + .expect("Pod has no name") + .into_owned(); + + match api + .exec(&pod_name, command, &AttachParams::default().stdout(true).stderr(true)) + .await + { + Err(e) => Err(e.to_string()), + Ok(mut process) => { + let status = process + .take_status() + .expect("No status handle") + .await + .expect("Status channel closed"); + + if let Some(s) = status.status { + let mut buf = String::new(); + if let Some(mut stdout) = process.stdout() { + stdout + .read_to_string(&mut buf) + .await + .map_err(|e| format!("Failed to read stdout: {e}"))?; + } + debug!("exec status: {} - {:?}", s, status.details); + if s == "Success" { Ok(buf) } else { Err(s) } + } else { + Err("No inner status from pod exec".to_string()) + } + } + } + } + + /// Execute a command in the first pod matching + /// `app.kubernetes.io/name={name}`. + pub async fn exec_app( + &self, + name: String, + namespace: Option<&str>, + command: Vec<&str>, + ) -> Result<(), String> { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + let pod_list = api + .list(&ListParams::default().labels(&format!("app.kubernetes.io/name={name}"))) + .await + .expect("Failed to list pods"); + + let pod_name = pod_list + .items + .first() + .expect("No matching pod") + .name() + .expect("Pod has no name") + .into_owned(); + + match api.exec(&pod_name, command, &AttachParams::default()).await { + Err(e) => Err(e.to_string()), + Ok(mut process) => { + let status = process + .take_status() + .expect("No status handle") + .await + .expect("Status channel closed"); + + if let Some(s) = status.status { + debug!("exec status: {} - {:?}", s, status.details); + if s == "Success" { Ok(()) } else { Err(s) } + } else { + Err("No inner status from pod exec".to_string()) + } + } + } + } +} diff --git a/harmony-k8s/src/resources.rs b/harmony-k8s/src/resources.rs new file mode 100644 index 0000000..054598e --- /dev/null +++ b/harmony-k8s/src/resources.rs @@ -0,0 +1,301 @@ +use std::collections::HashMap; + +use k8s_openapi::api::{ + apps::v1::Deployment, + core::v1::{Node, ServiceAccount}, +}; +use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; +use kube::{ + Error, Resource, + api::{Api, DynamicObject, GroupVersionKind, ListParams, ObjectList}, + runtime::conditions, + runtime::wait::await_condition, +}; +use kube::api::ApiResource; +use log::debug; +use serde::de::DeserializeOwned; +use serde_json::Value; +use std::time::Duration; + +use crate::client::K8sClient; +use crate::types::ScopeResolver; + +impl K8sClient { + pub async fn has_healthy_deployment_with_label( + &self, + namespace: &str, + label_selector: &str, + ) -> Result { + let api: Api = Api::namespaced(self.client.clone(), namespace); + let list = api.list(&ListParams::default().labels(label_selector)).await?; + for d in list.items { + let available = d.status.as_ref().and_then(|s| s.available_replicas).unwrap_or(0); + if available > 0 { + return Ok(true); + } + if let Some(conds) = d.status.as_ref().and_then(|s| s.conditions.as_ref()) { + if conds.iter().any(|c| c.type_ == "Available" && c.status == "True") { + return Ok(true); + } + } + } + Ok(false) + } + + pub async fn list_namespaces_with_healthy_deployments( + &self, + label_selector: &str, + ) -> Result, Error> { + let api: Api = Api::all(self.client.clone()); + let list = api.list(&ListParams::default().labels(label_selector)).await?; + + let mut healthy_ns: HashMap = HashMap::new(); + for d in list.items { + let ns = match d.metadata.namespace.clone() { + Some(n) => n, + None => continue, + }; + let available = d.status.as_ref().and_then(|s| s.available_replicas).unwrap_or(0); + let is_healthy = if available > 0 { + true + } else { + d.status + .as_ref() + .and_then(|s| s.conditions.as_ref()) + .map(|c| c.iter().any(|c| c.type_ == "Available" && c.status == "True")) + .unwrap_or(false) + }; + if is_healthy { + healthy_ns.insert(ns, true); + } + } + Ok(healthy_ns.into_keys().collect()) + } + + pub async fn get_controller_service_account_name( + &self, + ns: &str, + ) -> Result, Error> { + let api: Api = Api::namespaced(self.client.clone(), ns); + let list = api + .list(&ListParams::default().labels("app.kubernetes.io/component=controller")) + .await?; + if let Some(dep) = list.items.first() { + if let Some(sa) = dep + .spec + .as_ref() + .and_then(|s| s.template.spec.as_ref()) + .and_then(|s| s.service_account_name.clone()) + { + return Ok(Some(sa)); + } + } + Ok(None) + } + + pub async fn list_clusterrolebindings_json(&self) -> Result, Error> { + let gvk = GroupVersionKind::gvk("rbac.authorization.k8s.io", "v1", "ClusterRoleBinding"); + let ar = ApiResource::from_gvk(&gvk); + let api: Api = Api::all_with(self.client.clone(), &ar); + let list = api.list(&ListParams::default()).await?; + Ok(list + .items + .into_iter() + .map(|o| serde_json::to_value(&o).unwrap_or(Value::Null)) + .collect()) + } + + pub async fn is_service_account_cluster_wide( + &self, + sa: &str, + ns: &str, + ) -> Result { + let sa_user = format!("system:serviceaccount:{ns}:{sa}"); + for crb in self.list_clusterrolebindings_json().await? { + if let Some(subjects) = crb.get("subjects").and_then(|s| s.as_array()) { + for subj in subjects { + let kind = subj.get("kind").and_then(|v| v.as_str()).unwrap_or(""); + let name = subj.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let subj_ns = subj.get("namespace").and_then(|v| v.as_str()).unwrap_or(""); + if (kind == "ServiceAccount" && name == sa && subj_ns == ns) + || (kind == "User" && name == sa_user) + { + return Ok(true); + } + } + } + } + Ok(false) + } + + pub async fn has_crd(&self, name: &str) -> Result { + let api: Api = Api::all(self.client.clone()); + let crds = api + .list(&ListParams::default().fields(&format!("metadata.name={name}"))) + .await?; + Ok(!crds.items.is_empty()) + } + + pub async fn service_account_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + pub async fn get_resource_json_value( + &self, + name: &str, + namespace: Option<&str>, + gvk: &GroupVersionKind, + ) -> Result { + let ar = ApiResource::from_gvk(gvk); + let api: Api = match namespace { + Some(ns) => Api::namespaced_with(self.client.clone(), ns, &ar), + None => Api::default_namespaced_with(self.client.clone(), &ar), + }; + api.get(name).await + } + + pub async fn get_secret_json_value( + &self, + name: &str, + namespace: Option<&str>, + ) -> Result { + self.get_resource_json_value( + name, + namespace, + &GroupVersionKind { + group: String::new(), + version: "v1".to_string(), + kind: "Secret".to_string(), + }, + ) + .await + } + + pub async fn get_deployment( + &self, + name: &str, + namespace: Option<&str>, + ) -> Result, Error> { + let api: Api = match namespace { + Some(ns) => { + debug!("Getting namespaced deployment '{name}' in '{ns}'"); + Api::namespaced(self.client.clone(), ns) + } + None => { + debug!("Getting deployment '{name}' in default namespace"); + Api::default_namespaced(self.client.clone()) + } + }; + api.get_opt(name).await + } + + pub async fn scale_deployment( + &self, + name: &str, + namespace: Option<&str>, + replicas: u32, + ) -> Result<(), Error> { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + use kube::api::{Patch, PatchParams}; + use serde_json::json; + let patch = json!({ "spec": { "replicas": replicas } }); + api.patch_scale(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await?; + Ok(()) + } + + pub async fn delete_deployment( + &self, + name: &str, + namespace: Option<&str>, + ) -> Result<(), Error> { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + api.delete(name, &kube::api::DeleteParams::default()).await?; + Ok(()) + } + + pub async fn wait_until_deployment_ready( + &self, + name: &str, + namespace: Option<&str>, + timeout: Option, + ) -> Result<(), String> { + let api: Api = match namespace { + Some(ns) => Api::namespaced(self.client.clone(), ns), + None => Api::default_namespaced(self.client.clone()), + }; + let timeout = timeout.unwrap_or(Duration::from_secs(120)); + let establish = await_condition(api, name, conditions::is_deployment_completed()); + tokio::time::timeout(timeout, establish) + .await + .map(|_| ()) + .map_err(|_| "Timed out waiting for deployment".to_string()) + } + + /// Gets a single named resource, using the correct API scope for `K`. + pub async fn get_resource( + &self, + name: &str, + namespace: Option<&str>, + ) -> Result, Error> + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned, + ::Scope: ScopeResolver, + ::DynamicType: Default, + { + let api: Api = + <::Scope as ScopeResolver>::get_api(&self.client, namespace); + api.get_opt(name).await + } + + pub async fn list_resources( + &self, + namespace: Option<&str>, + list_params: Option, + ) -> Result, Error> + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned, + ::Scope: ScopeResolver, + ::DynamicType: Default, + { + let api: Api = + <::Scope as ScopeResolver>::get_api(&self.client, namespace); + api.list(&list_params.unwrap_or_default()).await + } + + pub async fn list_all_resources_with_labels(&self, labels: &str) -> Result, Error> + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned, + ::DynamicType: Default, + { + Api::::all(self.client.clone()) + .list(&ListParams::default().labels(labels)) + .await + .map(|l| l.items) + } + + pub async fn get_all_resource_in_all_namespace(&self) -> Result, Error> + where + K: Resource + Clone + std::fmt::Debug + DeserializeOwned, + ::Scope: ScopeResolver, + ::DynamicType: Default, + { + Api::::all(self.client.clone()) + .list(&Default::default()) + .await + .map(|l| l.items) + } + + pub async fn get_nodes( + &self, + list_params: Option, + ) -> Result, Error> { + self.list_resources(None, list_params).await + } +} diff --git a/harmony-k8s/src/types.rs b/harmony-k8s/src/types.rs new file mode 100644 index 0000000..8535331 --- /dev/null +++ b/harmony-k8s/src/types.rs @@ -0,0 +1,100 @@ +use std::time::Duration; + +use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope}; +use kube::{Api, Client, Resource}; +use serde::Serialize; + +/// Which Kubernetes distribution is running. Detected once at runtime via +/// [`crate::discovery::K8sClient::get_k8s_distribution`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub enum KubernetesDistribution { + Default, + OpenshiftFamily, + K3sFamily, +} + +/// A file to be written to a node's filesystem. +#[derive(Debug, Clone)] +pub struct NodeFile { + /// Absolute path on the host where the file should be written. + pub path: String, + /// Content of the file. + pub content: String, + /// UNIX permissions (e.g. `0o600`). + pub mode: u32, +} + +/// Options controlling the behaviour of a [`crate::K8sClient::drain_node`] operation. +#[derive(Debug, Clone)] +pub struct DrainOptions { + /// Evict pods that use `emptyDir` volumes (ephemeral data is lost). + /// Equivalent to `kubectl drain --delete-emptydir-data`. + pub delete_emptydir_data: bool, + /// Silently skip DaemonSet-managed pods instead of blocking the drain. + /// Equivalent to `kubectl drain --ignore-daemonsets`. + pub ignore_daemonsets: bool, + /// Maximum wall-clock time to wait for all evictions to complete. + pub timeout: Duration, +} + +impl Default for DrainOptions { + fn default() -> Self { + Self { + delete_emptydir_data: false, + ignore_daemonsets: true, + timeout: Duration::from_secs(1), + } + } +} + +impl DrainOptions { + pub fn default_ignore_daemonset_delete_emptydir_data() -> Self { + Self { + delete_emptydir_data: true, + ignore_daemonsets: true, + ..Self::default() + } + } +} + +/// Controls how [`crate::K8sClient::apply_with_strategy`] behaves when the +/// resource already exists (or does not). +pub enum WriteMode { + /// Server-side apply; create if absent, update if present (default). + CreateOrUpdate, + /// POST only; return an error if the resource already exists. + Create, + /// Server-side apply only; return an error if the resource does not exist. + Update, +} + +// ── Scope resolution trait ─────────────────────────────────────────────────── + +/// Resolves the correct [`kube::Api`] for a resource type based on its scope +/// (cluster-wide vs. namespace-scoped). +pub trait ScopeResolver { + fn get_api(client: &Client, ns: Option<&str>) -> Api; +} + +impl ScopeResolver for ClusterResourceScope +where + K: Resource, + ::DynamicType: Default, +{ + fn get_api(client: &Client, _ns: Option<&str>) -> Api { + Api::all(client.clone()) + } +} + +impl ScopeResolver for NamespaceResourceScope +where + K: Resource, + ::DynamicType: Default, +{ + fn get_api(client: &Client, ns: Option<&str>) -> Api { + match ns { + Some(ns) => Api::namespaced(client.clone(), ns), + None => Api::default_namespaced(client.clone()), + } + } +} diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index d154277..baa52be 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -21,6 +21,8 @@ semver = "1.0.23" serde.workspace = true serde_json.workspace = true tokio.workspace = true +tokio-retry.workspace = true +tokio-util.workspace = true derive-new.workspace = true log.workspace = true env_logger.workspace = true @@ -31,6 +33,7 @@ opnsense-config-xml = { path = "../opnsense-config-xml" } harmony_macros = { path = "../harmony_macros" } harmony_types = { path = "../harmony_types" } harmony_execution = { path = "../harmony_execution" } +harmony-k8s = { path = "../harmony-k8s" } uuid.workspace = true url.workspace = true kube = { workspace = true, features = ["derive"] } @@ -60,7 +63,6 @@ temp-dir = "0.1.14" dyn-clone = "1.0.19" similar.workspace = true futures-util = "0.3.31" -tokio-util = "0.7.15" strum = { version = "0.27.1", features = ["derive"] } tempfile.workspace = true serde_with = "3.14.0" @@ -80,7 +82,7 @@ sqlx.workspace = true inquire.workspace = true brocade = { path = "../brocade" } option-ext = "0.2.0" -tokio-retry = "0.3.0" +rand.workspace = true [dev-dependencies] pretty_assertions.workspace = true diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index ea4f2f8..e68e274 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use harmony_k8s::K8sClient; use harmony_macros::ip; use harmony_types::{ id::Id, @@ -16,7 +17,7 @@ use super::{ DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig, HttpServer, IpAddress, K8sclient, LoadBalancer, LoadBalancerService, LogicalHost, NetworkError, NetworkManager, PreparationError, PreparationOutcome, Router, Switch, SwitchClient, - SwitchError, TftpServer, Topology, k8s::K8sClient, + SwitchError, TftpServer, Topology, }; use std::{ process::Command, diff --git a/harmony/src/domain/topology/k8s/mod.rs b/harmony/src/domain/topology/k8s/mod.rs deleted file mode 100644 index a5d45ce..0000000 --- a/harmony/src/domain/topology/k8s/mod.rs +++ /dev/null @@ -1,2631 +0,0 @@ -pub mod bundle; -pub mod config; -pub mod helper; - -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use k8s_openapi::{ - ClusterResourceScope, NamespaceResourceScope, - api::{ - apps::v1::Deployment, - core::v1::{ - ConfigMap, ConfigMapVolumeSource, Node, Pod, ServiceAccount, Volume, VolumeMount, - }, - }, - apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition, - apimachinery::pkg::{apis::meta::v1::ObjectMeta, version::Info}, -}; -use kube::{ - Client, Config, Discovery, Error, Resource, - api::{ - Api, AttachParams, DeleteParams, EvictParams, ListParams, ObjectList, Patch, PatchParams, - PostParams, ResourceExt, - }, - config::{KubeConfigOptions, Kubeconfig}, - core::ErrorResponse, - discovery::{ApiCapabilities, Scope}, - error::DiscoveryError, - runtime::reflector::Lookup, -}; -use kube::{api::DynamicObject, runtime::conditions}; -use kube::{ - api::{ApiResource, GroupVersionKind}, - runtime::wait::await_condition, -}; -use log::{debug, error, info, trace, warn}; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; -use similar::TextDiff; -use tokio::{ - io::AsyncReadExt, - sync::{Mutex, OnceCell}, - time::sleep, -}; -use tokio_retry::{Retry, strategy::ExponentialBackoff}; -use url::Url; - -use crate::topology::{KubernetesDistribution, k8s::helper::PrivilegedPodConfig}; - -#[derive(Clone)] -pub struct K8sClient { - client: Client, - k8s_distribution: Arc>, - discovery: Arc>, -} - -impl Serialize for K8sClient { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -impl std::fmt::Debug for K8sClient { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // This is a poor man's debug implementation for now as kube::Client does not provide much - // useful information - f.write_fmt(format_args!( - "K8sClient {{ kube client using default namespace {} }}", - self.client.default_namespace() - )) - } -} - -/// A file to be written to a node's filesystem. -#[derive(Debug, Clone)] -pub struct NodeFile { - /// The absolute path on the host where the file should be written. - pub path: String, - /// The content of the file. - pub content: String, - /// The file permissions (e.g. 0o600). - pub mode: u32, -} - -/// Options controlling the behavior of a [`K8sClient::drain_node`] operation. -#[derive(Debug, Clone)] -pub struct DrainOptions { - /// If `true`, pods that use `emptyDir` volumes will be evicted (their - /// ephemeral data is lost). Equivalent to `kubectl drain - /// --delete-emptydir-data`. - pub delete_emptydir_data: bool, - /// If `true`, DaemonSet-managed pods are silently skipped instead of - /// blocking the drain. Equivalent to `kubectl drain --ignore-daemonsets`. - pub ignore_daemonsets: bool, - /// Maximum wall-clock time to wait for all evictions to complete before - /// returning an error. - pub timeout: Duration, -} - -pub enum WriteMode { - CreateOrUpdate, - Create, - Update, -} - -impl Default for DrainOptions { - fn default() -> Self { - Self { - delete_emptydir_data: false, - ignore_daemonsets: true, - // TODO sane timeout - timeout: Duration::from_secs(1), - } - } -} - -impl DrainOptions { - pub fn default_ignore_daemonset_delete_emptydir_data() -> DrainOptions { - let mut drain_opts = DrainOptions::default(); - drain_opts.delete_emptydir_data = true; - drain_opts.ignore_daemonsets = true; - drain_opts - } -} - -impl K8sClient { - pub fn new(client: Client) -> Self { - Self { - client, - k8s_distribution: Arc::new(OnceCell::new()), - discovery: Arc::new(OnceCell::new()), - } - } - - pub async fn try_default() -> Result { - let client = Self { - client: Client::try_default().await?, - k8s_distribution: Arc::new(OnceCell::new()), - discovery: Arc::new(OnceCell::new()), - }; - - Ok(client) - } - - /// Returns true if any deployment in the given namespace matching the label selector - /// has status.availableReplicas > 0 (or condition Available=True). - pub async fn has_healthy_deployment_with_label( - &self, - namespace: &str, - label_selector: &str, - ) -> Result { - let api: Api = Api::namespaced(self.client.clone(), namespace); - let lp = ListParams::default().labels(label_selector); - let list = api.list(&lp).await?; - for d in list.items { - // Check AvailableReplicas > 0 or Available condition - let available = d - .status - .as_ref() - .and_then(|s| s.available_replicas) - .unwrap_or(0); - if available > 0 { - return Ok(true); - } - // Fallback: scan conditions - if let Some(conds) = d.status.as_ref().and_then(|s| s.conditions.as_ref()) { - if conds - .iter() - .any(|c| c.type_ == "Available" && c.status == "True") - { - return Ok(true); - } - } - } - Ok(false) - } - - /// Cluster-wide: returns namespaces that have at least one healthy deployment - /// matching the label selector (equivalent to kubectl -A -l ...). - pub async fn list_namespaces_with_healthy_deployments( - &self, - label_selector: &str, - ) -> Result, Error> { - let api: Api = Api::all(self.client.clone()); - let lp = ListParams::default().labels(label_selector); - let list = api.list(&lp).await?; - - let mut healthy_ns: HashMap = HashMap::new(); - for d in list.items { - let ns = match d.metadata.namespace.clone() { - Some(n) => n, - None => continue, - }; - let available = d - .status - .as_ref() - .and_then(|s| s.available_replicas) - .unwrap_or(0); - let is_healthy = if available > 0 { - true - } else { - d.status - .as_ref() - .and_then(|s| s.conditions.as_ref()) - .map(|conds| { - conds - .iter() - .any(|c| c.type_ == "Available" && c.status == "True") - }) - .unwrap_or(false) - }; - if is_healthy { - healthy_ns.insert(ns, true); - } - } - - Ok(healthy_ns.into_keys().collect()) - } - - /// Get the application-controller ServiceAccount name (fallback to default) - pub async fn get_controller_service_account_name( - &self, - ns: &str, - ) -> Result, Error> { - let api: Api = Api::namespaced(self.client.clone(), ns); - let lp = ListParams::default().labels("app.kubernetes.io/component=controller"); - let list = api.list(&lp).await?; - if let Some(dep) = list.items.get(0) { - if let Some(sa) = dep - .spec - .as_ref() - .and_then(|ds| ds.template.spec.as_ref()) - .and_then(|ps| ps.service_account_name.clone()) - { - return Ok(Some(sa)); - } - } - Ok(None) - } - - // List ClusterRoleBindings dynamically and return as JSON values - pub async fn list_clusterrolebindings_json(&self) -> Result, Error> { - let gvk = kube::api::GroupVersionKind::gvk( - "rbac.authorization.k8s.io", - "v1", - "ClusterRoleBinding", - ); - let ar = kube::api::ApiResource::from_gvk(&gvk); - let api: Api = Api::all_with(self.client.clone(), &ar); - let crbs = api.list(&ListParams::default()).await?; - let mut out = Vec::new(); - for o in crbs { - let v = serde_json::to_value(&o).unwrap_or(Value::Null); - out.push(v); - } - Ok(out) - } - - /// Determine if Argo controller in ns has cluster-wide permissions via CRBs - // TODO This does not belong in the generic k8s client, should be refactored at some point - pub async fn is_service_account_cluster_wide(&self, sa: &str, ns: &str) -> Result { - let crbs = self.list_clusterrolebindings_json().await?; - let sa_user = format!("system:serviceaccount:{}:{}", ns, sa); - for crb in crbs { - if let Some(subjects) = crb.get("subjects").and_then(|s| s.as_array()) { - for subj in subjects { - let kind = subj.get("kind").and_then(|v| v.as_str()).unwrap_or(""); - let name = subj.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let subj_ns = subj.get("namespace").and_then(|v| v.as_str()).unwrap_or(""); - if (kind == "ServiceAccount" && name == sa && subj_ns == ns) - || (kind == "User" && name == sa_user) - { - return Ok(true); - } - } - } - } - Ok(false) - } - - pub async fn has_crd(&self, name: &str) -> Result { - let api: Api = Api::all(self.client.clone()); - let lp = ListParams::default().fields(&format!("metadata.name={}", name)); - let crds = api.list(&lp).await?; - Ok(!crds.items.is_empty()) - } - - pub async fn service_account_api(&self, namespace: &str) -> Api { - let api: Api = Api::namespaced(self.client.clone(), namespace); - api - } - - pub async fn get_apiserver_version(&self) -> Result { - let client: Client = self.client.clone(); - let version_info: Info = client.apiserver_version().await?; - Ok(version_info) - } - - pub async fn discovery(&self) -> Result<&Discovery, Error> { - // Retry with exponential backoff in case of API server load - let retry_strategy = ExponentialBackoff::from_millis(1000) - .max_delay(Duration::from_secs(32)) - .take(6); - - let attempt = Mutex::new(0); - Retry::spawn(retry_strategy, || async { - let mut alock = attempt.lock().await; - *alock += 1; - match self - .discovery - .get_or_try_init(async || { - debug!("Running Kubernetes API discovery (attempt {})", *alock); - let discovery = Discovery::new(self.client.clone()).run().await?; - debug!("Kubernetes API discovery completed"); - Ok(discovery) - }) - .await - { - Ok(discovery) => Ok(discovery), - Err(e) => { - warn!( - "Kubernetes API discovery failed (attempt {}): {}", - *alock, e - ); - Err(e) - } - } - }) - .await - .map_err(|e| { - error!("Kubernetes API discovery failed after all retries: {}", e); - e - }) - } - - pub async fn get_resource_json_value( - &self, - name: &str, - namespace: Option<&str>, - gvk: &GroupVersionKind, - ) -> Result { - let gvk = ApiResource::from_gvk(gvk); - let resource: Api = if let Some(ns) = namespace { - Api::namespaced_with(self.client.clone(), ns, &gvk) - } else { - Api::default_namespaced_with(self.client.clone(), &gvk) - }; - - resource.get(name).await - } - - pub async fn get_secret_json_value( - &self, - name: &str, - namespace: Option<&str>, - ) -> Result { - self.get_resource_json_value( - name, - namespace, - &GroupVersionKind { - group: "".to_string(), - version: "v1".to_string(), - kind: "Secret".to_string(), - }, - ) - .await - } - - pub async fn get_deployment( - &self, - name: &str, - namespace: Option<&str>, - ) -> Result, Error> { - let deps: Api = if let Some(ns) = namespace { - debug!("getting namespaced deployment"); - Api::namespaced(self.client.clone(), ns) - } else { - debug!("getting default namespace deployment"); - Api::default_namespaced(self.client.clone()) - }; - - debug!("getting deployment {} in ns {}", name, namespace.unwrap()); - deps.get_opt(name).await - } - - pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result, Error> { - let pods: Api = if let Some(ns) = namespace { - Api::namespaced(self.client.clone(), ns) - } else { - Api::default_namespaced(self.client.clone()) - }; - - pods.get_opt(name).await - } - - pub async fn scale_deployment( - &self, - name: &str, - namespace: Option<&str>, - replicas: u32, - ) -> Result<(), Error> { - let deployments: Api = if let Some(ns) = namespace { - Api::namespaced(self.client.clone(), ns) - } else { - Api::default_namespaced(self.client.clone()) - }; - - let patch = json!({ - "spec": { - "replicas": replicas - } - }); - let pp = PatchParams::default(); - let scale = Patch::Merge(&patch); - deployments.patch_scale(name, &pp, &scale).await?; - Ok(()) - } - - pub async fn delete_deployment( - &self, - name: &str, - namespace: Option<&str>, - ) -> Result<(), Error> { - let deployments: Api = if let Some(ns) = namespace { - Api::namespaced(self.client.clone(), ns) - } else { - Api::default_namespaced(self.client.clone()) - }; - let delete_params = DeleteParams::default(); - deployments.delete(name, &delete_params).await?; - Ok(()) - } - - pub async fn wait_until_deployment_ready( - &self, - name: &str, - namespace: Option<&str>, - timeout: Option, - ) -> Result<(), String> { - let api: Api; - - if let Some(ns) = namespace { - api = Api::namespaced(self.client.clone(), ns); - } else { - api = Api::default_namespaced(self.client.clone()); - } - - let establish = await_condition(api, name, conditions::is_deployment_completed()); - let timeout = timeout.unwrap_or(Duration::from_secs(120)); - let res = tokio::time::timeout(timeout, establish).await; - - if res.is_ok() { - Ok(()) - } else { - Err("timed out while waiting for deployment".to_string()) - } - } - - pub async fn wait_for_pod_ready( - &self, - pod_name: &str, - namespace: Option<&str>, - ) -> Result<(), Error> { - let mut elapsed = 0; - let interval = 5; // seconds between checks - let timeout_secs = 120; - loop { - let pod = self.get_pod(pod_name, namespace).await?; - - if let Some(p) = pod { - if let Some(status) = p.status { - if let Some(phase) = status.phase { - if phase.to_lowercase() == "running" { - return Ok(()); - } - } - } - } - - if elapsed >= timeout_secs { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "'{}' in ns '{}' did not become ready within {}s", - pod_name, - namespace.unwrap(), - timeout_secs - )))); - } - - sleep(Duration::from_secs(interval)).await; - elapsed += interval; - } - } - - /// Will execute a commond in the first pod found that matches the specified label - /// '{label}={name}' - pub async fn exec_app_capture_output( - &self, - name: String, - label: String, - namespace: Option<&str>, - command: Vec<&str>, - ) -> Result { - let api: Api; - - if let Some(ns) = namespace { - api = Api::namespaced(self.client.clone(), ns); - } else { - api = Api::default_namespaced(self.client.clone()); - } - let pod_list = api - .list(&ListParams::default().labels(format!("{label}={name}").as_str())) - .await - .expect("couldn't get list of pods"); - - let res = api - .exec( - pod_list - .items - .first() - .expect("couldn't get pod") - .name() - .expect("couldn't get pod name") - .into_owned() - .as_str(), - command, - &AttachParams::default().stdout(true).stderr(true), - ) - .await; - match res { - Err(e) => Err(e.to_string()), - Ok(mut process) => { - let status = process - .take_status() - .expect("Couldn't get status") - .await - .expect("Couldn't unwrap status"); - - if let Some(s) = status.status { - let mut stdout_buf = String::new(); - if let Some(mut stdout) = process.stdout() { - stdout - .read_to_string(&mut stdout_buf) - .await - .map_err(|e| format!("Failed to get status stdout {e}"))?; - } - debug!("Status: {} - {:?}", s, status.details); - if s == "Success" { - Ok(stdout_buf) - } else { - Err(s) - } - } else { - Err("Couldn't get inner status of pod exec".to_string()) - } - } - } - } - - /// Will execute a command in the first pod found that matches the label `app.kubernetes.io/name={name}` - pub async fn exec_app( - &self, - name: String, - namespace: Option<&str>, - command: Vec<&str>, - ) -> Result<(), String> { - let api: Api; - - if let Some(ns) = namespace { - api = Api::namespaced(self.client.clone(), ns); - } else { - api = Api::default_namespaced(self.client.clone()); - } - let pod_list = api - .list(&ListParams::default().labels(format!("app.kubernetes.io/name={name}").as_str())) - .await - .expect("couldn't get list of pods"); - - let res = api - .exec( - pod_list - .items - .first() - .expect("couldn't get pod") - .name() - .expect("couldn't get pod name") - .into_owned() - .as_str(), - command, - &AttachParams::default(), - ) - .await; - - match res { - Err(e) => Err(e.to_string()), - Ok(mut process) => { - let status = process - .take_status() - .expect("Couldn't get status") - .await - .expect("Couldn't unwrap status"); - - if let Some(s) = status.status { - debug!("Status: {} - {:?}", s, status.details); - if s == "Success" { Ok(()) } else { Err(s) } - } else { - Err("Couldn't get inner status of pod exec".to_string()) - } - } - } - } - - pub(crate) fn get_api_for_dynamic_object( - &self, - object: &DynamicObject, - ns: Option<&str>, - ) -> Result, Error> { - let api_resource = object - .types - .as_ref() - .and_then(|t| { - let parts: Vec<&str> = t.api_version.split('/').collect(); - match parts.as_slice() { - [version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk( - "", version, &t.kind, - ))), - [group, version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk( - group, version, &t.kind, - ))), - _ => None, - } - }) - .ok_or_else(|| { - Error::BuildRequest(kube::core::request::Error::Validation( - "Invalid apiVersion in DynamicObject {object:#?}".to_string(), - )) - })?; - - match ns { - Some(ns) => Ok(Api::namespaced_with(self.client.clone(), ns, &api_resource)), - None => Ok(Api::default_namespaced_with( - self.client.clone(), - &api_resource, - )), - } - } - - pub async fn apply_dynamic_many( - &self, - resource: &[DynamicObject], - namespace: Option<&str>, - force_conflicts: bool, - ) -> Result, Error> { - let mut result = Vec::new(); - for r in resource.iter() { - result.push(self.apply_dynamic(r, namespace, force_conflicts).await?); - } - - Ok(result) - } - - /// Apply DynamicObject resource to the cluster - pub async fn apply_dynamic( - &self, - resource: &DynamicObject, - namespace: Option<&str>, - force_conflicts: bool, - ) -> Result { - // Use discovery to determine the correct API scope - trace!( - "Apply dynamic resource {resource:#?} \n namespace :{namespace:?} force_conflicts {force_conflicts}" - ); - let discovery = self.discovery().await?; - - let type_meta = resource.types.as_ref().ok_or_else(|| { - Error::BuildRequest(kube::core::request::Error::Validation( - "DynamicObject must have types (apiVersion and kind)".to_string(), - )) - })?; - - let gvk = GroupVersionKind::try_from(type_meta).map_err(|_| { - Error::BuildRequest(kube::core::request::Error::Validation(format!( - "Invalid GroupVersionKind in DynamicObject: {:?}", - type_meta - ))) - })?; - - let (ar, caps) = discovery.resolve_gvk(&gvk).ok_or_else(|| { - Error::Discovery(DiscoveryError::MissingResource(format!( - "Cannot resolve GVK: {:?}", - gvk - ))) - })?; - - // Determine namespace based on resource scope - let effective_namespace = if caps.scope == Scope::Cluster { - None - } else { - namespace.or_else(|| resource.metadata.namespace.as_deref()) - }; - - trace!( - "Discovered information ar {ar:?}, caps {caps:?}, effective_namespace {effective_namespace:?}" - ); - - // Build API using discovered resource and capabilities - let api = get_dynamic_api(ar, caps, self.client.clone(), effective_namespace, false); - let name = resource - .metadata - .name - .as_ref() - .ok_or_else(|| { - Error::BuildRequest(kube::core::request::Error::Validation( - "DynamicObject must have metadata.name".to_string(), - )) - })? - .as_str(); - - debug!( - "Applying dynamic resource kind={:?} apiVersion={:?} name='{}' ns={:?}", - resource.types.as_ref().map(|t| &t.kind), - resource.types.as_ref().map(|t| &t.api_version), - name, - namespace - ); - trace!( - "Dynamic resource payload:\n{:#}", - serde_json::to_value(resource).unwrap_or(serde_json::Value::Null) - ); - - // Using same field manager as in apply() - let mut patch_params = PatchParams::apply("harmony"); - patch_params.force = force_conflicts; - - if *crate::config::DRY_RUN { - // Dry-run path: fetch current, show diff, and return appropriate object - match api.get(name).await { - Ok(current) => { - trace!("Received current dynamic value {current:#?}"); - - println!("\nPerforming dry-run for resource: '{}'", name); - - // Serialize current and new, and strip status from current if present - let mut current_yaml = - serde_yaml::to_value(¤t).unwrap_or_else(|_| serde_yaml::Value::Null); - if let Some(map) = current_yaml.as_mapping_mut() { - if map.contains_key(&serde_yaml::Value::String("status".to_string())) { - let removed = - map.remove(&serde_yaml::Value::String("status".to_string())); - trace!("Removed status from current dynamic object: {:?}", removed); - } else { - trace!( - "Did not find status entry for current dynamic object {}/{}", - current.metadata.namespace.as_deref().unwrap_or(""), - current.metadata.name.as_deref().unwrap_or("") - ); - } - } - - let current_yaml = serde_yaml::to_string(¤t_yaml) - .unwrap_or_else(|_| "Failed to serialize current resource".to_string()); - let new_yaml = serde_yaml::to_string(resource) - .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); - - if current_yaml == new_yaml { - println!("No changes detected."); - return Ok(current); - } - - println!("Changes detected:"); - let diff = TextDiff::from_lines(¤t_yaml, &new_yaml); - for change in diff.iter_all_changes() { - let sign = match change.tag() { - similar::ChangeTag::Delete => "-", - similar::ChangeTag::Insert => "+", - similar::ChangeTag::Equal => " ", - }; - print!("{}{}", sign, change); - } - - // Return the incoming resource as the would-be applied state - Ok(resource.clone()) - } - Err(Error::Api(ErrorResponse { code: 404, .. })) => { - println!("\nPerforming dry-run for new resource: '{}'", name); - println!( - "Resource does not exist. It would be created with the following content:" - ); - let new_yaml = serde_yaml::to_string(resource) - .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); - for line in new_yaml.lines() { - println!("+{}", line); - } - Ok(resource.clone()) - } - Err(e) => { - error!("Failed to get dynamic resource '{}': {}", name, e); - Err(e) - } - } - } else { - // Real apply via server-side apply - // Server-side apply works for both create and update operations - debug!("Applying (server-side apply) dynamic resource '{}'", name); - match api - .patch(name, &patch_params, &Patch::Apply(resource)) - .await - { - Ok(obj) => Ok(obj), - Err(Error::Api(ErrorResponse { code: 404, .. })) => { - // Resource doesn't exist, server-side apply should create it - // This can happen with some API servers, so we explicitly create - debug!("Resource '{}' not found, creating via POST", name); - trace!("{resource:#?}"); - api.create(&PostParams::default(), resource) - .await - .map_err(|e| { - error!("Failed to create dynamic resource '{}': {}", name, e); - e - }) - } - Err(e) => { - error!("Failed to apply dynamic resource '{}': {}", name, e); - Err(e) - } - } - } - } - - /// Apply a resource in namespace - /// - /// See `kubectl apply` for more information on the expected behavior of this function - pub async fn apply(&self, resource: &K, namespace: Option<&str>) -> Result - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, - ::DynamicType: Default, - { - self.apply_with_strategy(resource, namespace, WriteMode::CreateOrUpdate).await - } - - pub async fn apply_with_strategy(&self, resource: &K, namespace: Option<&str>, apply_strategy: WriteMode) -> Result - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, - ::DynamicType: Default, - { - todo!("Refactoring in progress: Handle the apply_strategy parameter and add utility functions like apply that set it for ease of use (create, update)"); - - debug!( - "Applying resource {:?} with ns {:?}", - resource.meta().name, - namespace - ); - trace!( - "{:#}", - serde_json::to_value(resource).unwrap_or(serde_json::Value::Null) - ); - - // ── 1. Extract GVK from compile-time type info ────────────────────────── - let dyntype = K::DynamicType::default(); - let gvk = GroupVersionKind { - group: K::group(&dyntype).to_string(), - version: K::version(&dyntype).to_string(), - kind: K::kind(&dyntype).to_string(), - }; - - // ── 2. Resolve scope at runtime via discovery ──────────────────────────── - let discovery = self.discovery().await?; - let (ar, caps) = discovery.resolve_gvk(&gvk).ok_or_else(|| { - Error::Discovery(DiscoveryError::MissingResource(format!( - "Cannot resolve GVK: {:?}", - gvk - ))) - })?; - - let effective_namespace = if caps.scope == Scope::Cluster { - None - } else { - // Prefer the caller-supplied namespace, fall back to the resource's own - namespace.or_else(|| resource.meta().namespace.as_deref()) - }; - - // ── 3. Determine the effective namespace based on the discovered scope ─── - let api: Api = - get_dynamic_api(ar, caps, self.client.clone(), effective_namespace, false); - - let patch_params = PatchParams::apply("harmony"); - let name = resource - .meta() - .name - .as_ref() - .expect("K8s Resource should have a name"); - - if *crate::config::DRY_RUN { - match api.get(name).await { - Ok(current) => { - trace!("Received current value {current:#?}"); - // The resource exists, so we calculate and display a diff. - println!("\nPerforming dry-run for resource: '{name}'"); - let mut current_yaml = serde_yaml::to_value(¤t).unwrap_or_else(|_| { - panic!("Could not serialize current value : {current:#?}") - }); - if current_yaml.is_mapping() && current_yaml.get("status").is_some() { - let map = current_yaml.as_mapping_mut().unwrap(); - let removed = map.remove_entry("status"); - trace!("Removed status {removed:?}"); - } else { - trace!( - "Did not find status entry for current object {}/{}", - current.meta().namespace.as_ref().unwrap_or(&"".to_string()), - current.meta().name.as_ref().unwrap_or(&"".to_string()) - ); - } - let current_yaml = serde_yaml::to_string(¤t_yaml) - .unwrap_or_else(|_| "Failed to serialize current resource".to_string()); - let new_yaml = serde_yaml::to_string(resource) - .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); - - if current_yaml == new_yaml { - println!("No changes detected."); - // Return the current resource state as there are no changes. - return helper::dyn_to_typed(current); - } - - println!("Changes detected:"); - let diff = TextDiff::from_lines(¤t_yaml, &new_yaml); - - // Iterate over the changes and print them in a git-like diff format. - for change in diff.iter_all_changes() { - let sign = match change.tag() { - similar::ChangeTag::Delete => "-", - similar::ChangeTag::Insert => "+", - similar::ChangeTag::Equal => " ", - }; - print!("{sign}{change}"); - } - // In a dry run, we return the new resource state that would have been applied. - Ok(resource.clone()) - } - Err(Error::Api(ErrorResponse { code: 404, .. })) => { - // The resource does not exist, so the "diff" is the entire new resource. - println!("\nPerforming dry-run for new resource: '{name}'"); - println!( - "Resource does not exist. It would be created with the following content:" - ); - let new_yaml = serde_yaml::to_string(resource) - .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); - - // Print each line of the new resource with a '+' prefix. - for line in new_yaml.lines() { - println!("+{line}"); - } - // In a dry run, we return the new resource state that would have been created. - Ok(resource.clone()) - } - Err(e) => { - // Another API error occurred. - error!("Failed to get resource '{name}': {e}"); - Err(e) - } - } - } else { - // Real apply via server-side apply - // Server-side apply works for both create and update operations - match api - .patch(name, &patch_params, &Patch::Apply(resource)) - .await - { - Ok(obj) => helper::dyn_to_typed(obj), - Err(Error::Api(ErrorResponse { code: 404, .. })) => { - // Resource doesn't exist, server-side apply should create it - // This can happen with some API servers, so we explicitly create - debug!("Resource '{}' not found, creating via POST", name); - let dyn_resource: DynamicObject = serde_json::from_value( - serde_json::to_value(resource).map_err(Error::SerdeError)?, - ) - .map_err(Error::SerdeError)?; - - api.create(&PostParams::default(), &dyn_resource) - .await - .and_then(helper::dyn_to_typed) - .map_err(|e| { - error!("Failed to create resource '{}': {}", name, e); - e - }) - } - Err(e) => { - error!("Failed to apply resource '{}': {}", name, e); - Err(e) - } - } - } - } - - pub async fn apply_many(&self, resource: &[K], ns: Option<&str>) -> Result, Error> - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, - ::DynamicType: Default, - { - let mut result = Vec::new(); - for r in resource.iter() { - let apply_result = self.apply(r, ns).await; - if apply_result.is_err() { - // NOTE : We should be careful about this one, it may leak sensitive information in - // logs - // Maybe just reducing it to debug would be enough as we already know debug logs - // are unsafe. - // But keeping it at warn makes it much easier to understand what is going on. So be it for now. - warn!( - "Failed to apply k8s resource : {}", - serde_json::to_string_pretty(r).map_err(|e| Error::SerdeError(e))? - ); - } - - result.push(apply_result?); - } - - Ok(result) - } - - pub async fn apply_yaml_many( - &self, - #[allow(clippy::ptr_arg)] yaml: &Vec, - ns: Option<&str>, - ) -> Result<(), Error> { - for y in yaml.iter() { - self.apply_yaml(y, ns).await?; - } - Ok(()) - } - - pub async fn apply_yaml( - &self, - yaml: &serde_yaml::Value, - ns: Option<&str>, - ) -> Result<(), Error> { - let obj: DynamicObject = serde_yaml::from_value(yaml.clone()).expect("TODO do not unwrap"); - let name = obj.metadata.name.as_ref().expect("YAML must have a name"); - - let api_version = yaml - .get("apiVersion") - .expect("couldn't get apiVersion from YAML") - .as_str() - .expect("couldn't get apiVersion as str"); - let kind = yaml - .get("kind") - .expect("couldn't get kind from YAML") - .as_str() - .expect("couldn't get kind as str"); - - let mut it = api_version.splitn(2, '/'); - let first = it.next().unwrap(); - let (g, v) = match it.next() { - Some(second) => (first, second), - None => ("", first), - }; - - let gvk = GroupVersionKind::gvk(g, v, kind); - let api_resource = ApiResource::from_gvk(&gvk); - - let namespace = match ns { - Some(n) => n, - None => obj - .metadata - .namespace - .as_ref() - .expect("YAML must have a namespace"), - }; - - // 5. Create a dynamic API client for this resource type. - let api: Api = - Api::namespaced_with(self.client.clone(), namespace, &api_resource); - - // 6. Apply the object to the cluster using Server-Side Apply. - // This will create the resource if it doesn't exist, or update it if it does. - println!("Applying '{name}' in namespace '{namespace}'...",); - let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name - let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?; - - println!("Successfully applied '{}'.", result.name_any()); - - Ok(()) - } - - /// Apply a resource from a URL - /// - /// It is the equivalent of `kubectl apply -f ` - pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> { - let patch_params = PatchParams::apply("harmony"); - let discovery = self.discovery().await?; - - let yaml = reqwest::get(url) - .await - .expect("Could not get URL") - .text() - .await - .expect("Could not get content from URL"); - - for doc in multidoc_deserialize(&yaml).expect("failed to parse YAML from file") { - let obj: DynamicObject = - serde_yaml::from_value(doc).expect("cannot apply without valid YAML"); - let namespace = obj.metadata.namespace.as_deref().or(ns); - let type_meta = obj - .types - .as_ref() - .expect("cannot apply object without valid TypeMeta"); - let gvk = GroupVersionKind::try_from(type_meta) - .expect("cannot apply object without valid GroupVersionKind"); - let name = obj.name_any(); - - if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) { - let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false); - trace!( - "Applying {}: \n{}", - gvk.kind, - serde_yaml::to_string(&obj).expect("Failed to serialize YAML") - ); - let data: serde_json::Value = - serde_json::to_value(&obj).expect("Failed to serialize JSON"); - let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?; - debug!("applied {} {}", gvk.kind, name); - } else { - warn!("Cannot apply document for unknown {gvk:?}"); - } - } - - Ok(()) - } - - /// Gets a single named resource of a specific type `K`. - /// - /// This function uses the `ApplyStrategy` trait to correctly determine - /// whether to look in a specific namespace or in the entire cluster. - /// - /// Returns `Ok(None)` if the resource is not found (404). - pub async fn get_resource( - &self, - name: &str, - namespace: Option<&str>, - ) -> Result, Error> - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned, - ::Scope: ApplyStrategy, - ::DynamicType: Default, - { - let api: Api = - <::Scope as ApplyStrategy>::get_api(&self.client, namespace); - - api.get_opt(name).await - } - - pub async fn list_all_resources_with_labels(&self, labels: &str) -> Result, Error> - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned, - ::DynamicType: Default, - { - let api: Api = Api::all(self.client.clone()); - - let lp = ListParams::default().labels(labels); - Ok(api.list(&lp).await?.items) - } - - pub async fn get_all_resource_in_all_namespace(&self) -> Result, Error> - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned, - ::Scope: ApplyStrategy, - ::DynamicType: Default, - { - let api: Api = Api::all(self.client.clone()); - Ok(api.list(&Default::default()).await?.items) - } - - /// Lists all resources of a specific type `K`. - /// - /// This function uses the `ApplyStrategy` trait to correctly determine - /// whether to list from a specific namespace or from the entire cluster. - pub async fn list_resources( - &self, - namespace: Option<&str>, - list_params: Option, - ) -> Result, Error> - where - K: Resource + Clone + std::fmt::Debug + DeserializeOwned, - ::Scope: ApplyStrategy, - ::DynamicType: Default, - { - let api: Api = - <::Scope as ApplyStrategy>::get_api(&self.client, namespace); - - let list_params = list_params.unwrap_or_default(); - api.list(&list_params).await - } - - /// Fetches a list of all Nodes in the cluster. - pub async fn get_nodes( - &self, - list_params: Option, - ) -> Result, Error> { - self.list_resources(None, list_params).await - } - - pub async fn from_kubeconfig(path: &str) -> Option { - Self::from_kubeconfig_with_opts(path, &KubeConfigOptions::default()).await - } - - pub async fn from_kubeconfig_with_context( - path: &str, - context: Option, - ) -> Option { - let mut opts = KubeConfigOptions::default(); - opts.context = context; - - Self::from_kubeconfig_with_opts(path, &opts).await - } - - pub async fn from_kubeconfig_with_opts( - path: &str, - opts: &KubeConfigOptions, - ) -> Option { - let k = match Kubeconfig::read_from(path) { - Ok(k) => k, - Err(e) => { - error!("Failed to load kubeconfig from {path} : {e}"); - return None; - } - }; - - Some(K8sClient::new( - Client::try_from(Config::from_custom_kubeconfig(k, &opts).await.unwrap()).unwrap(), - )) - } - - pub async fn cordon_node(&self, node_name: &str) -> Result<(), Error> { - let api: Api = Api::all(self.client.clone()); - - api.cordon(node_name).await?; - Ok(()) - } - - pub async fn uncordon_node(&self, node_name: &str) -> Result<(), Error> { - let api: Api = Api::all(self.client.clone()); - - api.uncordon(node_name).await?; - - Ok(()) - } - - /// Lists every pod currently scheduled on `node_name`. - async fn list_pods_on_node(&self, node_name: &str) -> Result, Error> { - let api: Api = Api::all(self.client.clone()); - let lp = ListParams::default().fields(&format!("spec.nodeName={}", node_name)); - Ok(api.list(&lp).await?.items) - } - - /// Returns `true` when the pod is a *mirror pod* (a static manifest - /// managed directly by the kubelet). - fn is_mirror_pod(pod: &Pod) -> bool { - pod.metadata - .annotations - .as_ref() - .map(|a| a.contains_key("kubernetes.io/config.mirror")) - .unwrap_or(false) - } - - /// Returns `true` when the pod is owned by a `DaemonSet`. - fn is_daemonset_pod(pod: &Pod) -> bool { - pod.metadata - .owner_references - .as_ref() - .map(|refs| refs.iter().any(|r| r.kind == "DaemonSet")) - .unwrap_or(false) - } - - /// Returns `true` when the pod spec contains at least one `emptyDir` - /// volume. - fn has_emptydir_volume(pod: &Pod) -> bool { - pod.spec - .as_ref() - .and_then(|s| s.volumes.as_ref()) - .map(|vols| vols.iter().any(|v| v.empty_dir.is_some())) - .unwrap_or(false) - } - - /// Returns `true` when the pod has already terminated (`Succeeded` or - /// `Failed`). - fn is_completed_pod(pod: &Pod) -> bool { - pod.status - .as_ref() - .and_then(|s| s.phase.as_deref()) - .map(|phase| phase == "Succeeded" || phase == "Failed") - .unwrap_or(false) - } - - /// Partitions `pods` into *(evictable, skipped_descriptions)*. - /// - /// Returns `Err` with a human-readable message when one or more pods would - /// block the drain (e.g. a `DaemonSet` pod with `ignore_daemonsets = - /// false`). - fn classify_pods_for_drain( - pods: &[Pod], - options: &DrainOptions, - ) -> Result<(Vec, Vec), String> { - let mut evictable: Vec = Vec::new(); - let mut skipped: Vec = Vec::new(); - let mut blocking: Vec = Vec::new(); - - for pod in pods { - let name = pod.metadata.name.as_deref().unwrap_or(""); - let ns = pod.metadata.namespace.as_deref().unwrap_or(""); - let qualified = format!("{}/{}", ns, name); - - // Mirror pods are managed by the kubelet — never evict. - if Self::is_mirror_pod(pod) { - skipped.push(format!("{} (mirror pod)", qualified)); - continue; - } - - // Already-terminated pods do not need eviction. - if Self::is_completed_pod(pod) { - skipped.push(format!("{} (completed)", qualified)); - continue; - } - - // DaemonSet pods: skip or block depending on options. - if Self::is_daemonset_pod(pod) { - if options.ignore_daemonsets { - skipped.push(format!("{} (DaemonSet-managed)", qualified)); - } else { - blocking.push(format!( - "{} is managed by a DaemonSet (set ignore_daemonsets to skip)", - qualified - )); - } - continue; - } - - // Pods with emptyDir data: block unless explicitly allowed. - if Self::has_emptydir_volume(pod) && !options.delete_emptydir_data { - blocking.push(format!( - "{} uses emptyDir volumes (set delete_emptydir_data to allow eviction)", - qualified - )); - continue; - } - - evictable.push(pod.clone()); - } - - if !blocking.is_empty() { - return Err(format!( - "Cannot drain node — the following pods block eviction:\n - {}", - blocking.join("\n - ") - )); - } - - Ok((evictable, skipped)) - } - - async fn wait_for_pod_completion(&self, name: &str, namespace: &str) -> Result { - let pod_api: Api = Api::namespaced(self.client.clone(), namespace); - let poll_interval = Duration::from_secs(2); - for _ in 0..60 { - // 2 minutes timeout - sleep(poll_interval).await; - let p = pod_api.get(name).await?; - if let Some(status) = p.status { - match status.phase.as_deref() { - Some("Succeeded") => { - // Capture pod logs as output - let logs = pod_api - .logs(name, &Default::default()) - .await - .unwrap_or_else(|_| String::new()); - - debug!("Retrieved pod {namespace}/{name} logs {logs}"); - - return Ok(logs); - } - Some("Failed") => { - let logs = pod_api - .logs(name, &Default::default()) - .await - .unwrap_or_else(|_| String::new()); - - debug!("Retrieved failed pod {namespace}/{name} logs {logs}"); - - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Pod {} failed. Logs:\n{}", - name, logs - )))); - } - _ => {} - } - } - } - Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Timed out waiting for pod {}", - name - )))) - } - - pub async fn get_k8s_distribution(&self) -> Result { - self.k8s_distribution - .get_or_try_init(async || { - debug!("Trying to detect k8s distribution"); - let api_groups = self.client.list_api_groups().await?; - trace!("list_api_groups {:?}", api_groups); - debug!("K8s discovery completed"); - - let version = self.get_apiserver_version().await?; - - // OpenShift / OKD - if api_groups - .groups - .iter() - .any(|g| g.name == "project.openshift.io") - { - info!("Found KubernetesDistribution OpenshiftFamily"); - return Ok(KubernetesDistribution::OpenshiftFamily); - } - - // K3d / K3s - if version.git_version.contains("k3s") { - info!("Found KubernetesDistribution K3sFamily"); - return Ok(KubernetesDistribution::K3sFamily); - } - - info!("Could not identify KubernetesDistribution, using Default"); - return Ok(KubernetesDistribution::Default); - }) - .await - .map(|k| k.clone()) - } - - /// Writes a set of files to a node's filesystem using a privileged ephemeral pod. - /// - /// This method creates a ConfigMap containing the file contents and a privileged Pod - /// that mounts the host filesystem. It then copies the files from the ConfigMap - /// to the specified paths on the host and sets the requested permissions. - /// - /// On OpenShift clusters, the required SCC binding is automatically created via - /// the ResourceBundle pattern. - /// - /// ## Use Case: Network Bonding Configuration (ADR-019) - /// - /// This method is designed to support operations like writing NetworkManager - /// configuration files to `/etc/NetworkManager/system-connections/` for - /// setting up LACP bonds on worker nodes, where interface names vary across - /// hardware. - /// - /// Files written via this method persist across reboots on Fedora CoreOS/SCOS. - /// - /// # Arguments - /// - /// * `node_name` - The name of the node to write files to - /// * `files` - A slice of [`NodeFile`] structs containing path, content, and permissions - /// - /// # Example - /// - /// ```rust,no_run - /// # use harmony::topology::k8s::{K8sClient, NodeFile}; - /// # async fn example(client: K8sClient) { - /// let bond_config = NodeFile { - /// path: "/etc/NetworkManager/system-connections/bond0.nmconnection".to_string(), - /// content: "[connection]\nid=bond0\n...".to_string(), - /// mode: 0o600, - /// }; - /// - /// client.write_files_to_node("worker-01", &[bond_config]).await.unwrap(); - /// # } - /// ``` - pub async fn write_files_to_node( - &self, - node_name: &str, - files: &[NodeFile], - ) -> Result { - let ns = self.client.default_namespace(); - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis(); - let name = format!("harmony-writer-{}", suffix); - - debug!( - "Preparing to write {} files to node '{}'", - files.len(), - node_name - ); - - // 1. Prepare ConfigMap data & Script - let mut data = BTreeMap::new(); - let mut script = String::from("set -e\n"); - - for (i, file) in files.iter().enumerate() { - let key = format!("f{}", i); - data.insert(key.clone(), file.content.clone()); - - // Ensure parent dir exists - script.push_str(&format!("mkdir -p \"$(dirname \"/host{}\")\"\n", file.path)); - // Copy file - script.push_str(&format!("cp \"/payload/{}\" \"/host{}\"\n", key, file.path)); - // Chmod (format as octal) - script.push_str(&format!("chmod {:o} \"/host{}\"\n", file.mode, file.path)); - } - - let cm = ConfigMap { - metadata: ObjectMeta { - name: Some(name.clone()), - namespace: Some(ns.to_string()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - - let cm_api: Api = Api::namespaced(self.client.clone(), ns); - cm_api.create(&PostParams::default(), &cm).await?; - debug!("Created ConfigMap {}", name); - - // 2. Build resource bundle with Pod and RBAC - let (host_vol, host_mount) = helper::host_root_volume(); - - let payload_vol = Volume { - name: "payload".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: name.clone(), - ..Default::default() - }), - ..Default::default() - }; - - let payload_mount = VolumeMount { - name: "payload".to_string(), - mount_path: "/payload".to_string(), - ..Default::default() - }; - - let bundle = helper::build_privileged_bundle( - PrivilegedPodConfig { - name: name.clone(), - namespace: ns.to_string(), - node_name: node_name.to_string(), - container_name: "writer".to_string(), - command: vec!["/bin/bash".to_string(), "-c".to_string(), script], - volumes: vec![payload_vol, host_vol], - volume_mounts: vec![payload_mount, host_mount], - host_pid: false, - host_network: false, - }, - &self.get_k8s_distribution().await?, - ); - - // 3. Apply bundle (RBAC + Pod) - bundle.apply(self).await?; - debug!("Created privileged pod bundle {}", name); - - // 4. Wait for completion - let result = self.wait_for_pod_completion(&name, ns).await; - - // 5. Cleanup - debug!("Cleaning up resources for {}", name); - let _ = bundle.delete(self).await; - let _ = cm_api.delete(&name, &DeleteParams::default()).await; - - result - } - - /// Runs a privileged command on a specific node using an ephemeral pod. - /// - /// This method creates a privileged pod with host PID and network namespaces - /// enabled, along with the host filesystem mounted at `/host`. The pod runs - /// the specified command and waits for completion. - /// - /// On OpenShift clusters, the required SCC binding is automatically created via - /// the ResourceBundle pattern. - /// - /// # Arguments - /// - /// * `node_name` - The name of the node to run the command on - /// * `command` - The shell command to execute (runs in `/bin/bash -c`) - /// - /// # Returns - /// - /// The stdout output from the command execution. - /// - /// # Example - /// - /// ```rust,no_run - /// # use harmony::topology::k8s::K8sClient; - /// # async fn example(client: K8sClient) { - /// // Reload NetworkManager configuration after writing .nmconnection files - /// let output = client.run_privileged_command_on_node( - /// "worker-01", - /// "nmcli connection reload" - /// ).await.unwrap(); - /// # } - /// ``` - pub async fn run_privileged_command_on_node( - &self, - node_name: &str, - command: &str, - ) -> Result { - let namespace = self.client.default_namespace(); - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis(); - let name = format!("harmony-cmd-{}", suffix); - - debug!( - "Running privileged command on node '{}': {}", - node_name, command - ); - - // Build resource bundle with Pod and RBAC - let (host_vol, host_mount) = helper::host_root_volume(); - trace!("Got host volume {host_vol:#?}"); - trace!("Got host volume mount {host_mount:#?}"); - let bundle = helper::build_privileged_bundle( - PrivilegedPodConfig { - name: name.clone(), - namespace: namespace.to_string(), - node_name: node_name.to_string(), - container_name: "runner".to_string(), - command: vec![ - "/bin/bash".to_string(), - "-c".to_string(), - command.to_string(), - ], - volumes: vec![host_vol], - volume_mounts: vec![host_mount], - host_pid: true, - host_network: true, - }, - &self.get_k8s_distribution().await?, - ); - - debug!("Built privileged bundle {bundle:#?}"); - debug!("Built privileged bundle for command : {command}"); - - // Apply bundle (RBAC + Pod) - bundle.apply(self).await?; - debug!("Created privileged pod bundle {}", name); - - // Wait for completion - let result = self.wait_for_pod_completion(&name, namespace).await; - - // Cleanup - debug!("Cleaning up resources for {}", name); - let _ = bundle.delete(self).await; - - result - } - - /// Reboots a Kubernetes node safely with proper drain/uncordon cycle. - /// - /// This method implements a robust node reboot procedure: - /// 1. Records the current boot ID from node status - /// 2. Drains the node (cordons + evicts all pods) - /// 3. Issues a delayed reboot command (fire-and-forget) - /// 4. Waits for the node to go NotReady (confirms shutdown started) - /// 5. Waits for the node to become Ready again - /// 6. Verifies the boot ID changed (confirms actual reboot occurred) - /// 7. Uncordons the node - /// - /// # Arguments - /// - /// * `node_name` - The name of the node to reboot - /// * `drain_options` - Options controlling pod eviction behavior - /// * `timeout` - Maximum time to wait for the entire reboot cycle - /// - /// # Example - /// - /// ```rust,no_run - /// # use harmony::topology::k8s::{K8sClient, DrainOptions}; - /// # use std::time::Duration; - /// # async fn example(client: K8sClient) { - /// client.reboot_node( - /// "worker-01", - /// &DrainOptions::default_ignore_daemonset_delete_emptydir_data(), - /// Duration::from_secs(3600) // 1 hour timeout - /// ).await.unwrap(); - /// # } - /// ``` - pub async fn reboot_node( - &self, - node_name: &str, - drain_options: &DrainOptions, - timeout: Duration, - ) -> Result<(), Error> { - info!("Starting reboot procedure for node '{}'", node_name); - - // 1. Get current boot ID from node status - let node_api: Api = Api::all(self.client.clone()); - let node = node_api.get(node_name).await?; - let boot_id_before = node - .status - .as_ref() - .and_then(|s| s.node_info.as_ref()) - .and_then(|ni| Some(ni.boot_id.clone())) - .ok_or_else(|| { - Error::Discovery(DiscoveryError::MissingResource(format!( - "Node '{}' does not have boot_id in status", - node_name - ))) - })?; - debug!( - "Current boot_id for node '{}': {}", - node_name, boot_id_before - ); - - // 2. Drain the node - info!("Draining node '{}'...", node_name); - self.drain_node(node_name, drain_options).await?; - - let start_time = tokio::time::Instant::now(); - - // 3. Issue delayed reboot command (fire-and-forget) - info!("Scheduling reboot for node '{}'...", node_name); - let reboot_cmd = - "echo rebooting ; nohup bash -c 'sleep 5 && nsenter -t 1 -m -- systemctl reboot'"; - - // Ignore errors - the pod will die during shutdown and we can't wait for completion - match self - .run_privileged_command_on_node(node_name, reboot_cmd) - .await - { - Ok(_) => debug!("Reboot command scheduled successfully"), - Err(e) => { - // This is expected - the node may start shutting down before we can read the pod status - debug!( - "Reboot command scheduling completed with error (expected): {}", - e - ); - } - } - - // 4. Wait for node to go NotReady (proves shutdown started) - info!("Waiting for node '{}' to begin shutdown...", node_name); - let remaining_timeout = timeout.saturating_sub(start_time.elapsed()); - self.wait_for_node_not_ready(node_name, remaining_timeout) - .await?; - - if start_time.elapsed() > timeout { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Timeout during node '{}' reboot (shutdown detection phase)", - node_name - )))); - } - - // 5. Wait for node to become Ready again - info!("Waiting for node '{}' to come back online...", node_name); - let remaining_timeout = timeout.saturating_sub(start_time.elapsed()); - self.wait_for_node_ready_with_timeout(node_name, remaining_timeout) - .await?; - - if start_time.elapsed() > timeout { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Timeout during node '{}' reboot (ready phase)", - node_name - )))); - } - - // 6. Verify boot ID changed (confirms actual reboot) - info!("Verifying node '{}' actually rebooted...", node_name); - let node = node_api.get(node_name).await?; - let boot_id_after = node - .status - .as_ref() - .and_then(|s| s.node_info.as_ref()) - .and_then(|ni| Some(ni.boot_id.clone())) - .ok_or_else(|| { - Error::Discovery(DiscoveryError::MissingResource(format!( - "Node '{}' does not have boot_id in status after reboot", - node_name - ))) - })?; - - if boot_id_before == boot_id_after { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Node '{}' did not actually reboot (boot_id unchanged: {})", - node_name, boot_id_before - )))); - } - - debug!( - "Node '{}' boot_id changed: {} -> {}", - node_name, boot_id_before, boot_id_after - ); - - // 7. Uncordon the node - info!("Uncordoning node '{}'...", node_name); - self.uncordon_node(node_name).await?; - - info!( - "Successfully rebooted node '{}' (took {:?})", - node_name, - start_time.elapsed() - ); - - Ok(()) - } - - /// Waits for a node to transition to NotReady status. - /// - /// This is useful for detecting when a node shutdown has begun. - async fn wait_for_node_not_ready( - &self, - node_name: &str, - timeout: Duration, - ) -> Result<(), Error> { - let api: Api = Api::all(self.client.clone()); - let poll_interval = Duration::from_secs(5); - let start = tokio::time::Instant::now(); - - loop { - if start.elapsed() > timeout { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Node '{}' did not become NotReady within {:?}", - node_name, timeout - )))); - } - - match api.get(node_name).await { - Ok(node) => { - if let Some(status) = node.status { - if let Some(conditions) = status.conditions { - let is_ready = conditions - .iter() - .any(|cond| cond.type_ == "Ready" && cond.status == "True"); - - if !is_ready { - debug!("Node '{}' is now NotReady", node_name); - return Ok(()); - } - } - } - } - Err(e) => { - debug!("Error checking node '{}' status: {}", node_name, e); - } - } - - sleep(poll_interval).await; - } - } - - pub async fn wait_for_node_ready(&self, node_name: &str) -> Result<(), Error> { - // Default 10 minute timeout for backwards compatibility - self.wait_for_node_ready_with_timeout(node_name, Duration::from_secs(600)) - .await - } - - /// Waits for a node to become Ready with a custom timeout. - async fn wait_for_node_ready_with_timeout( - &self, - node_name: &str, - timeout: Duration, - ) -> Result<(), Error> { - let api: Api = Api::all(self.client.clone()); - let poll_interval = Duration::from_secs(5); - let start = tokio::time::Instant::now(); - - loop { - if start.elapsed() > timeout { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Node '{}' did not become ready within {:?}", - node_name, timeout - )))); - } - - match api.get(node_name).await { - Ok(node) => { - if let Some(status) = node.status { - if let Some(conditions) = status.conditions { - for cond in conditions { - if cond.type_ == "Ready" && cond.status == "True" { - debug!("Node '{}' is now Ready", node_name); - return Ok(()); - } - } - } - } - } - Err(e) => { - debug!("Failed to get node '{}': {}", node_name, e); - } - } - - sleep(poll_interval).await; - } - } - - /// Sends a single eviction request for `pod`. - async fn evict_pod(&self, pod: &Pod) -> Result<(), Error> { - let name = pod.metadata.name.as_deref().unwrap_or_default(); - let ns = pod.metadata.namespace.as_deref().unwrap_or_default(); - let api: Api = Api::namespaced(self.client.clone(), ns); - debug!("Sending eviction for pod {}/{}", ns, name); - api.evict(name, &EvictParams::default()).await.map(|_| ()) - } - - /// Drains a node by cordoning it, evicting eligible pods, and waiting for - /// them to terminate. - /// - /// The operation mirrors `kubectl drain`: - /// 1. **Cordon** — marks the node as unschedulable. - /// 2. **Classify** — separates pods into evictable / skipped / blocking. - /// 3. **Evict & wait** — sends eviction requests and re-tries on each - /// polling interval until every pod is gone or the timeout expires. - /// - /// Re-sending eviction requests each iteration ensures that pods - /// previously blocked by a `PodDisruptionBudget` are retried once budget - /// becomes available. - /// - /// # Errors - /// Returns an error if the node cannot be cordoned, if any pod blocks - /// eviction (see [`DrainOptions`]), or if evictions do not complete within - /// the configured timeout. - pub async fn drain_node(&self, node_name: &str, options: &DrainOptions) -> Result<(), Error> { - // ── 1. Cordon ────────────────────────────────────────────────── - debug!("Cordoning node '{}'", node_name); - self.cordon_node(node_name).await?; - - // ── 2. List & classify pods ──────────────────────────────────── - let pods = self.list_pods_on_node(node_name).await?; - debug!("Found {} pod(s) on node '{}'", pods.len(), node_name); - - let (evictable, skipped) = - Self::classify_pods_for_drain(&pods, options).map_err(|msg| { - error!("{}", msg); - Error::Discovery(DiscoveryError::MissingResource(msg)) - })?; - - for s in &skipped { - info!("Skipping pod: {}", s); - } - - if evictable.is_empty() { - info!("No pods to evict on node '{}'", node_name); - return Ok(()); - } - - info!( - "Evicting {} pod(s) from node '{}'", - evictable.len(), - node_name - ); - - // ── 3. Evict & wait loop ────────────────────────────────────── - let mut start = tokio::time::Instant::now(); - let poll_interval = Duration::from_secs(5); - let mut pending = evictable; - - loop { - // Send (or re-send) eviction requests for all pending pods. - for pod in &pending { - match self.evict_pod(pod).await { - Ok(()) => {} - // Pod already gone — will be filtered out below. - Err(Error::Api(ErrorResponse { code: 404, .. })) => {} - // PDB is blocking — will retry next iteration. - Err(Error::Api(ErrorResponse { code: 429, .. })) => { - warn!( - "PDB prevented eviction of {}/{}; will retry", - pod.metadata.namespace.as_deref().unwrap_or(""), - pod.metadata.name.as_deref().unwrap_or("") - ); - } - Err(e) => { - error!( - "Failed to evict pod {}/{}: {}", - pod.metadata.namespace.as_deref().unwrap_or(""), - pod.metadata.name.as_deref().unwrap_or(""), - e - ); - return Err(e); - } - } - } - - // Wait before polling pod presence. - sleep(poll_interval).await; - - // Check which pods are still present on the API server. - let mut still_present: Vec = Vec::new(); - for pod in pending { - let ns = pod.metadata.namespace.as_deref().unwrap_or_default(); - let name = pod.metadata.name.as_deref().unwrap_or_default(); - match self.get_pod(name, Some(ns)).await? { - Some(_) => still_present.push(pod), - None => debug!("Pod {}/{} evicted successfully", ns, name), - } - } - - pending = still_present; - - if pending.is_empty() { - break; - } - - if start.elapsed() > options.timeout { - let names: Vec = pending - .iter() - .map(|p| { - format!( - "{}/{}", - p.metadata.namespace.as_deref().unwrap_or(""), - p.metadata.name.as_deref().unwrap_or("") - ) - }) - .collect(); - let msg = format!( - "Timed out after {:?} waiting for pod evictions on node '{}'. Remaining:\n - {}", - options.timeout, - node_name, - names.join("\n - ") - ); - - warn!("{}", msg); - - // Prompt user for action - match helper::prompt_drain_timeout_action( - node_name, - pending.len(), - options.timeout, - )? { - helper::DrainTimeoutAction::Accept => { - // User confirmed acceptance - break the loop and continue - break; - } - helper::DrainTimeoutAction::Retry => { - // Reset the start time to retry for another full timeout period - start = tokio::time::Instant::now(); - continue; - } - helper::DrainTimeoutAction::Abort => { - return Err(Error::Discovery(DiscoveryError::MissingResource(format!( - "Drain operation aborted. {} pods remaining on node '{}'", - pending.len(), - node_name - )))); - } - } - } - - debug!( - "Waiting for {} pod(s) to terminate on node '{}'", - pending.len(), - node_name - ); - } - - debug!("Node '{}' drained successfully", node_name); - Ok(()) - } -} - -fn get_dynamic_api( - resource: ApiResource, - capabilities: ApiCapabilities, - client: Client, - ns: Option<&str>, - all: bool, -) -> Api { - if capabilities.scope == Scope::Cluster || all { - Api::all_with(client, &resource) - } else if let Some(namespace) = ns { - Api::namespaced_with(client, namespace, &resource) - } else { - Api::default_namespaced_with(client, &resource) - } -} - -fn multidoc_deserialize(data: &str) -> Result, serde_yaml::Error> { - use serde::Deserialize; - let mut docs = vec![]; - for de in serde_yaml::Deserializer::from_str(data) { - docs.push(serde_yaml::Value::deserialize(de)?); - } - Ok(docs) -} - -pub trait ApplyStrategy { - fn get_api(client: &Client, ns: Option<&str>) -> Api; -} - -/// Implementation for all resources that are cluster-scoped. -/// It will always use `Api::all` and ignore the namespace parameter. -impl ApplyStrategy for ClusterResourceScope -where - K: Resource, - ::DynamicType: Default, -{ - fn get_api(client: &Client, _ns: Option<&str>) -> Api { - Api::all(client.clone()) - } -} - -/// Implementation for all resources that are namespace-scoped. -/// It will use `Api::namespaced` if a namespace is provided, otherwise -/// it falls back to the default namespace configured in your kubeconfig. -impl ApplyStrategy for NamespaceResourceScope -where - K: Resource, - ::DynamicType: Default, -{ - fn get_api(client: &Client, ns: Option<&str>) -> Api { - match ns { - Some(ns) => Api::namespaced(client.clone(), ns), - None => Api::default_namespaced(client.clone()), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use k8s_openapi::api::core::v1::{EmptyDirVolumeSource, Pod, PodSpec, PodStatus, Volume}; - use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; - use std::collections::BTreeMap; - - // ── Test helpers ──────────────────────────────────────────────────── - - /// Builds a minimal pod with the given name/namespace and no special - /// annotations, owner refs, volumes, or status. - fn base_pod(name: &str, ns: &str) -> Pod { - Pod { - metadata: ObjectMeta { - name: Some(name.to_string()), - namespace: Some(ns.to_string()), - ..Default::default() - }, - spec: Some(PodSpec::default()), - status: Some(PodStatus { - phase: Some("Running".to_string()), - ..Default::default() - }), - } - } - - fn mirror_pod(name: &str, ns: &str) -> Pod { - let mut pod = base_pod(name, ns); - let mut annotations = BTreeMap::new(); - annotations.insert( - "kubernetes.io/config.mirror".to_string(), - "abc123".to_string(), - ); - pod.metadata.annotations = Some(annotations); - pod - } - - fn daemonset_pod(name: &str, ns: &str) -> Pod { - let mut pod = base_pod(name, ns); - pod.metadata.owner_references = Some(vec![OwnerReference { - api_version: "apps/v1".to_string(), - kind: "DaemonSet".to_string(), - name: "some-ds".to_string(), - uid: "uid-ds".to_string(), - ..Default::default() - }]); - pod - } - - fn emptydir_pod(name: &str, ns: &str) -> Pod { - let mut pod = base_pod(name, ns); - pod.spec = Some(PodSpec { - volumes: Some(vec![Volume { - name: "scratch".to_string(), - empty_dir: Some(EmptyDirVolumeSource::default()), - ..Default::default() - }]), - ..Default::default() - }); - pod - } - - fn completed_pod(name: &str, ns: &str, phase: &str) -> Pod { - let mut pod = base_pod(name, ns); - pod.status = Some(PodStatus { - phase: Some(phase.to_string()), - ..Default::default() - }); - pod - } - - fn default_opts() -> DrainOptions { - DrainOptions::default() - } - - // ── Tests ─────────────────────────────────────────────────────────── - - #[test] - fn empty_pod_list_returns_empty_vecs() { - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&[], &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert!(skipped.is_empty()); - } - - #[test] - fn normal_pod_is_evictable() { - let pods = vec![base_pod("web", "default")]; - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - assert_eq!(evictable.len(), 1); - assert_eq!(evictable[0].metadata.name.as_deref(), Some("web")); - assert!(skipped.is_empty()); - } - - #[test] - fn mirror_pod_is_skipped() { - let pods = vec![mirror_pod("kube-apiserver", "kube-system")]; - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!(skipped[0].contains("mirror pod")); - } - - #[test] - fn completed_succeeded_pod_is_skipped() { - let pods = vec![completed_pod("job-xyz", "batch", "Succeeded")]; - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!(skipped[0].contains("completed")); - } - - #[test] - fn completed_failed_pod_is_skipped() { - let pods = vec![completed_pod("job-fail", "batch", "Failed")]; - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!(skipped[0].contains("completed")); - } - - #[test] - fn daemonset_pod_skipped_when_ignore_daemonsets_true() { - let pods = vec![daemonset_pod("fluentd", "logging")]; - let opts = DrainOptions { - ignore_daemonsets: true, - ..default_opts() - }; - let (evictable, skipped) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!(skipped[0].contains("DaemonSet-managed")); - } - - #[test] - fn daemonset_pod_blocks_when_ignore_daemonsets_false() { - let pods = vec![daemonset_pod("fluentd", "logging")]; - let opts = DrainOptions { - ignore_daemonsets: false, - ..default_opts() - }; - let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); - assert!(err.contains("DaemonSet")); - assert!(err.contains("logging/fluentd")); - } - - #[test] - fn emptydir_pod_blocks_when_delete_emptydir_data_false() { - let pods = vec![emptydir_pod("cache", "default")]; - let opts = DrainOptions { - delete_emptydir_data: false, - ..default_opts() - }; - let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); - assert!(err.contains("emptyDir")); - assert!(err.contains("default/cache")); - } - - #[test] - fn emptydir_pod_evictable_when_delete_emptydir_data_true() { - let pods = vec![emptydir_pod("cache", "default")]; - let opts = DrainOptions { - delete_emptydir_data: true, - ..default_opts() - }; - let (evictable, skipped) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); - assert_eq!(evictable.len(), 1); - assert_eq!(evictable[0].metadata.name.as_deref(), Some("cache")); - assert!(skipped.is_empty()); - } - - #[test] - fn multiple_blocking_pods_all_reported() { - let pods = vec![daemonset_pod("ds-a", "ns1"), emptydir_pod("ed-b", "ns2")]; - let opts = DrainOptions { - ignore_daemonsets: false, - delete_emptydir_data: false, - ..default_opts() - }; - let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); - assert!(err.contains("ns1/ds-a")); - assert!(err.contains("ns2/ed-b")); - } - - #[test] - fn mixed_pods_classified_correctly() { - let pods = vec![ - base_pod("web", "default"), - mirror_pod("kube-apiserver", "kube-system"), - daemonset_pod("fluentd", "logging"), - completed_pod("job-done", "batch", "Succeeded"), - base_pod("api", "default"), - ]; - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - - let evict_names: Vec<&str> = evictable - .iter() - .map(|p| p.metadata.name.as_deref().unwrap()) - .collect(); - assert_eq!(evict_names, vec!["web", "api"]); - assert_eq!(skipped.len(), 3); - } - - #[test] - fn classification_priority_mirror_before_completed() { - // A mirror pod that also has phase=Succeeded should still be - // classified as "mirror pod" (the first check wins). - let mut pod = mirror_pod("static-etcd", "kube-system"); - pod.status = Some(PodStatus { - phase: Some("Succeeded".to_string()), - ..Default::default() - }); - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!( - skipped[0].contains("mirror pod"), - "expected mirror-pod label, got: {}", - skipped[0] - ); - } - - #[test] - fn classification_priority_completed_before_daemonset() { - // A completed DaemonSet pod should be skipped as "completed", - // not as "DaemonSet-managed". - let mut pod = daemonset_pod("collector", "monitoring"); - pod.status = Some(PodStatus { - phase: Some("Failed".to_string()), - ..Default::default() - }); - let (evictable, skipped) = - K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); - assert!(evictable.is_empty()); - assert_eq!(skipped.len(), 1); - assert!( - skipped[0].contains("completed"), - "expected completed label, got: {}", - skipped[0] - ); - } - - #[test] - fn pod_with_no_metadata_names_uses_unknown_placeholder() { - let pod = Pod { - metadata: ObjectMeta::default(), - spec: Some(PodSpec::default()), - status: Some(PodStatus { - phase: Some("Running".to_string()), - ..Default::default() - }), - }; - let (evictable, _) = K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); - assert_eq!(evictable.len(), 1); - } -} - -#[cfg(test)] -mod apply_tests { - //! Integration tests for apply() and apply_dynamic() functions. - //! - //! ## Testing Strategy - //! - //! These functions interact with the Kubernetes API server, making them difficult - //! to unit test. We recommend a multi-layered testing approach: - //! - //! ### 1. **Integration Tests with Real Cluster (Recommended)** - //! - Use a local development cluster (kind, k3d, minikube) - //! - Place tests in `tests/` directory for optional execution - //! - Run with: `cargo test --test k8s_apply_integration -- --ignored` - //! - //! ### 2. **Contract Tests with Mock Server** - //! - Use `wiremock` or `mockito` to simulate Kubernetes API responses - //! - Test specific scenarios: 404 → create, 200 → update, error cases - //! - Fast, deterministic, no cluster required - //! - //! ### 3. **Property-Based Tests** - //! - Use `proptest` to generate various resource configurations - //! - Verify idempotency: apply(x) → apply(x) should not error - //! - //! ### 4. **Example Tests Below** - //! - These demonstrate the testing patterns - //! - Marked with `#[ignore]` to require opt-in execution - //! - Can be run in CI with proper cluster setup - //! - //! ## Running Tests - //! - //! ```bash - //! # Setup test cluster - //! kind create cluster --name harmony-test - //! - //! # Run integration tests - //! cargo test --test k8s_apply_integration - //! - //! # Or run ignored tests in this module - //! cargo test apply_tests -- --ignored --nocapture - //! ``` - - use kube::api::TypeMeta; - - use super::*; - - /// Example integration test for apply() with ConfigMap creation. - /// - /// This test requires a real Kubernetes cluster and is marked as ignored. - /// Run with: `cargo test apply_creates_new_configmap -- --ignored` - #[tokio::test] - #[ignore = "requires kubernetes cluster"] - async fn apply_creates_new_configmap() { - let client = K8sClient::try_default() - .await - .expect("failed to create client"); - let test_ns = "default"; - let cm_name = format!( - "test-cm-{}", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() - ); - - let mut data = BTreeMap::new(); - data.insert("key1".to_string(), "value1".to_string()); - - let configmap = ConfigMap { - metadata: ObjectMeta { - name: Some(cm_name.clone()), - namespace: Some(test_ns.to_string()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - - // Apply should create the resource - let result = client.apply(&configmap, Some(test_ns)).await; - assert!( - result.is_ok(), - "failed to apply new ConfigMap: {:?}", - result.err() - ); - - // Verify it exists - let fetched: Option = - client.get_resource(&cm_name, Some(test_ns)).await.unwrap(); - assert!(fetched.is_some(), "ConfigMap was not created"); - assert_eq!( - fetched.unwrap().data.unwrap().get("key1").unwrap(), - "value1" - ); - - // Cleanup - let api: Api = Api::namespaced(client.client.clone(), test_ns); - let _ = api.delete(&cm_name, &DeleteParams::default()).await; - } - - /// Example integration test for apply() updating an existing resource. - #[tokio::test] - #[ignore = "requires kubernetes cluster"] - async fn apply_updates_existing_configmap() { - let client = K8sClient::try_default() - .await - .expect("failed to create client"); - let test_ns = "default"; - let cm_name = format!( - "test-cm-{}", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() - ); - - // Create initial ConfigMap - let mut data = BTreeMap::new(); - data.insert("key1".to_string(), "value1".to_string()); - let configmap = ConfigMap { - metadata: ObjectMeta { - name: Some(cm_name.clone()), - namespace: Some(test_ns.to_string()), - ..Default::default() - }, - data: Some(data.clone()), - ..Default::default() - }; - - client.apply(&configmap, Some(test_ns)).await.unwrap(); - - // Update the ConfigMap - data.insert("key2".to_string(), "value2".to_string()); - let updated_cm = ConfigMap { - metadata: ObjectMeta { - name: Some(cm_name.clone()), - namespace: Some(test_ns.to_string()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - - let result = client.apply(&updated_cm, Some(test_ns)).await; - assert!( - result.is_ok(), - "failed to update ConfigMap: {:?}", - result.err() - ); - - // Verify the update - let fetched: Option = - client.get_resource(&cm_name, Some(test_ns)).await.unwrap(); - let fetched_data = fetched.unwrap().data.unwrap(); - assert_eq!(fetched_data.get("key1").unwrap(), "value1"); - assert_eq!(fetched_data.get("key2").unwrap(), "value2"); - - // Cleanup - let api: Api = Api::namespaced(client.client.clone(), test_ns); - let _ = api.delete(&cm_name, &DeleteParams::default()).await; - } - - /// Example integration test for apply_dynamic() with new resource. - #[tokio::test] - #[ignore = "requires kubernetes cluster"] - async fn apply_dynamic_creates_new_resource() { - let client = K8sClient::try_default() - .await - .expect("failed to create client"); - let test_ns = "default"; - let cm_name = format!( - "test-dyn-cm-{}", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() - ); - - let mut data = BTreeMap::new(); - data.insert("foo".to_string(), serde_json::json!("bar")); - - let dynamic_obj = DynamicObject { - types: Some(TypeMeta { - api_version: "v1".to_string(), - kind: "ConfigMap".to_string(), - }), - metadata: ObjectMeta { - name: Some(cm_name.clone()), - namespace: Some(test_ns.to_string()), - ..Default::default() - }, - data: serde_json::json!(data), - }; - - let result = client - .apply_dynamic(&dynamic_obj, Some(test_ns), false) - .await; - assert!( - result.is_ok(), - "failed to apply dynamic object: {:?}", - result.err() - ); - - // Verify it exists - let api: Api = Api::namespaced(client.client.clone(), test_ns); - let fetched = api.get_opt(&cm_name).await.unwrap(); - assert!(fetched.is_some(), "Dynamic resource was not created"); - - // Cleanup - let _ = api.delete(&cm_name, &DeleteParams::default()).await; - } - - /// Example showing idempotency: applying same resource twice should succeed. - #[tokio::test] - #[ignore = "requires kubernetes cluster"] - async fn apply_is_idempotent() { - let client = K8sClient::try_default() - .await - .expect("failed to create client"); - let test_ns = "default"; - let cm_name = format!( - "test-idem-{}", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis() - ); - - let configmap = ConfigMap { - metadata: ObjectMeta { - name: Some(cm_name.clone()), - namespace: Some(test_ns.to_string()), - ..Default::default() - }, - data: Some(BTreeMap::from([("key".to_string(), "value".to_string())])), - ..Default::default() - }; - - // Apply twice - let result1 = client.apply(&configmap, Some(test_ns)).await; - let result2 = client.apply(&configmap, Some(test_ns)).await; - - assert!(result1.is_ok(), "first apply failed"); - assert!(result2.is_ok(), "second apply failed (not idempotent)"); - - // Cleanup - let api: Api = Api::namespaced(client.client.clone(), test_ns); - let _ = api.delete(&cm_name, &DeleteParams::default()).await; - } -} diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 55091d2..0f93a18 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -2,6 +2,7 @@ use std::{collections::BTreeMap, process::Command, sync::Arc, time::Duration}; use async_trait::async_trait; use base64::{Engine, engine::general_purpose}; +use harmony_k8s::{K8sClient, KubernetesDistribution}; use harmony_types::rfc1123::Rfc1123Name; use k8s_openapi::api::{ core::v1::{Pod, Secret}, @@ -58,7 +59,6 @@ use crate::{ use super::super::{ DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, PreparationError, PreparationOutcome, Topology, - k8s::K8sClient, oberservability::monitoring::AlertReceiver, tenant::{ TenantConfig, TenantManager, @@ -76,13 +76,6 @@ struct K8sState { message: String, } -#[derive(Debug, Clone, Serialize)] -pub enum KubernetesDistribution { - OpenshiftFamily, - K3sFamily, - Default, -} - #[derive(Debug, Clone)] enum K8sSource { LocalK3d, diff --git a/harmony/src/domain/topology/mod.rs b/harmony/src/domain/topology/mod.rs index 28161fc..c661e8a 100644 --- a/harmony/src/domain/topology/mod.rs +++ b/harmony/src/domain/topology/mod.rs @@ -16,7 +16,6 @@ pub mod tenant; use derive_new::new; pub use k8s_anywhere::*; pub use localhost::*; -pub mod k8s; mod load_balancer; pub mod router; mod tftp; diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index 1ec3cf3..8e55484 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -9,6 +9,7 @@ use std::{ use async_trait::async_trait; use brocade::PortOperatingMode; use derive_new::new; +use harmony_k8s::K8sClient; use harmony_types::{ id::Id, net::{IpAddress, MacAddress}, @@ -18,7 +19,7 @@ use serde::Serialize; use crate::executors::ExecutorError; -use super::{LogicalHost, k8s::K8sClient}; +use super::{LogicalHost}; #[derive(Debug)] pub struct DHCPStaticEntry { diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index d7d99c0..b4c8b17 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -1,10 +1,8 @@ use std::sync::Arc; -use crate::{ - executors::ExecutorError, - topology::k8s::{ApplyStrategy, K8sClient}, -}; +use crate::executors::ExecutorError; use async_trait::async_trait; +use harmony_k8s::K8sClient; use k8s_openapi::{ api::{ core::v1::{LimitRange, Namespace, ResourceQuota}, @@ -14,7 +12,7 @@ use k8s_openapi::{ }, apimachinery::pkg::util::intstr::IntOrString, }; -use kube::{Resource, api::DynamicObject}; +use kube::Resource; use log::debug; use serde::de::DeserializeOwned; use serde_json::json; @@ -59,7 +57,6 @@ impl K8sTenantManager { ) -> Result where ::DynamicType: Default, - ::Scope: ApplyStrategy, { self.apply_labels(&mut resource, config); self.k8s_client diff --git a/harmony/src/infra/network_manager.rs b/harmony/src/infra/network_manager.rs index 6b7a342..338f1ee 100644 --- a/harmony/src/infra/network_manager.rs +++ b/harmony/src/infra/network_manager.rs @@ -5,6 +5,7 @@ use std::{ use askama::Template; use async_trait::async_trait; +use harmony_k8s::{DrainOptions, K8sClient, NodeFile}; use harmony_types::id::Id; use k8s_openapi::api::core::v1::Node; use kube::{ @@ -17,7 +18,6 @@ use crate::{ modules::okd::crd::nmstate, topology::{ HostNetworkConfig, NetworkError, NetworkManager, - k8s::{DrainOptions, K8sClient, NodeFile}, }, }; diff --git a/harmony/src/modules/application/features/helm_argocd_score.rs b/harmony/src/modules/application/features/helm_argocd_score.rs index 4a65a1e..6f02a22 100644 --- a/harmony/src/modules/application/features/helm_argocd_score.rs +++ b/harmony/src/modules/application/features/helm_argocd_score.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use harmony_k8s::K8sClient; use harmony_macros::hurl; use log::{debug, info, trace, warn}; use non_blank_string_rs::NonBlankString; @@ -14,7 +15,7 @@ use crate::{ helm::chart::{HelmChartScore, HelmRepository}, }, score::Score, - topology::{HelmCommand, K8sclient, Topology, ingress::Ingress, k8s::K8sClient}, + topology::{HelmCommand, K8sclient, Topology, ingress::Ingress}, }; use harmony_types::id::Id; diff --git a/harmony/src/modules/argocd/mod.rs b/harmony/src/modules/argocd/mod.rs index 402c90f..4f4329d 100644 --- a/harmony/src/modules/argocd/mod.rs +++ b/harmony/src/modules/argocd/mod.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use harmony_k8s::K8sClient; use log::{debug, info}; -use crate::{interpret::InterpretError, topology::k8s::K8sClient}; +use crate::interpret::InterpretError; #[derive(Clone, Debug, PartialEq, Eq)] pub enum ArgoScope { diff --git a/harmony/src/modules/cert_manager/cluster_issuer.rs b/harmony/src/modules/cert_manager/cluster_issuer.rs index 70294fe..38f9e6b 100644 --- a/harmony/src/modules/cert_manager/cluster_issuer.rs +++ b/harmony/src/modules/cert_manager/cluster_issuer.rs @@ -1,3 +1,4 @@ +use harmony_k8s::K8sClient; use std::sync::Arc; use async_trait::async_trait; @@ -11,7 +12,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{K8sclient, Topology, k8s::K8sClient}, + topology::{K8sclient, Topology}, }; #[derive(Clone, Debug, Serialize)] diff --git a/harmony/src/modules/k8s/failover.rs b/harmony/src/modules/k8s/failover.rs index 939d9ab..b660e07 100644 --- a/harmony/src/modules/k8s/failover.rs +++ b/harmony/src/modules/k8s/failover.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use async_trait::async_trait; use log::warn; -use crate::topology::{FailoverTopology, K8sclient, k8s::K8sClient}; +use crate::topology::{FailoverTopology, K8sclient}; +use harmony_k8s::K8sClient; #[async_trait] impl K8sclient for FailoverTopology { diff --git a/harmony/src/modules/k8s/resource.rs b/harmony/src/modules/k8s/resource.rs index 2e22f60..bff8183 100644 --- a/harmony/src/modules/k8s/resource.rs +++ b/harmony/src/modules/k8s/resource.rs @@ -109,7 +109,7 @@ where topology .k8s_client() .await - .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}"))) + .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}")))? .apply_many(&self.score.resource, self.score.namespace.as_deref()) .await?; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 88ec745..a547dce 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -6,7 +6,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::{ - interpret::{InterpretError, Outcome}, + interpret::InterpretError, inventory::Inventory, modules::{ monitoring::{ @@ -17,10 +17,10 @@ use crate::{ topology::{ K8sclient, Topology, installable::Installable, - k8s::K8sClient, oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget}, }, }; +use harmony_k8s::K8sClient; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs index a53b24e..4e6e68e 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs @@ -4,10 +4,8 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::topology::{ - k8s::K8sClient, - oberservability::monitoring::{AlertReceiver, AlertSender}, -}; +use crate::topology::oberservability::monitoring::{AlertReceiver, AlertSender}; +use harmony_k8s::K8sClient; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/ntfy/ntfy.rs b/harmony/src/modules/monitoring/ntfy/ntfy.rs index f82aaf7..9b796d9 100644 --- a/harmony/src/modules/monitoring/ntfy/ntfy.rs +++ b/harmony/src/modules/monitoring/ntfy/ntfy.rs @@ -11,8 +11,9 @@ use crate::{ inventory::Inventory, modules::monitoring::ntfy::helm::ntfy_helm_chart::ntfy_helm_chart_score, score::Score, - topology::{HelmCommand, K8sclient, MultiTargetTopology, Topology, k8s::K8sClient}, + topology::{HelmCommand, K8sclient, MultiTargetTopology, Topology}, }; +use harmony_k8s::K8sClient; use harmony_types::id::Id; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/okd/config.rs b/harmony/src/modules/monitoring/okd/config.rs index b86c5f0..4e53eb8 100644 --- a/harmony/src/modules/monitoring/okd/config.rs +++ b/harmony/src/modules/monitoring/okd/config.rs @@ -1,9 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; -use crate::{ - interpret::{InterpretError, Outcome}, - topology::k8s::K8sClient, -}; +use crate::interpret::{InterpretError, Outcome}; +use harmony_k8s::K8sClient; use k8s_openapi::api::core::v1::ConfigMap; use kube::api::ObjectMeta; diff --git a/harmony/src/modules/nats/score_nats_k8s.rs b/harmony/src/modules/nats/score_nats_k8s.rs index cad35e3..4aac85f 100644 --- a/harmony/src/modules/nats/score_nats_k8s.rs +++ b/harmony/src/modules/nats/score_nats_k8s.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, str::FromStr}; use async_trait::async_trait; +use harmony_k8s::KubernetesDistribution; use harmony_macros::hurl; use harmony_secret::{Secret, SecretManager}; use harmony_types::id::Id; @@ -25,7 +26,7 @@ use crate::{ }, }, score::Score, - topology::{HelmCommand, K8sclient, KubernetesDistribution, TlsRouter, Topology}, + topology::{HelmCommand, K8sclient, TlsRouter, Topology}, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/postgresql/operator.rs b/harmony/src/modules/postgresql/operator.rs index 6298ddb..53d7d0a 100644 --- a/harmony/src/modules/postgresql/operator.rs +++ b/harmony/src/modules/postgresql/operator.rs @@ -53,7 +53,7 @@ pub struct CloudNativePgOperatorScore { } impl CloudNativePgOperatorScore { - fn default_openshift() -> Self { + pub fn default_openshift() -> Self { Self { namespace: "openshift-operators".to_string(), channel: "stable-v1".to_string(), diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index 7093ee8..586029b 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -12,8 +12,7 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::C use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules; use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{ Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, - GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec, - GrafanaValueFrom, GrafanaValueSource, + GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSpec, }; use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{ PrometheusRule, PrometheusRuleSpec, RuleGroup, @@ -23,7 +22,7 @@ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, }; use crate::topology::oberservability::monitoring::AlertReceiver; -use crate::topology::{K8sclient, Topology, k8s::K8sClient}; +use crate::topology::{K8sclient, Topology}; use crate::{ data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, @@ -38,6 +37,7 @@ use crate::{ }, score::Score, }; +use harmony_k8s::K8sClient; use harmony_types::id::Id; use super::prometheus::PrometheusMonitoring; diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index 644e6f9..8a85d1b 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -30,12 +30,13 @@ use crate::modules::monitoring::kube_prometheus::crd::rhob_service_monitor::{ use crate::score::Score; use crate::topology::ingress::Ingress; use crate::topology::oberservability::monitoring::AlertReceiver; -use crate::topology::{K8sclient, Topology, k8s::K8sClient}; +use crate::topology::{K8sclient, Topology}; use crate::{ data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, }; +use harmony_k8s::K8sClient; use harmony_types::id::Id; use super::prometheus::PrometheusMonitoring; diff --git a/harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs b/harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs index 787f9cc..eca87b0 100644 --- a/harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs +++ b/harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs @@ -4,6 +4,7 @@ use std::{ }; use async_trait::async_trait; +use harmony_k8s::K8sClient; use log::{debug, warn}; use serde::{Deserialize, Serialize}; use tokio::time::sleep; @@ -13,7 +14,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{K8sclient, Topology, k8s::K8sClient}, + topology::{K8sclient, Topology}, }; use harmony_types::id::Id; diff --git a/harmony/src/modules/storage/ceph/ceph_validate_health_score.rs b/harmony/src/modules/storage/ceph/ceph_validate_health_score.rs index ee331bc..067d9f9 100644 --- a/harmony/src/modules/storage/ceph/ceph_validate_health_score.rs +++ b/harmony/src/modules/storage/ceph/ceph_validate_health_score.rs @@ -9,8 +9,9 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{K8sclient, Topology, k8s::K8sClient}, + topology::{K8sclient, Topology}, }; +use harmony_k8s::K8sClient; use harmony_types::id::Id; #[derive(Clone, Debug, Serialize)] diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index f4ebcc0..d121d39 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -1,9 +1,9 @@ use base64::{Engine, prelude::BASE64_STANDARD}; -use rand::{thread_rng, Rng}; -use rand::distributions::Alphanumeric; use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::{ByteString, api::core::v1::Secret}; +use rand::distr::Alphanumeric; +use rand::{rng, Rng}; use std::collections::BTreeMap; use std::str::FromStr; @@ -165,9 +165,7 @@ impl Interpret for Zitade let db_port = endpoint.port; let host = &self.host; - debug!( - "[Zitadel] DB credentials source — secret: '{pg_user_secret}', key: 'password'" - ); + debug!("[Zitadel] DB credentials source — secret: '{pg_user_secret}', key: 'password'"); debug!( "[Zitadel] DB credentials source — superuser secret: '{pg_superuser_secret}', key: 'password'" ); @@ -179,9 +177,8 @@ impl Interpret for Zitade MASTERKEY_SECRET_NAME, NAMESPACE ); - // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes. - let masterkey: String = thread_rng() + let masterkey: String = rng() .sample_iter(&Alphanumeric) .take(32) .map(char::from) @@ -204,8 +201,8 @@ impl Interpret for Zitade topology .k8s_client() .await - .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}"))) - .create(masterkey_secret) + .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}")))? + .create(&masterkey_secret, Some(NAMESPACE)) .await?; K8sResourceScore::single(masterkey_secret, Some(NAMESPACE.to_string())) -- 2.39.5 From f5aac67af896b5885ebcf32faa2ca424fb34c5fb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Mar 2026 15:15:35 -0500 Subject: [PATCH 3/6] feat: k8s client works fine, added version config in zitadel and fix master key secret existence handling --- brocade/src/fast_iron.rs | 2 +- brocade/src/network_operating_system.rs | 2 +- examples/cert_manager/src/main.rs | 4 +- examples/openbao/src/main.rs | 2 +- examples/operatorhub_catalog/src/main.rs | 2 - examples/opnsense_node_exporter/src/main.rs | 15 ++---- examples/public_postgres/src/main.rs | 3 +- .../rhob_application_monitoring/src/main.rs | 2 +- examples/rust/src/main.rs | 2 +- examples/zitadel/src/main.rs | 1 + harmony/src/domain/interpret/mod.rs | 2 - .../src/modules/application/backend_app.rs | 2 +- harmony/src/modules/postgresql/cnpg/crd.rs | 1 + harmony/src/modules/zitadel/mod.rs | 53 ++++++++++++------- 14 files changed, 47 insertions(+), 46 deletions(-) diff --git a/brocade/src/fast_iron.rs b/brocade/src/fast_iron.rs index cd425dc..d9d4588 100644 --- a/brocade/src/fast_iron.rs +++ b/brocade/src/fast_iron.rs @@ -1,7 +1,7 @@ use super::BrocadeClient; use crate::{ BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, MacAddressEntry, - PortChannelId, PortOperatingMode, SecurityLevel, parse_brocade_mac_address, + PortChannelId, PortOperatingMode, parse_brocade_mac_address, shell::BrocadeShell, }; diff --git a/brocade/src/network_operating_system.rs b/brocade/src/network_operating_system.rs index 994dbee..fa99bf6 100644 --- a/brocade/src/network_operating_system.rs +++ b/brocade/src/network_operating_system.rs @@ -8,7 +8,7 @@ use regex::Regex; use crate::{ BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, InterfaceStatus, InterfaceType, MacAddressEntry, PortChannelId, PortOperatingMode, - SecurityLevel, parse_brocade_mac_address, shell::BrocadeShell, + parse_brocade_mac_address, shell::BrocadeShell, }; #[derive(Debug)] diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index e024725..1aa2655 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -1,8 +1,8 @@ use harmony::{ inventory::Inventory, modules::cert_manager::{ - capability::CertificateManagementConfig, score_cert_management::CertificateManagementScore, - score_certificate::CertificateScore, score_issuer::CertificateIssuerScore, + capability::CertificateManagementConfig, score_certificate::CertificateScore, + score_issuer::CertificateIssuerScore, }, topology::K8sAnywhereTopology, }; diff --git a/examples/openbao/src/main.rs b/examples/openbao/src/main.rs index 1d5653b..200d0f7 100644 --- a/examples/openbao/src/main.rs +++ b/examples/openbao/src/main.rs @@ -5,7 +5,7 @@ use harmony::{ #[tokio::main] async fn main() { let openbao = OpenbaoScore { - host: String::new(), + host: "openbao.sebastien.sto1.nationtech.io".to_string(), }; harmony_cli::run( diff --git a/examples/operatorhub_catalog/src/main.rs b/examples/operatorhub_catalog/src/main.rs index 09ef182..e4e3ac6 100644 --- a/examples/operatorhub_catalog/src/main.rs +++ b/examples/operatorhub_catalog/src/main.rs @@ -1,5 +1,3 @@ -use std::str::FromStr; - use harmony::{ inventory::Inventory, modules::{k8s::apps::OperatorHubCatalogSourceScore, postgresql::CloudNativePgOperatorScore}, diff --git a/examples/opnsense_node_exporter/src/main.rs b/examples/opnsense_node_exporter/src/main.rs index d71d2ed..75480e8 100644 --- a/examples/opnsense_node_exporter/src/main.rs +++ b/examples/opnsense_node_exporter/src/main.rs @@ -1,22 +1,13 @@ -use std::{ - net::{IpAddr, Ipv4Addr}, - sync::Arc, -}; +use std::sync::Arc; use async_trait::async_trait; -use cidr::Ipv4Cidr; use harmony::{ executors::ExecutorError, - hardware::{HostCategory, Location, PhysicalHost, SwitchGroup}, - infra::opnsense::OPNSenseManagementInterface, inventory::Inventory, modules::opnsense::node_exporter::NodeExporterScore, - topology::{ - HAClusterTopology, LogicalHost, PreparationError, PreparationOutcome, Topology, - UnmanagedRouter, node_exporter::NodeExporter, - }, + topology::{PreparationError, PreparationOutcome, Topology, node_exporter::NodeExporter}, }; -use harmony_macros::{ip, ipv4, mac_address}; +use harmony_macros::ip; #[derive(Debug)] struct OpnSenseTopology { diff --git a/examples/public_postgres/src/main.rs b/examples/public_postgres/src/main.rs index 029080e..772e801 100644 --- a/examples/public_postgres/src/main.rs +++ b/examples/public_postgres/src/main.rs @@ -1,8 +1,7 @@ use harmony::{ inventory::Inventory, modules::postgresql::{ - K8sPostgreSQLScore, PostgreSQLConnectionScore, PublicPostgreSQLScore, - capability::PostgreSQLConfig, + PostgreSQLConnectionScore, PublicPostgreSQLScore, capability::PostgreSQLConfig, }, topology::K8sAnywhereTopology, }; diff --git a/examples/rhob_application_monitoring/src/main.rs b/examples/rhob_application_monitoring/src/main.rs index 6eeaea2..a2596fa 100644 --- a/examples/rhob_application_monitoring/src/main.rs +++ b/examples/rhob_application_monitoring/src/main.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{path::PathBuf, sync::Arc}; use harmony::{ inventory::Inventory, diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs index a100606..f100b41 100644 --- a/examples/rust/src/main.rs +++ b/examples/rust/src/main.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{path::PathBuf, sync::Arc}; use harmony::{ inventory::Inventory, diff --git a/examples/zitadel/src/main.rs b/examples/zitadel/src/main.rs index 78bef1d..e2d24ba 100644 --- a/examples/zitadel/src/main.rs +++ b/examples/zitadel/src/main.rs @@ -6,6 +6,7 @@ use harmony::{ async fn main() { let zitadel = ZitadelScore { host: "sso.sto1.nationtech.io".to_string(), + zitadel_version: "v4.12.1".to_string(), }; harmony_cli::run( diff --git a/harmony/src/domain/interpret/mod.rs b/harmony/src/domain/interpret/mod.rs index c5e92ce..4a9855a 100644 --- a/harmony/src/domain/interpret/mod.rs +++ b/harmony/src/domain/interpret/mod.rs @@ -4,8 +4,6 @@ use std::error::Error; use async_trait::async_trait; use derive_new::new; -use crate::inventory::HostRole; - use super::{ data::Version, executors::ExecutorError, inventory::Inventory, topology::PreparationError, }; diff --git a/harmony/src/modules/application/backend_app.rs b/harmony/src/modules/application/backend_app.rs index d11feaa..4183e30 100644 --- a/harmony/src/modules/application/backend_app.rs +++ b/harmony/src/modules/application/backend_app.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use log::{debug, info, trace}; +use log::{debug, info}; use serde::Serialize; use std::path::PathBuf; diff --git a/harmony/src/modules/postgresql/cnpg/crd.rs b/harmony/src/modules/postgresql/cnpg/crd.rs index 76bbeae..5b54fd0 100644 --- a/harmony/src/modules/postgresql/cnpg/crd.rs +++ b/harmony/src/modules/postgresql/cnpg/crd.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; #[serde(rename_all = "camelCase")] pub struct ClusterSpec { pub instances: u32, + #[serde(skip_serializing_if = "Option::is_none")] pub image_name: Option, pub storage: Storage, pub bootstrap: Bootstrap, diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index d121d39..3a9e829 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -2,8 +2,9 @@ use base64::{Engine, prelude::BASE64_STANDARD}; use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::{ByteString, api::core::v1::Secret}; +use kube::{Error as KubeError, core::ErrorResponse}; use rand::distr::Alphanumeric; -use rand::{rng, Rng}; +use rand::{Rng, rng}; use std::collections::BTreeMap; use std::str::FromStr; @@ -57,6 +58,7 @@ const MASTERKEY_SECRET_NAME: &str = "zitadel-masterkey"; pub struct ZitadelScore { /// External domain (e.g. `"auth.example.com"`). pub host: String, + pub zitadel_version: String, } impl Score for ZitadelScore { @@ -68,6 +70,7 @@ impl Score for ZitadelSco fn create_interpret(&self) -> Box> { Box::new(ZitadelInterpret { host: self.host.clone(), + zitadel_version: self.zitadel_version.clone(), }) } } @@ -77,6 +80,7 @@ impl Score for ZitadelSco #[derive(Debug, Clone)] struct ZitadelInterpret { host: String, + zitadel_version: String, } #[async_trait] @@ -183,10 +187,8 @@ impl Interpret for Zitade .take(32) .map(char::from) .collect(); - let masterkey_bytes = BASE64_STANDARD.encode(&masterkey); - let mut masterkey_data: BTreeMap = BTreeMap::new(); - masterkey_data.insert("masterkey".to_string(), ByteString(masterkey_bytes.into())); + masterkey_data.insert("masterkey".to_string(), ByteString(masterkey.into())); let masterkey_secret = Secret { metadata: ObjectMeta { @@ -198,26 +200,34 @@ impl Interpret for Zitade ..Secret::default() }; - topology + match topology .k8s_client() .await .map_err(|e| InterpretError::new(format!("Failed to get k8s client : {e}")))? .create(&masterkey_secret, Some(NAMESPACE)) - .await?; - - K8sResourceScore::single(masterkey_secret, Some(NAMESPACE.to_string())) - .interpret(inventory, topology) .await - .map_err(|e| { - let msg = format!("[Zitadel] Failed to create masterkey secret: {e}"); + { + Ok(_) => { + info!( + "[Zitadel] Masterkey secret '{}' created", + MASTERKEY_SECRET_NAME + ); + } + Err(KubeError::Api(ErrorResponse { code: 409, .. })) => { + info!( + "[Zitadel] Masterkey secret '{}' already exists, leaving it untouched", + MASTERKEY_SECRET_NAME + ); + } + Err(other) => { + let msg = format!( + "[Zitadel] Failed to create masterkey secret '{}': {other}", + MASTERKEY_SECRET_NAME + ); error!("{msg}"); - InterpretError::new(msg) - })?; - - info!( - "[Zitadel] Masterkey secret '{}' created", - MASTERKEY_SECRET_NAME - ); + return Err(InterpretError::new(msg)); + } + }; // --- Step 4: Build Helm values ------------------------------------ @@ -230,7 +240,9 @@ impl Interpret for Zitade ); let values_yaml = format!( - r#"zitadel: + r#"image: + tag: {zitadel_version} +zitadel: masterkeySecretName: "{MASTERKEY_SECRET_NAME}" configmapConfig: ExternalDomain: "{host}" @@ -361,7 +373,8 @@ login: - host: "{host}" paths: - path: /ui/v2/login - pathType: Prefix"# + pathType: Prefix"#, + zitadel_version = self.zitadel_version ); trace!("[Zitadel] Helm values YAML:\n{values_yaml}"); -- 2.39.5 From a98113dd4056a75999fba85a2aa54c69116bb5ad Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 6 Mar 2026 15:28:21 -0500 Subject: [PATCH 4/6] wip: zitadel ingress https not working yet --- harmony/src/modules/zitadel/mod.rs | 47 +++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 3a9e829..85b96c4 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -39,12 +39,17 @@ const MASTERKEY_SECRET_NAME: &str = "zitadel-masterkey"; /// Photos, and others. /// /// # Ingress annotations -/// No controller-specific ingress annotations are set. The Zitadel service -/// already carries the Traefik h2c annotation for k3s/k3d by default. -/// Add annotations via `values_overrides` depending on your distribution: +/// No controller-specific ingress annotations are set by default. On +/// OKD/OpenShift, the ingress should request TLS so the generated Route is +/// edge-terminated instead of HTTP-only. Optional cert-manager annotations are +/// included for clusters that have cert-manager installed; clusters without +/// cert-manager will ignore them. +/// Add or adjust annotations via `values_overrides` depending on your +/// distribution: /// - NGINX: `nginx.ingress.kubernetes.io/backend-protocol: GRPC` -/// - OpenShift HAProxy: `haproxy.router.openshift.io/*` or use OpenShift Routes +/// - OpenShift HAProxy: `route.openshift.io/termination: edge` /// - AWS ALB: set `ingress.controller: aws` + /// /// # Database credentials /// CNPG creates a `-superuser` secret with key `password`. Because @@ -232,11 +237,9 @@ impl Interpret for Zitade // --- Step 4: Build Helm values ------------------------------------ warn!( - "[Zitadel] No ingress controller annotations are set. \ - Add controller-specific annotations for your distribution: \ - NGINX → 'nginx.ingress.kubernetes.io/backend-protocol: GRPC'; \ - OpenShift HAProxy → 'haproxy.router.openshift.io/*' or use Routes; \ - AWS ALB → set ingress.controller=aws." + "[Zitadel] Applying TLS-enabled ingress defaults for OKD/OpenShift. \ + cert-manager annotations are included as optional hints and are \ + ignored on clusters without cert-manager." ); let values_yaml = format!( @@ -248,7 +251,7 @@ zitadel: ExternalDomain: "{host}" ExternalSecure: true TLS: - Enabled: false + Enabled: true Database: Postgres: Host: "{db_host}" @@ -342,12 +345,21 @@ setupJob: type: RuntimeDefault ingress: enabled: true - annotations: {{}} + tls: + - secretName: zitadel-tls + hosts: + - "{host}" + annotations: + route.openshift.io/termination: edge + route.openshift.io/insecureEdgeTerminationPolicy: Redirect + cert-manager.io/cluster-issuer: selfsigned-cluster-issuer + kubernetes.io/tls-acme: "true" hosts: - host: "{host}" - paths: + paths:: - path: / pathType: Prefix + login: enabled: true podSecurityContext: @@ -368,12 +380,21 @@ login: type: RuntimeDefault ingress: enabled: true - annotations: {{}} + tls: + - secretName: zitadel-login-tls + hosts: + - "{host}" + annotations: + route.openshift.io/termination: edge + route.openshift.io/insecureEdgeTerminationPolicy: Redirect + cert-manager.io/cluster-issuer: selfsigned-cluster-issuer + kubernetes.io/tls-acme: "true" hosts: - host: "{host}" paths: - path: /ui/v2/login pathType: Prefix"#, + zitadel_version = self.zitadel_version ); -- 2.39.5 From ce041f495b93ff6cc50df35385ab49f33a6f9b2d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 7 Mar 2026 15:29:26 -0500 Subject: [PATCH 5/6] fix(zitadel): include admin@zitadel.{host} username, secure password with symbol/number, and cert-manager TLS configuration Update Zitadel deployment to use correct username format (admin@zitadel.{host}), generate secure passwords with required complexity (uppercase, lowercase, digit, symbol), configure edge TLS termination for OpenShift, and add cert-manager annotations. Also refactor password generation to ensure all complexity requirements are met. --- harmony/src/modules/zitadel/mod.rs | 120 ++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 26 deletions(-) diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 85b96c4..aebacf8 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -1,10 +1,9 @@ -use base64::{Engine, prelude::BASE64_STANDARD}; use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::{ByteString, api::core::v1::Secret}; use kube::{Error as KubeError, core::ErrorResponse}; -use rand::distr::Alphanumeric; -use rand::{Rng, rng}; +use rand::distr::Distribution; +use rand::{Rng, rng, seq::SliceRandom}; use std::collections::BTreeMap; use std::str::FromStr; @@ -179,6 +178,50 @@ impl Interpret for Zitade "[Zitadel] DB credentials source — superuser secret: '{pg_superuser_secret}', key: 'password'" ); + // Zitadel requires one symbol, one number and more. So let's force it. + fn generate_secure_password(length: usize) -> String { + const ALPHA_UPPER: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const ALPHA_LOWER: &[u8] = b"abcdefghijklmnopqrstuvwxyz"; + const DIGITS: &[u8] = b"0123456789"; + const SYMBOLS: &[u8] = b"!@#$%^&*()_+-=[]{}|;:',.<>?/"; + + let mut rng = rand::rng(); + let uniform_alpha_upper = rand::distr::Uniform::new(0, ALPHA_UPPER.len()) + .expect("Failed to create distribution"); + let uniform_alpha_lower = rand::distr::Uniform::new(0, ALPHA_LOWER.len()) + .expect("Failed to create distribution"); + let uniform_digits = + rand::distr::Uniform::new(0, DIGITS.len()).expect("Failed to create distribution"); + let uniform_symbols = + rand::distr::Uniform::new(0, SYMBOLS.len()).expect("Failed to create distribution"); + + let mut chars: Vec = Vec::with_capacity(length); + + // Ensure at least one of each: upper, lower, digit, symbol + chars.push(ALPHA_UPPER[uniform_alpha_upper.sample(&mut rng)] as char); + chars.push(ALPHA_LOWER[uniform_alpha_lower.sample(&mut rng)] as char); + chars.push(DIGITS[uniform_digits.sample(&mut rng)] as char); + chars.push(SYMBOLS[uniform_symbols.sample(&mut rng)] as char); + + // Fill remaining with random from all categories + let all_chars: Vec = [ALPHA_UPPER, ALPHA_LOWER, DIGITS, SYMBOLS].concat(); + + let uniform_all = rand::distr::Uniform::new(0, all_chars.len()) + .expect("Failed to create distribution"); + + for _ in 0..(length - 4) { + chars.push(all_chars[uniform_all.sample(&mut rng)] as char); + } + + // Shuffle + let mut shuffled = chars; + shuffled.shuffle(&mut rng); + + return shuffled.iter().collect(); + } + + let admin_password = generate_secure_password(16); + // --- Step 3: Create masterkey secret ------------------------------------ debug!( @@ -186,12 +229,18 @@ impl Interpret for Zitade MASTERKEY_SECRET_NAME, NAMESPACE ); - // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes. - let masterkey: String = rng() - .sample_iter(&Alphanumeric) + // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes (alphanumeric only). + let masterkey = rng() + .sample_iter(&rand::distr::Alphanumeric) .take(32) .map(char::from) - .collect(); + .collect::(); + + debug!( + "[Zitadel] Created masterkey secret '{}' in namespace '{}'", + MASTERKEY_SECRET_NAME, NAMESPACE + ); + let mut masterkey_data: BTreeMap = BTreeMap::new(); masterkey_data.insert("masterkey".to_string(), ByteString(masterkey.into())); @@ -234,6 +283,11 @@ impl Interpret for Zitade } }; + debug!( + "[Zitadel] Masterkey secret '{}' created successfully", + MASTERKEY_SECRET_NAME + ); + // --- Step 4: Build Helm values ------------------------------------ warn!( @@ -250,8 +304,17 @@ zitadel: configmapConfig: ExternalDomain: "{host}" ExternalSecure: true + FirstInstance: + Org: + Human: + UserName: "admin" + Password: "{admin_password}" + FirstName: "Zitadel" + LastName: "Admin" + Email: "admin@zitadel.example.com" + PasswordChangeRequired: true TLS: - Enabled: true + Enabled: false Database: Postgres: Host: "{db_host}" @@ -345,20 +408,18 @@ setupJob: type: RuntimeDefault ingress: enabled: true - tls: - - secretName: zitadel-tls - hosts: - - "{host}" annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod route.openshift.io/termination: edge - route.openshift.io/insecureEdgeTerminationPolicy: Redirect - cert-manager.io/cluster-issuer: selfsigned-cluster-issuer - kubernetes.io/tls-acme: "true" hosts: - host: "{host}" - paths:: + paths: - path: / pathType: Prefix + tls: + - hosts: + - "{host}" + secretName: "{host}-tls" login: enabled: true @@ -380,21 +441,18 @@ login: type: RuntimeDefault ingress: enabled: true - tls: - - secretName: zitadel-login-tls - hosts: - - "{host}" annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod route.openshift.io/termination: edge - route.openshift.io/insecureEdgeTerminationPolicy: Redirect - cert-manager.io/cluster-issuer: selfsigned-cluster-issuer - kubernetes.io/tls-acme: "true" hosts: - host: "{host}" paths: - path: /ui/v2/login - pathType: Prefix"#, - + pathType: Prefix + tls: + - hosts: + - "{host}" + secretName: "{host}-tls""#, zitadel_version = self.zitadel_version ); @@ -425,7 +483,17 @@ login: .await; match &result { - Ok(_) => info!("[Zitadel] Helm chart deployed successfully"), + Ok(_) => info!( + "[Zitadel] Helm chart deployed successfully\n\n\ + ===== ZITADEL DEPLOYMENT COMPLETE =====\n\ + Login URL: https://{host}\n\ + Username: admin@zitadel.{host}\n\ + Password: {admin_password}\n\n\ + IMPORTANT: The password is saved in ConfigMap 'zitadel-config-yaml'\n\ + and must be changed on first login. Save the credentials in a\n\ + secure location after changing them.\n\ + =========================================" + ), Err(e) => error!("[Zitadel] Helm chart deployment failed: {e}"), } -- 2.39.5 From 787cc8feabc454b88f563d311ce8b0626d37d6a3 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 7 Mar 2026 15:43:30 -0500 Subject: [PATCH 6/6] Fix doc tests for harmony-k8s crate refactoring - Updated harmony-k8s doc tests to import from harmony_k8s instead of harmony - Changed CloudNativePgOperatorScore::default() to default_openshift() This ensures doc tests work correctly after moving K8sClient to the harmony-k8s crate. --- brocade/src/fast_iron.rs | 3 +- harmony-k8s/src/apply.rs | 145 +++++++++++++-------- harmony-k8s/src/bundle.rs | 6 +- harmony-k8s/src/client.rs | 22 ++-- harmony-k8s/src/discovery.rs | 6 +- harmony-k8s/src/helper.rs | 6 +- harmony-k8s/src/node.rs | 105 +++++++++++---- harmony-k8s/src/pod.rs | 22 ++-- harmony-k8s/src/resources.rs | 41 ++++-- harmony/src/domain/topology/network.rs | 2 +- harmony/src/infra/network_manager.rs | 4 +- harmony/src/modules/postgresql/operator.rs | 2 +- 12 files changed, 235 insertions(+), 129 deletions(-) diff --git a/brocade/src/fast_iron.rs b/brocade/src/fast_iron.rs index d9d4588..371c265 100644 --- a/brocade/src/fast_iron.rs +++ b/brocade/src/fast_iron.rs @@ -1,8 +1,7 @@ use super::BrocadeClient; use crate::{ BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, MacAddressEntry, - PortChannelId, PortOperatingMode, parse_brocade_mac_address, - shell::BrocadeShell, + PortChannelId, PortOperatingMode, parse_brocade_mac_address, shell::BrocadeShell, }; use async_trait::async_trait; diff --git a/harmony-k8s/src/apply.rs b/harmony-k8s/src/apply.rs index e624766..cf1e7bf 100644 --- a/harmony-k8s/src/apply.rs +++ b/harmony-k8s/src/apply.rs @@ -1,8 +1,8 @@ use kube::{ Client, Error, Resource, api::{ - Api, ApiResource, DynamicObject, GroupVersionKind, Patch, PatchParams, - PostParams, ResourceExt, + Api, ApiResource, DynamicObject, GroupVersionKind, Patch, PatchParams, PostParams, + ResourceExt, }, core::ErrorResponse, discovery::Scope, @@ -44,8 +44,7 @@ async fn show_dry_run( match api.get(name).await { Ok(current) => { println!("\nDry-run for resource: '{name}'"); - let mut current_val = - serde_yaml::to_value(¤t).unwrap_or(serde_yaml::Value::Null); + let mut current_val = serde_yaml::to_value(¤t).unwrap_or(serde_yaml::Value::Null); if let Some(map) = current_val.as_mapping_mut() { map.remove(&serde_yaml::Value::String("status".to_string())); } @@ -101,10 +100,12 @@ async fn do_apply( Err(Error::Api(ErrorResponse { code: 404, .. })) => { debug!("Resource '{name}' not found via SSA, falling back to POST"); let dyn_obj = to_dynamic(payload)?; - api.create(&PostParams::default(), &dyn_obj).await.map_err(|e| { - error!("Failed to create '{name}': {e}"); - e - }) + api.create(&PostParams::default(), &dyn_obj) + .await + .map_err(|e| { + error!("Failed to create '{name}': {e}"); + e + }) } Err(e) => { error!("Failed to apply '{name}': {e}"); @@ -114,28 +115,26 @@ async fn do_apply( } WriteMode::Create => { let dyn_obj = to_dynamic(payload)?; - api.create(&PostParams::default(), &dyn_obj).await.map_err(|e| { - error!("Failed to create '{name}': {e}"); - e - }) + api.create(&PostParams::default(), &dyn_obj) + .await + .map_err(|e| { + error!("Failed to create '{name}': {e}"); + e + }) } - WriteMode::Update => { - match api.patch(name, patch_params, &Patch::Apply(payload)).await { - Ok(obj) => Ok(obj), - Err(Error::Api(ErrorResponse { code: 404, .. })) => Err(Error::Api(ErrorResponse { - code: 404, - message: format!( - "Resource '{name}' not found and WriteMode is UpdateOnly" - ), - reason: "NotFound".to_string(), - status: "Failure".to_string(), - })), - Err(e) => { - error!("Failed to update '{name}': {e}"); - Err(e) - } + WriteMode::Update => match api.patch(name, patch_params, &Patch::Apply(payload)).await { + Ok(obj) => Ok(obj), + Err(Error::Api(ErrorResponse { code: 404, .. })) => Err(Error::Api(ErrorResponse { + code: 404, + message: format!("Resource '{name}' not found and WriteMode is UpdateOnly"), + reason: "NotFound".to_string(), + status: "Failure".to_string(), + })), + Err(e) => { + error!("Failed to update '{name}': {e}"); + Err(e) } - } + }, } } @@ -286,15 +285,11 @@ impl K8sClient { }; let api = get_dynamic_api(ar, caps, self.client.clone(), effective_ns, false); - let name = resource - .metadata - .name - .as_deref() - .ok_or_else(|| { - Error::BuildRequest(kube::core::request::Error::Validation( - "DynamicObject must have metadata.name".to_string(), - )) - })?; + let name = resource.metadata.name.as_deref().ok_or_else(|| { + Error::BuildRequest(kube::core::request::Error::Validation( + "DynamicObject must have metadata.name".to_string(), + )) + })?; debug!( "apply_dynamic kind={:?} name='{name}' ns={effective_ns:?}", @@ -311,7 +306,14 @@ impl K8sClient { let mut patch_params = PatchParams::apply(FIELD_MANAGER); patch_params.force = force_conflicts; - do_apply(&api, name, resource, &patch_params, &WriteMode::CreateOrUpdate).await + do_apply( + &api, + name, + resource, + &patch_params, + &WriteMode::CreateOrUpdate, + ) + .await } pub async fn apply_dynamic_many( @@ -394,8 +396,8 @@ impl K8sClient { serde_yaml::from_value(doc).expect("YAML document is not a valid object"); let namespace = obj.metadata.namespace.as_deref().or(ns); let type_meta = obj.types.as_ref().expect("Object is missing TypeMeta"); - let gvk = GroupVersionKind::try_from(type_meta) - .expect("Object has invalid GroupVersionKind"); + let gvk = + GroupVersionKind::try_from(type_meta).expect("Object has invalid GroupVersionKind"); let name = obj.name_any(); if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) { @@ -437,9 +439,9 @@ impl K8sClient { } }) .ok_or_else(|| { - Error::BuildRequest(kube::core::request::Error::Validation( - format!("Invalid apiVersion in DynamicObject: {object:#?}"), - )) + Error::BuildRequest(kube::core::request::Error::Validation(format!( + "Invalid apiVersion in DynamicObject: {object:#?}" + ))) })?; Ok(match ns { @@ -496,10 +498,20 @@ mod apply_tests { async fn apply_creates_new_configmap() { let client = K8sClient::try_default().await.unwrap(); let ns = "default"; - let name = format!("test-cm-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + let name = format!( + "test-cm-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + ); let cm = ConfigMap { - metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(ns.to_string()), + ..Default::default() + }, data: Some(BTreeMap::from([("key1".to_string(), "value1".to_string())])), ..Default::default() }; @@ -515,16 +527,32 @@ mod apply_tests { async fn apply_is_idempotent() { let client = K8sClient::try_default().await.unwrap(); let ns = "default"; - let name = format!("test-idem-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + let name = format!( + "test-idem-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + ); let cm = ConfigMap { - metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(ns.to_string()), + ..Default::default() + }, data: Some(BTreeMap::from([("key".to_string(), "value".to_string())])), ..Default::default() }; - assert!(client.apply(&cm, Some(ns)).await.is_ok(), "first apply failed"); - assert!(client.apply(&cm, Some(ns)).await.is_ok(), "second apply failed (not idempotent)"); + assert!( + client.apply(&cm, Some(ns)).await.is_ok(), + "first apply failed" + ); + assert!( + client.apply(&cm, Some(ns)).await.is_ok(), + "second apply failed (not idempotent)" + ); let api: Api = Api::namespaced(client.client.clone(), ns); let _ = api.delete(&name, &DeleteParams::default()).await; @@ -535,11 +563,24 @@ mod apply_tests { async fn apply_dynamic_creates_new_resource() { let client = K8sClient::try_default().await.unwrap(); let ns = "default"; - let name = format!("test-dyn-{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()); + let name = format!( + "test-dyn-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + ); let obj = DynamicObject { - types: Some(TypeMeta { api_version: "v1".to_string(), kind: "ConfigMap".to_string() }), - metadata: ObjectMeta { name: Some(name.clone()), namespace: Some(ns.to_string()), ..Default::default() }, + types: Some(TypeMeta { + api_version: "v1".to_string(), + kind: "ConfigMap".to_string(), + }), + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(ns.to_string()), + ..Default::default() + }, data: serde_json::json!({}), }; diff --git a/harmony-k8s/src/bundle.rs b/harmony-k8s/src/bundle.rs index ee77cc7..0a15341 100644 --- a/harmony-k8s/src/bundle.rs +++ b/harmony-k8s/src/bundle.rs @@ -25,9 +25,9 @@ //! //! ## Example //! -//! ```rust,no_run -//! use harmony::topology::k8s::{K8sClient, helper}; -//! use harmony::topology::KubernetesDistribution; +//! ``` +//! use harmony_k8s::{K8sClient, helper}; +//! use harmony_k8s::KubernetesDistribution; //! //! async fn write_network_config(client: &K8sClient, node: &str) { //! // Create a bundle with platform-specific RBAC diff --git a/harmony-k8s/src/client.rs b/harmony-k8s/src/client.rs index 1e828ac..9b22602 100644 --- a/harmony-k8s/src/client.rs +++ b/harmony-k8s/src/client.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use kube::{Client, Config, Discovery, Error}; use kube::config::{KubeConfigOptions, Kubeconfig}; +use kube::{Client, Config, Discovery, Error}; use log::error; use serde::Serialize; use tokio::sync::OnceCell; @@ -59,7 +59,10 @@ impl K8sClient { /// Create a client that always operates in dry-run mode, regardless of /// the environment variable. pub fn new_dry_run(client: Client) -> Self { - Self { dry_run: true, ..Self::new(client) } + Self { + dry_run: true, + ..Self::new(client) + } } /// Returns `true` if this client is operating in dry-run mode. @@ -75,19 +78,13 @@ impl K8sClient { Self::from_kubeconfig_with_opts(path, &KubeConfigOptions::default()).await } - pub async fn from_kubeconfig_with_context( - path: &str, - context: Option, - ) -> Option { + pub async fn from_kubeconfig_with_context(path: &str, context: Option) -> Option { let mut opts = KubeConfigOptions::default(); opts.context = context; Self::from_kubeconfig_with_opts(path, &opts).await } - pub async fn from_kubeconfig_with_opts( - path: &str, - opts: &KubeConfigOptions, - ) -> Option { + pub async fn from_kubeconfig_with_opts(path: &str, opts: &KubeConfigOptions) -> Option { let k = match Kubeconfig::read_from(path) { Ok(k) => k, Err(e) => { @@ -96,10 +93,7 @@ impl K8sClient { } }; Some(Self::new( - Client::try_from( - Config::from_custom_kubeconfig(k, opts).await.unwrap(), - ) - .unwrap(), + Client::try_from(Config::from_custom_kubeconfig(k, opts).await.unwrap()).unwrap(), )) } } diff --git a/harmony-k8s/src/discovery.rs b/harmony-k8s/src/discovery.rs index e65f85a..3acc487 100644 --- a/harmony-k8s/src/discovery.rs +++ b/harmony-k8s/src/discovery.rs @@ -60,7 +60,11 @@ impl K8sClient { let version = self.get_apiserver_version().await?; - if api_groups.groups.iter().any(|g| g.name == "project.openshift.io") { + if api_groups + .groups + .iter() + .any(|g| g.name == "project.openshift.io") + { info!("Detected distribution: OpenshiftFamily"); return Ok(KubernetesDistribution::OpenshiftFamily); } diff --git a/harmony-k8s/src/helper.rs b/harmony-k8s/src/helper.rs index 808ea12..9b64034 100644 --- a/harmony-k8s/src/helper.rs +++ b/harmony-k8s/src/helper.rs @@ -133,9 +133,9 @@ pub fn host_root_volume() -> (Volume, VolumeMount) { /// /// # Example /// -/// ```rust,no_run -/// # use harmony::topology::k8s::helper::{build_privileged_bundle, PrivilegedPodConfig}; -/// # use harmony::topology::KubernetesDistribution; +/// ``` +/// use harmony_k8s::helper::{build_privileged_bundle, PrivilegedPodConfig}; +/// use harmony_k8s::KubernetesDistribution; /// let bundle = build_privileged_bundle( /// PrivilegedPodConfig { /// name: "network-setup".to_string(), diff --git a/harmony-k8s/src/node.rs b/harmony-k8s/src/node.rs index c7a7c85..34f2af7 100644 --- a/harmony-k8s/src/node.rs +++ b/harmony-k8s/src/node.rs @@ -20,12 +20,16 @@ use crate::types::{DrainOptions, NodeFile}; impl K8sClient { pub async fn cordon_node(&self, node_name: &str) -> Result<(), Error> { - Api::::all(self.client.clone()).cordon(node_name).await?; + Api::::all(self.client.clone()) + .cordon(node_name) + .await?; Ok(()) } pub async fn uncordon_node(&self, node_name: &str) -> Result<(), Error> { - Api::::all(self.client.clone()).uncordon(node_name).await?; + Api::::all(self.client.clone()) + .uncordon(node_name) + .await?; Ok(()) } @@ -54,7 +58,11 @@ impl K8sClient { .status .as_ref() .and_then(|s| s.conditions.as_ref()) - .map(|conds| conds.iter().any(|c| c.type_ == "Ready" && c.status == "True")) + .map(|conds| { + conds + .iter() + .any(|c| c.type_ == "Ready" && c.status == "True") + }) .unwrap_or(false) { debug!("Node '{node_name}' is Ready"); @@ -87,7 +95,11 @@ impl K8sClient { .status .as_ref() .and_then(|s| s.conditions.as_ref()) - .map(|conds| conds.iter().any(|c| c.type_ == "Ready" && c.status == "True")) + .map(|conds| { + conds + .iter() + .any(|c| c.type_ == "Ready" && c.status == "True") + }) .unwrap_or(false); if !is_ready { debug!("Node '{node_name}' is NotReady"); @@ -200,11 +212,7 @@ impl K8sClient { } /// Drains a node: cordon → classify → evict & wait. - pub async fn drain_node( - &self, - node_name: &str, - options: &DrainOptions, - ) -> Result<(), Error> { + pub async fn drain_node(&self, node_name: &str, options: &DrainOptions) -> Result<(), Error> { debug!("Cordoning '{node_name}'"); self.cordon_node(node_name).await?; @@ -327,7 +335,10 @@ impl K8sClient { info!("Scheduling reboot for '{node_name}'"); let reboot_cmd = "echo rebooting ; nohup bash -c 'sleep 5 && nsenter -t 1 -m -- systemctl reboot'"; - match self.run_privileged_command_on_node(node_name, reboot_cmd).await { + match self + .run_privileged_command_on_node(node_name, reboot_cmd) + .await + { Ok(_) => debug!("Reboot command dispatched"), Err(e) => debug!("Reboot command error (expected if node began shutdown): {e}"), } @@ -343,11 +354,8 @@ impl K8sClient { } info!("Waiting for '{node_name}' to come back online"); - self.wait_for_node_ready_with_timeout( - node_name, - timeout.saturating_sub(start.elapsed()), - ) - .await?; + self.wait_for_node_ready_with_timeout(node_name, timeout.saturating_sub(start.elapsed())) + .await?; if start.elapsed() > timeout { return Err(Error::Discovery(DiscoveryError::MissingResource(format!( @@ -387,7 +395,10 @@ impl K8sClient { files: &[NodeFile], ) -> Result { let ns = self.client.default_namespace(); - let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); let name = format!("harmony-k8s-writer-{suffix}"); debug!("Writing {} file(s) to '{node_name}'", files.len()); @@ -465,7 +476,10 @@ impl K8sClient { command: &str, ) -> Result { let namespace = self.client.default_namespace(); - let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); let name = format!("harmony-k8s-cmd-{suffix}"); debug!("Running privileged command on '{node_name}': {command}"); @@ -477,7 +491,11 @@ impl K8sClient { namespace: namespace.to_string(), node_name: node_name.to_string(), container_name: "runner".to_string(), - command: vec!["/bin/bash".to_string(), "-c".to_string(), command.to_string()], + command: vec![ + "/bin/bash".to_string(), + "-c".to_string(), + command.to_string(), + ], volumes: vec![host_vol], volume_mounts: vec![host_mount], host_pid: true, @@ -515,7 +533,10 @@ mod tests { ..Default::default() }, spec: Some(PodSpec::default()), - status: Some(PodStatus { phase: Some("Running".to_string()), ..Default::default() }), + status: Some(PodStatus { + phase: Some("Running".to_string()), + ..Default::default() + }), } } @@ -555,7 +576,10 @@ mod tests { fn completed_pod(name: &str, ns: &str, phase: &str) -> Pod { let mut pod = base_pod(name, ns); - pod.status = Some(PodStatus { phase: Some(phase.to_string()), ..Default::default() }); + pod.status = Some(PodStatus { + phase: Some(phase.to_string()), + ..Default::default() + }); pod } @@ -601,7 +625,10 @@ mod tests { #[test] fn daemonset_skipped_when_ignored() { let pods = vec![daemonset_pod("fluentd", "logging")]; - let opts = DrainOptions { ignore_daemonsets: true, ..default_opts() }; + let opts = DrainOptions { + ignore_daemonsets: true, + ..default_opts() + }; let (e, s) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); assert!(e.is_empty()); assert!(s[0].contains("DaemonSet-managed")); @@ -610,7 +637,10 @@ mod tests { #[test] fn daemonset_blocks_when_not_ignored() { let pods = vec![daemonset_pod("fluentd", "logging")]; - let opts = DrainOptions { ignore_daemonsets: false, ..default_opts() }; + let opts = DrainOptions { + ignore_daemonsets: false, + ..default_opts() + }; let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); assert!(err.contains("DaemonSet") && err.contains("logging/fluentd")); } @@ -618,7 +648,10 @@ mod tests { #[test] fn emptydir_blocks_without_flag() { let pods = vec![emptydir_pod("cache", "default")]; - let opts = DrainOptions { delete_emptydir_data: false, ..default_opts() }; + let opts = DrainOptions { + delete_emptydir_data: false, + ..default_opts() + }; let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); assert!(err.contains("emptyDir") && err.contains("default/cache")); } @@ -626,7 +659,10 @@ mod tests { #[test] fn emptydir_evictable_with_flag() { let pods = vec![emptydir_pod("cache", "default")]; - let opts = DrainOptions { delete_emptydir_data: true, ..default_opts() }; + let opts = DrainOptions { + delete_emptydir_data: true, + ..default_opts() + }; let (e, s) = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap(); assert_eq!(e.len(), 1); assert!(s.is_empty()); @@ -635,7 +671,11 @@ mod tests { #[test] fn multiple_blocking_all_reported() { let pods = vec![daemonset_pod("ds", "ns1"), emptydir_pod("ed", "ns2")]; - let opts = DrainOptions { ignore_daemonsets: false, delete_emptydir_data: false, ..default_opts() }; + let opts = DrainOptions { + ignore_daemonsets: false, + delete_emptydir_data: false, + ..default_opts() + }; let err = K8sClient::classify_pods_for_drain(&pods, &opts).unwrap_err(); assert!(err.contains("ns1/ds") && err.contains("ns2/ed")); } @@ -650,7 +690,10 @@ mod tests { base_pod("api", "default"), ]; let (e, s) = K8sClient::classify_pods_for_drain(&pods, &default_opts()).unwrap(); - let names: Vec<&str> = e.iter().map(|p| p.metadata.name.as_deref().unwrap()).collect(); + let names: Vec<&str> = e + .iter() + .map(|p| p.metadata.name.as_deref().unwrap()) + .collect(); assert_eq!(names, vec!["web", "api"]); assert_eq!(s.len(), 3); } @@ -658,7 +701,10 @@ mod tests { #[test] fn mirror_checked_before_completed() { let mut pod = mirror_pod("static-etcd", "kube-system"); - pod.status = Some(PodStatus { phase: Some("Succeeded".to_string()), ..Default::default() }); + pod.status = Some(PodStatus { + phase: Some("Succeeded".to_string()), + ..Default::default() + }); let (_, s) = K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); assert!(s[0].contains("mirror pod"), "got: {}", s[0]); } @@ -666,7 +712,10 @@ mod tests { #[test] fn completed_checked_before_daemonset() { let mut pod = daemonset_pod("collector", "monitoring"); - pod.status = Some(PodStatus { phase: Some("Failed".to_string()), ..Default::default() }); + pod.status = Some(PodStatus { + phase: Some("Failed".to_string()), + ..Default::default() + }); let (_, s) = K8sClient::classify_pods_for_drain(&[pod], &default_opts()).unwrap(); assert!(s[0].contains("completed"), "got: {}", s[0]); } diff --git a/harmony-k8s/src/pod.rs b/harmony-k8s/src/pod.rs index 353fa5b..3c1efbd 100644 --- a/harmony-k8s/src/pod.rs +++ b/harmony-k8s/src/pod.rs @@ -14,11 +14,7 @@ use tokio::time::sleep; use crate::client::K8sClient; impl K8sClient { - pub async fn get_pod( - &self, - name: &str, - namespace: Option<&str>, - ) -> Result, Error> { + pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result, Error> { let api: Api = match namespace { Some(ns) => Api::namespaced(self.client.clone(), ns), None => Api::default_namespaced(self.client.clone()), @@ -68,12 +64,18 @@ impl K8sClient { let p = api.get(name).await?; match p.status.and_then(|s| s.phase).as_deref() { Some("Succeeded") => { - let logs = api.logs(name, &Default::default()).await.unwrap_or_default(); + let logs = api + .logs(name, &Default::default()) + .await + .unwrap_or_default(); debug!("Pod {namespace}/{name} succeeded. Logs: {logs}"); return Ok(logs); } Some("Failed") => { - let logs = api.logs(name, &Default::default()).await.unwrap_or_default(); + let logs = api + .logs(name, &Default::default()) + .await + .unwrap_or_default(); debug!("Pod {namespace}/{name} failed. Logs: {logs}"); return Err(Error::Discovery(DiscoveryError::MissingResource(format!( "Pod '{name}' failed.\n{logs}" @@ -113,7 +115,11 @@ impl K8sClient { .into_owned(); match api - .exec(&pod_name, command, &AttachParams::default().stdout(true).stderr(true)) + .exec( + &pod_name, + command, + &AttachParams::default().stdout(true).stderr(true), + ) .await { Err(e) => Err(e.to_string()), diff --git a/harmony-k8s/src/resources.rs b/harmony-k8s/src/resources.rs index 054598e..b788318 100644 --- a/harmony-k8s/src/resources.rs +++ b/harmony-k8s/src/resources.rs @@ -5,13 +5,13 @@ use k8s_openapi::api::{ core::v1::{Node, ServiceAccount}, }; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; +use kube::api::ApiResource; use kube::{ Error, Resource, api::{Api, DynamicObject, GroupVersionKind, ListParams, ObjectList}, runtime::conditions, runtime::wait::await_condition, }; -use kube::api::ApiResource; use log::debug; use serde::de::DeserializeOwned; use serde_json::Value; @@ -27,14 +27,23 @@ impl K8sClient { label_selector: &str, ) -> Result { let api: Api = Api::namespaced(self.client.clone(), namespace); - let list = api.list(&ListParams::default().labels(label_selector)).await?; + let list = api + .list(&ListParams::default().labels(label_selector)) + .await?; for d in list.items { - let available = d.status.as_ref().and_then(|s| s.available_replicas).unwrap_or(0); + let available = d + .status + .as_ref() + .and_then(|s| s.available_replicas) + .unwrap_or(0); if available > 0 { return Ok(true); } if let Some(conds) = d.status.as_ref().and_then(|s| s.conditions.as_ref()) { - if conds.iter().any(|c| c.type_ == "Available" && c.status == "True") { + if conds + .iter() + .any(|c| c.type_ == "Available" && c.status == "True") + { return Ok(true); } } @@ -47,7 +56,9 @@ impl K8sClient { label_selector: &str, ) -> Result, Error> { let api: Api = Api::all(self.client.clone()); - let list = api.list(&ListParams::default().labels(label_selector)).await?; + let list = api + .list(&ListParams::default().labels(label_selector)) + .await?; let mut healthy_ns: HashMap = HashMap::new(); for d in list.items { @@ -55,14 +66,21 @@ impl K8sClient { Some(n) => n, None => continue, }; - let available = d.status.as_ref().and_then(|s| s.available_replicas).unwrap_or(0); + let available = d + .status + .as_ref() + .and_then(|s| s.available_replicas) + .unwrap_or(0); let is_healthy = if available > 0 { true } else { d.status .as_ref() .and_then(|s| s.conditions.as_ref()) - .map(|c| c.iter().any(|c| c.type_ == "Available" && c.status == "True")) + .map(|c| { + c.iter() + .any(|c| c.type_ == "Available" && c.status == "True") + }) .unwrap_or(false) }; if is_healthy { @@ -105,11 +123,7 @@ impl K8sClient { .collect()) } - pub async fn is_service_account_cluster_wide( - &self, - sa: &str, - ns: &str, - ) -> Result { + pub async fn is_service_account_cluster_wide(&self, sa: &str, ns: &str) -> Result { let sa_user = format!("system:serviceaccount:{ns}:{sa}"); for crb in self.list_clusterrolebindings_json().await? { if let Some(subjects) = crb.get("subjects").and_then(|s| s.as_array()) { @@ -216,7 +230,8 @@ impl K8sClient { Some(ns) => Api::namespaced(self.client.clone(), ns), None => Api::default_namespaced(self.client.clone()), }; - api.delete(name, &kube::api::DeleteParams::default()).await?; + api.delete(name, &kube::api::DeleteParams::default()) + .await?; Ok(()) } diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index 8e55484..b80b3e0 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -19,7 +19,7 @@ use serde::Serialize; use crate::executors::ExecutorError; -use super::{LogicalHost}; +use super::LogicalHost; #[derive(Debug)] pub struct DHCPStaticEntry { diff --git a/harmony/src/infra/network_manager.rs b/harmony/src/infra/network_manager.rs index 338f1ee..8323eba 100644 --- a/harmony/src/infra/network_manager.rs +++ b/harmony/src/infra/network_manager.rs @@ -16,9 +16,7 @@ use log::{debug, info, warn}; use crate::{ modules::okd::crd::nmstate, - topology::{ - HostNetworkConfig, NetworkError, NetworkManager, - }, + topology::{HostNetworkConfig, NetworkError, NetworkManager}, }; /// NetworkManager bond configuration template diff --git a/harmony/src/modules/postgresql/operator.rs b/harmony/src/modules/postgresql/operator.rs index 53d7d0a..1eb2af9 100644 --- a/harmony/src/modules/postgresql/operator.rs +++ b/harmony/src/modules/postgresql/operator.rs @@ -20,7 +20,7 @@ use crate::topology::{K8sclient, Topology}; /// # Usage /// ``` /// use harmony::modules::postgresql::CloudNativePgOperatorScore; -/// let score = CloudNativePgOperatorScore::default(); +/// let score = CloudNativePgOperatorScore::default_openshift(); /// ``` /// /// Or, you can take control of most relevant fiedls this way : -- 2.39.5