From fbcd3e4f7f2e52cbc763c26ac7fbf35017ed7c64 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 24 Apr 2025 17:36:01 -0400 Subject: [PATCH] feat: implement k3d cluster management - Adds functionality to download, install, and manage k3d clusters. - Includes methods for downloading the latest release, creating clusters, and verifying cluster existence. - Implements `ensure_k3d_installed`, `get_latest_release_tag`, `download_latest_release`, `is_k3d_installed`, `verify_cluster_exists`, `create_cluster` and `create_kubernetes_client`. - Provides a `get_client` method to access the Kubernetes client. - Includes unit tests for download and installation. - Adds handling for different operating systems. - Improves error handling and logging. - Introduces a `K3d` struct to encapsulate k3d cluster management logic. - Adds the ability to specify the cluster name during K3d initialization. --- Cargo.lock | 52 +++++ examples/lamp/src/main.rs | 8 +- harmony/Cargo.toml | 3 + harmony/src/domain/config.rs | 9 + harmony/src/domain/maestro/mod.rs | 21 -- harmony/src/domain/mod.rs | 1 + harmony/src/domain/topology/k8s.rs | 2 + harmony/src/domain/topology/k8s_anywhere.rs | 21 +- harmony/src/modules/k3d/install.rs | 50 +++-- harmony_cli/src/lib.rs | 2 +- k3d/Cargo.toml | 1 + k3d/src/lib.rs | 221 +++++++++++++++++++- 12 files changed, 335 insertions(+), 56 deletions(-) create mode 100644 harmony/src/domain/config.rs diff --git a/Cargo.lock b/Cargo.lock index 197da40..5d6c373 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -795,6 +795,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.59.0", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1341,14 +1362,17 @@ dependencies = [ "async-trait", "cidr", "derive-new", + "directories", "env_logger", "harmony_macros", "harmony_types", "helm-wrapper-rs", "http 1.3.1", "inquire", + "k3d-rs", "k8s-openapi", "kube", + "lazy_static", "libredfish", "log", "non-blank-string-rs", @@ -2085,6 +2109,7 @@ dependencies = [ "env_logger", "futures-util", "httptest", + "kube", "log", "octocrab", "pretty_assertions", @@ -2207,6 +2232,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2572,6 +2607,12 @@ dependencies = [ "yaserde_derive", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -3051,6 +3092,17 @@ dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "redox_users" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 2.0.12", +] + [[package]] name = "regex" version = "1.11.1" diff --git a/examples/lamp/src/main.rs b/examples/lamp/src/main.rs index 3075e31..d070871 100644 --- a/examples/lamp/src/main.rs +++ b/examples/lamp/src/main.rs @@ -1,5 +1,6 @@ use harmony::{ data::Version, + inventory::Inventory, maestro::Maestro, modules::lamp::{LAMPConfig, LAMPScore}, topology::{K8sAnywhereTopology, Url}, @@ -17,7 +18,12 @@ async fn main() { }, }; - let mut maestro = Maestro::::load_from_env(); + let mut maestro = Maestro::::initialize( + Inventory::autoload(), + K8sAnywhereTopology::new(), + ) + .await + .unwrap(); maestro.register_all(vec![Box::new(lamp_stack)]); harmony_tui::init(maestro).await.unwrap(); } diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index aae188d..5c36335 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -33,3 +33,6 @@ serde-value = { workspace = true } inquire.workspace = true helm-wrapper-rs = "0.4.0" non-blank-string-rs = "1.0.4" +k3d-rs = { path = "../k3d" } +directories = "6.0.0" +lazy_static = "1.5.0" diff --git a/harmony/src/domain/config.rs b/harmony/src/domain/config.rs new file mode 100644 index 0000000..320e9a0 --- /dev/null +++ b/harmony/src/domain/config.rs @@ -0,0 +1,9 @@ +use lazy_static::lazy_static; +use std::path::PathBuf; + +lazy_static! { + pub static ref HARMONY_CONFIG_DIR: PathBuf = directories::BaseDirs::new() + .unwrap() + .data_dir() + .join("harmony"); +} diff --git a/harmony/src/domain/maestro/mod.rs b/harmony/src/domain/maestro/mod.rs index 27932d2..f53fbee 100644 --- a/harmony/src/domain/maestro/mod.rs +++ b/harmony/src/domain/maestro/mod.rs @@ -52,27 +52,6 @@ impl Maestro { Ok(outcome) } - // Load the inventory and inventory from environment. - // This function is able to discover the context that it is running in, such as k8s clusters, aws cloud, linux host, etc. - // When the HARMONY_TOPOLOGY environment variable is not set, it will default to install k3s - // locally (lazily, if not installed yet, when the first execution occurs) and use that as a topology - // So, by default, the inventory is a single host that the binary is running on, and the - // topology is a single node k3s - // - // By default : - // - Linux => k3s - // - macos, windows => docker compose - // - // To run more complex cases like OKDHACluster, either provide the default target in the - // harmony infrastructure as code or as an environment variable - pub fn load_from_env() -> Self { - // Load env var HARMONY_TOPOLOGY - match std::env::var("HARMONY_TOPOLOGY") { - Ok(_) => todo!(), - Err(_) => todo!(), - } - } - pub fn register_all(&mut self, mut scores: ScoreVec) { let mut score_mut = self.scores.write().expect("Should acquire lock"); score_mut.append(&mut scores); diff --git a/harmony/src/domain/mod.rs b/harmony/src/domain/mod.rs index ece55ef..349191a 100644 --- a/harmony/src/domain/mod.rs +++ b/harmony/src/domain/mod.rs @@ -1,3 +1,4 @@ +pub mod config; pub mod data; pub mod executors; pub mod filter; diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index ed345ee..beecbf0 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -1,7 +1,9 @@ +use derive_new::new; use k8s_openapi::NamespaceResourceScope; use kube::{Api, Client, Error, Resource, api::PostParams}; use serde::de::DeserializeOwned; +#[derive(new)] pub struct K8sClient { client: Client, } diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 18a2ccc..780bc92 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -61,13 +61,15 @@ impl K8sAnywhereTopology { todo!("Use kube-rs to load kubeconfig at path {path}"); } - async fn try_install_k3d(&self) -> Result { + fn get_k3d_installation_score(&self) -> K3DInstallationScore { + K3DInstallationScore::default() + } + + async fn try_install_k3d(&self) -> Result<(), InterpretError> { let maestro = Maestro::initialize(Inventory::autoload(), LocalhostTopology::new()).await?; - let k3d_score = K3DInstallationScore::new(); + let k3d_score = self.get_k3d_installation_score(); maestro.interpret(Box::new(k3d_score)).await?; - todo!( - "Create Maestro with LocalDockerTopology or something along these lines and run a K3dInstallationScore on it" - ); + Ok(()) } async fn try_get_or_install_k8s_client(&self) -> Result, InterpretError> { @@ -112,9 +114,14 @@ impl K8sAnywhereTopology { } info!("Starting K8sAnywhere installation"); - match self.try_install_k3d().await { + self.try_install_k3d().await?; + let k3d_score = self.get_k3d_installation_score(); + match k3d_rs::K3d::new(k3d_score.installation_path, Some(k3d_score.cluster_name)) + .get_client() + .await + { Ok(client) => Ok(Some(K8sState { - _client: client, + _client: K8sClient::new(client), _source: K8sSource::LocalK3d, message: "Successfully installed K3D cluster and acquired client".to_string(), })), diff --git a/harmony/src/modules/k3d/install.rs b/harmony/src/modules/k3d/install.rs index 5317996..f825f2e 100644 --- a/harmony/src/modules/k3d/install.rs +++ b/harmony/src/modules/k3d/install.rs @@ -1,7 +1,11 @@ +use std::path::PathBuf; + use async_trait::async_trait; +use log::info; use serde::Serialize; use crate::{ + config::HARMONY_CONFIG_DIR, data::{Id, Version}, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, @@ -10,26 +14,25 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct K3DInstallationScore {} +pub struct K3DInstallationScore { + pub installation_path: PathBuf, + pub cluster_name: String, +} -impl K3DInstallationScore { - pub fn new() -> Self { - Self {} +impl Default for K3DInstallationScore { + fn default() -> Self { + Self { + installation_path: HARMONY_CONFIG_DIR.join("k3d"), + cluster_name: "harmony".to_string(), + } } } impl Score for K3DInstallationScore { fn create_interpret(&self) -> Box> { - todo!(" - 1. Decide if I create a new crate for k3d management, especially to avoid the ocrtograb dependency - 2. Implement k3d management - 3. Find latest tag - 4. Download k3d to some path managed by harmony (or not?) - 5. Bootstrap cluster - 6. Get kubeconfig - 7. Load kubeconfig in k8s anywhere - 8. Complete k8sanywhere setup - ") + Box::new(K3dInstallationInterpret { + score: self.clone(), + }) } fn name(&self) -> String { @@ -38,7 +41,9 @@ impl Score for K3DInstallationScore { } #[derive(Debug)] -pub struct K3dInstallationInterpret {} +pub struct K3dInstallationInterpret { + score: K3DInstallationScore, +} #[async_trait] impl Interpret for K3dInstallationInterpret { @@ -47,7 +52,20 @@ impl Interpret for K3dInstallationInterpret { _inventory: &Inventory, _topology: &T, ) -> Result { - todo!() + let k3d = k3d_rs::K3d::new( + self.score.installation_path.clone(), + Some(self.score.cluster_name.clone()), + ); + match k3d.ensure_installed().await { + Ok(_client) => { + let msg = format!("k3d cluster {} is installed ", self.score.cluster_name); + info!("{msg}"); + Ok(Outcome::success(msg)) + } + Err(msg) => Err(InterpretError::new(format!( + "K3dInstallationInterpret failed to ensure k3d is installed : {msg}" + ))), + } } fn get_name(&self) -> InterpretName { InterpretName::K3dInstallation diff --git a/harmony_cli/src/lib.rs b/harmony_cli/src/lib.rs index a47df06..33759fa 100644 --- a/harmony_cli/src/lib.rs +++ b/harmony_cli/src/lib.rs @@ -99,7 +99,7 @@ pub async fn init( return Err("Not compiled with interactive support".into()); } - env_logger::builder().init(); + let _ = env_logger::builder().try_init(); let scores_vec = maestro_scores_filter(&maestro, args.all, args.filter, args.number); diff --git a/k3d/Cargo.toml b/k3d/Cargo.toml index 1124d75..aaa15ce 100644 --- a/k3d/Cargo.toml +++ b/k3d/Cargo.toml @@ -15,6 +15,7 @@ reqwest = { version = "0.12", features = ["stream"] } url.workspace = true sha2 = "0.10.8" futures-util = "0.3.31" +kube.workspace = true [dev-dependencies] env_logger = { workspace = true } diff --git a/k3d/src/lib.rs b/k3d/src/lib.rs index 8e7fb72..88d5963 100644 --- a/k3d/src/lib.rs +++ b/k3d/src/lib.rs @@ -1,18 +1,23 @@ mod downloadable_asset; use downloadable_asset::*; -use log::{debug, info}; +use kube::Client; +use log::{debug, info, warn}; use std::path::PathBuf; const K3D_BIN_FILE_NAME: &str = "k3d"; pub struct K3d { base_dir: PathBuf, + cluster_name: Option, } impl K3d { - pub fn new(base_dir: PathBuf) -> Self { - Self { base_dir } + pub fn new(base_dir: PathBuf, cluster_name: Option) -> Self { + Self { + base_dir, + cluster_name, + } } async fn get_binary_for_current_platform( @@ -24,7 +29,6 @@ impl K3d { debug!("Detecting platform: OS={}, ARCH={}", os, arch); - // 2. Construct the binary name pattern based on platform let binary_pattern = match (os, arch) { ("linux", "x86") => "k3d-linux-386", ("linux", "x86_64") => "k3d-linux-amd64", @@ -38,7 +42,6 @@ impl K3d { debug!("Looking for binary matching pattern: {}", binary_pattern); - // 3. Find the matching binary in release assets let binary_asset = latest_release .assets .iter() @@ -47,14 +50,12 @@ impl K3d { let binary_url = binary_asset.browser_download_url.clone(); - // 4. Find and parse the checksums file let checksums_asset = latest_release .assets .iter() .find(|asset| asset.name == "checksums.txt") .expect("Checksums file not found in release assets"); - // 5. Download and parse checksums file let checksums_url = checksums_asset.browser_download_url.clone(); let body = reqwest::get(checksums_url) @@ -65,7 +66,6 @@ impl K3d { .unwrap(); println!("body: {body}"); - // 6. Find the checksum for our binary let checksum = body .lines() .find_map(|line| { @@ -109,6 +109,207 @@ impl K3d { Ok(latest_release) } + + /// Checks if k3d binary exists and is executable + /// + /// Verifies that: + /// 1. The k3d binary exists in the base directory + /// 2. It has proper executable permissions (on Unix systems) + /// 3. It responds correctly to a simple command (`k3d --version`) + pub fn is_installed(&self) -> bool { + let binary_path = self.base_dir.join(K3D_BIN_FILE_NAME); + + if !binary_path.exists() { + debug!("K3d binary not found at {:?}", binary_path); + return false; + } + + if !self.ensure_binary_executable(&binary_path) { + return false; + } + + self.can_execute_binary_check(&binary_path) + } + + /// Verifies if the specified cluster is already running + /// + /// Executes `k3d cluster list ` and checks for a successful response, + /// indicating that the cluster exists and is registered with k3d. + pub fn is_cluster_initialized(&self) -> bool { + let cluster_name = match &self.cluster_name { + Some(name) => name, + None => { + debug!("No cluster name specified, can't verify if cluster is initialized"); + return false; + } + }; + + let binary_path = self.base_dir.join(K3D_BIN_FILE_NAME); + if !binary_path.exists() { + return false; + } + + self.verify_cluster_exists(&binary_path, cluster_name) + } + + /// Creates a new k3d cluster with the specified name + /// + /// This method: + /// 1. Creates a new k3d cluster using `k3d cluster create ` + /// 2. Waits for the cluster to initialize + /// 3. Returns a configured Kubernetes client connected to the cluster + /// + /// # Returns + /// - `Ok(Client)` - Successfully created cluster and connected client + /// - `Err(String)` - Error message detailing what went wrong + pub async fn initialize_cluster(&self) -> Result { + let cluster_name = match &self.cluster_name { + Some(name) => name, + None => return Err("No cluster name specified for initialization".to_string()), + }; + + let binary_path = self.base_dir.join(K3D_BIN_FILE_NAME); + if !binary_path.exists() { + return Err(format!("K3d binary not found at {:?}", binary_path)); + } + + info!("Initializing k3d cluster '{}'", cluster_name); + + self.create_cluster(&binary_path, cluster_name)?; + self.create_kubernetes_client().await + } + + /// Ensures k3d is installed and the cluster is initialized + /// + /// This method provides a complete setup flow: + /// 1. Checks if k3d is installed, downloads and installs it if needed + /// 2. Verifies if the specified cluster exists, creates it if not + /// 3. Returns a Kubernetes client connected to the cluster + /// + /// # Returns + /// - `Ok(Client)` - Successfully ensured k3d and cluster are ready + /// - `Err(String)` - Error message if any step failed + pub async fn ensure_installed(&self) -> Result { + if !self.is_installed() { + info!("K3d is not installed, downloading latest release"); + self.download_latest_release() + .await + .map_err(|e| format!("Failed to download k3d: {}", e))?; + + if !self.is_installed() { + return Err("Failed to install k3d properly".to_string()); + } + } + + if !self.is_cluster_initialized() { + info!("Cluster is not initialized, initializing now"); + return self.initialize_cluster().await; + } + + info!("K3d and cluster are already properly set up"); + self.create_kubernetes_client().await + } + + // Private helper methods + + #[cfg(not(target_os = "windows"))] + fn ensure_binary_executable(&self, binary_path: &PathBuf) -> bool { + use std::os::unix::fs::PermissionsExt; + + let mut perms = match std::fs::metadata(binary_path) { + Ok(metadata) => metadata.permissions(), + Err(e) => { + debug!("Failed to get binary metadata: {}", e); + return false; + } + }; + + perms.set_mode(0o755); + + if let Err(e) = std::fs::set_permissions(binary_path, perms) { + debug!("Failed to set executable permissions on k3d binary: {}", e); + return false; + } + + true + } + + #[cfg(target_os = "windows")] + fn ensure_binary_executable(&self, _binary_path: &PathBuf) -> bool { + // Windows doesn't use executable file permissions + true + } + + fn can_execute_binary_check(&self, binary_path: &PathBuf) -> bool { + match std::process::Command::new(binary_path) + .arg("--version") + .output() + { + Ok(output) => { + if output.status.success() { + debug!("K3d binary is installed and working"); + true + } else { + debug!("K3d binary check failed: {:?}", output); + false + } + } + Err(e) => { + debug!("Failed to execute K3d binary: {}", e); + false + } + } + } + + fn verify_cluster_exists(&self, binary_path: &PathBuf, cluster_name: &str) -> bool { + match std::process::Command::new(binary_path) + .args(["cluster", "list", cluster_name, "--no-headers"]) + .output() + { + Ok(output) => { + if output.status.success() && !output.stdout.is_empty() { + debug!("Cluster '{}' is initialized", cluster_name); + true + } else { + debug!("Cluster '{}' is not initialized", cluster_name); + false + } + } + Err(e) => { + debug!("Failed to check cluster initialization: {}", e); + false + } + } + } + + fn create_cluster(&self, binary_path: &PathBuf, cluster_name: &str) -> Result<(), String> { + let output = std::process::Command::new(binary_path) + .args(["cluster", "create", cluster_name]) + .output() + .map_err(|e| format!("Failed to execute k3d command: {}", e))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("Failed to create cluster: {}", stderr)); + } + + info!("Successfully created k3d cluster '{}'", cluster_name); + Ok(()) + } + + async fn create_kubernetes_client(&self) -> Result { + warn!("TODO this method is way too dumb, it should make sure that the client is connected to the k3d cluster actually represented by this instance, not just any default client"); + Client::try_default() + .await + .map_err(|e| format!("Failed to create Kubernetes client: {}", e)) + } + + pub async fn get_client(&self) -> Result { + match self.is_cluster_initialized() { + true => Ok(self.create_kubernetes_client().await?), + false => Err("Cannot get client! Cluster not initialized yet".to_string()), + } + } } #[cfg(test)] @@ -124,7 +325,7 @@ mod test { assert_eq!(dir.join(K3D_BIN_FILE_NAME).exists(), false); - let k3d = K3d::new(dir.clone()); + let k3d = K3d::new(dir.clone(), None); let latest_release = k3d.get_latest_release_tag().await.unwrap(); let tag_regex = Regex::new(r"^v\d+\.\d+\.\d+$").unwrap(); @@ -138,7 +339,7 @@ mod test { assert_eq!(dir.join(K3D_BIN_FILE_NAME).exists(), false); - let k3d = K3d::new(dir.clone()); + let k3d = K3d::new(dir.clone(), None); let bin_file_path = k3d.download_latest_release().await.unwrap(); assert_eq!(bin_file_path, dir.join(K3D_BIN_FILE_NAME)); assert_eq!(dir.join(K3D_BIN_FILE_NAME).exists(), true);