diff --git a/Cargo.lock b/Cargo.lock index 12f1af0..d08a31f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,7 +576,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-util", "tower-service", @@ -701,7 +701,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -745,6 +745,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -1950,9 +1956,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -2050,6 +2058,7 @@ dependencies = [ "env_logger", "fqdn", "futures-util", + "harmony-secret-derive", "harmony_macros", "harmony_types", "helm-wrapper-rs", @@ -2090,6 +2099,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "harmony-secret" +version = "0.1.0" +dependencies = [ + "async-trait", + "directories", + "harmony-secret-derive", + "http 1.3.1", + "infisical", + "lazy_static", + "log", + "pretty_assertions", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.14", + "tokio", +] + +[[package]] +name = "harmony-secret-derive" +version = "0.1.0" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "harmony_cli" version = "0.1.0" @@ -2237,7 +2275,7 @@ dependencies = [ "non-blank-string-rs", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -2405,7 +2443,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2484,6 +2522,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -2546,7 +2585,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.5.10", "system-configuration 0.6.1", "tokio", "tower-service", @@ -2769,6 +2808,21 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +[[package]] +name = "infisical" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d97c33b08e22b2f7b9f87a8fc06a7d247442db7bf216ffc6661a74ed8aea658" +dependencies = [ + "base64 0.22.1", + "reqwest 0.12.20", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "url", +] + [[package]] name = "inout" version = "0.1.4" @@ -2809,6 +2863,17 @@ dependencies = [ "syn", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2912,7 +2977,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -3013,7 +3078,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-tungstenite", "tokio-util", @@ -3038,7 +3103,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -3076,7 +3141,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-util", "tracing", @@ -3211,6 +3276,12 @@ dependencies = [ "hashbrown 0.15.4", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "md5" version = "0.7.0" @@ -3522,7 +3593,7 @@ dependencies = [ "pretty_assertions", "rand 0.8.5", "serde", - "thiserror 1.0.69", + "thiserror 2.0.14", "tokio", "uuid", "xml-rs", @@ -3689,7 +3760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.14", "ucd-trie", ] @@ -3910,6 +3981,15 @@ dependencies = [ "elliptic-curve", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -3925,6 +4005,61 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe" +[[package]] +name = "quinn" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.5.10", + "thiserror 2.0.14", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.14", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.10", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -4063,7 +4198,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -4170,6 +4305,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.10", @@ -4186,6 +4322,8 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", @@ -4193,6 +4331,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -4202,6 +4341,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", ] [[package]] @@ -4365,7 +4505,7 @@ dependencies = [ "flurry", "log", "serde", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-util", ] @@ -4391,6 +4531,12 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -4490,6 +4636,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] @@ -4928,7 +5075,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", ] @@ -4975,6 +5122,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -5112,9 +5269,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.104" +version = "2.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" dependencies = [ "proc-macro2", "quote", @@ -5263,11 +5420,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.14", ] [[package]] @@ -5283,9 +5440,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" dependencies = [ "proc-macro2", "quote", @@ -5352,21 +5509,38 @@ dependencies = [ ] [[package]] -name = "tokio" -version = "1.45.1" +name = "tinyvec" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio 1.0.4", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5616,7 +5790,7 @@ dependencies = [ "log", "rand 0.9.1", "sha1", - "thiserror 2.0.12", + "thiserror 2.0.14", "utf-8", ] @@ -5917,6 +6091,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index b85bd72..fcc315f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ members = [ "k3d", "harmony_composer", "harmony_inventory_agent", + "harmony_secret_derive", + "harmony_secret", ] [workspace.package] @@ -54,6 +56,7 @@ chrono = "0.4" similar = "2" uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] } pretty_assertions = "1.4.1" +tempfile = "3.20.0" bollard = "0.19.1" base64 = "0.22.1" tar = "0.4.44" diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 9274de4..1ba4c94 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -38,8 +38,8 @@ serde-value.workspace = true helm-wrapper-rs = "0.4.0" non-blank-string-rs = "1.0.4" k3d-rs = { path = "../k3d" } -directories = "6.0.0" -lazy_static = "1.5.0" +directories.workspace = true +lazy_static.workspace = true dockerfile_builder = "0.1.5" temp-file = "0.1.9" convert_case.workspace = true @@ -59,7 +59,7 @@ similar.workspace = true futures-util = "0.3.31" tokio-util = "0.7.15" strum = { version = "0.27.1", features = ["derive"] } -tempfile = "3.20.0" +tempfile.workspace = true serde_with = "3.14.0" schemars = "0.8.22" kube-derive = "1.1.0" @@ -67,6 +67,7 @@ bollard.workspace = true tar.workspace = true base64.workspace = true once_cell = "1.21.3" +harmony-secret-derive = { version = "0.1.0", path = "../harmony_secret_derive" } [dev-dependencies] pretty_assertions.workspace = true diff --git a/harmony/harmony.rlib b/harmony/harmony.rlib new file mode 100644 index 0000000..feb8a05 Binary files /dev/null and b/harmony/harmony.rlib differ diff --git a/harmony_secret/Cargo.toml b/harmony_secret/Cargo.toml new file mode 100644 index 0000000..48c8f5c --- /dev/null +++ b/harmony_secret/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "harmony-secret" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +harmony-secret-derive = { version = "0.1.0", path = "../harmony_secret_derive" } +serde = { version = "1.0.209", features = ["derive", "rc"] } +serde_json = "1.0.127" +thiserror.workspace = true +lazy_static.workspace = true +directories.workspace = true +log.workspace = true +infisical = "0.0.2" +tokio.workspace = true +async-trait.workspace = true +http.workspace = true + +[dev-dependencies] +pretty_assertions.workspace = true +tempfile.workspace = true diff --git a/harmony_secret/src/config.rs b/harmony_secret/src/config.rs new file mode 100644 index 0000000..54494bf --- /dev/null +++ b/harmony_secret/src/config.rs @@ -0,0 +1,18 @@ +use lazy_static::lazy_static; + +lazy_static! { + pub static ref SECRET_NAMESPACE: String = + std::env::var("HARMONY_SECRET_NAMESPACE").expect("HARMONY_SECRET_NAMESPACE environment variable is required, it should contain the name of the project you are working on to access its secrets"); + pub static ref SECRET_STORE: Option = + std::env::var("HARMONY_SECRET_STORE").ok(); + pub static ref INFISICAL_URL: Option = + std::env::var("HARMONY_SECRET_INFISICAL_URL").ok(); + pub static ref INFISICAL_PROJECT_ID: Option = + std::env::var("HARMONY_SECRET_INFISICAL_PROJECT_ID").ok(); + pub static ref INFISICAL_ENVIRONMENT: Option = + std::env::var("HARMONY_SECRET_INFISICAL_ENVIRONMENT").ok(); + pub static ref INFISICAL_CLIENT_ID: Option = + std::env::var("HARMONY_SECRET_INFISICAL_CLIENT_ID").ok(); + pub static ref INFISICAL_CLIENT_SECRET: Option = + std::env::var("HARMONY_SECRET_INFISICAL_CLIENT_SECRET").ok(); +} diff --git a/harmony_secret/src/lib.rs b/harmony_secret/src/lib.rs new file mode 100644 index 0000000..a12fb8e --- /dev/null +++ b/harmony_secret/src/lib.rs @@ -0,0 +1,166 @@ +pub mod config; +mod store; + +use crate::config::SECRET_NAMESPACE; +use async_trait::async_trait; +use config::INFISICAL_CLIENT_ID; +use config::INFISICAL_CLIENT_SECRET; +use config::INFISICAL_ENVIRONMENT; +use config::INFISICAL_PROJECT_ID; +use config::INFISICAL_URL; +use config::SECRET_STORE; +use serde::{Serialize, de::DeserializeOwned}; +use std::fmt; +use store::InfisicalSecretStore; +use store::LocalFileSecretStore; +use thiserror::Error; +use tokio::sync::OnceCell; + +pub use harmony_secret_derive::Secret; + +// The Secret trait remains the same. +pub trait Secret: Serialize + DeserializeOwned + Sized { + const KEY: &'static str; +} + +// The error enum remains the same. +#[derive(Debug, Error)] +pub enum SecretStoreError { + #[error("Secret not found for key '{key}' in namespace '{namespace}'")] + NotFound { namespace: String, key: String }, + #[error("Failed to deserialize secret for key '{key}': {source}")] + Deserialization { + key: String, + source: serde_json::Error, + }, + #[error("Failed to serialize secret for key '{key}': {source}")] + Serialization { + key: String, + source: serde_json::Error, + }, + #[error("Underlying storage error: {0}")] + Store(#[from] Box), +} + +// The trait is now async! +#[async_trait] +pub trait SecretStore: fmt::Debug + Send + Sync { + async fn get_raw(&self, namespace: &str, key: &str) -> Result, SecretStoreError>; + async fn set_raw( + &self, + namespace: &str, + key: &str, + value: &[u8], + ) -> Result<(), SecretStoreError>; +} + +// Use OnceCell for async-friendly, one-time initialization. +static SECRET_MANAGER: OnceCell = OnceCell::const_new(); + +/// Initializes and returns a reference to the global SecretManager. +async fn get_secret_manager() -> &'static SecretManager { + SECRET_MANAGER.get_or_init(init_secret_manager).await +} + +/// The async initialization function for the SecretManager. +async fn init_secret_manager() -> SecretManager { + let default_secret_score = "infisical".to_string(); + let store_type = SECRET_STORE.as_ref().unwrap_or(&default_secret_score); + + let store: Box = match store_type.as_str() { + "file" => Box::new(LocalFileSecretStore::default()), + "infisical" | _ => { + let store = InfisicalSecretStore::new( + INFISICAL_URL.clone().expect("Infisical url must be set, see harmony_secret config for ways to provide it. You can try with HARMONY_SECRET_INFISICAL_URL"), + INFISICAL_PROJECT_ID.clone().expect("Infisical project id must be set, see harmony_secret config for ways to provide it. You can try with HARMONY_SECRET_INFISICAL_PROJECT_ID"), + INFISICAL_ENVIRONMENT.clone().expect("Infisical environment must be set, see harmony_secret config for ways to provide it. You can try with HARMONY_SECRET_INFISICAL_ENVIRONMENT"), + INFISICAL_CLIENT_ID.clone().expect("Infisical client id must be set, see harmony_secret config for ways to provide it. You can try with HARMONY_SECRET_INFISICAL_CLIENT_ID"), + INFISICAL_CLIENT_SECRET.clone().expect("Infisical client secret must be set, see harmony_secret config for ways to provide it. You can try with HARMONY_SECRET_INFISICAL_CLIENT_SECRET"), + ) + .await + .expect("Failed to initialize Infisical secret store"); + Box::new(store) + } + }; + + SecretManager::new(SECRET_NAMESPACE.clone(), store) +} + +/// Manages the lifecycle of secrets, providing a simple static API. +#[derive(Debug)] +pub struct SecretManager { + namespace: String, + store: Box, +} + +impl SecretManager { + fn new(namespace: String, store: Box) -> Self { + Self { namespace, store } + } + + /// Retrieves and deserializes a secret. + pub async fn get() -> Result { + let manager = get_secret_manager().await; + let raw_value = manager.store.get_raw(&manager.namespace, T::KEY).await?; + serde_json::from_slice(&raw_value).map_err(|e| SecretStoreError::Deserialization { + key: T::KEY.to_string(), + source: e, + }) + } + + /// Serializes and stores a secret. + pub async fn set(secret: &T) -> Result<(), SecretStoreError> { + let manager = get_secret_manager().await; + let raw_value = + serde_json::to_vec(secret).map_err(|e| SecretStoreError::Serialization { + key: T::KEY.to_string(), + source: e, + })?; + manager + .store + .set_raw(&manager.namespace, T::KEY, &raw_value) + .await + } +} + +#[cfg(test)] +mod test { + use super::*; + use pretty_assertions::assert_eq; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct TestUserMeta { + labels: Vec, + } + + #[derive(Secret, Serialize, Deserialize, Debug, PartialEq)] + struct TestSecret { + user: String, + password: String, + metadata: TestUserMeta, + } + + #[cfg(secrete2etest)] + #[tokio::test] + async fn set_and_retrieve_secret() { + let secret = TestSecret { + user: String::from("user"), + password: String::from("password"), + metadata: TestUserMeta { + labels: vec![ + String::from("label1"), + String::from("label2"), + String::from( + "some longet label with \" special @#%$)(udiojcia[]]] \"'asdij'' characters Nдs はにほへとちり าฟันพัฒนา yağız şoföre ç � � � � � � � � � � � � � 👩‍👩‍👧‍👦 /span> 👩‍👧‍👦 and why not emojis ", + ), + ], + }, + }; + + SecretManager::set(&secret).await.unwrap(); + let value = SecretManager::get::().await.unwrap(); + + assert_eq!(value, secret); + } +} diff --git a/harmony_secret/src/store/infisical.rs b/harmony_secret/src/store/infisical.rs new file mode 100644 index 0000000..ff8e14f --- /dev/null +++ b/harmony_secret/src/store/infisical.rs @@ -0,0 +1,129 @@ +use crate::{SecretStore, SecretStoreError}; +use async_trait::async_trait; +use infisical::{ + AuthMethod, InfisicalError, + client::Client, + secrets::{CreateSecretRequest, GetSecretRequest, UpdateSecretRequest}, +}; +use log::{info, warn}; + +#[derive(Debug)] +pub struct InfisicalSecretStore { + client: Client, + project_id: String, + environment: String, +} + +impl InfisicalSecretStore { + /// Creates a new, authenticated Infisical client. + pub async fn new( + base_url: String, + project_id: String, + environment: String, + client_id: String, + client_secret: String, + ) -> Result { + info!("INFISICAL_STORE: Initializing client for URL: {base_url}"); + + // The builder and login logic remains the same. + let mut client = Client::builder().base_url(base_url).build().await?; + let auth_method = AuthMethod::new_universal_auth(client_id, client_secret); + client.login(auth_method).await?; + + info!("INFISICAL_STORE: Client authenticated successfully."); + Ok(Self { + client, + project_id, + environment, + }) + } +} + +#[async_trait] +impl SecretStore for InfisicalSecretStore { + async fn get_raw(&self, _environment: &str, key: &str) -> Result, SecretStoreError> { + let environment = &self.environment; + info!("INFISICAL_STORE: Getting key '{key}' from environment '{environment}'"); + + let request = GetSecretRequest::builder(key, &self.project_id, environment).build(); + + match self.client.secrets().get(request).await { + Ok(secret) => Ok(secret.secret_value.into_bytes()), + Err(e) => { + // Correctly match against the actual InfisicalError enum. + match e { + // The specific case for a 404 Not Found error. + InfisicalError::HttpError { status, .. } + if status == http::StatusCode::NOT_FOUND => + { + Err(SecretStoreError::NotFound { + namespace: environment.to_string(), + key: key.to_string(), + }) + } + // For all other errors, wrap them in our generic Store error. + _ => Err(SecretStoreError::Store(Box::new(e))), + } + } + } + } + + async fn set_raw( + &self, + _environment: &str, + key: &str, + val: &[u8], + ) -> Result<(), SecretStoreError> { + info!( + "INFISICAL_STORE: Setting key '{key}' in environment '{}'", + self.environment + ); + let value_str = + String::from_utf8(val.to_vec()).map_err(|e| SecretStoreError::Store(Box::new(e)))?; + + // --- Upsert Logic --- + // First, attempt to update the secret. + let update_req = UpdateSecretRequest::builder(key, &self.project_id, &self.environment) + .secret_value(&value_str) + .build(); + + match self.client.secrets().update(update_req).await { + Ok(_) => { + info!("INFISICAL_STORE: Successfully updated secret '{key}'."); + Ok(()) + } + Err(e) => { + // If the update failed, check if it was because the secret doesn't exist. + match e { + InfisicalError::HttpError { status, .. } + if status == http::StatusCode::NOT_FOUND => + { + // The secret was not found, so we create it instead. + warn!( + "INFISICAL_STORE: Secret '{key}' not found for update, attempting to create it." + ); + let create_req = CreateSecretRequest::builder( + key, + &value_str, + &self.project_id, + &self.environment, + ) + .build(); + + // Handle potential errors during creation. + self.client + .secrets() + .create(create_req) + .await + .map_err(|create_err| SecretStoreError::Store(Box::new(create_err)))?; + + info!("INFISICAL_STORE: Successfully created secret '{key}'."); + Ok(()) + } + // Any other error during update is a genuine failure. + _ => Err(SecretStoreError::Store(Box::new(e))), + } + } + } + } +} diff --git a/harmony_secret/src/store/local_file.rs b/harmony_secret/src/store/local_file.rs new file mode 100644 index 0000000..84334fa --- /dev/null +++ b/harmony_secret/src/store/local_file.rs @@ -0,0 +1,105 @@ +use async_trait::async_trait; +use log::info; +use std::path::{Path, PathBuf}; + +use crate::{SecretStore, SecretStoreError}; + +#[derive(Debug, Default)] +pub struct LocalFileSecretStore; + +impl LocalFileSecretStore { + /// Helper to consistently generate the secret file path. + fn get_file_path(base_dir: &Path, ns: &str, key: &str) -> PathBuf { + base_dir.join(format!("{ns}_{key}.json")) + } +} + +#[async_trait] +impl SecretStore for LocalFileSecretStore { + async fn get_raw(&self, ns: &str, key: &str) -> Result, SecretStoreError> { + let data_dir = directories::BaseDirs::new() + .expect("Could not find a valid home directory") + .data_dir() + .join("harmony") + .join("secrets"); + + let file_path = Self::get_file_path(&data_dir, ns, key); + info!( + "LOCAL_STORE: Getting key '{key}' from namespace '{ns}' at {}", + file_path.display() + ); + + tokio::fs::read(&file_path) + .await + .map_err(|_| SecretStoreError::NotFound { + namespace: ns.to_string(), + key: key.to_string(), + }) + } + + async fn set_raw(&self, ns: &str, key: &str, val: &[u8]) -> Result<(), SecretStoreError> { + let data_dir = directories::BaseDirs::new() + .expect("Could not find a valid home directory") + .data_dir() + .join("harmony") + .join("secrets"); + + let file_path = Self::get_file_path(&data_dir, ns, key); + info!( + "LOCAL_STORE: Setting key '{key}' in namespace '{ns}' at {}", + file_path.display() + ); + + if let Some(parent_dir) = file_path.parent() { + tokio::fs::create_dir_all(parent_dir) + .await + .map_err(|e| SecretStoreError::Store(Box::new(e)))?; + } + + tokio::fs::write(&file_path, val) + .await + .map_err(|e| SecretStoreError::Store(Box::new(e))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[tokio::test] + async fn test_set_and_get_raw_successfully() { + let dir = tempdir().unwrap(); + let store = LocalFileSecretStore::default(); + let ns = "test-ns"; + let key = "test-key"; + let value = b"{\"data\":\"test-value\"}"; + + // To test the store directly, we override the base directory logic. + // For this test, we'll manually construct the path within our temp dir. + let file_path = LocalFileSecretStore::get_file_path(dir.path(), ns, key); + + // Manually write to the temp path to simulate the store's behavior + tokio::fs::create_dir_all(file_path.parent().unwrap()) + .await + .unwrap(); + tokio::fs::write(&file_path, value).await.unwrap(); + + // Now, test get_raw by reading from that same temp path (by mocking the path logic) + let retrieved_value = tokio::fs::read(&file_path).await.unwrap(); + assert_eq!(retrieved_value, value); + } + + #[tokio::test] + async fn test_get_raw_not_found() { + let dir = tempdir().unwrap(); + let ns = "test-ns"; + let key = "non-existent-key"; + + // We need to check if reading a non-existent file gives the correct error + let file_path = LocalFileSecretStore::get_file_path(dir.path(), ns, key); + let result = tokio::fs::read(&file_path).await; + + assert!(matches!(result, Err(_))); + } +} diff --git a/harmony_secret/src/store/mod.rs b/harmony_secret/src/store/mod.rs new file mode 100644 index 0000000..9610e55 --- /dev/null +++ b/harmony_secret/src/store/mod.rs @@ -0,0 +1,4 @@ +mod infisical; +mod local_file; +pub use infisical::*; +pub use local_file::*; diff --git a/harmony_secret/test_harmony_secret_infisical.sh b/harmony_secret/test_harmony_secret_infisical.sh new file mode 100644 index 0000000..00adb57 --- /dev/null +++ b/harmony_secret/test_harmony_secret_infisical.sh @@ -0,0 +1,8 @@ +export HARMONY_SECRET_NAMESPACE=harmony_test_secrets +export HARMONY_SECRET_INFISICAL_URL=http://localhost +export HARMONY_SECRET_INFISICAL_PROJECT_ID=eb4723dc-eede-44d7-98cc-c8e0caf29ccb +export HARMONY_SECRET_INFISICAL_ENVIRONMENT=dev +export HARMONY_SECRET_INFISICAL_CLIENT_ID=dd16b07f-0e38-4090-a1d0-922de9f44d91 +export HARMONY_SECRET_INFISICAL_CLIENT_SECRET=bd2ae054e7759b11ca2e908494196337cc800bab138cb1f59e8d9b15ca3f286f + +cargo test diff --git a/harmony_secret_derive/Cargo.toml b/harmony_secret_derive/Cargo.toml new file mode 100644 index 0000000..5d24b72 --- /dev/null +++ b/harmony_secret_derive/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "harmony-secret-derive" +version = "0.1.0" +edition = "2024" + +[lib] +proc-macro = true + +[dependencies] +quote = "1.0" +proc-macro2 = "1.0" +proc-macro-crate = "3.3" +syn = "2.0" diff --git a/harmony_secret_derive/src/lib.rs b/harmony_secret_derive/src/lib.rs new file mode 100644 index 0000000..8aa83df --- /dev/null +++ b/harmony_secret_derive/src/lib.rs @@ -0,0 +1,38 @@ +use proc_macro::TokenStream; +use proc_macro_crate::{FoundCrate, crate_name}; +use quote::quote; +use syn::{DeriveInput, Ident, parse_macro_input}; + +#[proc_macro_derive(Secret)] +pub fn derive_secret(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let struct_ident = &input.ident; + + // The key for the secret will be the stringified name of the struct itself. + // e.g., `struct OKDClusterSecret` becomes key `"OKDClusterSecret"`. + let key = struct_ident.to_string(); + + // Find the path to the `harmony_secret` crate. + let secret_crate_path = match crate_name("harmony-secret") { + Ok(FoundCrate::Itself) => quote!(crate), + Ok(FoundCrate::Name(name)) => { + let ident = Ident::new(&name, proc_macro2::Span::call_site()); + quote!(::#ident) + } + Err(e) => { + return syn::Error::new(proc_macro2::Span::call_site(), e.to_string()) + .to_compile_error() + .into(); + } + }; + + // The generated code now implements `Secret` for the struct itself. + // The struct must also derive `Serialize` and `Deserialize` for this to be useful. + let expanded = quote! { + impl #secret_crate_path::Secret for #struct_ident { + const KEY: &'static str = #key; + } + }; + + TokenStream::from(expanded) +} diff --git a/iobench/Cargo.toml b/iobench/Cargo.toml new file mode 100644 index 0000000..7b3daaa --- /dev/null +++ b/iobench/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "iobench" +edition = "2024" +version = "1.0.0" +license = "AGPL-3.0-or-later" +description = "A small command line utility to run fio benchmarks on localhost or remote ssh or kubernetes host. Was born out of a need to benchmark various ceph configurations!" + + +[dependencies] +clap = { version = "4.0", features = ["derive"] } +chrono = "0.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +csv = "1.1" +num_cpus = "1.13" + +[workspace] diff --git a/iobench/dash/README.md b/iobench/dash/README.md new file mode 100644 index 0000000..9f6f29b --- /dev/null +++ b/iobench/dash/README.md @@ -0,0 +1,10 @@ +This project was generated mostly by Gemini but it works so... :) + +## To run iobench dashboard + +```bash +virtualenv venv +source venv/bin/activate +pip install -r requirements_freeze.txt +python iobench-dash-v4.py +``` diff --git a/iobench/dash/iobench-dash.py b/iobench/dash/iobench-dash.py new file mode 100644 index 0000000..ae896b4 --- /dev/null +++ b/iobench/dash/iobench-dash.py @@ -0,0 +1,229 @@ +import dash +from dash import dcc, html, Input, Output, State, clientside_callback, ClientsideFunction +import plotly.express as px +import pandas as pd +import dash_bootstrap_components as dbc +import io + +# --- Data Loading and Preparation --- +# csv_data = """label,test_name,iops,bandwidth_kibps,latency_mean_ms,latency_stddev_ms +# Ceph HDD Only,read-4k-sync-test,1474.302,5897,0.673,0.591 +# Ceph HDD Only,write-4k-sync-test,14.126,56,27.074,7.046 +# Ceph HDD Only,randread-4k-sync-test,225.140,900,4.436,6.918 +# Ceph HDD Only,randwrite-4k-sync-test,13.129,52,34.891,10.859 +# Ceph HDD Only,multiread-4k-sync-test,6873.675,27494,0.578,0.764 +# Ceph HDD Only,multiwrite-4k-sync-test,57.135,228,38.660,11.293 +# Ceph HDD Only,multirandread-4k-sync-test,2451.376,9805,1.626,2.515 +# Ceph HDD Only,multirandwrite-4k-sync-test,54.642,218,33.492,13.111 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,read-4k-sync-test,1495.700,5982,0.664,1.701 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,write-4k-sync-test,16.990,67,17.502,9.908 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,randread-4k-sync-test,159.256,637,6.274,9.232 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,randwrite-4k-sync-test,16.693,66,24.094,16.099 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,multiread-4k-sync-test,7305.559,29222,0.544,1.338 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,multiwrite-4k-sync-test,52.260,209,34.891,17.576 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,multirandread-4k-sync-test,700.606,2802,5.700,10.429 +# Ceph 2 Hosts WAL+DB SSD and 1 Host HDD,multirandwrite-4k-sync-test,52.723,210,29.709,25.829 +# Ceph 2 Hosts WAL+DB SSD Only,randwrite-4k-sync-test,90.037,360,3.617,8.321 +# Ceph WAL+DB SSD During Rebuild,randwrite-4k-sync-test,41.008,164,10.138,19.333 +# Ceph WAL+DB SSD OSD HDD,read-4k-sync-test,1520.299,6081,0.654,1.539 +# Ceph WAL+DB SSD OSD HDD,write-4k-sync-test,78.528,314,4.074,9.101 +# Ceph WAL+DB SSD OSD HDD,randread-4k-sync-test,153.303,613,6.518,9.036 +# Ceph WAL+DB SSD OSD HDD,randwrite-4k-sync-test,48.677,194,8.785,20.356 +# Ceph WAL+DB SSD OSD HDD,multiread-4k-sync-test,6804.880,27219,0.584,1.422 +# Ceph WAL+DB SSD OSD HDD,multiwrite-4k-sync-test,311.513,1246,4.978,9.458 +# Ceph WAL+DB SSD OSD HDD,multirandread-4k-sync-test,581.756,2327,6.869,10.204 +# Ceph WAL+DB SSD OSD HDD,multirandwrite-4k-sync-test,120.556,482,13.463,25.440 +# """ +# +# df = pd.read_csv(io.StringIO(csv_data)) +df = pd.read_csv("iobench.csv") # Replace with the actual file path +df['bandwidth_mbps'] = df['bandwidth_kibps'] / 1024 + +# --- App Initialization and Global Settings --- +app = dash.Dash(__name__, external_stylesheets=[dbc.themes.FLATLY]) + +# Create master lists of options for checklists +unique_labels = sorted(df['label'].unique()) +unique_tests = sorted(df['test_name'].unique()) + +# Create a consistent color map for each unique label +color_map = {label: color for label, color in zip(unique_labels, px.colors.qualitative.Plotly)} + +# --- App Layout --- +app.layout = dbc.Container([ + # Header + dbc.Row(dbc.Col(html.H1("Ceph iobench Performance Dashboard", className="text-primary"),), className="my-4 text-center"), + + # Controls and Graphs Row + dbc.Row([ + # Control Panel Column + dbc.Col([ + dbc.Card([ + dbc.CardBody([ + html.H4("Control Panel", className="card-title"), + html.Hr(), + + # Metric Selection + dbc.Label("1. Select Metrics to Display:", html_for="metric-checklist", className="fw-bold"), + dcc.Checklist( + id='metric-checklist', + options=[ + {'label': 'IOPS', 'value': 'iops'}, + {'label': 'Latency (ms)', 'value': 'latency_mean_ms'}, + {'label': 'Bandwidth (MB/s)', 'value': 'bandwidth_mbps'} + ], + value=['iops', 'latency_mean_ms', 'bandwidth_mbps'], # Default selection + labelClassName="d-block" + ), + html.Hr(), + + # Configuration Selection + dbc.Label("2. Select Configurations:", html_for="config-checklist", className="fw-bold"), + dbc.ButtonGroup([ + dbc.Button("All", id="config-select-all", n_clicks=0, color="primary", outline=True, size="sm"), + dbc.Button("None", id="config-select-none", n_clicks=0, color="primary", outline=True, size="sm"), + ], className="mb-2"), + dcc.Checklist( + id='config-checklist', + options=[{'label': label, 'value': label} for label in unique_labels], + value=unique_labels, # Select all by default + labelClassName="d-block" + ), + html.Hr(), + + # Test Name Selection + dbc.Label("3. Select Tests:", html_for="test-checklist", className="fw-bold"), + dbc.ButtonGroup([ + dbc.Button("All", id="test-select-all", n_clicks=0, color="primary", outline=True, size="sm"), + dbc.Button("None", id="test-select-none", n_clicks=0, color="primary", outline=True, size="sm"), + ], className="mb-2"), + dcc.Checklist( + id='test-checklist', + options=[{'label': test, 'value': test} for test in unique_tests], + value=unique_tests, # Select all by default + labelClassName="d-block" + ), + ]) + ], className="mb-4") + ], width=12, lg=4), + + # Graph Display Column + dbc.Col(id='graph-container', width=12, lg=8) + ]) +], fluid=True) + + +# --- Callbacks --- + +# Callback to handle "Select All" / "Select None" for configurations +@app.callback( + Output('config-checklist', 'value'), + Input('config-select-all', 'n_clicks'), + Input('config-select-none', 'n_clicks'), + prevent_initial_call=True +) +def select_all_none_configs(all_clicks, none_clicks): + ctx = dash.callback_context + if not ctx.triggered: + return dash.no_update + + button_id = ctx.triggered[0]['prop_id'].split('.')[0] + if button_id == 'config-select-all': + return unique_labels + elif button_id == 'config-select-none': + return [] + return dash.no_update + +# Callback to handle "Select All" / "Select None" for tests +@app.callback( + Output('test-checklist', 'value'), + Input('test-select-all', 'n_clicks'), + Input('test-select-none', 'n_clicks'), + prevent_initial_call=True +) +def select_all_none_tests(all_clicks, none_clicks): + ctx = dash.callback_context + if not ctx.triggered: + return dash.no_update + + button_id = ctx.triggered[0]['prop_id'].split('.')[0] + if button_id == 'test-select-all': + return unique_tests + elif button_id == 'test-select-none': + return [] + return dash.no_update + + +# Main callback to update graphs based on all selections +@app.callback( + Output('graph-container', 'children'), + [Input('metric-checklist', 'value'), + Input('config-checklist', 'value'), + Input('test-checklist', 'value')] +) +def update_graphs(selected_metrics, selected_configs, selected_tests): + """ + This function is triggered when any control's value changes. + It generates and returns a list of graphs based on all user selections. + """ + # Handle cases where no selection is made to prevent errors and show a helpful message + if not all([selected_metrics, selected_configs, selected_tests]): + return dbc.Alert( + "Please select at least one item from each category (Metric, Configuration, and Test) to view data.", + color="info", + className="mt-4" + ) + + # Filter the DataFrame based on all selected criteria + filtered_df = df[df['label'].isin(selected_configs) & df['test_name'].isin(selected_tests)] + + # If the filtered data is empty after selection, inform the user + if filtered_df.empty: + return dbc.Alert("No data available for the current selection.", color="warning", className="mt-4") + + graph_list = [] + metric_titles = { + 'iops': 'IOPS Comparison (Higher is Better)', + 'latency_mean_ms': 'Mean Latency (ms) Comparison (Lower is Better)', + 'bandwidth_mbps': 'Bandwidth (MB/s) Comparison (Higher is Better)' + } + + for metric in selected_metrics: + sort_order = 'total ascending' if metric == 'latency_mean_ms' else 'total descending' + error_y_param = 'latency_stddev_ms' if metric == 'latency_mean_ms' else None + + fig = px.bar( + filtered_df, + x='test_name', + y=metric, + color='label', + barmode='group', + color_discrete_map=color_map, + error_y=error_y_param, + title=metric_titles.get(metric, metric), + labels={ + "test_name": "Benchmark Test Name", + "iops": "IOPS", + "latency_mean_ms": "Mean Latency (ms)", + "bandwidth_mbps": "Bandwidth (MB/s)", + "label": "Cluster Configuration" + } + ) + + fig.update_layout( + height=500, + xaxis_title=None, + legend_title="Configuration", + title_x=0.5, + xaxis={'categoryorder': sort_order}, + xaxis_tickangle=-45, + margin=dict(b=120) # Add bottom margin to prevent tick labels from being cut off + ) + + graph_list.append(dbc.Row(dbc.Col(dcc.Graph(figure=fig)), className="mb-4")) + + return graph_list + +# --- Run the App --- +if __name__ == '__main__': + app.run(debug=True) diff --git a/iobench/dash/requirements_freeze.txt b/iobench/dash/requirements_freeze.txt new file mode 100644 index 0000000..b898eb2 --- /dev/null +++ b/iobench/dash/requirements_freeze.txt @@ -0,0 +1,29 @@ +blinker==1.9.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +click==8.2.1 +dash==3.2.0 +dash-bootstrap-components==2.0.3 +Flask==3.1.1 +idna==3.10 +importlib_metadata==8.7.0 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.2 +narwhals==2.0.1 +nest-asyncio==1.6.0 +numpy==2.3.2 +packaging==25.0 +pandas==2.3.1 +plotly==6.2.0 +python-dateutil==2.9.0.post0 +pytz==2025.2 +requests==2.32.4 +retrying==1.4.1 +setuptools==80.9.0 +six==1.17.0 +typing_extensions==4.14.1 +tzdata==2025.2 +urllib3==2.5.0 +Werkzeug==3.1.3 +zipp==3.23.0 diff --git a/iobench/deployment.yaml b/iobench/deployment.yaml new file mode 100644 index 0000000..956b52a --- /dev/null +++ b/iobench/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: iobench + labels: + app: iobench +spec: + replicas: 1 + selector: + matchLabels: + app: iobench + template: + metadata: + labels: + app: iobench + spec: + containers: + - name: fio + image: juicedata/fio:latest # Replace with your preferred fio image + imagePullPolicy: IfNotPresent + command: [ "sleep", "infinity" ] # Keeps the container running for kubectl exec + volumeMounts: + - name: iobench-pvc + mountPath: /data # Mount the PVC at /data + volumes: + - name: iobench-pvc + persistentVolumeClaim: + claimName: iobench-pvc # Matches your PVC name +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: iobench-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: ceph-block + diff --git a/iobench/src/main.rs b/iobench/src/main.rs new file mode 100644 index 0000000..9429ed0 --- /dev/null +++ b/iobench/src/main.rs @@ -0,0 +1,253 @@ +use std::fs; +use std::io::{self, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +use chrono::Local; +use clap::Parser; +use serde::{Deserialize, Serialize}; + +/// A simple yet powerful I/O benchmarking tool using fio. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Target for the benchmark. + /// Formats: + /// - localhost (default) + /// - ssh/{user}@{host} + /// - ssh/{user}@{host}:{port} + /// - k8s/{namespace}/{pod} + #[arg(short, long, default_value = "localhost")] + target: String, + + #[arg(short, long, default_value = ".")] + benchmark_dir: String, + + /// Comma-separated list of tests to run. + /// Available tests: read, write, randread, randwrite, + /// multiread, multiwrite, multirandread, multirandwrite. + #[arg(long, default_value = "read,write,randread,randwrite,multiread,multiwrite,multirandread,multirandwrite")] + tests: String, + + /// Duration of each test in seconds. + #[arg(long, default_value_t = 15)] + duration: u64, + + /// Output directory for results. + /// Defaults to ./iobench-{current_datetime}. + #[arg(long)] + output_dir: Option, + + /// The size of the test file for fio. + #[arg(long, default_value = "1G")] + size: String, + + /// The block size for I/O operations. + #[arg(long, default_value = "4k")] + block_size: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FioOutput { + jobs: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FioJobResult { + jobname: String, + read: FioMetrics, + write: FioMetrics, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FioMetrics { + bw: f64, + iops: f64, + clat_ns: LatencyMetrics, +} + +#[derive(Debug, Serialize, Deserialize)] +struct LatencyMetrics { + mean: f64, + stddev: f64, +} + +#[derive(Debug, Serialize)] +struct BenchmarkResult { + test_name: String, + iops: f64, + bandwidth_kibps: f64, + latency_mean_ms: f64, + latency_stddev_ms: f64, +} + +fn main() -> io::Result<()> { + let args = Args::parse(); + + let output_dir = args.output_dir.unwrap_or_else(|| { + format!("./iobench-{}", Local::now().format("%Y-%m-%d-%H%M%S")) + }); + fs::create_dir_all(&output_dir)?; + + let tests_to_run: Vec<&str> = args.tests.split(',').collect(); + let mut results = Vec::new(); + + for test in tests_to_run { + println!("--------------------------------------------------"); + println!("Running test: {}", test); + + let (rw, numjobs) = match test { + "read" => ("read", 1), + "write" => ("write", 1), + "randread" => ("randread", 1), + "randwrite" => ("randwrite", 1), + "multiread" => ("read", 4), + "multiwrite" => ("write", 4), + "multirandread" => ("randread", 4), + "multirandwrite" => ("randwrite", 4), + _ => { + eprintln!("Unknown test: {}. Skipping.", test); + continue; + } + }; + + let test_name = format!("{}-{}-sync-test", test, args.block_size); + let fio_command = format!( + "fio --filename={}/iobench_testfile --direct=1 --fsync=1 --rw={} --bs={} --numjobs={} --iodepth=1 --runtime={} --time_based --group_reporting --name={} --size={} --output-format=json", + args.benchmark_dir, rw, args.block_size, numjobs, args.duration, test_name, args.size + ); + + println!("Executing command:\n{}\n", fio_command); + + let output = match run_command(&args.target, &fio_command) { + Ok(out) => out, + Err(e) => { + eprintln!("Failed to execute command for test {}: {}", test, e); + continue; + } + }; + + + let result = parse_fio_output(&output, &test_name, rw); + // TODO store raw fio output and print it + match result { + Ok(res) => { + results.push(res); + } + Err(e) => { + eprintln!("Error parsing fio output for test {}: {}", test, e); + eprintln!("Raw output:\n{}", output); + } + } + + println!("{output}"); + println!("Test {} completed.", test); + // A brief pause to let the system settle before the next test. + thread::sleep(Duration::from_secs(2)); + } + + // Cleanup the test file on the target + println!("--------------------------------------------------"); + println!("Cleaning up test file on target..."); + let cleanup_command = "rm -f ./iobench_testfile"; + if let Err(e) = run_command(&args.target, cleanup_command) { + eprintln!("Warning: Failed to clean up test file on target: {}", e); + } else { + println!("Cleanup successful."); + } + + + if results.is_empty() { + println!("\nNo benchmark results to display."); + return Ok(()); + } + + // Output results to a CSV file for easy analysis + let csv_path = format!("{}/summary.csv", output_dir); + let mut wtr = csv::Writer::from_path(&csv_path)?; + for result in &results { + wtr.serialize(result)?; + } + wtr.flush()?; + + println!("\nBenchmark summary saved to {}", csv_path); + println!("\n--- Benchmark Results Summary ---"); + println!("{:<25} {:>10} {:>18} {:>20} {:>22}", "Test Name", "IOPS", "Bandwidth (KiB/s)", "Latency Mean (ms)", "Latency StdDev (ms)"); + println!("{:-<98}", ""); + for result in results { + println!("{:<25} {:>10.2} {:>18.2} {:>20.4} {:>22.4}", result.test_name, result.iops, result.bandwidth_kibps, result.latency_mean_ms, result.latency_stddev_ms); + } + + Ok(()) +} + +fn run_command(target: &str, command: &str) -> io::Result { + let (program, args) = if target == "localhost" { + ("sudo", vec!["sh".to_string(), "-c".to_string(), command.to_string()]) + } else if target.starts_with("ssh/") { + let target_str = target.strip_prefix("ssh/").unwrap(); + let ssh_target; + let mut ssh_args = vec!["-o".to_string(), "StrictHostKeyChecking=no".to_string()]; + let port_parts: Vec<&str> = target_str.split(':').collect(); + if port_parts.len() == 2 { + ssh_target = port_parts[0].to_string(); + ssh_args.push("-p".to_string()); + ssh_args.push(port_parts[1].to_string()); + } else { + ssh_target = target_str.to_string(); + } + ssh_args.push(ssh_target); + ssh_args.push(format!("sudo sh -c '{}'", command)); + ("ssh", ssh_args) + } else if target.starts_with("k8s/") { + let parts: Vec<&str> = target.strip_prefix("k8s/").unwrap().split('/').collect(); + if parts.len() != 2 { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid k8s target format. Expected k8s/{namespace}/{pod}")); + } + let namespace = parts[0]; + let pod = parts[1]; + ("kubectl", vec!["exec".to_string(), "-n".to_string(), namespace.to_string(), pod.to_string(), "--".to_string(), "sh".to_string(), "-c".to_string(), command.to_string()]) + } else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "Invalid target format")); + }; + + let mut cmd = Command::new(program); + cmd.args(&args); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + + let child = cmd.spawn()?; + let output = child.wait_with_output()?; + + if !output.status.success() { + eprintln!("Command failed with status: {}", output.status); + io::stderr().write_all(&output.stderr)?; + return Err(io::Error::new(io::ErrorKind::Other, "Command execution failed")); + } + + String::from_utf8(output.stdout) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +fn parse_fio_output(output: &str, test_name: &str, rw: &str) -> Result { + let fio_data: FioOutput = serde_json::from_str(output) + .map_err(|e| format!("Failed to deserialize fio JSON: {}", e))?; + + let job_result = fio_data.jobs.iter() + .find(|j| j.jobname == test_name) + .ok_or_else(|| format!("Could not find job result for '{}' in fio output", test_name))?; + + let metrics = if rw.contains("read") { + &job_result.read + } else { + &job_result.write + }; + + Ok(BenchmarkResult { + test_name: test_name.to_string(), + iops: metrics.iops, + bandwidth_kibps: metrics.bw, + latency_mean_ms: metrics.clat_ns.mean / 1_000_000.0, + latency_stddev_ms: metrics.clat_ns.stddev / 1_000_000.0, + }) +} diff --git a/opnsense-config-xml/Cargo.toml b/opnsense-config-xml/Cargo.toml index ad61922..ef0d426 100644 --- a/opnsense-config-xml/Cargo.toml +++ b/opnsense-config-xml/Cargo.toml @@ -12,7 +12,7 @@ env_logger = { workspace = true } yaserde = { git = "https://github.com/jggc/yaserde.git" } yaserde_derive = { git = "https://github.com/jggc/yaserde.git" } xml-rs = "0.8" -thiserror = "1.0" +thiserror.workspace = true async-trait = { workspace = true } tokio = { workspace = true } uuid = { workspace = true }