diff --git a/examples/nats-supercluster/Cargo.toml b/examples/nats-supercluster/Cargo.toml new file mode 100644 index 0000000..fd1591a --- /dev/null +++ b/examples/nats-supercluster/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "example-nats-supercluster" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } +cidr = { workspace = true } +tokio = { workspace = true } +harmony_macros = { path = "../../harmony_macros" } +log = { workspace = true } +env_logger = { workspace = true } +url = { workspace = true } diff --git a/examples/nats-supercluster/env_example.sh b/examples/nats-supercluster/env_example.sh new file mode 100644 index 0000000..cb23647 --- /dev/null +++ b/examples/nats-supercluster/env_example.sh @@ -0,0 +1,6 @@ +# Cluster 1 +export HARMONY_NATS_SITE_1="kubeconfig=$HOME/.config/nt/kube/config,context=your_cluster_1_kube_context_name" +export HARMONY_NATS_SITE_1_DOMAIN="your_cluster_1_public_domain" +# Cluster 2 +export HARMONY_NATS_SITE_2="kubeconfig=$HOME/.config/nt/kube/config,context=your_cluster_2_kube_context_name" +export HARMONY_NATS_SITE_2_DOMAIN="your_cluster_2_public_domain" diff --git a/examples/nats-supercluster/src/main.rs b/examples/nats-supercluster/src/main.rs new file mode 100644 index 0000000..9af3948 --- /dev/null +++ b/examples/nats-supercluster/src/main.rs @@ -0,0 +1,196 @@ +use std::str::FromStr; + +use harmony::{ + inventory::Inventory, + modules::helm::chart::{HelmChartScore, HelmRepository, NonBlankString}, + topology::{HelmCommand, K8sAnywhereConfig, K8sAnywhereTopology, TlsRouter, Topology}, +}; +use harmony_macros::hurl; +use log::{debug, info}; + +#[tokio::main] +async fn main() { + let site1_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_1", + )); + let site2_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_2", + )); + let (t1, t2) = tokio::join!(site1_topo.ensure_ready(), site2_topo.ensure_ready(),); + + t1.unwrap(); + t2.unwrap(); + + let site1_domain = std::env::var("HARMONY_NATS_SITE_1_DOMAIN") + .expect("HARMONY_NATS_SITE_1_DOMAIN env var not found"); + let site2_domain = std::env::var("HARMONY_NATS_SITE_2_DOMAIN") + .expect("HARMONY_NATS_SITE_2_DOMAIN env var not found"); + + // TODO automate creation of this ca bundle + // It is simply a secret that contains one key ca.crt + // And the value is the base64 with each clusters ca.crt concatenated + let supercluster_ca_secret_name = "nats-supercluster-ca-bundle"; + + let nats_site_1 = NatsCluster { + replicas: 1, + name: "nats-site1", + gateway_advertise: format!("nats-site1-gw.{site1_domain}:443"), + supercluster_ca_secret_name, + tls_secret_name: "nats-gateway-tls", + jetstream_enabled: "false", + }; + + let nats_site_2 = NatsCluster { + replicas: 1, + name: "nats-site2", + gateway_advertise: format!("nats-site2-gw.{site2_domain}:443"), + supercluster_ca_secret_name, + tls_secret_name: "nats-gateway-tls", + jetstream_enabled: "false", + }; + + tokio::join!( + deploy_nats( + site1_topo, + &nats_site_1, + vec![&nats_site_2] + ), + deploy_nats( + site2_topo, + &nats_site_2, + vec![&nats_site_1] + ), + ); +} + +struct NatsCluster { + replicas: usize, + name: &'static str, + gateway_advertise: String, + supercluster_ca_secret_name: &'static str, + tls_secret_name: &'static str, + jetstream_enabled: &'static str, +} + +async fn deploy_nats( + topology: T, + cluster: &NatsCluster, + peers: Vec<&NatsCluster>, +) { + let mut gateway_gateways = String::new(); + for peer in peers { + // Construct wss:// URLs on port 443 for the remote gateways + gateway_gateways.push_str(&format!( + r#" + - name: {} + urls: + - nats://{}"#, + peer.name, peer.gateway_advertise + )); + } + let domain = topology.get_internal_domain().await.unwrap().unwrap(); + + // Inject gateway config into the 'merge' block to comply with chart structure + let values_yaml = Some(format!( + r#"config: + merge: + authorization: + default_permissions: + publish: ["TEST.*"] + subscribe: ["PUBLIC.>"] + users: + # - user: "admin" + # password: "admin_1" + # permissions: + # publish: ">" + # subscribe: ">" + - password: "enGk0cgZUabM6bN6FXHT" + user: "testUser" + accounts: + system: + users: + - user: "admin" + password: "admin_2" + logtime: true + debug: true + trace: true + system_account: system + cluster: + name: {cluster_name} + enabled: true + replicas: {replicas} + jetstream: + enabled: {jetstream_enabled} + fileStorage: + enabled: true + size: 10Gi + storageDirectory: /data/jetstream + leafnodes: + enabled: false + websocket: + enabled: false + ingress: + enabled: true + className: openshift-default + pathType: Prefix + hosts: + - nats-ws.{domain} + gateway: + enabled: true + port: 7222 + name: {cluster_name} + merge: + advertise: {gateway_advertise} + gateways: {gateway_gateways} + tls: + enabled: true + secretName: {tls_secret_name} + # merge: + # ca_file: "/etc/nats-certs/gateway/ca.crt" +service: + ports: + gateway: + enabled: true +tlsCA: + enabled: true + secretName: {supercluster_ca_secret_name} +natsBox: + container: + image: + tag: nonroot"#, + cluster_name = cluster.name, + replicas = cluster.replicas, + domain = domain, + gateway_gateways = gateway_gateways, + gateway_advertise = cluster.gateway_advertise, + tls_secret_name = cluster.tls_secret_name, + jetstream_enabled = cluster.jetstream_enabled, +supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, + )); + let namespace = "harmony-nats"; + + debug!("Prepared Helm Chart values : \n{values_yaml:#?}"); + let nats = HelmChartScore { + namespace: Some(NonBlankString::from_str(namespace).unwrap()), + release_name: NonBlankString::from_str(&cluster.name).unwrap(), + chart_name: NonBlankString::from_str("nats/nats").unwrap(), + chart_version: None, + values_overrides: None, + values_yaml, + create_namespace: true, + install_only: false, + repository: Some(HelmRepository::new( + "nats".to_string(), + hurl!("https://nats-io.github.io/k8s/helm/charts/"), + true, + )), + }; + + harmony_cli::run(Inventory::autoload(), topology, vec![Box::new(nats)], None) + .await + .unwrap(); + + info!( + "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`" + ); +}