NationTech/harmony
6
0
Fork 1
Code Issues 32 Pull Requests 7 Actions 7 Packages Projects Releases 1 Wiki Activity

feat: OKD bootstrap automation pretty much complete with a few prompt for manual steps
Some checks failed
Run Check Script / check (pull_request) Failing after 1m12s
Details

Browse Source
This commit is contained in:
Jean-Gabriel Gill-Couture 2025-09-03 00:00:35 -04:00
parent 6f746d4c88
commit f1209b3823
14 changed files with 174 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/okd_installation/src/topology.rs
View File

@ -54,7 +54,7 @@ pub async fn get_topology() -> HAClusterTopology {
name: "master".to_string(), name: "master".to_string(),
}], }],
bootstrap_host: LogicalHost { bootstrap_host: LogicalHost {
ip: ip!("192.168.1.20"), ip: ip!("192.168.1.10"),
name: "bootstrap".to_string(), name: "bootstrap".to_string(),
}, },
workers: vec![], workers: vec![],

12
harmony/src/domain/topology/ha_cluster.rs
View File

@ -237,7 +237,11 @@ impl Router for HAClusterTopology {
#[async_trait] #[async_trait]
impl HttpServer for HAClusterTopology { impl HttpServer for HAClusterTopology {
async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError> { async fn serve_files(
&self,
url: &Url,
remote_path: &Option<String>,
) -> Result<(), ExecutorError> {
self.http_server.serve_files(url, remote_path).await self.http_server.serve_files(url, remote_path).await
} }
@ -397,7 +401,11 @@ impl TftpServer for DummyInfra {
#[async_trait] #[async_trait]
impl HttpServer for DummyInfra { impl HttpServer for DummyInfra {
async fn serve_files(&self, _url: &Url, _remote_path: &Option<String>) -> Result<(), ExecutorError> { async fn serve_files(
&self,
_url: &Url,
_remote_path: &Option<String>,
) -> Result<(), ExecutorError> {
unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA) unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
} }
async fn serve_file_content(&self, _file: &FileContent) -> Result<(), ExecutorError> { async fn serve_file_content(&self, _file: &FileContent) -> Result<(), ExecutorError> {

6
harmony/src/domain/topology/http.rs
View File

@ -5,7 +5,11 @@ use harmony_types::net::IpAddress;
use harmony_types::net::Url; use harmony_types::net::Url;
#[async_trait] #[async_trait]
pub trait HttpServer: Send + Sync { pub trait HttpServer: Send + Sync {
async fn serve_files(&self, url: &Url, remote_path: &Option<String>) -> Result<(), ExecutorError>; async fn serve_files(
&self,
url: &Url,
remote_path: &Option<String>,
) -> Result<(), ExecutorError>;
async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError>; async fn serve_file_content(&self, file: &FileContent) -> Result<(), ExecutorError>;
fn get_ip(&self) -> IpAddress; fn get_ip(&self) -> IpAddress;

81
harmony/src/infra/opnsense/dns.rs
View File

@ -12,21 +12,22 @@ use super::OPNSenseFirewall;
#[async_trait] #[async_trait]
impl DnsServer for OPNSenseFirewall { impl DnsServer for OPNSenseFirewall {
async fn register_hosts(&self, hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> { async fn register_hosts(&self, hosts: Vec<DnsRecord>) -> Result<(), ExecutorError> {
let mut writable_opnsense = self.opnsense_config.write().await; todo!("Refactor this to use dnsmasq")
let mut dns = writable_opnsense.dns(); // let mut writable_opnsense = self.opnsense_config.write().await;
let hosts = hosts // let mut dns = writable_opnsense.dns();
.iter() // let hosts = hosts
.map(|h| { // .iter()
Host::new( // .map(|h| {
h.host.clone(), // Host::new(
h.domain.clone(), // h.host.clone(),
h.record_type.to_string(), // h.domain.clone(),
h.value.to_string(), // h.record_type.to_string(),
) // h.value.to_string(),
}) // )
.collect(); // })
dns.register_hosts(hosts); // .collect();
Ok(()) // dns.add_static_mapping(hosts);
// Ok(())
} }
fn remove_record( fn remove_record(
@ -38,25 +39,26 @@ impl DnsServer for OPNSenseFirewall {
} }
async fn list_records(&self) -> Vec<crate::topology::DnsRecord> { async fn list_records(&self) -> Vec<crate::topology::DnsRecord> {
self.opnsense_config todo!("Refactor this to use dnsmasq")
.write() // self.opnsense_config
.await // .write()
.dns() // .await
.get_hosts() // .dns()
.iter() // .get_hosts()
.map(|h| DnsRecord { // .iter()
host: h.hostname.clone(), // .map(|h| DnsRecord {
domain: h.domain.clone(), // host: h.hostname.clone(),
record_type: h // domain: h.domain.clone(),
.rr // record_type: h
.parse() // .rr
.expect("received invalid record type {h.rr} from opnsense"), // .parse()
value: h // .expect("received invalid record type {h.rr} from opnsense"),
.server // value: h
.parse() // .server
.expect("received invalid ipv4 record from opnsense {h.server}"), // .parse()
}) // .expect("received invalid ipv4 record from opnsense {h.server}"),
.collect() // })
// .collect()
} }
fn get_ip(&self) -> IpAddress { fn get_ip(&self) -> IpAddress {
@ -68,11 +70,12 @@ impl DnsServer for OPNSenseFirewall {
} }
async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> { async fn register_dhcp_leases(&self, register: bool) -> Result<(), ExecutorError> {
let mut writable_opnsense = self.opnsense_config.write().await; todo!("Refactor this to use dnsmasq")
let mut dns = writable_opnsense.dns(); // let mut writable_opnsense = self.opnsense_config.write().await;
dns.register_dhcp_leases(register); // let mut dns = writable_opnsense.dns();
// dns.register_dhcp_leases(register);
Ok(()) //
// Ok(())
} }
async fn commit_config(&self) -> Result<(), ExecutorError> { async fn commit_config(&self) -> Result<(), ExecutorError> {

9
harmony/src/modules/http.rs
View File

@ -24,8 +24,9 @@ use harmony_types::{id::Id, net::MacAddress};
/// } /// }
/// ``` /// ```
#[derive(Debug, new, Clone, Serialize)] #[derive(Debug, new, Clone, Serialize)]
pub struct StaticFilesHttpScore { // TODO this should be split in two scores, one for folder and pub struct StaticFilesHttpScore {
// other for files // TODO this should be split in two scores, one for folder and
// other for files
pub folder_to_serve: Option<Url>, pub folder_to_serve: Option<Url>,
pub files: Vec<FileContent>, pub files: Vec<FileContent>,
pub remote_path: Option<String>, pub remote_path: Option<String>,
@ -56,7 +57,9 @@ impl<T: Topology + HttpServer> Interpret<T> for StaticFilesHttpInterpret {
http_server.ensure_initialized().await?; http_server.ensure_initialized().await?;
// http_server.set_ip(topology.router.get_gateway()).await?; // http_server.set_ip(topology.router.get_gateway()).await?;
if let Some(folder) = self.score.folder_to_serve.as_ref() { if let Some(folder) = self.score.folder_to_serve.as_ref() {
http_server.serve_files(folder, &self.score.remote_path).await?; http_server
.serve_files(folder, &self.score.remote_path)
.await?;
} }
for f in self.score.files.iter() { for f in self.score.files.iter() {

9
harmony/src/modules/okd/bootstrap_load_balancer.rs
View File

@ -54,6 +54,7 @@ impl OKDBootstrapLoadBalancerScore {
}, },
} }
} }
fn topology_to_backend_server(topology: &HAClusterTopology, port: u16) -> Vec<BackendServer> { fn topology_to_backend_server(topology: &HAClusterTopology, port: u16) -> Vec<BackendServer> {
let mut backend: Vec<_> = topology let mut backend: Vec<_> = topology
.control_plane .control_plane
@ -63,6 +64,14 @@ impl OKDBootstrapLoadBalancerScore {
port, port,
}) })
.collect(); .collect();
topology.workers.iter().for_each(|worker| {
backend.push(BackendServer {
address: worker.ip.to_string(),
port,
})
});
backend.push(BackendServer { backend.push(BackendServer {
address: topology.bootstrap_host.ip.to_string(), address: topology.bootstrap_host.ip.to_string(),
port, port,

101
harmony/src/modules/okd/installation.rs
View File

@ -73,7 +73,11 @@ use crate::{
dhcp::DhcpHostBindingScore, dhcp::DhcpHostBindingScore,
http::{IPxeMacBootFileScore, StaticFilesHttpScore}, http::{IPxeMacBootFileScore, StaticFilesHttpScore},
inventory::LaunchDiscoverInventoryAgentScore, inventory::LaunchDiscoverInventoryAgentScore,
okd::{dns::OKDDnsScore, templates::InstallConfigYaml}, okd::{
bootstrap_load_balancer::OKDBootstrapLoadBalancerScore,
dns::OKDDnsScore,
templates::{BootstrapIpxeTpl, InstallConfigYaml},
},
}, },
score::Score, score::Score,
topology::{HAClusterTopology, HostBinding}, topology::{HAClusterTopology, HostBinding},
@ -207,7 +211,7 @@ impl Interpret<HAClusterTopology> for OKDInstallationInterpret {
info!("Starting OKD installation pipeline",); info!("Starting OKD installation pipeline",);
// self.run_inventory_phase(inventory, topology).await?; self.run_inventory_phase(inventory, topology).await?;
self.run_bootstrap_phase(inventory, topology).await?; self.run_bootstrap_phase(inventory, topology).await?;
@ -289,9 +293,23 @@ impl Interpret<HAClusterTopology> for OKDSetup01InventoryInterpret {
topology: &HAClusterTopology, topology: &HAClusterTopology,
) -> Result<Outcome, InterpretError> { ) -> Result<Outcome, InterpretError> {
info!("Setting up base DNS config for OKD"); info!("Setting up base DNS config for OKD");
OKDDnsScore::new(topology) let cluster_domain = &topology.domain_name;
.interpret(inventory, topology) let load_balancer_ip = &topology.load_balancer.get_ip();
.await?; inquire::Confirm::new(&format!(
"Set hostnames manually in your opnsense dnsmasq config :
*.apps.{cluster_domain} -> {load_balancer_ip}
api.{cluster_domain} -> {load_balancer_ip}
api-int.{cluster_domain} -> {load_balancer_ip}
When you can dig them, confirm to continue.
"
))
.prompt()
.expect("Prompt error");
// TODO reactivate automatic dns config
// OKDDnsScore::new(topology)
// .interpret(inventory, topology)
// .await?;
info!( info!(
"Launching discovery agent, make sure that your nodes are successfully PXE booted and running inventory agent. They should answer on `http://<node_ip>:8080/inventory`" "Launching discovery agent, make sure that your nodes are successfully PXE booted and running inventory agent. They should answer on `http://<node_ip>:8080/inventory`"
@ -368,7 +386,7 @@ struct OKDSetup02BootstrapScore {}
impl Score<HAClusterTopology> for OKDSetup02BootstrapScore { impl Score<HAClusterTopology> for OKDSetup02BootstrapScore {
fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> { fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
Box::new(OKDSetup02BootstrapInterpret::new(self.clone())) Box::new(OKDSetup02BootstrapInterpret::new())
} }
fn name(&self) -> String { fn name(&self) -> String {
@ -378,17 +396,15 @@ impl Score<HAClusterTopology> for OKDSetup02BootstrapScore {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct OKDSetup02BootstrapInterpret { struct OKDSetup02BootstrapInterpret {
score: OKDSetup02BootstrapScore,
version: Version, version: Version,
status: InterpretStatus, status: InterpretStatus,
} }
impl OKDSetup02BootstrapInterpret { impl OKDSetup02BootstrapInterpret {
pub fn new(score: OKDSetup02BootstrapScore) -> Self { pub fn new() -> Self {
let version = Version::from("1.0.0").unwrap(); let version = Version::from("1.0.0").unwrap();
Self { Self {
version, version,
score,
status: InterpretStatus::QUEUED, status: InterpretStatus::QUEUED,
} }
} }
@ -572,20 +588,30 @@ impl OKDSetup02BootstrapInterpret {
Ok(output) Ok(output)
}; };
info!("Successfully prepared ignition files for OKD installation");
// ignition_files_http_path // = PathBuf::from("okd_ignition_files"); // ignition_files_http_path // = PathBuf::from("okd_ignition_files");
let scos_http_path = PathBuf::from("scos");
info!( info!(
r#"Uploading images, they can be refreshed with a command similar to this one: openshift-install coreos print-stream-json | grep -Eo '"https.*(kernel.|initramfs.|rootfs.)\w+(\.img)?"' | grep x86_64 | xargs -n 1 curl -LO"# r#"Uploading images, they can be refreshed with a command similar to this one: openshift-install coreos print-stream-json | grep -Eo '"https.*(kernel.|initramfs.|rootfs.)\w+(\.img)?"' | grep x86_64 | xargs -n 1 curl -LO"#
); );
StaticFilesHttpScore {
folder_to_serve: Some(Url::LocalFolder(okd_images_path.to_string_lossy().to_string())),
remote_path: Some(scos_http_path.to_string_lossy().to_string()),
files: vec![],
}
.interpret(inventory, topology)
.await?;
todo!("What's up next?") warn!(
"TODO push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved"
);
inquire::Confirm::new(
"push installer image files with `scp -r data/okd/installer_image/* root@192.168.1.1:/usr/local/http/scos/` until performance issue is resolved").prompt().expect("Prompt error");
// let scos_http_path = PathBuf::from("scos");
// StaticFilesHttpScore {
// folder_to_serve: Some(Url::LocalFolder(
// okd_images_path.to_string_lossy().to_string(),
// )),
// remote_path: Some(scos_http_path.to_string_lossy().to_string()),
// files: vec![],
// }
// .interpret(inventory, topology)
// .await?;
Ok(())
} }
async fn configure_host_binding( async fn configure_host_binding(
@ -613,12 +639,25 @@ impl OKDSetup02BootstrapInterpret {
inventory: &Inventory, inventory: &Inventory,
topology: &HAClusterTopology, topology: &HAClusterTopology,
) -> Result<(), InterpretError> { ) -> Result<(), InterpretError> {
// Placeholder: use Harmony templates to emit {MAC}.ipxe selecting SCOS live + bootstrap ignition. let content = BootstrapIpxeTpl {
info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node"); http_ip: &topology.http_server.get_ip().to_string(),
scos_path: "scos", // TODO use some constant
installation_device: "/dev/sda", // TODO do something smart based on the host drives
// topology. Something like use the smallest device
// above 200G that is an ssd
}
.to_string();
let bootstrap_node = self.get_bootstrap_node().await?; let bootstrap_node = self.get_bootstrap_node().await?;
let mac_address = bootstrap_node.get_mac_address();
info!("[Bootstrap] Rendering per-MAC PXE for bootstrap node");
debug!("bootstrap ipxe content : {content}");
debug!("bootstrap mac addresses : {mac_address:?}");
IPxeMacBootFileScore { IPxeMacBootFileScore {
mac_address: bootstrap_node.get_mac_address(), mac_address,
content: todo!("templace for bootstrap node"), content,
} }
.interpret(inventory, topology) .interpret(inventory, topology)
.await?; .await?;
@ -630,15 +669,25 @@ impl OKDSetup02BootstrapInterpret {
inventory: &Inventory, inventory: &Inventory,
topology: &HAClusterTopology, topology: &HAClusterTopology,
) -> Result<(), InterpretError> { ) -> Result<(), InterpretError> {
todo!( let outcome = OKDBootstrapLoadBalancerScore::new(topology)
"OKD loadbalancer score already exists, just call it here probably? 6443 22623, 80 and 443 \n\nhttps://docs.okd.io/latest/installing/installing_bare_metal/upi/installing-bare-metal.html#installation-load-balancing-user-infra_installing-bare-metal" .interpret(inventory, topology)
); .await?;
info!("Successfully executed OKDBootstrapLoadBalancerScore : {outcome:?}");
Ok(())
} }
async fn reboot_target(&self) -> Result<(), InterpretError> { async fn reboot_target(&self) -> Result<(), InterpretError> {
// Placeholder: ssh reboot using the inventory ephemeral key // Placeholder: ssh reboot using the inventory ephemeral key
info!("[Bootstrap] Rebooting bootstrap node via SSH"); info!("[Bootstrap] Rebooting bootstrap node via SSH");
todo!("[Bootstrap] Rebooting bootstrap node via SSH") // TODO reboot programatically, there are some logical checks and refactoring to do such as
// accessing the bootstrap node config (ip address) from the inventory
let confirmation = inquire::Confirm::new(
"Now reboot the bootstrap node so it picks up its pxe boot file. Press enter when ready.",
)
.with_default(true)
.prompt()
.expect("Unexpected prompt error");
Ok(())
} }
async fn wait_for_bootstrap_complete(&self) -> Result<(), InterpretError> { async fn wait_for_bootstrap_complete(&self) -> Result<(), InterpretError> {

1
harmony/src/modules/okd/ipxe.rs
View File

@ -120,6 +120,7 @@ impl<T: Topology + DhcpServer + TftpServer + HttpServer + Router> Interpret<T> f
Err(e) => return Err(e), Err(e) => return Err(e),
}; };
} }
inquire::Confirm::new("Execute the copy : `scp -r data/pxe/okd/http_files/* root@192.168.1.1:/usr/local/http/` and confirm when done to continue").prompt().expect("Prompt error");
Ok(Outcome::success("Ipxe installed".to_string())) Ok(Outcome::success("Ipxe installed".to_string()))
} }

2
harmony/src/modules/okd/mod.rs
View File

@ -5,5 +5,5 @@ pub mod dns;
pub mod installation; pub mod installation;
pub mod ipxe; pub mod ipxe;
pub mod load_balancer; pub mod load_balancer;
pub mod upgrade;
pub mod templates; pub mod templates;
pub mod upgrade;

8
harmony/src/modules/okd/templates.rs
View File

@ -8,3 +8,11 @@ pub struct InstallConfigYaml<'a> {
pub ssh_public_key: &'a str, pub ssh_public_key: &'a str,
pub cluster_name: &'a str, pub cluster_name: &'a str,
} }
#[derive(Template)]
#[template(path = "okd/bootstrap.ipxe.j2")]
pub struct BootstrapIpxeTpl<'a> {
pub http_ip: &'a str,
pub scos_path: &'a str,
pub installation_device: &'a str,
}

7
harmony/templates/okd/bootstrap.ipxe.j2 Normal file
View File

@ -0,0 +1,7 @@
set base-url http://{{ http_ip }}:8080
set scos-base-url = ${base-url}/{{ scos_path }}
set installation-device = {{ installation_device }}
kernel ${scos-base-url}/scos-live-kernel.x86_64 initrd=main coreos.live.rootfs_url=${scos-base-url}/scos-live-rootfs.x86_64.img coreos.inst.install_dev=${installation-device} coreos.inst.ignition_url=${base-url}/bootstrap.ign
initrd --name main ${scos-base-url}/scos-live-initramfs.x86_64.img
boot

6
opnsense-config/src/config/config.rs
View File

@ -4,7 +4,7 @@ use crate::{
config::{SshConfigManager, SshCredentials, SshOPNSenseShell}, config::{SshConfigManager, SshCredentials, SshOPNSenseShell},
error::Error, error::Error,
modules::{ modules::{
caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::DnsConfig, caddy::CaddyConfig, dhcp_legacy::DhcpConfigLegacyISC, dns::UnboundDnsConfig,
dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig, tftp::TftpConfig, dnsmasq::DhcpConfigDnsMasq, load_balancer::LoadBalancerConfig, tftp::TftpConfig,
}, },
}; };
@ -51,8 +51,8 @@ impl Config {
DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone()) DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone())
} }
pub fn dns(&mut self) -> DnsConfig<'_> { pub fn dns(&mut self) -> DhcpConfigDnsMasq<'_> {
DnsConfig::new(&mut self.opnsense) DhcpConfigDnsMasq::new(&mut self.opnsense, self.shell.clone())
} }
pub fn tftp(&mut self) -> TftpConfig<'_> { pub fn tftp(&mut self) -> TftpConfig<'_> {

4
opnsense-config/src/modules/dns.rs
View File

@ -1,10 +1,10 @@
use opnsense_config_xml::{Host, OPNsense}; use opnsense_config_xml::{Host, OPNsense};
pub struct DnsConfig<'a> { pub struct UnboundDnsConfig<'a> {
opnsense: &'a mut OPNsense, opnsense: &'a mut OPNsense,
} }
impl<'a> DnsConfig<'a> { impl<'a> UnboundDnsConfig<'a> {
pub fn new(opnsense: &'a mut OPNsense) -> Self { pub fn new(opnsense: &'a mut OPNsense) -> Self {
Self { opnsense } Self { opnsense }
} }

6
opnsense-config/src/modules/dnsmasq.rs
View File

@ -144,14 +144,16 @@ impl<'a> DhcpConfigDnsMasq<'a> {
let host_to_modify_ip = host_to_modify.ip.content_string(); let host_to_modify_ip = host_to_modify.ip.content_string();
if host_to_modify_ip != ip_str { if host_to_modify_ip != ip_str {
warn!( warn!(
"Hostname '{}' already exists with a different IP ({}). Appending MAC {}.", "Hostname '{}' already exists with a different IP ({}). Setting new IP {ip_str}. Appending MAC {}.",
hostname, host_to_modify_ip, mac hostname, host_to_modify_ip, mac
); );
host_to_modify.ip.content = Some(ip_str);
} else if host_to_modify.host != hostname { } else if host_to_modify.host != hostname {
warn!( warn!(
"IP {} already exists with a different hostname ('{}'). Appending MAC {}.", "IP {} already exists with a different hostname ('{}'). Setting hostname to {hostname}. Appending MAC {}.",
ipaddr, host_to_modify.host, mac ipaddr, host_to_modify.host, mac
); );
host_to_modify.host = hostname.to_string();
} }
if !host_to_modify if !host_to_modify