All checks were successful
Run Check Script / check (pull_request) Successful in 2m6s
- Updated harmony-k8s doc tests to import from harmony_k8s instead of harmony - Changed CloudNativePgOperatorScore::default() to default_openshift() This ensures doc tests work correctly after moving K8sClient to the harmony-k8s crate.
614 lines
20 KiB
Rust
614 lines
20 KiB
Rust
use std::collections::BTreeMap;
|
|
use std::time::Duration;
|
|
|
|
use crate::KubernetesDistribution;
|
|
|
|
use super::bundle::ResourceBundle;
|
|
use super::config::PRIVILEGED_POD_IMAGE;
|
|
use k8s_openapi::api::core::v1::{
|
|
Container, HostPathVolumeSource, Pod, PodSpec, SecurityContext, Volume, VolumeMount,
|
|
};
|
|
use k8s_openapi::api::rbac::v1::{ClusterRoleBinding, RoleRef, Subject};
|
|
use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
|
|
use kube::api::DynamicObject;
|
|
use kube::error::DiscoveryError;
|
|
use log::{debug, error, info, warn};
|
|
use serde::de::DeserializeOwned;
|
|
|
|
#[derive(Debug)]
|
|
pub struct PrivilegedPodConfig {
|
|
pub name: String,
|
|
pub namespace: String,
|
|
pub node_name: String,
|
|
pub container_name: String,
|
|
pub command: Vec<String>,
|
|
pub volumes: Vec<Volume>,
|
|
pub volume_mounts: Vec<VolumeMount>,
|
|
pub host_pid: bool,
|
|
pub host_network: bool,
|
|
}
|
|
|
|
impl Default for PrivilegedPodConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
name: "privileged-pod".to_string(),
|
|
namespace: "harmony".to_string(),
|
|
node_name: "".to_string(),
|
|
container_name: "privileged-container".to_string(),
|
|
command: vec![],
|
|
volumes: vec![],
|
|
volume_mounts: vec![],
|
|
host_pid: false,
|
|
host_network: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn build_privileged_pod(
|
|
config: PrivilegedPodConfig,
|
|
k8s_distribution: &KubernetesDistribution,
|
|
) -> Pod {
|
|
let annotations = match k8s_distribution {
|
|
KubernetesDistribution::OpenshiftFamily => Some(BTreeMap::from([
|
|
("openshift.io/scc".to_string(), "privileged".to_string()),
|
|
(
|
|
"openshift.io/required-scc".to_string(),
|
|
"privileged".to_string(),
|
|
),
|
|
])),
|
|
_ => None,
|
|
};
|
|
|
|
Pod {
|
|
metadata: ObjectMeta {
|
|
name: Some(config.name),
|
|
namespace: Some(config.namespace),
|
|
annotations,
|
|
..Default::default()
|
|
},
|
|
spec: Some(PodSpec {
|
|
node_name: Some(config.node_name),
|
|
restart_policy: Some("Never".to_string()),
|
|
host_pid: Some(config.host_pid),
|
|
host_network: Some(config.host_network),
|
|
containers: vec![Container {
|
|
name: config.container_name,
|
|
image: Some(PRIVILEGED_POD_IMAGE.to_string()),
|
|
command: Some(config.command),
|
|
security_context: Some(SecurityContext {
|
|
privileged: Some(true),
|
|
..Default::default()
|
|
}),
|
|
volume_mounts: Some(config.volume_mounts),
|
|
..Default::default()
|
|
}],
|
|
volumes: Some(config.volumes),
|
|
..Default::default()
|
|
}),
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
pub fn host_root_volume() -> (Volume, VolumeMount) {
|
|
(
|
|
Volume {
|
|
name: "host".to_string(),
|
|
host_path: Some(HostPathVolumeSource {
|
|
path: "/".to_string(),
|
|
..Default::default()
|
|
}),
|
|
..Default::default()
|
|
},
|
|
VolumeMount {
|
|
name: "host".to_string(),
|
|
mount_path: "/host".to_string(),
|
|
..Default::default()
|
|
},
|
|
)
|
|
}
|
|
|
|
/// Build a ResourceBundle containing a privileged pod and any required RBAC.
|
|
///
|
|
/// This function implements the Resource Bundle pattern to encapsulate platform-specific
|
|
/// security requirements for running privileged operations on nodes.
|
|
///
|
|
/// # Platform-Specific Behavior
|
|
///
|
|
/// - **OpenShift**: Creates a ClusterRoleBinding to grant the default ServiceAccount
|
|
/// access to the `system:openshift:scc:privileged` ClusterRole, which allows the pod
|
|
/// to use the privileged Security Context Constraint (SCC).
|
|
/// - **Standard Kubernetes/K3s**: Only creates the Pod resource, as these distributions
|
|
/// use standard PodSecurityPolicy or don't enforce additional security constraints.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `config` - Configuration for the privileged pod (name, namespace, command, etc.)
|
|
/// * `k8s_distribution` - The detected Kubernetes distribution to determine RBAC requirements
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A `ResourceBundle` containing 1-2 resources:
|
|
/// - ClusterRoleBinding (OpenShift only)
|
|
/// - Pod (all distributions)
|
|
///
|
|
/// # Example
|
|
///
|
|
/// ```
|
|
/// use harmony_k8s::helper::{build_privileged_bundle, PrivilegedPodConfig};
|
|
/// use harmony_k8s::KubernetesDistribution;
|
|
/// let bundle = build_privileged_bundle(
|
|
/// PrivilegedPodConfig {
|
|
/// name: "network-setup".to_string(),
|
|
/// namespace: "default".to_string(),
|
|
/// node_name: "worker-01".to_string(),
|
|
/// container_name: "setup".to_string(),
|
|
/// command: vec!["nmcli".to_string(), "connection".to_string(), "reload".to_string()],
|
|
/// ..Default::default()
|
|
/// },
|
|
/// &KubernetesDistribution::OpenshiftFamily,
|
|
/// );
|
|
/// // Bundle now contains ClusterRoleBinding + Pod
|
|
/// ```
|
|
pub fn build_privileged_bundle(
|
|
config: PrivilegedPodConfig,
|
|
k8s_distribution: &KubernetesDistribution,
|
|
) -> ResourceBundle {
|
|
debug!(
|
|
"Building privileged bundle for config {config:#?} on distribution {k8s_distribution:?}"
|
|
);
|
|
let mut bundle = ResourceBundle::new();
|
|
let pod_name = config.name.clone();
|
|
let namespace = config.namespace.clone();
|
|
|
|
// 1. On OpenShift, create RBAC binding to privileged SCC
|
|
if let KubernetesDistribution::OpenshiftFamily = k8s_distribution {
|
|
// The default ServiceAccount needs to be bound to the privileged SCC
|
|
// via the system:openshift:scc:privileged ClusterRole
|
|
let crb = ClusterRoleBinding {
|
|
metadata: ObjectMeta {
|
|
name: Some(format!("{}-scc-binding", pod_name)),
|
|
..Default::default()
|
|
},
|
|
role_ref: RoleRef {
|
|
api_group: "rbac.authorization.k8s.io".to_string(),
|
|
kind: "ClusterRole".to_string(),
|
|
name: "system:openshift:scc:privileged".to_string(),
|
|
},
|
|
subjects: Some(vec![Subject {
|
|
kind: "ServiceAccount".to_string(),
|
|
name: "default".to_string(),
|
|
namespace: Some(namespace.clone()),
|
|
api_group: None,
|
|
..Default::default()
|
|
}]),
|
|
};
|
|
bundle.add(crb);
|
|
}
|
|
|
|
// 2. Build the privileged pod
|
|
let pod = build_privileged_pod(config, k8s_distribution);
|
|
bundle.add(pod);
|
|
|
|
bundle
|
|
}
|
|
|
|
/// Action to take when a drain operation times out.
|
|
pub enum DrainTimeoutAction {
|
|
/// Accept the partial drain and continue
|
|
Accept,
|
|
/// Retry the drain for another timeout period
|
|
Retry,
|
|
/// Abort the drain operation
|
|
Abort,
|
|
}
|
|
|
|
/// Prompts the user to confirm acceptance of a partial drain.
|
|
///
|
|
/// Returns `Ok(true)` if the user confirms acceptance, `Ok(false)` if the user
|
|
/// chooses to retry or abort, and `Err` if the prompt system fails entirely.
|
|
pub fn prompt_drain_timeout_action(
|
|
node_name: &str,
|
|
pending_count: usize,
|
|
timeout_duration: Duration,
|
|
) -> Result<DrainTimeoutAction, kube::Error> {
|
|
let prompt_msg = format!(
|
|
"Drain operation timed out on node '{}' with {} pod(s) remaining. What would you like to do?",
|
|
node_name, pending_count
|
|
);
|
|
|
|
loop {
|
|
let choices = vec![
|
|
"Accept drain failure (requires confirmation)".to_string(),
|
|
format!("Retry drain for another {:?}", timeout_duration),
|
|
"Abort operation".to_string(),
|
|
];
|
|
|
|
let selection = inquire::Select::new(&prompt_msg, choices)
|
|
.with_help_message("Use arrow keys to navigate, Enter to select")
|
|
.prompt()
|
|
.map_err(|e| {
|
|
kube::Error::Discovery(DiscoveryError::MissingResource(format!(
|
|
"Prompt failed: {}",
|
|
e
|
|
)))
|
|
})?;
|
|
|
|
if selection.starts_with("Accept") {
|
|
// Require typed confirmation - retry until correct or user cancels
|
|
let required_confirmation = format!("yes-accept-drain:{}={}", node_name, pending_count);
|
|
|
|
let confirmation_prompt = format!(
|
|
"To accept this partial drain, type exactly: {}",
|
|
required_confirmation
|
|
);
|
|
|
|
match inquire::Text::new(&confirmation_prompt)
|
|
.with_help_message(&format!(
|
|
"This action acknowledges {} pods will remain on the node",
|
|
pending_count
|
|
))
|
|
.prompt()
|
|
{
|
|
Ok(input) if input == required_confirmation => {
|
|
warn!(
|
|
"User accepted partial drain of node '{}' with {} pods remaining (confirmation: {})",
|
|
node_name, pending_count, required_confirmation
|
|
);
|
|
return Ok(DrainTimeoutAction::Accept);
|
|
}
|
|
Ok(input) => {
|
|
warn!(
|
|
"Confirmation failed. Expected '{}', got '{}'. Please try again.",
|
|
required_confirmation, input
|
|
);
|
|
}
|
|
Err(e) => {
|
|
// User cancelled (Ctrl+C) or prompt system failed
|
|
error!("Confirmation prompt cancelled or failed: {}", e);
|
|
return Ok(DrainTimeoutAction::Abort);
|
|
}
|
|
}
|
|
} else if selection.starts_with("Retry") {
|
|
info!(
|
|
"User chose to retry drain operation for another {:?}",
|
|
timeout_duration
|
|
);
|
|
return Ok(DrainTimeoutAction::Retry);
|
|
} else {
|
|
error!("Drain operation aborted by user");
|
|
return Ok(DrainTimeoutAction::Abort);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// JSON round-trip: DynamicObject → K
|
|
///
|
|
/// Safe because the DynamicObject was produced by the apiserver from a
|
|
/// payload that was originally serialized from K, so the schema is identical.
|
|
pub(crate) fn dyn_to_typed<K: DeserializeOwned>(obj: DynamicObject) -> Result<K, kube::Error> {
|
|
serde_json::to_value(obj)
|
|
.and_then(serde_json::from_value)
|
|
.map_err(kube::Error::SerdeError)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use pretty_assertions::assert_eq;
|
|
|
|
#[test]
|
|
fn test_host_root_volume() {
|
|
let (volume, mount) = host_root_volume();
|
|
|
|
assert_eq!(volume.name, "host");
|
|
assert_eq!(volume.host_path.as_ref().unwrap().path, "/");
|
|
|
|
assert_eq!(mount.name, "host");
|
|
assert_eq!(mount.mount_path, "/host");
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_pod_minimal() {
|
|
let pod = build_privileged_pod(
|
|
PrivilegedPodConfig {
|
|
name: "minimal-pod".to_string(),
|
|
namespace: "kube-system".to_string(),
|
|
node_name: "node-123".to_string(),
|
|
container_name: "debug-container".to_string(),
|
|
command: vec!["sleep".to_string(), "3600".to_string()],
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::Default,
|
|
);
|
|
|
|
assert_eq!(pod.metadata.name, Some("minimal-pod".to_string()));
|
|
assert_eq!(pod.metadata.namespace, Some("kube-system".to_string()));
|
|
|
|
let spec = pod.spec.as_ref().expect("Pod spec should be present");
|
|
assert_eq!(spec.node_name, Some("node-123".to_string()));
|
|
assert_eq!(spec.restart_policy, Some("Never".to_string()));
|
|
assert_eq!(spec.host_pid, Some(false));
|
|
assert_eq!(spec.host_network, Some(false));
|
|
|
|
assert_eq!(spec.containers.len(), 1);
|
|
let container = &spec.containers[0];
|
|
assert_eq!(container.name, "debug-container");
|
|
assert_eq!(container.image, Some(PRIVILEGED_POD_IMAGE.to_string()));
|
|
assert_eq!(
|
|
container.command,
|
|
Some(vec!["sleep".to_string(), "3600".to_string()])
|
|
);
|
|
|
|
// Security context check
|
|
let sec_ctx = container
|
|
.security_context
|
|
.as_ref()
|
|
.expect("Security context missing");
|
|
assert_eq!(sec_ctx.privileged, Some(true));
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_pod_with_volumes_and_host_access() {
|
|
let (host_vol, host_mount) = host_root_volume();
|
|
|
|
let pod = build_privileged_pod(
|
|
PrivilegedPodConfig {
|
|
name: "full-pod".to_string(),
|
|
namespace: "default".to_string(),
|
|
node_name: "node-1".to_string(),
|
|
container_name: "runner".to_string(),
|
|
command: vec!["/bin/sh".to_string()],
|
|
volumes: vec![host_vol.clone()],
|
|
volume_mounts: vec![host_mount.clone()],
|
|
host_pid: true,
|
|
host_network: true,
|
|
},
|
|
&KubernetesDistribution::Default,
|
|
);
|
|
|
|
let spec = pod.spec.as_ref().expect("Pod spec should be present");
|
|
assert_eq!(spec.host_pid, Some(true));
|
|
assert_eq!(spec.host_network, Some(true));
|
|
|
|
// Check volumes in Spec
|
|
let volumes = spec.volumes.as_ref().expect("Volumes should be present");
|
|
assert_eq!(volumes.len(), 1);
|
|
assert_eq!(volumes[0].name, "host");
|
|
|
|
// Check mounts in Container
|
|
let container = &spec.containers[0];
|
|
let mounts = container
|
|
.volume_mounts
|
|
.as_ref()
|
|
.expect("Mounts should be present");
|
|
assert_eq!(mounts.len(), 1);
|
|
assert_eq!(mounts[0].name, "host");
|
|
assert_eq!(mounts[0].mount_path, "/host");
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_pod_structure_correctness() {
|
|
// This test validates that the construction logic puts things in the right places
|
|
// effectively validating the "template".
|
|
|
|
let custom_vol = Volume {
|
|
name: "custom-vol".to_string(),
|
|
..Default::default()
|
|
};
|
|
let custom_mount = VolumeMount {
|
|
name: "custom-vol".to_string(),
|
|
mount_path: "/custom".to_string(),
|
|
..Default::default()
|
|
};
|
|
|
|
let pod = build_privileged_pod(
|
|
PrivilegedPodConfig {
|
|
name: "structure-test".to_string(),
|
|
namespace: "test-ns".to_string(),
|
|
node_name: "test-node".to_string(),
|
|
container_name: "test-container".to_string(),
|
|
command: vec!["cmd".to_string()],
|
|
volumes: vec![custom_vol],
|
|
volume_mounts: vec![custom_mount],
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::Default,
|
|
);
|
|
|
|
// Validate structure depth
|
|
let spec = pod.spec.as_ref().unwrap();
|
|
|
|
// 1. Spec level fields
|
|
assert!(spec.node_name.is_some());
|
|
assert!(spec.volumes.is_some());
|
|
|
|
// 2. Container level fields
|
|
let container = &spec.containers[0];
|
|
assert!(container.security_context.is_some());
|
|
assert!(container.volume_mounts.is_some());
|
|
|
|
// 3. Nested fields
|
|
assert!(
|
|
container
|
|
.security_context
|
|
.as_ref()
|
|
.unwrap()
|
|
.privileged
|
|
.unwrap()
|
|
);
|
|
assert_eq!(spec.volumes.as_ref().unwrap()[0].name, "custom-vol");
|
|
assert_eq!(
|
|
container.volume_mounts.as_ref().unwrap()[0].mount_path,
|
|
"/custom"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_bundle_default_distribution() {
|
|
let bundle = build_privileged_bundle(
|
|
PrivilegedPodConfig {
|
|
name: "test-bundle".to_string(),
|
|
namespace: "test-ns".to_string(),
|
|
node_name: "node-1".to_string(),
|
|
container_name: "test-container".to_string(),
|
|
command: vec!["echo".to_string(), "hello".to_string()],
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::Default,
|
|
);
|
|
|
|
// For Default distribution, only the Pod should be in the bundle
|
|
assert_eq!(bundle.resources.len(), 1);
|
|
|
|
let pod_obj = &bundle.resources[0];
|
|
assert_eq!(pod_obj.metadata.name.as_deref(), Some("test-bundle"));
|
|
assert_eq!(pod_obj.metadata.namespace.as_deref(), Some("test-ns"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_bundle_openshift_distribution() {
|
|
let bundle = build_privileged_bundle(
|
|
PrivilegedPodConfig {
|
|
name: "test-bundle-ocp".to_string(),
|
|
namespace: "test-ns".to_string(),
|
|
node_name: "node-1".to_string(),
|
|
container_name: "test-container".to_string(),
|
|
command: vec!["echo".to_string(), "hello".to_string()],
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::OpenshiftFamily,
|
|
);
|
|
|
|
// For OpenShift, both ClusterRoleBinding and Pod should be in the bundle
|
|
assert_eq!(bundle.resources.len(), 2);
|
|
|
|
// First resource should be the ClusterRoleBinding
|
|
let crb_obj = &bundle.resources[0];
|
|
assert_eq!(
|
|
crb_obj.metadata.name.as_deref(),
|
|
Some("test-bundle-ocp-scc-binding")
|
|
);
|
|
|
|
// Verify it's targeting the privileged SCC
|
|
if let Some(role_ref) = crb_obj.data.get("roleRef") {
|
|
assert_eq!(
|
|
role_ref.get("name").and_then(|v| v.as_str()),
|
|
Some("system:openshift:scc:privileged")
|
|
);
|
|
}
|
|
|
|
// Second resource should be the Pod
|
|
let pod_obj = &bundle.resources[1];
|
|
assert_eq!(pod_obj.metadata.name.as_deref(), Some("test-bundle-ocp"));
|
|
assert_eq!(pod_obj.metadata.namespace.as_deref(), Some("test-ns"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_build_privileged_bundle_k3s_distribution() {
|
|
let bundle = build_privileged_bundle(
|
|
PrivilegedPodConfig {
|
|
name: "test-bundle-k3s".to_string(),
|
|
namespace: "test-ns".to_string(),
|
|
node_name: "node-1".to_string(),
|
|
container_name: "test-container".to_string(),
|
|
command: vec!["echo".to_string(), "hello".to_string()],
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::K3sFamily,
|
|
);
|
|
|
|
// For K3s, only the Pod should be in the bundle (no special SCC)
|
|
assert_eq!(bundle.resources.len(), 1);
|
|
|
|
let pod_obj = &bundle.resources[0];
|
|
assert_eq!(pod_obj.metadata.name.as_deref(), Some("test-bundle-k3s"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_pod_yaml_rendering_expected() {
|
|
let pod = build_privileged_pod(
|
|
PrivilegedPodConfig {
|
|
name: "pod_name".to_string(),
|
|
namespace: "pod_namespace".to_string(),
|
|
node_name: "node name".to_string(),
|
|
container_name: "container name".to_string(),
|
|
command: vec!["command".to_string(), "argument".to_string()],
|
|
host_pid: true,
|
|
host_network: true,
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::Default,
|
|
);
|
|
|
|
assert_eq!(
|
|
&serde_yaml::to_string(&pod).unwrap(),
|
|
"apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: pod_name
|
|
namespace: pod_namespace
|
|
spec:
|
|
containers:
|
|
- command:
|
|
- command
|
|
- argument
|
|
image: hub.nationtech.io/redhat/ubi10:latest
|
|
name: container name
|
|
securityContext:
|
|
privileged: true
|
|
volumeMounts: []
|
|
hostNetwork: true
|
|
hostPID: true
|
|
nodeName: node name
|
|
restartPolicy: Never
|
|
volumes: []
|
|
"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_pod_yaml_rendering_openshift() {
|
|
let pod = build_privileged_pod(
|
|
PrivilegedPodConfig {
|
|
name: "pod_name".to_string(),
|
|
namespace: "pod_namespace".to_string(),
|
|
node_name: "node name".to_string(),
|
|
container_name: "container name".to_string(),
|
|
command: vec!["command".to_string(), "argument".to_string()],
|
|
host_pid: true,
|
|
host_network: true,
|
|
..Default::default()
|
|
},
|
|
&KubernetesDistribution::OpenshiftFamily,
|
|
);
|
|
|
|
assert_eq!(
|
|
&serde_yaml::to_string(&pod).unwrap(),
|
|
"apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
annotations:
|
|
openshift.io/required-scc: privileged
|
|
openshift.io/scc: privileged
|
|
name: pod_name
|
|
namespace: pod_namespace
|
|
spec:
|
|
containers:
|
|
- command:
|
|
- command
|
|
- argument
|
|
image: hub.nationtech.io/redhat/ubi10:latest
|
|
name: container name
|
|
securityContext:
|
|
privileged: true
|
|
volumeMounts: []
|
|
hostNetwork: true
|
|
hostPID: true
|
|
nodeName: node name
|
|
restartPolicy: Never
|
|
volumes: []
|
|
"
|
|
);
|
|
}
|
|
}
|