feat/fleet-cd-staging-deploy #310
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -4093,12 +4093,14 @@ dependencies = [
|
||||
"harmony-fleet-auth",
|
||||
"harmony-reconciler-contracts",
|
||||
"harmony_cli",
|
||||
"harmony_config",
|
||||
"harmony_macros",
|
||||
"harmony_types",
|
||||
"k8s-openapi",
|
||||
"kube",
|
||||
"log",
|
||||
"non-blank-string-rs",
|
||||
"schemars 0.8.22",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
|
||||
@@ -245,8 +245,10 @@ async fn main() -> anyhow::Result<()> {
|
||||
// Deploy + configure OpenBao (no JWT auth yet -- Zitadel isn't up)
|
||||
cleanup_openbao_webhook(&k8s).await?;
|
||||
OpenbaoScore {
|
||||
instance: Default::default(),
|
||||
host: OPENBAO_HOST.to_string(),
|
||||
openshift: false,
|
||||
tls_issuer: None,
|
||||
}
|
||||
.interpret(&Inventory::autoload(), &topology)
|
||||
.await
|
||||
|
||||
@@ -5,8 +5,10 @@ use harmony::{
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let openbao = OpenbaoScore {
|
||||
instance: Default::default(),
|
||||
host: "openbao.sebastien.sto1.nationtech.io".to_string(),
|
||||
openshift: false,
|
||||
tls_issuer: None,
|
||||
};
|
||||
|
||||
harmony_cli::run(
|
||||
|
||||
@@ -102,23 +102,17 @@ k3d cluster delete fleet-e2e
|
||||
|
||||
## Production deploys
|
||||
|
||||
`harmony-fleet-deploy` is the binary that puts the fleet stack on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach). It composes `FleetNatsScore` + `FleetOperatorScore` + `FleetAgentScore` against the topology you point it at.
|
||||
`harmony-fleet-deploy` puts the published operator chart on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach) — the `harmony apply` / CD path. It loads `FleetDeploySecrets` from config (Env → OpenBao) and runs one `FleetOperatorScore`; auth is Zitadel-SSO-only. The full bring-up stack (`FleetNatsScore` + `FleetAgentScore` + …) is composed by the e2e harness directly over the same lib Scores, not by this binary.
|
||||
|
||||
```bash
|
||||
# Default: K8sAnywhereTopology against whatever KUBECONFIG points at
|
||||
# Deploy a released tag (version parsed from it in Rust):
|
||||
cargo run -p harmony-fleet-deploy -- \
|
||||
--namespace fleet-system \
|
||||
--operator-image hub.nationtech.io/harmony/harmony-fleet-operator:dev \
|
||||
--agent-image hub.nationtech.io/harmony/harmony-fleet-agent:dev \
|
||||
--agent-device-id fleet-agent-01
|
||||
|
||||
# Pick a single component with the harmony_cli filter
|
||||
cargo run -p harmony-fleet-deploy -- \
|
||||
--namespace fleet-system \
|
||||
-- --filter FleetOperatorScore --all
|
||||
--filter FleetOperatorScore \
|
||||
--from-tag harmony-fleet-operator-v0.0.2 \
|
||||
--namespace fleet-system --yes
|
||||
```
|
||||
|
||||
`harmony-fleet-deploy` reads its full config from CLI flags + env vars (`FLEET_NAMESPACE`, `FLEET_OPERATOR_IMAGE`, …). The minimal-CLI surface is deliberate — per ADR-023 the long-term answer is a plugin-discovery layer over `harmony-*` binaries; until that lands, deploy crates stay small and use the existing `harmony_cli`.
|
||||
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
|
||||
|
||||
### Connecting to the operator
|
||||
|
||||
|
||||
@@ -28,43 +28,46 @@ cargo run -p harmony-fleet-deploy --bin harmony-fleet-release -- \
|
||||
--from-tag harmony-fleet-operator-v0.0.2 --no-push
|
||||
```
|
||||
|
||||
## 2. Deploy a published version (`harmony apply`)
|
||||
## 2. Deploy a published version to staging (manual, for now)
|
||||
|
||||
`--operator-chart-version` switches the operator to install the
|
||||
published `oci://hub.nationtech.io/harmony/harmony-fleet-operator:0.0.2`
|
||||
chart instead of rendering one from local source. Same command
|
||||
bootstraps and upgrades; re-running with the same version is a no-op.
|
||||
Push to staging is manual until headless OpenBao auth (Zitadel machine
|
||||
identity) lands; secrets still come from shared OpenBao config. Point at
|
||||
your staging kube context and OpenBao, then run the operator deploy:
|
||||
|
||||
```sh
|
||||
export HARMONY_FLEET_NATS_ADMIN_USER=… HARMONY_FLEET_NATS_ADMIN_PASS=…
|
||||
export HARMONY_FLEET_NATS_DEVICE_USER=… HARMONY_FLEET_NATS_DEVICE_PASS=…
|
||||
|
||||
harmony-fleet-deploy \
|
||||
--filter FleetOperatorScore \
|
||||
--operator-chart-version 0.0.2 \
|
||||
--namespace fleet-system \
|
||||
--yes
|
||||
export OPENBAO_URL=<your OpenBao URL>
|
||||
export OPENBAO_TOKEN=<scoped read token for secret/<ns>/*>
|
||||
harmony-fleet-deploy --filter FleetOperatorScore \
|
||||
--from-tag <release-tag> --namespace fleet-staging --yes
|
||||
```
|
||||
|
||||
It installs the published
|
||||
`oci://hub.nationtech.io/harmony/harmony-fleet-operator:<version>` chart;
|
||||
the version is parsed from the tag in Rust (the tag is the only source
|
||||
of truth). Same command bootstraps and upgrades; re-running the same tag
|
||||
is a no-op. Auth is Zitadel-SSO-only: the operator gets its zitadel-jwt
|
||||
`operator_credentials_toml` from `FleetDeploySecrets` in OpenBao (no
|
||||
user/pass on the published-chart path). For manual deploy, store that
|
||||
config **without** a `kubeconfig` field so your own kube context is used.
|
||||
|
||||
## 3. Roll forward
|
||||
|
||||
Same command, a newer (or previous-good) version:
|
||||
|
||||
```sh
|
||||
harmony-fleet-deploy --filter FleetOperatorScore --operator-chart-version 0.0.3 --namespace fleet-system --yes
|
||||
```
|
||||
|
||||
`helm upgrade --install` applies it and fails loudly if convergence
|
||||
fails — no automatic rollback. Fix the spec, bump, re-run.
|
||||
Re-run with a newer (or previous-good) tag. `helm upgrade --install`
|
||||
applies it and fails loudly if convergence fails — no automatic
|
||||
rollback. Fix the spec, bump, re-run.
|
||||
|
||||
## Automated vs. manual
|
||||
|
||||
| Step | Where |
|
||||
|---|---|
|
||||
| Build + push image + chart on tag | CI (`release` job) |
|
||||
| Deploy a published version + roll forward | Manual `harmony apply` (above) |
|
||||
| Build + push image + chart on tag | CI (`release` job, on tag) |
|
||||
| Push to staging + roll forward | Manual (operator runs the deploy) |
|
||||
|
||||
A staging-auto / production-gated CD job is a follow-up — it needs the
|
||||
cluster `KUBECONFIG` + NATS secrets provisioned, which is out of scope
|
||||
for the initial CD branch (ADR-012-2). The release job is fully
|
||||
functional today.
|
||||
## Future: in-cluster CD (blocked on headless OpenBao auth)
|
||||
|
||||
Once `harmony_config` can authenticate to OpenBao headlessly (Zitadel
|
||||
machine identity), these exports become a `deploy-staging` workflow on an
|
||||
in-cluster, permissionless Gitea runner that pulls a `fleet-deployer`
|
||||
`kubeconfig` + operator credentials from OpenBao at job time (provisioned
|
||||
via a `TenantScore` with one extra egress CIDR to the OpenBao/Zitadel
|
||||
ingress). Production-gated promotion is a later step (ADR-012-2).
|
||||
|
||||
@@ -9,9 +9,8 @@ description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, ca
|
||||
[lib]
|
||||
path = "src/lib.rs"
|
||||
|
||||
# CLI entry point. `harmony-fleet-deploy <component>` picks a subcommand
|
||||
# per fleet component plus an `all` composite. Built on harmony_cli the
|
||||
# way the rest of the workspace's *-deploy crates are.
|
||||
# CLI entry point: deploy the published operator chart (harmony apply).
|
||||
# Built on harmony_cli like the rest of the workspace's *-deploy crates.
|
||||
[[bin]]
|
||||
name = "harmony-fleet-deploy"
|
||||
path = "src/main.rs"
|
||||
@@ -25,6 +24,7 @@ path = "src/bin/harmony-fleet-release.rs"
|
||||
[dependencies]
|
||||
harmony = { path = "../../harmony", features = ["podman"] }
|
||||
harmony_cli = { path = "../../harmony_cli" }
|
||||
harmony_config = { path = "../../harmony_config" }
|
||||
harmony_types = { path = "../../harmony_types" }
|
||||
harmony_macros = { path = "../../harmony_macros" }
|
||||
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
||||
@@ -38,6 +38,7 @@ kube = { workspace = true, features = ["runtime", "derive"] }
|
||||
log = { workspace = true }
|
||||
env_logger = { workspace = true }
|
||||
non-blank-string-rs = "1"
|
||||
schemars = "0.8"
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
|
||||
@@ -32,6 +32,7 @@ pub mod companion;
|
||||
pub mod nats;
|
||||
pub mod operator;
|
||||
pub mod release;
|
||||
pub mod secrets;
|
||||
pub mod server;
|
||||
|
||||
pub use agent::{FleetAgentScore, PodTarget};
|
||||
@@ -39,4 +40,5 @@ pub use companion::AgentObservation;
|
||||
pub use nats::{FleetNatsScore, UserPassCredentials};
|
||||
pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart};
|
||||
pub use release::{release_operator, version_from_tag};
|
||||
pub use secrets::FleetDeploySecrets;
|
||||
pub use server::FleetServerScore;
|
||||
|
||||
@@ -1,39 +1,28 @@
|
||||
//! `harmony-fleet-deploy` — deploy the fleet stack to a cluster.
|
||||
//! `harmony-fleet-deploy` — deploy the published fleet operator chart
|
||||
//! (`harmony apply`). Loads its secrets from config and runs one
|
||||
//! [`FleetOperatorScore`] against [`K8sAnywhereTopology`].
|
||||
//!
|
||||
//! Built on `harmony_cli::run` like the rest of the workspace's
|
||||
//! deploy binaries (`harmony_agent_deploy`, …). The CLI offers a
|
||||
//! minimal env-driven config and hands off to `harmony_cli`, which
|
||||
//! provides the standard `--filter` / `--all` / `--list` selection
|
||||
//! surface; pick a single component by filter or run them all.
|
||||
//!
|
||||
//! Topology default is [`K8sAnywhereTopology::from_env`] — local k3d
|
||||
//! when `HARMONY_USE_LOCAL_K3D` flips, otherwise whatever cluster
|
||||
//! `KUBECONFIG` points at. Per ADR-023 the binary's full set of
|
||||
//! topologies is compile-time; for the moment only `K8sAnywhere` is
|
||||
//! wired in, with `K8sBareTopology` planned for the next iteration.
|
||||
//!
|
||||
//! What the binary owns: assembling the Scores from environment
|
||||
//! input. What it does **not** own: any handrolled k8s manifests,
|
||||
//! any imperative bring-up loops, any auth secret rendering — that
|
||||
//! all sits inside the `*Score` impls in [`harmony_fleet_deploy`].
|
||||
//! The full dev/e2e stack (NATS + agent, user/pass) is composed by the
|
||||
//! e2e harness directly over the `harmony_fleet_deploy` Scores — not
|
||||
//! here. This binary is the prod/CD path: Zitadel-SSO-only, no
|
||||
//! handrolled manifests.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::io::Write;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use harmony::inventory::Inventory;
|
||||
use harmony::score::Score;
|
||||
use harmony::topology::K8sAnywhereTopology;
|
||||
use harmony_cli::Args as HarmonyCliArgs;
|
||||
use harmony_fleet_deploy::nats::UserPassCredentials;
|
||||
use harmony_fleet_deploy::{FleetAgentScore, FleetNatsScore, FleetOperatorScore, agent::PodTarget};
|
||||
use harmony_config::ConfigClient;
|
||||
use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, version_from_tag};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "harmony-fleet-deploy",
|
||||
about = "Deploy the harmony fleet stack to a Kubernetes cluster"
|
||||
about = "Deploy the published harmony fleet operator chart"
|
||||
)]
|
||||
struct CliConfig {
|
||||
/// Namespace every component lands in. Override with
|
||||
/// `HARMONY_FLEET_NAMESPACE`.
|
||||
#[arg(
|
||||
long,
|
||||
env = "HARMONY_FLEET_NAMESPACE",
|
||||
@@ -41,59 +30,13 @@ struct CliConfig {
|
||||
)]
|
||||
namespace: String,
|
||||
|
||||
/// Operator image to pull. Defaults to the dev tag the
|
||||
/// `harmony-fleet-operator/Dockerfile` produces.
|
||||
#[arg(
|
||||
long,
|
||||
env = "HARMONY_FLEET_OPERATOR_IMAGE",
|
||||
default_value = "localhost/harmony-fleet-operator:dev"
|
||||
)]
|
||||
operator_image: String,
|
||||
/// Release tag to deploy (e.g. `harmony-fleet-operator-v0.0.2`); the
|
||||
/// version is parsed from it in Rust so the workflow passes a tag and
|
||||
/// never parses one in YAML. Wins over `--operator-chart-version`.
|
||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_TAG")]
|
||||
from_tag: Option<String>,
|
||||
|
||||
/// Agent image to pull. The e2e harness sideloads
|
||||
/// `localhost/harmony-fleet-agent:e2e`; production override via
|
||||
/// the env var.
|
||||
#[arg(
|
||||
long,
|
||||
env = "HARMONY_FLEET_AGENT_IMAGE",
|
||||
default_value = "localhost/harmony-fleet-agent:e2e"
|
||||
)]
|
||||
agent_image: String,
|
||||
|
||||
/// Device id for the agent Pod this deploy provisions. Production
|
||||
/// will likely deploy multiple agents; for the minimal CLI v1 the
|
||||
/// caller runs the binary once per device.
|
||||
#[arg(
|
||||
long,
|
||||
env = "HARMONY_FLEET_AGENT_DEVICE_ID",
|
||||
default_value = "fleet-agent-00"
|
||||
)]
|
||||
agent_device_id: String,
|
||||
|
||||
/// Host-side NodePort the NATS Service exposes.
|
||||
#[arg(long, env = "HARMONY_FLEET_NATS_NODE_PORT", default_value_t = 30423)]
|
||||
nats_node_port: u16,
|
||||
|
||||
/// NATS admin user (full pub/sub on the `APP` account). Required.
|
||||
#[arg(long, env = "HARMONY_FLEET_NATS_ADMIN_USER")]
|
||||
nats_admin_user: Option<String>,
|
||||
|
||||
/// NATS admin password. Required.
|
||||
#[arg(long, env = "HARMONY_FLEET_NATS_ADMIN_PASS")]
|
||||
nats_admin_pass: Option<String>,
|
||||
|
||||
/// NATS device user (limited pub/sub). Required.
|
||||
#[arg(long, env = "HARMONY_FLEET_NATS_DEVICE_USER")]
|
||||
nats_device_user: Option<String>,
|
||||
|
||||
/// NATS device password. Required.
|
||||
#[arg(long, env = "HARMONY_FLEET_NATS_DEVICE_PASS")]
|
||||
nats_device_pass: Option<String>,
|
||||
|
||||
/// Deploy the published operator chart at this version
|
||||
/// (`oci://<registry>/<project>/harmony-fleet-operator:<version>`)
|
||||
/// instead of rendering one from source — the CD `harmony apply`
|
||||
/// path. Re-run with a newer version to roll forward.
|
||||
/// Bare chart version, for the laptop path.
|
||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")]
|
||||
operator_chart_version: Option<String>,
|
||||
|
||||
@@ -111,74 +54,63 @@ struct CliConfig {
|
||||
)]
|
||||
operator_chart_project: String,
|
||||
|
||||
// Flattened so CI can pass `--yes` / `--filter` etc.
|
||||
/// Config namespace `FleetDeploySecrets` resolves under (Env → OpenBao).
|
||||
#[arg(long, env = "HARMONY_SECRET_NAMESPACE", default_value = "harmony")]
|
||||
config_namespace: String,
|
||||
|
||||
#[command(flatten)]
|
||||
harmony_cli: HarmonyCliArgs,
|
||||
}
|
||||
|
||||
impl CliConfig {
|
||||
/// Build the NATS credentials struct or fail with an actionable
|
||||
/// error message naming the missing env var. No dev defaults —
|
||||
/// the deploy binary refuses to ship known-weak passwords.
|
||||
fn nats_creds(&self) -> Result<UserPassCredentials> {
|
||||
Ok(UserPassCredentials {
|
||||
admin_user: self
|
||||
.nats_admin_user
|
||||
.clone()
|
||||
.context("HARMONY_FLEET_NATS_ADMIN_USER must be set")?,
|
||||
admin_pass: self
|
||||
.nats_admin_pass
|
||||
.clone()
|
||||
.context("HARMONY_FLEET_NATS_ADMIN_PASS must be set")?,
|
||||
device_user: self
|
||||
.nats_device_user
|
||||
.clone()
|
||||
.context("HARMONY_FLEET_NATS_DEVICE_USER must be set")?,
|
||||
device_pass: self
|
||||
.nats_device_pass
|
||||
.clone()
|
||||
.context("HARMONY_FLEET_NATS_DEVICE_PASS must be set")?,
|
||||
})
|
||||
fn chart_version(&self) -> Result<String> {
|
||||
match (&self.from_tag, &self.operator_chart_version) {
|
||||
(Some(tag), _) => version_from_tag(tag),
|
||||
(None, Some(v)) => Ok(v.clone()),
|
||||
(None, None) => bail!("set --from-tag or --operator-chart-version"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let cli = CliConfig::parse();
|
||||
let version = cli.chart_version()?;
|
||||
|
||||
let creds = cli.nats_creds()?;
|
||||
let device_user = creds.device_user.clone();
|
||||
let device_pass = creds.device_pass.clone();
|
||||
let nats = FleetNatsScore::user_pass(cli.namespace.clone(), cli.nats_node_port, creds);
|
||||
let mut operator = FleetOperatorScore::new()
|
||||
.namespace(cli.namespace.clone())
|
||||
.image(cli.operator_image.clone())
|
||||
.nats_url(nats.in_cluster_url());
|
||||
if let Some(version) = cli.operator_chart_version.clone() {
|
||||
operator = operator.published_chart(
|
||||
cli.operator_chart_registry.clone(),
|
||||
cli.operator_chart_project.clone(),
|
||||
let secrets: FleetDeploySecrets = ConfigClient::for_namespace(&cli.config_namespace)
|
||||
.await
|
||||
.get()
|
||||
.await
|
||||
.context("loading FleetDeploySecrets (set HARMONY_CONFIG_FleetDeploySecrets or OpenBao)")?;
|
||||
|
||||
// Point KUBECONFIG at the scoped deployer credential before the
|
||||
// topology reads it, so the runner pod needs no standing permissions.
|
||||
// Held to end of scope so the tempfile outlives the deploy.
|
||||
let _kubeconfig = match &secrets.kubeconfig {
|
||||
Some(kubeconfig) => {
|
||||
let mut f = tempfile::NamedTempFile::new().context("kubeconfig tempfile")?;
|
||||
f.write_all(kubeconfig.as_bytes())
|
||||
.context("writing kubeconfig")?;
|
||||
// SAFETY: single-threaded startup, before any topology reads it.
|
||||
unsafe { std::env::set_var("KUBECONFIG", f.path()) };
|
||||
Some(f)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let operator = FleetOperatorScore::new()
|
||||
.namespace(cli.namespace)
|
||||
.credentials(secrets.operator_credentials_toml)
|
||||
.published_chart(
|
||||
cli.operator_chart_registry,
|
||||
cli.operator_chart_project,
|
||||
version,
|
||||
);
|
||||
}
|
||||
let agent = FleetAgentScore::pod(
|
||||
cli.namespace.clone(),
|
||||
PodTarget::user_pass(
|
||||
cli.agent_device_id.clone(),
|
||||
cli.agent_image.clone(),
|
||||
nats.in_cluster_url(),
|
||||
device_user,
|
||||
device_pass,
|
||||
),
|
||||
);
|
||||
|
||||
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> =
|
||||
vec![Box::new(nats), Box::new(operator), Box::new(agent)];
|
||||
|
||||
harmony_cli::run(
|
||||
Inventory::autoload(),
|
||||
K8sAnywhereTopology::from_env(),
|
||||
scores,
|
||||
vec![Box::new(operator)],
|
||||
Some(cli.harmony_cli),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -133,6 +133,16 @@ impl FleetOperatorScore {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the operator's NATS auth-callout credentials (zitadel-jwt
|
||||
/// `[credentials]` TOML). Applied as the operator Secret before the
|
||||
/// helm install — including on the published-chart (CD) path.
|
||||
pub fn credentials(mut self, credentials_toml: impl Into<String>) -> Self {
|
||||
self.credentials = Some(OperatorCredentials {
|
||||
credentials_toml: credentials_toml.into(),
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
pub fn log_level(mut self, level: impl Into<String>) -> Self {
|
||||
self.log_level = level.into();
|
||||
self
|
||||
@@ -306,4 +316,15 @@ mod tests {
|
||||
assert_eq!(s.nats_url, "nats://nats:4222");
|
||||
assert_eq!(s.log_level, "debug");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_set_on_published_chart_path() {
|
||||
let s = FleetOperatorScore::new()
|
||||
.credentials("type = \"zitadel-jwt\"")
|
||||
.published_chart("hub.example", "proj", "1.2.3");
|
||||
let creds = s.credentials.expect("credentials wired");
|
||||
assert_eq!(creds.credentials_toml, "type = \"zitadel-jwt\"");
|
||||
// The published-chart interpret applies this Secret before install.
|
||||
assert!(s.published_chart.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
40
fleet/harmony-fleet-deploy/src/secrets.rs
Normal file
40
fleet/harmony-fleet-deploy/src/secrets.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
//! Secrets for the published-chart (CD) operator deploy, via
|
||||
//! [`harmony_config::ConfigClient`]. SSO-only by construction: no
|
||||
//! user/pass field exists, so dev-only user/pass auth can't reach a prod
|
||||
//! deploy. Resolved EnvSource → OpenBao, so the in-cluster runner pulls
|
||||
//! them at job time with no plaintext on the pod.
|
||||
|
||||
use harmony_config::Config;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// `#[config(secret)]` keeps every field out of cleartext SQLite and
|
||||
/// masks it in logs.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
|
||||
#[config(secret)]
|
||||
pub struct FleetDeploySecrets {
|
||||
/// Operator's zitadel-jwt `[credentials]` TOML — its NATS auth. The
|
||||
/// CD path's only credential; there is no user/pass fallback.
|
||||
pub operator_credentials_toml: String,
|
||||
|
||||
/// Scoped fleet-deployer credential, fetched at job time so the
|
||||
/// runner pod holds no standing permissions. `None` uses the ambient
|
||||
/// `KUBECONFIG`.
|
||||
#[serde(default)]
|
||||
pub kubeconfig: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use harmony_config::ConfigClass;
|
||||
|
||||
#[test]
|
||||
fn secret_class_and_optional_kubeconfig() {
|
||||
let s: FleetDeploySecrets =
|
||||
serde_json::from_str(r#"{ "operator_credentials_toml": "x" }"#).unwrap();
|
||||
assert!(s.kubeconfig.is_none());
|
||||
// Secrets must never land in cleartext SQLite.
|
||||
assert_eq!(FleetDeploySecrets::CLASS, ConfigClass::Secret);
|
||||
}
|
||||
}
|
||||
@@ -15,26 +15,72 @@ use crate::{
|
||||
|
||||
pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser};
|
||||
|
||||
const DEFAULT_NAMESPACE: &str = "openbao";
|
||||
const DEFAULT_RELEASE: &str = "openbao";
|
||||
|
||||
/// Where one OpenBao instance lives — the single authority both the
|
||||
/// deploy ([`OpenbaoScore`]) and the setup ([`OpenbaoSetupScore`]) take,
|
||||
/// so namespace and release can't drift apart.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OpenbaoInstance {
|
||||
pub namespace: String,
|
||||
/// Helm release name; the chart names its StatefulSet after it.
|
||||
pub release: String,
|
||||
}
|
||||
|
||||
impl OpenbaoInstance {
|
||||
/// `{release}-0` — the chart deploys a StatefulSet, so a stored pod
|
||||
/// literal would rot the moment `release` changes.
|
||||
pub fn pod(&self) -> String {
|
||||
format!("{}-0", self.release)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for OpenbaoInstance {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
namespace: DEFAULT_NAMESPACE.to_string(),
|
||||
release: DEFAULT_RELEASE.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone)]
|
||||
pub struct OpenbaoScore {
|
||||
/// Where this OpenBao is deployed (namespace + helm release).
|
||||
#[serde(default)]
|
||||
pub instance: OpenbaoInstance,
|
||||
/// Host used for external access (ingress)
|
||||
pub host: String,
|
||||
/// Set to true when deploying to OpenShift. Defaults to false for k3d/Kubernetes.
|
||||
#[serde(default)]
|
||||
pub openshift: bool,
|
||||
/// cert-manager `ClusterIssuer` for ingress TLS. `None` serves plain
|
||||
/// HTTP (TLS terminated elsewhere); `Some` adds the annotation + a
|
||||
/// `tls` block so cert-manager issues and renews the cert. Carries the
|
||||
/// issuer name because a bare bool can't address an issuer.
|
||||
#[serde(default)]
|
||||
pub tls_issuer: Option<String>,
|
||||
}
|
||||
|
||||
impl<T: Topology + K8sclient + HelmCommand> Score<T> for OpenbaoScore {
|
||||
fn name(&self) -> String {
|
||||
"OpenbaoScore".to_string()
|
||||
}
|
||||
impl OpenbaoScore {
|
||||
fn values(&self) -> String {
|
||||
let Self {
|
||||
host,
|
||||
openshift,
|
||||
tls_issuer,
|
||||
instance: _,
|
||||
} = self;
|
||||
// Edge TLS: the listener stays plain HTTP behind the ingress, which
|
||||
// terminates with the cert-manager-issued cert.
|
||||
let ingress_tls = match tls_issuer {
|
||||
Some(issuer) => format!(
|
||||
"\n annotations:\n cert-manager.io/cluster-issuer: {issuer}\n tls:\n - hosts: [{host}]\n secretName: openbao-tls"
|
||||
),
|
||||
None => String::new(),
|
||||
};
|
||||
|
||||
#[doc(hidden)]
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||
let host = &self.host;
|
||||
let openshift = self.openshift;
|
||||
|
||||
let values_yaml = Some(format!(
|
||||
format!(
|
||||
r#"global:
|
||||
openshift: {openshift}
|
||||
server:
|
||||
@@ -65,7 +111,7 @@ server:
|
||||
ingress:
|
||||
enabled: true
|
||||
hosts:
|
||||
- host: {host}
|
||||
- host: {host}{ingress_tls}
|
||||
dataStorage:
|
||||
enabled: true
|
||||
size: 10Gi
|
||||
@@ -79,15 +125,24 @@ server:
|
||||
accessMode: ReadWriteOnce
|
||||
ui:
|
||||
enabled: true"#
|
||||
));
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Topology + K8sclient + HelmCommand> Score<T> for OpenbaoScore {
|
||||
fn name(&self) -> String {
|
||||
"OpenbaoScore".to_string()
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||
HelmChartScore {
|
||||
namespace: Some(NonBlankString::from_str("openbao").unwrap()),
|
||||
release_name: NonBlankString::from_str("openbao").unwrap(),
|
||||
namespace: Some(NonBlankString::from_str(&self.instance.namespace).unwrap()),
|
||||
release_name: NonBlankString::from_str(&self.instance.release).unwrap(),
|
||||
chart_name: NonBlankString::from_str("openbao/openbao").unwrap(),
|
||||
chart_version: None,
|
||||
values_overrides: None,
|
||||
values_yaml,
|
||||
values_yaml: Some(self.values()),
|
||||
create_namespace: true,
|
||||
install_only: false,
|
||||
repository: Some(HelmRepository::new(
|
||||
@@ -99,3 +154,35 @@ ui:
|
||||
.create_interpret()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn no_issuer_renders_plain_ingress() {
|
||||
let v = OpenbaoScore {
|
||||
instance: Default::default(),
|
||||
host: "bao.example".into(),
|
||||
openshift: false,
|
||||
tls_issuer: None,
|
||||
}
|
||||
.values();
|
||||
assert!(!v.contains("cert-manager.io/cluster-issuer"));
|
||||
assert!(!v.contains("secretName"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issuer_renders_certmanager_tls_for_host() {
|
||||
let v = OpenbaoScore {
|
||||
instance: Default::default(),
|
||||
host: "bao.example".into(),
|
||||
openshift: false,
|
||||
tls_issuer: Some("letsencrypt".into()),
|
||||
}
|
||||
.values();
|
||||
assert!(v.contains("cert-manager.io/cluster-issuer: letsencrypt"));
|
||||
assert!(v.contains("- hosts: [bao.example]"));
|
||||
assert!(v.contains("secretName: openbao-tls"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ use crate::{
|
||||
};
|
||||
use harmony_types::id::Id;
|
||||
|
||||
const DEFAULT_NAMESPACE: &str = "openbao";
|
||||
const DEFAULT_POD: &str = "openbao-0";
|
||||
use super::OpenbaoInstance;
|
||||
|
||||
const DEFAULT_KV_MOUNT: &str = "secret";
|
||||
|
||||
/// A policy to create in OpenBao.
|
||||
@@ -72,13 +72,9 @@ pub struct OpenbaoJwtAuth {
|
||||
/// deployments should use auto-unseal (Transit, cloud KMS, etc.).
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OpenbaoSetupScore {
|
||||
/// Kubernetes namespace where OpenBao is deployed.
|
||||
#[serde(default = "default_namespace")]
|
||||
pub namespace: String,
|
||||
|
||||
/// StatefulSet pod name to exec into.
|
||||
#[serde(default = "default_pod")]
|
||||
pub pod: String,
|
||||
/// Where the target OpenBao is deployed (namespace + release).
|
||||
#[serde(default)]
|
||||
pub instance: OpenbaoInstance,
|
||||
|
||||
/// KV v2 mount path to enable.
|
||||
#[serde(default = "default_kv_mount")]
|
||||
@@ -97,12 +93,6 @@ pub struct OpenbaoSetupScore {
|
||||
pub jwt_auth: Option<OpenbaoJwtAuth>,
|
||||
}
|
||||
|
||||
fn default_namespace() -> String {
|
||||
DEFAULT_NAMESPACE.to_string()
|
||||
}
|
||||
fn default_pod() -> String {
|
||||
DEFAULT_POD.to_string()
|
||||
}
|
||||
fn default_kv_mount() -> String {
|
||||
DEFAULT_KV_MOUNT.to_string()
|
||||
}
|
||||
@@ -110,8 +100,7 @@ fn default_kv_mount() -> String {
|
||||
impl Default for OpenbaoSetupScore {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
namespace: default_namespace(),
|
||||
pod: default_pod(),
|
||||
instance: OpenbaoInstance::default(),
|
||||
kv_mount: default_kv_mount(),
|
||||
policies: Vec::new(),
|
||||
users: Vec::new(),
|
||||
@@ -164,8 +153,12 @@ impl OpenbaoSetupInterpret {
|
||||
k8s: &harmony_k8s::K8sClient,
|
||||
command: Vec<&str>,
|
||||
) -> Result<String, String> {
|
||||
k8s.exec_pod_capture_output(&self.score.pod, Some(&self.score.namespace), command)
|
||||
.await
|
||||
k8s.exec_pod_capture_output(
|
||||
&self.score.instance.pod(),
|
||||
Some(&self.score.instance.namespace),
|
||||
command,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn bao_command(
|
||||
@@ -279,8 +272,8 @@ impl OpenbaoSetupInterpret {
|
||||
// status and parse the `sealed` field authoritatively.
|
||||
let sealed = match k8s
|
||||
.exec_pod_capture(
|
||||
&self.score.pod,
|
||||
Some(&self.score.namespace),
|
||||
&self.score.instance.pod(),
|
||||
Some(&self.score.instance.namespace),
|
||||
vec!["bao", "status", "-format=json"],
|
||||
)
|
||||
.await
|
||||
@@ -514,14 +507,18 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenbaoSetupInterpret {
|
||||
.map_err(|e| InterpretError::new(format!("Failed to get K8s client: {e}")))?;
|
||||
|
||||
// Wait for the pod to be running before attempting any operations.
|
||||
k8s.wait_for_pod_ready(&self.score.pod, Some(&self.score.namespace))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
InterpretError::new(format!(
|
||||
"Pod {}/{} not ready: {e}",
|
||||
self.score.namespace, self.score.pod
|
||||
))
|
||||
})?;
|
||||
k8s.wait_for_pod_ready(
|
||||
&self.score.instance.pod(),
|
||||
Some(&self.score.instance.namespace),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
InterpretError::new(format!(
|
||||
"Pod {}/{} not ready: {e}",
|
||||
self.score.instance.namespace,
|
||||
self.score.instance.pod()
|
||||
))
|
||||
})?;
|
||||
|
||||
let root_token = self.init(&k8s).await?;
|
||||
self.unseal(&k8s).await?;
|
||||
@@ -574,8 +571,8 @@ mod tests {
|
||||
#[test]
|
||||
fn default_score_carries_expected_mounts() {
|
||||
let s = OpenbaoSetupScore::default();
|
||||
assert_eq!(s.namespace, "openbao");
|
||||
assert_eq!(s.pod, "openbao-0");
|
||||
assert_eq!(s.instance.namespace, "openbao");
|
||||
assert_eq!(s.instance.pod(), "openbao-0");
|
||||
assert_eq!(s.kv_mount, "secret");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,6 +124,9 @@ async fn init_secret_manager() -> SecretManager {
|
||||
}
|
||||
|
||||
/// Manages the lifecycle of secrets, providing a simple static API.
|
||||
#[deprecated(
|
||||
note = "Use harmony_config::ConfigClient instead; it unifies config + secrets over the same stores."
|
||||
)]
|
||||
#[derive(Debug)]
|
||||
pub struct SecretManager {
|
||||
namespace: String,
|
||||
|
||||
Reference in New Issue
Block a user