feat/fleet-cd-staging-deploy #310
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -4093,12 +4093,14 @@ dependencies = [
|
|||||||
"harmony-fleet-auth",
|
"harmony-fleet-auth",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
"harmony_cli",
|
"harmony_cli",
|
||||||
|
"harmony_config",
|
||||||
"harmony_macros",
|
"harmony_macros",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
"log",
|
"log",
|
||||||
"non-blank-string-rs",
|
"non-blank-string-rs",
|
||||||
|
"schemars 0.8.22",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
|
|||||||
@@ -245,8 +245,10 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
// Deploy + configure OpenBao (no JWT auth yet -- Zitadel isn't up)
|
// Deploy + configure OpenBao (no JWT auth yet -- Zitadel isn't up)
|
||||||
cleanup_openbao_webhook(&k8s).await?;
|
cleanup_openbao_webhook(&k8s).await?;
|
||||||
OpenbaoScore {
|
OpenbaoScore {
|
||||||
|
instance: Default::default(),
|
||||||
host: OPENBAO_HOST.to_string(),
|
host: OPENBAO_HOST.to_string(),
|
||||||
openshift: false,
|
openshift: false,
|
||||||
|
tls_issuer: None,
|
||||||
}
|
}
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
.interpret(&Inventory::autoload(), &topology)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ use harmony::{
|
|||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
let openbao = OpenbaoScore {
|
let openbao = OpenbaoScore {
|
||||||
|
instance: Default::default(),
|
||||||
host: "openbao.sebastien.sto1.nationtech.io".to_string(),
|
host: "openbao.sebastien.sto1.nationtech.io".to_string(),
|
||||||
openshift: false,
|
openshift: false,
|
||||||
|
tls_issuer: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
harmony_cli::run(
|
harmony_cli::run(
|
||||||
|
|||||||
@@ -102,23 +102,17 @@ k3d cluster delete fleet-e2e
|
|||||||
|
|
||||||
## Production deploys
|
## Production deploys
|
||||||
|
|
||||||
`harmony-fleet-deploy` is the binary that puts the fleet stack on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach). It composes `FleetNatsScore` + `FleetOperatorScore` + `FleetAgentScore` against the topology you point it at.
|
`harmony-fleet-deploy` puts the published operator chart on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach) — the `harmony apply` / CD path. It loads `FleetDeploySecrets` from config (Env → OpenBao) and runs one `FleetOperatorScore`; auth is Zitadel-SSO-only. The full bring-up stack (`FleetNatsScore` + `FleetAgentScore` + …) is composed by the e2e harness directly over the same lib Scores, not by this binary.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Default: K8sAnywhereTopology against whatever KUBECONFIG points at
|
# Deploy a released tag (version parsed from it in Rust):
|
||||||
cargo run -p harmony-fleet-deploy -- \
|
cargo run -p harmony-fleet-deploy -- \
|
||||||
--namespace fleet-system \
|
--filter FleetOperatorScore \
|
||||||
--operator-image hub.nationtech.io/harmony/harmony-fleet-operator:dev \
|
--from-tag harmony-fleet-operator-v0.0.2 \
|
||||||
--agent-image hub.nationtech.io/harmony/harmony-fleet-agent:dev \
|
--namespace fleet-system --yes
|
||||||
--agent-device-id fleet-agent-01
|
|
||||||
|
|
||||||
# Pick a single component with the harmony_cli filter
|
|
||||||
cargo run -p harmony-fleet-deploy -- \
|
|
||||||
--namespace fleet-system \
|
|
||||||
-- --filter FleetOperatorScore --all
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`harmony-fleet-deploy` reads its full config from CLI flags + env vars (`FLEET_NAMESPACE`, `FLEET_OPERATOR_IMAGE`, …). The minimal-CLI surface is deliberate — per ADR-023 the long-term answer is a plugin-discovery layer over `harmony-*` binaries; until that lands, deploy crates stay small and use the existing `harmony_cli`.
|
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
|
||||||
|
|
||||||
### Connecting to the operator
|
### Connecting to the operator
|
||||||
|
|
||||||
|
|||||||
@@ -28,43 +28,46 @@ cargo run -p harmony-fleet-deploy --bin harmony-fleet-release -- \
|
|||||||
--from-tag harmony-fleet-operator-v0.0.2 --no-push
|
--from-tag harmony-fleet-operator-v0.0.2 --no-push
|
||||||
```
|
```
|
||||||
|
|
||||||
## 2. Deploy a published version (`harmony apply`)
|
## 2. Deploy a published version to staging (manual, for now)
|
||||||
|
|
||||||
`--operator-chart-version` switches the operator to install the
|
Push to staging is manual until headless OpenBao auth (Zitadel machine
|
||||||
published `oci://hub.nationtech.io/harmony/harmony-fleet-operator:0.0.2`
|
identity) lands; secrets still come from shared OpenBao config. Point at
|
||||||
chart instead of rendering one from local source. Same command
|
your staging kube context and OpenBao, then run the operator deploy:
|
||||||
bootstraps and upgrades; re-running with the same version is a no-op.
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
export HARMONY_FLEET_NATS_ADMIN_USER=… HARMONY_FLEET_NATS_ADMIN_PASS=…
|
export OPENBAO_URL=<your OpenBao URL>
|
||||||
export HARMONY_FLEET_NATS_DEVICE_USER=… HARMONY_FLEET_NATS_DEVICE_PASS=…
|
export OPENBAO_TOKEN=<scoped read token for secret/<ns>/*>
|
||||||
|
harmony-fleet-deploy --filter FleetOperatorScore \
|
||||||
harmony-fleet-deploy \
|
--from-tag <release-tag> --namespace fleet-staging --yes
|
||||||
--filter FleetOperatorScore \
|
|
||||||
--operator-chart-version 0.0.2 \
|
|
||||||
--namespace fleet-system \
|
|
||||||
--yes
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
It installs the published
|
||||||
|
`oci://hub.nationtech.io/harmony/harmony-fleet-operator:<version>` chart;
|
||||||
|
the version is parsed from the tag in Rust (the tag is the only source
|
||||||
|
of truth). Same command bootstraps and upgrades; re-running the same tag
|
||||||
|
is a no-op. Auth is Zitadel-SSO-only: the operator gets its zitadel-jwt
|
||||||
|
`operator_credentials_toml` from `FleetDeploySecrets` in OpenBao (no
|
||||||
|
user/pass on the published-chart path). For manual deploy, store that
|
||||||
|
config **without** a `kubeconfig` field so your own kube context is used.
|
||||||
|
|
||||||
## 3. Roll forward
|
## 3. Roll forward
|
||||||
|
|
||||||
Same command, a newer (or previous-good) version:
|
Re-run with a newer (or previous-good) tag. `helm upgrade --install`
|
||||||
|
applies it and fails loudly if convergence fails — no automatic
|
||||||
```sh
|
rollback. Fix the spec, bump, re-run.
|
||||||
harmony-fleet-deploy --filter FleetOperatorScore --operator-chart-version 0.0.3 --namespace fleet-system --yes
|
|
||||||
```
|
|
||||||
|
|
||||||
`helm upgrade --install` applies it and fails loudly if convergence
|
|
||||||
fails — no automatic rollback. Fix the spec, bump, re-run.
|
|
||||||
|
|
||||||
## Automated vs. manual
|
## Automated vs. manual
|
||||||
|
|
||||||
| Step | Where |
|
| Step | Where |
|
||||||
|---|---|
|
|---|---|
|
||||||
| Build + push image + chart on tag | CI (`release` job) |
|
| Build + push image + chart on tag | CI (`release` job, on tag) |
|
||||||
| Deploy a published version + roll forward | Manual `harmony apply` (above) |
|
| Push to staging + roll forward | Manual (operator runs the deploy) |
|
||||||
|
|
||||||
A staging-auto / production-gated CD job is a follow-up — it needs the
|
## Future: in-cluster CD (blocked on headless OpenBao auth)
|
||||||
cluster `KUBECONFIG` + NATS secrets provisioned, which is out of scope
|
|
||||||
for the initial CD branch (ADR-012-2). The release job is fully
|
Once `harmony_config` can authenticate to OpenBao headlessly (Zitadel
|
||||||
functional today.
|
machine identity), these exports become a `deploy-staging` workflow on an
|
||||||
|
in-cluster, permissionless Gitea runner that pulls a `fleet-deployer`
|
||||||
|
`kubeconfig` + operator credentials from OpenBao at job time (provisioned
|
||||||
|
via a `TenantScore` with one extra egress CIDR to the OpenBao/Zitadel
|
||||||
|
ingress). Production-gated promotion is a later step (ADR-012-2).
|
||||||
|
|||||||
@@ -9,9 +9,8 @@ description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, ca
|
|||||||
[lib]
|
[lib]
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
# CLI entry point. `harmony-fleet-deploy <component>` picks a subcommand
|
# CLI entry point: deploy the published operator chart (harmony apply).
|
||||||
# per fleet component plus an `all` composite. Built on harmony_cli the
|
# Built on harmony_cli like the rest of the workspace's *-deploy crates.
|
||||||
# way the rest of the workspace's *-deploy crates are.
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "harmony-fleet-deploy"
|
name = "harmony-fleet-deploy"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
@@ -25,6 +24,7 @@ path = "src/bin/harmony-fleet-release.rs"
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", features = ["podman"] }
|
harmony = { path = "../../harmony", features = ["podman"] }
|
||||||
harmony_cli = { path = "../../harmony_cli" }
|
harmony_cli = { path = "../../harmony_cli" }
|
||||||
|
harmony_config = { path = "../../harmony_config" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
harmony_macros = { path = "../../harmony_macros" }
|
harmony_macros = { path = "../../harmony_macros" }
|
||||||
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
||||||
@@ -38,6 +38,7 @@ kube = { workspace = true, features = ["runtime", "derive"] }
|
|||||||
log = { workspace = true }
|
log = { workspace = true }
|
||||||
env_logger = { workspace = true }
|
env_logger = { workspace = true }
|
||||||
non-blank-string-rs = "1"
|
non-blank-string-rs = "1"
|
||||||
|
schemars = "0.8"
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
serde_yaml = { workspace = true }
|
serde_yaml = { workspace = true }
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ pub mod companion;
|
|||||||
pub mod nats;
|
pub mod nats;
|
||||||
pub mod operator;
|
pub mod operator;
|
||||||
pub mod release;
|
pub mod release;
|
||||||
|
pub mod secrets;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
|
|
||||||
pub use agent::{FleetAgentScore, PodTarget};
|
pub use agent::{FleetAgentScore, PodTarget};
|
||||||
@@ -39,4 +40,5 @@ pub use companion::AgentObservation;
|
|||||||
pub use nats::{FleetNatsScore, UserPassCredentials};
|
pub use nats::{FleetNatsScore, UserPassCredentials};
|
||||||
pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart};
|
pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart};
|
||||||
pub use release::{release_operator, version_from_tag};
|
pub use release::{release_operator, version_from_tag};
|
||||||
|
pub use secrets::FleetDeploySecrets;
|
||||||
pub use server::FleetServerScore;
|
pub use server::FleetServerScore;
|
||||||
|
|||||||
@@ -1,39 +1,28 @@
|
|||||||
//! `harmony-fleet-deploy` — deploy the fleet stack to a cluster.
|
//! `harmony-fleet-deploy` — deploy the published fleet operator chart
|
||||||
|
//! (`harmony apply`). Loads its secrets from config and runs one
|
||||||
|
//! [`FleetOperatorScore`] against [`K8sAnywhereTopology`].
|
||||||
//!
|
//!
|
||||||
//! Built on `harmony_cli::run` like the rest of the workspace's
|
//! The full dev/e2e stack (NATS + agent, user/pass) is composed by the
|
||||||
//! deploy binaries (`harmony_agent_deploy`, …). The CLI offers a
|
//! e2e harness directly over the `harmony_fleet_deploy` Scores — not
|
||||||
//! minimal env-driven config and hands off to `harmony_cli`, which
|
//! here. This binary is the prod/CD path: Zitadel-SSO-only, no
|
||||||
//! provides the standard `--filter` / `--all` / `--list` selection
|
//! handrolled manifests.
|
||||||
//! surface; pick a single component by filter or run them all.
|
|
||||||
//!
|
|
||||||
//! Topology default is [`K8sAnywhereTopology::from_env`] — local k3d
|
|
||||||
//! when `HARMONY_USE_LOCAL_K3D` flips, otherwise whatever cluster
|
|
||||||
//! `KUBECONFIG` points at. Per ADR-023 the binary's full set of
|
|
||||||
//! topologies is compile-time; for the moment only `K8sAnywhere` is
|
|
||||||
//! wired in, with `K8sBareTopology` planned for the next iteration.
|
|
||||||
//!
|
|
||||||
//! What the binary owns: assembling the Scores from environment
|
|
||||||
//! input. What it does **not** own: any handrolled k8s manifests,
|
|
||||||
//! any imperative bring-up loops, any auth secret rendering — that
|
|
||||||
//! all sits inside the `*Score` impls in [`harmony_fleet_deploy`].
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use std::io::Write;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony::topology::K8sAnywhereTopology;
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
use harmony_cli::Args as HarmonyCliArgs;
|
use harmony_cli::Args as HarmonyCliArgs;
|
||||||
use harmony_fleet_deploy::nats::UserPassCredentials;
|
use harmony_config::ConfigClient;
|
||||||
use harmony_fleet_deploy::{FleetAgentScore, FleetNatsScore, FleetOperatorScore, agent::PodTarget};
|
use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, version_from_tag};
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(
|
#[command(
|
||||||
name = "harmony-fleet-deploy",
|
name = "harmony-fleet-deploy",
|
||||||
about = "Deploy the harmony fleet stack to a Kubernetes cluster"
|
about = "Deploy the published harmony fleet operator chart"
|
||||||
)]
|
)]
|
||||||
struct CliConfig {
|
struct CliConfig {
|
||||||
/// Namespace every component lands in. Override with
|
|
||||||
/// `HARMONY_FLEET_NAMESPACE`.
|
|
||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
env = "HARMONY_FLEET_NAMESPACE",
|
env = "HARMONY_FLEET_NAMESPACE",
|
||||||
@@ -41,59 +30,13 @@ struct CliConfig {
|
|||||||
)]
|
)]
|
||||||
namespace: String,
|
namespace: String,
|
||||||
|
|
||||||
/// Operator image to pull. Defaults to the dev tag the
|
/// Release tag to deploy (e.g. `harmony-fleet-operator-v0.0.2`); the
|
||||||
/// `harmony-fleet-operator/Dockerfile` produces.
|
/// version is parsed from it in Rust so the workflow passes a tag and
|
||||||
#[arg(
|
/// never parses one in YAML. Wins over `--operator-chart-version`.
|
||||||
long,
|
#[arg(long, env = "HARMONY_FLEET_OPERATOR_TAG")]
|
||||||
env = "HARMONY_FLEET_OPERATOR_IMAGE",
|
from_tag: Option<String>,
|
||||||
default_value = "localhost/harmony-fleet-operator:dev"
|
|
||||||
)]
|
|
||||||
operator_image: String,
|
|
||||||
|
|
||||||
/// Agent image to pull. The e2e harness sideloads
|
/// Bare chart version, for the laptop path.
|
||||||
/// `localhost/harmony-fleet-agent:e2e`; production override via
|
|
||||||
/// the env var.
|
|
||||||
#[arg(
|
|
||||||
long,
|
|
||||||
env = "HARMONY_FLEET_AGENT_IMAGE",
|
|
||||||
default_value = "localhost/harmony-fleet-agent:e2e"
|
|
||||||
)]
|
|
||||||
agent_image: String,
|
|
||||||
|
|
||||||
/// Device id for the agent Pod this deploy provisions. Production
|
|
||||||
/// will likely deploy multiple agents; for the minimal CLI v1 the
|
|
||||||
/// caller runs the binary once per device.
|
|
||||||
#[arg(
|
|
||||||
long,
|
|
||||||
env = "HARMONY_FLEET_AGENT_DEVICE_ID",
|
|
||||||
default_value = "fleet-agent-00"
|
|
||||||
)]
|
|
||||||
agent_device_id: String,
|
|
||||||
|
|
||||||
/// Host-side NodePort the NATS Service exposes.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NATS_NODE_PORT", default_value_t = 30423)]
|
|
||||||
nats_node_port: u16,
|
|
||||||
|
|
||||||
/// NATS admin user (full pub/sub on the `APP` account). Required.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NATS_ADMIN_USER")]
|
|
||||||
nats_admin_user: Option<String>,
|
|
||||||
|
|
||||||
/// NATS admin password. Required.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NATS_ADMIN_PASS")]
|
|
||||||
nats_admin_pass: Option<String>,
|
|
||||||
|
|
||||||
/// NATS device user (limited pub/sub). Required.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NATS_DEVICE_USER")]
|
|
||||||
nats_device_user: Option<String>,
|
|
||||||
|
|
||||||
/// NATS device password. Required.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NATS_DEVICE_PASS")]
|
|
||||||
nats_device_pass: Option<String>,
|
|
||||||
|
|
||||||
/// Deploy the published operator chart at this version
|
|
||||||
/// (`oci://<registry>/<project>/harmony-fleet-operator:<version>`)
|
|
||||||
/// instead of rendering one from source — the CD `harmony apply`
|
|
||||||
/// path. Re-run with a newer version to roll forward.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")]
|
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")]
|
||||||
operator_chart_version: Option<String>,
|
operator_chart_version: Option<String>,
|
||||||
|
|
||||||
@@ -111,74 +54,63 @@ struct CliConfig {
|
|||||||
)]
|
)]
|
||||||
operator_chart_project: String,
|
operator_chart_project: String,
|
||||||
|
|
||||||
// Flattened so CI can pass `--yes` / `--filter` etc.
|
/// Config namespace `FleetDeploySecrets` resolves under (Env → OpenBao).
|
||||||
|
#[arg(long, env = "HARMONY_SECRET_NAMESPACE", default_value = "harmony")]
|
||||||
|
config_namespace: String,
|
||||||
|
|
||||||
#[command(flatten)]
|
#[command(flatten)]
|
||||||
harmony_cli: HarmonyCliArgs,
|
harmony_cli: HarmonyCliArgs,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CliConfig {
|
impl CliConfig {
|
||||||
/// Build the NATS credentials struct or fail with an actionable
|
fn chart_version(&self) -> Result<String> {
|
||||||
/// error message naming the missing env var. No dev defaults —
|
match (&self.from_tag, &self.operator_chart_version) {
|
||||||
/// the deploy binary refuses to ship known-weak passwords.
|
(Some(tag), _) => version_from_tag(tag),
|
||||||
fn nats_creds(&self) -> Result<UserPassCredentials> {
|
(None, Some(v)) => Ok(v.clone()),
|
||||||
Ok(UserPassCredentials {
|
(None, None) => bail!("set --from-tag or --operator-chart-version"),
|
||||||
admin_user: self
|
}
|
||||||
.nats_admin_user
|
|
||||||
.clone()
|
|
||||||
.context("HARMONY_FLEET_NATS_ADMIN_USER must be set")?,
|
|
||||||
admin_pass: self
|
|
||||||
.nats_admin_pass
|
|
||||||
.clone()
|
|
||||||
.context("HARMONY_FLEET_NATS_ADMIN_PASS must be set")?,
|
|
||||||
device_user: self
|
|
||||||
.nats_device_user
|
|
||||||
.clone()
|
|
||||||
.context("HARMONY_FLEET_NATS_DEVICE_USER must be set")?,
|
|
||||||
device_pass: self
|
|
||||||
.nats_device_pass
|
|
||||||
.clone()
|
|
||||||
.context("HARMONY_FLEET_NATS_DEVICE_PASS must be set")?,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
let cli = CliConfig::parse();
|
let cli = CliConfig::parse();
|
||||||
|
let version = cli.chart_version()?;
|
||||||
|
|
||||||
let creds = cli.nats_creds()?;
|
let secrets: FleetDeploySecrets = ConfigClient::for_namespace(&cli.config_namespace)
|
||||||
let device_user = creds.device_user.clone();
|
.await
|
||||||
let device_pass = creds.device_pass.clone();
|
.get()
|
||||||
let nats = FleetNatsScore::user_pass(cli.namespace.clone(), cli.nats_node_port, creds);
|
.await
|
||||||
let mut operator = FleetOperatorScore::new()
|
.context("loading FleetDeploySecrets (set HARMONY_CONFIG_FleetDeploySecrets or OpenBao)")?;
|
||||||
.namespace(cli.namespace.clone())
|
|
||||||
.image(cli.operator_image.clone())
|
// Point KUBECONFIG at the scoped deployer credential before the
|
||||||
.nats_url(nats.in_cluster_url());
|
// topology reads it, so the runner pod needs no standing permissions.
|
||||||
if let Some(version) = cli.operator_chart_version.clone() {
|
// Held to end of scope so the tempfile outlives the deploy.
|
||||||
operator = operator.published_chart(
|
let _kubeconfig = match &secrets.kubeconfig {
|
||||||
cli.operator_chart_registry.clone(),
|
Some(kubeconfig) => {
|
||||||
cli.operator_chart_project.clone(),
|
let mut f = tempfile::NamedTempFile::new().context("kubeconfig tempfile")?;
|
||||||
|
f.write_all(kubeconfig.as_bytes())
|
||||||
|
.context("writing kubeconfig")?;
|
||||||
|
// SAFETY: single-threaded startup, before any topology reads it.
|
||||||
|
unsafe { std::env::set_var("KUBECONFIG", f.path()) };
|
||||||
|
Some(f)
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let operator = FleetOperatorScore::new()
|
||||||
|
.namespace(cli.namespace)
|
||||||
|
.credentials(secrets.operator_credentials_toml)
|
||||||
|
.published_chart(
|
||||||
|
cli.operator_chart_registry,
|
||||||
|
cli.operator_chart_project,
|
||||||
version,
|
version,
|
||||||
);
|
);
|
||||||
}
|
|
||||||
let agent = FleetAgentScore::pod(
|
|
||||||
cli.namespace.clone(),
|
|
||||||
PodTarget::user_pass(
|
|
||||||
cli.agent_device_id.clone(),
|
|
||||||
cli.agent_image.clone(),
|
|
||||||
nats.in_cluster_url(),
|
|
||||||
device_user,
|
|
||||||
device_pass,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> =
|
|
||||||
vec![Box::new(nats), Box::new(operator), Box::new(agent)];
|
|
||||||
|
|
||||||
harmony_cli::run(
|
harmony_cli::run(
|
||||||
Inventory::autoload(),
|
Inventory::autoload(),
|
||||||
K8sAnywhereTopology::from_env(),
|
K8sAnywhereTopology::from_env(),
|
||||||
scores,
|
vec![Box::new(operator)],
|
||||||
Some(cli.harmony_cli),
|
Some(cli.harmony_cli),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -133,6 +133,16 @@ impl FleetOperatorScore {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set the operator's NATS auth-callout credentials (zitadel-jwt
|
||||||
|
/// `[credentials]` TOML). Applied as the operator Secret before the
|
||||||
|
/// helm install — including on the published-chart (CD) path.
|
||||||
|
pub fn credentials(mut self, credentials_toml: impl Into<String>) -> Self {
|
||||||
|
self.credentials = Some(OperatorCredentials {
|
||||||
|
credentials_toml: credentials_toml.into(),
|
||||||
|
});
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn log_level(mut self, level: impl Into<String>) -> Self {
|
pub fn log_level(mut self, level: impl Into<String>) -> Self {
|
||||||
self.log_level = level.into();
|
self.log_level = level.into();
|
||||||
self
|
self
|
||||||
@@ -306,4 +316,15 @@ mod tests {
|
|||||||
assert_eq!(s.nats_url, "nats://nats:4222");
|
assert_eq!(s.nats_url, "nats://nats:4222");
|
||||||
assert_eq!(s.log_level, "debug");
|
assert_eq!(s.log_level, "debug");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn credentials_set_on_published_chart_path() {
|
||||||
|
let s = FleetOperatorScore::new()
|
||||||
|
.credentials("type = \"zitadel-jwt\"")
|
||||||
|
.published_chart("hub.example", "proj", "1.2.3");
|
||||||
|
let creds = s.credentials.expect("credentials wired");
|
||||||
|
assert_eq!(creds.credentials_toml, "type = \"zitadel-jwt\"");
|
||||||
|
// The published-chart interpret applies this Secret before install.
|
||||||
|
assert!(s.published_chart.is_some());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
40
fleet/harmony-fleet-deploy/src/secrets.rs
Normal file
40
fleet/harmony-fleet-deploy/src/secrets.rs
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
//! Secrets for the published-chart (CD) operator deploy, via
|
||||||
|
//! [`harmony_config::ConfigClient`]. SSO-only by construction: no
|
||||||
|
//! user/pass field exists, so dev-only user/pass auth can't reach a prod
|
||||||
|
//! deploy. Resolved EnvSource → OpenBao, so the in-cluster runner pulls
|
||||||
|
//! them at job time with no plaintext on the pod.
|
||||||
|
|
||||||
|
use harmony_config::Config;
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// `#[config(secret)]` keeps every field out of cleartext SQLite and
|
||||||
|
/// masks it in logs.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
|
||||||
|
#[config(secret)]
|
||||||
|
pub struct FleetDeploySecrets {
|
||||||
|
/// Operator's zitadel-jwt `[credentials]` TOML — its NATS auth. The
|
||||||
|
/// CD path's only credential; there is no user/pass fallback.
|
||||||
|
pub operator_credentials_toml: String,
|
||||||
|
|
||||||
|
/// Scoped fleet-deployer credential, fetched at job time so the
|
||||||
|
/// runner pod holds no standing permissions. `None` uses the ambient
|
||||||
|
/// `KUBECONFIG`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub kubeconfig: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use harmony_config::ConfigClass;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn secret_class_and_optional_kubeconfig() {
|
||||||
|
let s: FleetDeploySecrets =
|
||||||
|
serde_json::from_str(r#"{ "operator_credentials_toml": "x" }"#).unwrap();
|
||||||
|
assert!(s.kubeconfig.is_none());
|
||||||
|
// Secrets must never land in cleartext SQLite.
|
||||||
|
assert_eq!(FleetDeploySecrets::CLASS, ConfigClass::Secret);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,26 +15,72 @@ use crate::{
|
|||||||
|
|
||||||
pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser};
|
pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser};
|
||||||
|
|
||||||
|
const DEFAULT_NAMESPACE: &str = "openbao";
|
||||||
|
const DEFAULT_RELEASE: &str = "openbao";
|
||||||
|
|
||||||
|
/// Where one OpenBao instance lives — the single authority both the
|
||||||
|
/// deploy ([`OpenbaoScore`]) and the setup ([`OpenbaoSetupScore`]) take,
|
||||||
|
/// so namespace and release can't drift apart.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct OpenbaoInstance {
|
||||||
|
pub namespace: String,
|
||||||
|
/// Helm release name; the chart names its StatefulSet after it.
|
||||||
|
pub release: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenbaoInstance {
|
||||||
|
/// `{release}-0` — the chart deploys a StatefulSet, so a stored pod
|
||||||
|
/// literal would rot the moment `release` changes.
|
||||||
|
pub fn pod(&self) -> String {
|
||||||
|
format!("{}-0", self.release)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for OpenbaoInstance {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
namespace: DEFAULT_NAMESPACE.to_string(),
|
||||||
|
release: DEFAULT_RELEASE.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Clone)]
|
#[derive(Debug, Serialize, Clone)]
|
||||||
pub struct OpenbaoScore {
|
pub struct OpenbaoScore {
|
||||||
|
/// Where this OpenBao is deployed (namespace + helm release).
|
||||||
|
#[serde(default)]
|
||||||
|
pub instance: OpenbaoInstance,
|
||||||
/// Host used for external access (ingress)
|
/// Host used for external access (ingress)
|
||||||
pub host: String,
|
pub host: String,
|
||||||
/// Set to true when deploying to OpenShift. Defaults to false for k3d/Kubernetes.
|
/// Set to true when deploying to OpenShift. Defaults to false for k3d/Kubernetes.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub openshift: bool,
|
pub openshift: bool,
|
||||||
|
/// cert-manager `ClusterIssuer` for ingress TLS. `None` serves plain
|
||||||
|
/// HTTP (TLS terminated elsewhere); `Some` adds the annotation + a
|
||||||
|
/// `tls` block so cert-manager issues and renews the cert. Carries the
|
||||||
|
/// issuer name because a bare bool can't address an issuer.
|
||||||
|
#[serde(default)]
|
||||||
|
pub tls_issuer: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: Topology + K8sclient + HelmCommand> Score<T> for OpenbaoScore {
|
impl OpenbaoScore {
|
||||||
fn name(&self) -> String {
|
fn values(&self) -> String {
|
||||||
"OpenbaoScore".to_string()
|
let Self {
|
||||||
}
|
host,
|
||||||
|
openshift,
|
||||||
|
tls_issuer,
|
||||||
|
instance: _,
|
||||||
|
} = self;
|
||||||
|
// Edge TLS: the listener stays plain HTTP behind the ingress, which
|
||||||
|
// terminates with the cert-manager-issued cert.
|
||||||
|
let ingress_tls = match tls_issuer {
|
||||||
|
Some(issuer) => format!(
|
||||||
|
"\n annotations:\n cert-manager.io/cluster-issuer: {issuer}\n tls:\n - hosts: [{host}]\n secretName: openbao-tls"
|
||||||
|
),
|
||||||
|
None => String::new(),
|
||||||
|
};
|
||||||
|
|
||||||
#[doc(hidden)]
|
format!(
|
||||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
|
||||||
let host = &self.host;
|
|
||||||
let openshift = self.openshift;
|
|
||||||
|
|
||||||
let values_yaml = Some(format!(
|
|
||||||
r#"global:
|
r#"global:
|
||||||
openshift: {openshift}
|
openshift: {openshift}
|
||||||
server:
|
server:
|
||||||
@@ -65,7 +111,7 @@ server:
|
|||||||
ingress:
|
ingress:
|
||||||
enabled: true
|
enabled: true
|
||||||
hosts:
|
hosts:
|
||||||
- host: {host}
|
- host: {host}{ingress_tls}
|
||||||
dataStorage:
|
dataStorage:
|
||||||
enabled: true
|
enabled: true
|
||||||
size: 10Gi
|
size: 10Gi
|
||||||
@@ -79,15 +125,24 @@ server:
|
|||||||
accessMode: ReadWriteOnce
|
accessMode: ReadWriteOnce
|
||||||
ui:
|
ui:
|
||||||
enabled: true"#
|
enabled: true"#
|
||||||
));
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Topology + K8sclient + HelmCommand> Score<T> for OpenbaoScore {
|
||||||
|
fn name(&self) -> String {
|
||||||
|
"OpenbaoScore".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[doc(hidden)]
|
||||||
|
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||||
HelmChartScore {
|
HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str("openbao").unwrap()),
|
namespace: Some(NonBlankString::from_str(&self.instance.namespace).unwrap()),
|
||||||
release_name: NonBlankString::from_str("openbao").unwrap(),
|
release_name: NonBlankString::from_str(&self.instance.release).unwrap(),
|
||||||
chart_name: NonBlankString::from_str("openbao/openbao").unwrap(),
|
chart_name: NonBlankString::from_str("openbao/openbao").unwrap(),
|
||||||
chart_version: None,
|
chart_version: None,
|
||||||
values_overrides: None,
|
values_overrides: None,
|
||||||
values_yaml,
|
values_yaml: Some(self.values()),
|
||||||
create_namespace: true,
|
create_namespace: true,
|
||||||
install_only: false,
|
install_only: false,
|
||||||
repository: Some(HelmRepository::new(
|
repository: Some(HelmRepository::new(
|
||||||
@@ -99,3 +154,35 @@ ui:
|
|||||||
.create_interpret()
|
.create_interpret()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_issuer_renders_plain_ingress() {
|
||||||
|
let v = OpenbaoScore {
|
||||||
|
instance: Default::default(),
|
||||||
|
host: "bao.example".into(),
|
||||||
|
openshift: false,
|
||||||
|
tls_issuer: None,
|
||||||
|
}
|
||||||
|
.values();
|
||||||
|
assert!(!v.contains("cert-manager.io/cluster-issuer"));
|
||||||
|
assert!(!v.contains("secretName"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn issuer_renders_certmanager_tls_for_host() {
|
||||||
|
let v = OpenbaoScore {
|
||||||
|
instance: Default::default(),
|
||||||
|
host: "bao.example".into(),
|
||||||
|
openshift: false,
|
||||||
|
tls_issuer: Some("letsencrypt".into()),
|
||||||
|
}
|
||||||
|
.values();
|
||||||
|
assert!(v.contains("cert-manager.io/cluster-issuer: letsencrypt"));
|
||||||
|
assert!(v.contains("- hosts: [bao.example]"));
|
||||||
|
assert!(v.contains("secretName: openbao-tls"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ use crate::{
|
|||||||
};
|
};
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
|
|
||||||
const DEFAULT_NAMESPACE: &str = "openbao";
|
use super::OpenbaoInstance;
|
||||||
const DEFAULT_POD: &str = "openbao-0";
|
|
||||||
const DEFAULT_KV_MOUNT: &str = "secret";
|
const DEFAULT_KV_MOUNT: &str = "secret";
|
||||||
|
|
||||||
/// A policy to create in OpenBao.
|
/// A policy to create in OpenBao.
|
||||||
@@ -72,13 +72,9 @@ pub struct OpenbaoJwtAuth {
|
|||||||
/// deployments should use auto-unseal (Transit, cloud KMS, etc.).
|
/// deployments should use auto-unseal (Transit, cloud KMS, etc.).
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct OpenbaoSetupScore {
|
pub struct OpenbaoSetupScore {
|
||||||
/// Kubernetes namespace where OpenBao is deployed.
|
/// Where the target OpenBao is deployed (namespace + release).
|
||||||
#[serde(default = "default_namespace")]
|
#[serde(default)]
|
||||||
pub namespace: String,
|
pub instance: OpenbaoInstance,
|
||||||
|
|
||||||
/// StatefulSet pod name to exec into.
|
|
||||||
#[serde(default = "default_pod")]
|
|
||||||
pub pod: String,
|
|
||||||
|
|
||||||
/// KV v2 mount path to enable.
|
/// KV v2 mount path to enable.
|
||||||
#[serde(default = "default_kv_mount")]
|
#[serde(default = "default_kv_mount")]
|
||||||
@@ -97,12 +93,6 @@ pub struct OpenbaoSetupScore {
|
|||||||
pub jwt_auth: Option<OpenbaoJwtAuth>,
|
pub jwt_auth: Option<OpenbaoJwtAuth>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_namespace() -> String {
|
|
||||||
DEFAULT_NAMESPACE.to_string()
|
|
||||||
}
|
|
||||||
fn default_pod() -> String {
|
|
||||||
DEFAULT_POD.to_string()
|
|
||||||
}
|
|
||||||
fn default_kv_mount() -> String {
|
fn default_kv_mount() -> String {
|
||||||
DEFAULT_KV_MOUNT.to_string()
|
DEFAULT_KV_MOUNT.to_string()
|
||||||
}
|
}
|
||||||
@@ -110,8 +100,7 @@ fn default_kv_mount() -> String {
|
|||||||
impl Default for OpenbaoSetupScore {
|
impl Default for OpenbaoSetupScore {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
namespace: default_namespace(),
|
instance: OpenbaoInstance::default(),
|
||||||
pod: default_pod(),
|
|
||||||
kv_mount: default_kv_mount(),
|
kv_mount: default_kv_mount(),
|
||||||
policies: Vec::new(),
|
policies: Vec::new(),
|
||||||
users: Vec::new(),
|
users: Vec::new(),
|
||||||
@@ -164,8 +153,12 @@ impl OpenbaoSetupInterpret {
|
|||||||
k8s: &harmony_k8s::K8sClient,
|
k8s: &harmony_k8s::K8sClient,
|
||||||
command: Vec<&str>,
|
command: Vec<&str>,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
k8s.exec_pod_capture_output(&self.score.pod, Some(&self.score.namespace), command)
|
k8s.exec_pod_capture_output(
|
||||||
.await
|
&self.score.instance.pod(),
|
||||||
|
Some(&self.score.instance.namespace),
|
||||||
|
command,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn bao_command(
|
async fn bao_command(
|
||||||
@@ -279,8 +272,8 @@ impl OpenbaoSetupInterpret {
|
|||||||
// status and parse the `sealed` field authoritatively.
|
// status and parse the `sealed` field authoritatively.
|
||||||
let sealed = match k8s
|
let sealed = match k8s
|
||||||
.exec_pod_capture(
|
.exec_pod_capture(
|
||||||
&self.score.pod,
|
&self.score.instance.pod(),
|
||||||
Some(&self.score.namespace),
|
Some(&self.score.instance.namespace),
|
||||||
vec!["bao", "status", "-format=json"],
|
vec!["bao", "status", "-format=json"],
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -514,14 +507,18 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenbaoSetupInterpret {
|
|||||||
.map_err(|e| InterpretError::new(format!("Failed to get K8s client: {e}")))?;
|
.map_err(|e| InterpretError::new(format!("Failed to get K8s client: {e}")))?;
|
||||||
|
|
||||||
// Wait for the pod to be running before attempting any operations.
|
// Wait for the pod to be running before attempting any operations.
|
||||||
k8s.wait_for_pod_ready(&self.score.pod, Some(&self.score.namespace))
|
k8s.wait_for_pod_ready(
|
||||||
.await
|
&self.score.instance.pod(),
|
||||||
.map_err(|e| {
|
Some(&self.score.instance.namespace),
|
||||||
InterpretError::new(format!(
|
)
|
||||||
"Pod {}/{} not ready: {e}",
|
.await
|
||||||
self.score.namespace, self.score.pod
|
.map_err(|e| {
|
||||||
))
|
InterpretError::new(format!(
|
||||||
})?;
|
"Pod {}/{} not ready: {e}",
|
||||||
|
self.score.instance.namespace,
|
||||||
|
self.score.instance.pod()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
let root_token = self.init(&k8s).await?;
|
let root_token = self.init(&k8s).await?;
|
||||||
self.unseal(&k8s).await?;
|
self.unseal(&k8s).await?;
|
||||||
@@ -574,8 +571,8 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn default_score_carries_expected_mounts() {
|
fn default_score_carries_expected_mounts() {
|
||||||
let s = OpenbaoSetupScore::default();
|
let s = OpenbaoSetupScore::default();
|
||||||
assert_eq!(s.namespace, "openbao");
|
assert_eq!(s.instance.namespace, "openbao");
|
||||||
assert_eq!(s.pod, "openbao-0");
|
assert_eq!(s.instance.pod(), "openbao-0");
|
||||||
assert_eq!(s.kv_mount, "secret");
|
assert_eq!(s.kv_mount, "secret");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -124,6 +124,9 @@ async fn init_secret_manager() -> SecretManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Manages the lifecycle of secrets, providing a simple static API.
|
/// Manages the lifecycle of secrets, providing a simple static API.
|
||||||
|
#[deprecated(
|
||||||
|
note = "Use harmony_config::ConfigClient instead; it unifies config + secrets over the same stores."
|
||||||
|
)]
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct SecretManager {
|
pub struct SecretManager {
|
||||||
namespace: String,
|
namespace: String,
|
||||||
|
|||||||
Reference in New Issue
Block a user