docs: Score composition via refs, major architectural progress for simple yet type safe infrastructure component composition #342
10
.harmony/contexts.toml
Normal file
10
.harmony/contexts.toml
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[contexts.local]
|
||||||
|
profile = "local"
|
||||||
|
autoprovision = true
|
||||||
|
domain = "localhost"
|
||||||
|
|
||||||
|
# Customer contexts belong in their private repository and can be selected with
|
||||||
|
# `--config path/to/contexts.toml`. A production context also sets `registry`,
|
||||||
|
# `project`, `domain`, `image_pull_secret`, and brokered OpenBao cluster access.
|
||||||
|
# Its cluster contains the immutable `fleet-system/fleet-callout-credentials`
|
||||||
|
# Secret described in `fleet/README.md`.
|
||||||
104
Cargo.lock
generated
104
Cargo.lock
generated
@@ -2872,33 +2872,9 @@ name = "example-fleet-e2e-demo"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-nats",
|
|
||||||
"clap",
|
"clap",
|
||||||
"directories",
|
"harmony-fleet-e2e",
|
||||||
"env_logger",
|
|
||||||
"example-fleet-auth-callout",
|
|
||||||
"futures-util",
|
|
||||||
"harmony",
|
|
||||||
"harmony-fleet-auth",
|
|
||||||
"harmony-fleet-deploy",
|
|
||||||
"harmony-fleet-operator",
|
|
||||||
"harmony-k8s",
|
|
||||||
"harmony-nats-callout",
|
|
||||||
"harmony-reconciler-contracts",
|
|
||||||
"harmony_types",
|
|
||||||
"k3d-rs",
|
|
||||||
"k8s-openapi",
|
|
||||||
"kube",
|
|
||||||
"log",
|
|
||||||
"nkeys",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"tempfile",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-test",
|
|
||||||
"tracing",
|
|
||||||
"tracing-subscriber",
|
|
||||||
"url",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2917,31 +2893,6 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "example-fleet-staging-deploy"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"anyhow",
|
|
||||||
"async-nats",
|
|
||||||
"clap",
|
|
||||||
"env_logger",
|
|
||||||
"harmony",
|
|
||||||
"harmony-k8s",
|
|
||||||
"harmony-nats-callout",
|
|
||||||
"harmony_types",
|
|
||||||
"k8s-openapi",
|
|
||||||
"kube",
|
|
||||||
"log",
|
|
||||||
"nkeys",
|
|
||||||
"reqwest 0.12.28",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"tokio",
|
|
||||||
"tracing",
|
|
||||||
"tracing-subscriber",
|
|
||||||
"url",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "example-grafana"
|
name = "example-grafana"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -3447,6 +3398,7 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"env_logger",
|
"env_logger",
|
||||||
"harmony",
|
"harmony",
|
||||||
|
"harmony-fleet-deploy",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"log",
|
"log",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -3460,7 +3412,7 @@ dependencies = [
|
|||||||
"async-nats",
|
"async-nats",
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"harmony",
|
"harmony-fleet-operator",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
@@ -3489,6 +3441,7 @@ dependencies = [
|
|||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"clap",
|
"clap",
|
||||||
"harmony",
|
"harmony",
|
||||||
|
"harmony-fleet-deploy",
|
||||||
"harmony_cli",
|
"harmony_cli",
|
||||||
"harmony_secret",
|
"harmony_secret",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
@@ -3511,28 +3464,6 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "example_fleet_staging_install"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"anyhow",
|
|
||||||
"base64 0.22.1",
|
|
||||||
"clap",
|
|
||||||
"harmony",
|
|
||||||
"harmony-fleet-deploy",
|
|
||||||
"harmony-k8s",
|
|
||||||
"harmony-nats-callout",
|
|
||||||
"harmony_cli",
|
|
||||||
"harmony_config",
|
|
||||||
"harmony_secret",
|
|
||||||
"nkeys",
|
|
||||||
"rand 0.9.4",
|
|
||||||
"schemars 0.8.22",
|
|
||||||
"serde",
|
|
||||||
"tokio",
|
|
||||||
"tracing",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "example_fleet_vm_setup"
|
name = "example_fleet_vm_setup"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -3541,6 +3472,7 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"env_logger",
|
"env_logger",
|
||||||
"harmony",
|
"harmony",
|
||||||
|
"harmony-fleet-deploy",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"log",
|
"log",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -3552,7 +3484,8 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"clap",
|
"clap",
|
||||||
"harmony",
|
"harmony-fleet-operator",
|
||||||
|
"harmony-reconciler-contracts",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -4029,13 +3962,13 @@ dependencies = [
|
|||||||
"kube-derive",
|
"kube-derive",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
"log",
|
||||||
|
"nkeys",
|
||||||
"non-blank-string-rs",
|
"non-blank-string-rs",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"opnsense-api",
|
"opnsense-api",
|
||||||
"opnsense-config",
|
"opnsense-config",
|
||||||
"opnsense-config-xml",
|
"opnsense-config-xml",
|
||||||
"option-ext",
|
"option-ext",
|
||||||
"podman-api",
|
|
||||||
"pretty_assertions",
|
"pretty_assertions",
|
||||||
"rand 0.9.4",
|
"rand 0.9.4",
|
||||||
"reqwest 0.11.27",
|
"reqwest 0.11.27",
|
||||||
@@ -4079,12 +4012,13 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"harmony",
|
|
||||||
"harmony-fleet-auth",
|
"harmony-fleet-auth",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
"harmony_secret",
|
"harmony_secret",
|
||||||
|
"podman-api",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2 0.10.9",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
"toml",
|
"toml",
|
||||||
@@ -4121,12 +4055,14 @@ dependencies = [
|
|||||||
"fqdn",
|
"fqdn",
|
||||||
"harmony",
|
"harmony",
|
||||||
"harmony-fleet-auth",
|
"harmony-fleet-auth",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-fleet-operator",
|
||||||
|
"harmony_app",
|
||||||
"harmony_cli",
|
"harmony_cli",
|
||||||
"harmony_config",
|
"harmony_config",
|
||||||
"harmony_macros",
|
"harmony_macros",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"harmony_zitadel_auth",
|
"harmony_zitadel_auth",
|
||||||
|
"inquire 0.7.5",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
"log",
|
"log",
|
||||||
@@ -4135,11 +4071,13 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
|
"similar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
"toml",
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4157,6 +4095,7 @@ dependencies = [
|
|||||||
"harmony-fleet-operator",
|
"harmony-fleet-operator",
|
||||||
"harmony-k8s",
|
"harmony-k8s",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
|
"harmony_secret",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"jsonwebtoken",
|
"jsonwebtoken",
|
||||||
"k3d-rs",
|
"k3d-rs",
|
||||||
@@ -4188,10 +4127,10 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"harmony",
|
|
||||||
"harmony-fleet-auth",
|
"harmony-fleet-auth",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
"harmony_config",
|
"harmony_config",
|
||||||
|
"harmony_secret",
|
||||||
"harmony_zitadel_auth",
|
"harmony_zitadel_auth",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
@@ -4269,6 +4208,7 @@ dependencies = [
|
|||||||
name = "harmony-reconciler-contracts"
|
name = "harmony-reconciler-contracts"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
"chrono",
|
"chrono",
|
||||||
"harmony_types",
|
"harmony_types",
|
||||||
"schemars 0.8.22",
|
"schemars 0.8.22",
|
||||||
@@ -4317,7 +4257,6 @@ dependencies = [
|
|||||||
name = "harmony_app"
|
name = "harmony_app"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"docker-compose-types",
|
"docker-compose-types",
|
||||||
"fqdn",
|
"fqdn",
|
||||||
@@ -4327,8 +4266,10 @@ dependencies = [
|
|||||||
"harmony_types",
|
"harmony_types",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"log",
|
"log",
|
||||||
|
"reqwest 0.12.28",
|
||||||
"schemars 0.8.22",
|
"schemars 0.8.22",
|
||||||
"serde",
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
@@ -4377,6 +4318,8 @@ dependencies = [
|
|||||||
"indicatif",
|
"indicatif",
|
||||||
"inquire 0.7.5",
|
"inquire 0.7.5",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
@@ -4518,6 +4461,7 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"directories",
|
"directories",
|
||||||
|
"harmony-reconciler-contracts",
|
||||||
"harmony_secret_derive",
|
"harmony_secret_derive",
|
||||||
"harmony_zitadel_jwt",
|
"harmony_zitadel_jwt",
|
||||||
"http 1.4.0",
|
"http 1.4.0",
|
||||||
@@ -4584,10 +4528,12 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
|
"async-trait",
|
||||||
"axum",
|
"axum",
|
||||||
"axum-extra",
|
"axum-extra",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"harmony-reconciler-contracts",
|
||||||
"harmony_config",
|
"harmony_config",
|
||||||
"harmony_zitadel_jwt",
|
"harmony_zitadel_jwt",
|
||||||
"jsonwebtoken",
|
"jsonwebtoken",
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ members = [
|
|||||||
"fleet/harmony-fleet-e2e",
|
"fleet/harmony-fleet-e2e",
|
||||||
"harmony-reconciler-contracts",
|
"harmony-reconciler-contracts",
|
||||||
"examples/fleet_server_install",
|
"examples/fleet_server_install",
|
||||||
"examples/fleet_staging_install",
|
|
||||||
"nats/jwt",
|
"nats/jwt",
|
||||||
"nats/callout",
|
"nats/callout",
|
||||||
"nats/integration-test-callout",
|
"nats/integration-test-callout",
|
||||||
@@ -97,6 +96,7 @@ futures-util = "0.3"
|
|||||||
thiserror = "2.0.14"
|
thiserror = "2.0.14"
|
||||||
serde = { version = "1.0.209", features = ["derive", "rc"] }
|
serde = { version = "1.0.209", features = ["derive", "rc"] }
|
||||||
serde_json = "1.0.127"
|
serde_json = "1.0.127"
|
||||||
|
sha2 = "0.10"
|
||||||
askama = "0.14"
|
askama = "0.14"
|
||||||
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
|
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
|
||||||
reqwest = { version = "0.12", features = [
|
reqwest = { version = "0.12", features = [
|
||||||
|
|||||||
221
ROADMAP/fleet_platform/public_e2e_test_cluster.md
Normal file
221
ROADMAP/fleet_platform/public_e2e_test_cluster.md
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
# Public Fleet E2E environment
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Run public-facing Fleet regression tests directly on the production-adjacent
|
||||||
|
OKD cluster. Keep the environment definition, identities, context, and CI
|
||||||
|
workflow in a standalone private repository checked out under
|
||||||
|
`private_repos/`. The public Harmony repository contains only reusable Scores,
|
||||||
|
Refs, lifecycle code, and generic test harnesses.
|
||||||
|
|
||||||
|
Build and publish run on the same CI runner. Deployment receives only
|
||||||
|
digest-pinned image references published to an OCI registry.
|
||||||
|
|
||||||
|
## Private repository
|
||||||
|
|
||||||
|
The private repository owns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Cargo.toml
|
||||||
|
src/main.rs # Fleet app + compiled context catalog
|
||||||
|
.github/workflows/e2e.yml # or the equivalent CI system
|
||||||
|
README.md # operator setup and recovery
|
||||||
|
```
|
||||||
|
|
||||||
|
It is checked out beside Harmony under `private_repos/`, which the public
|
||||||
|
repository ignores. No customer or private project identifier belongs in a
|
||||||
|
tracked Harmony file, fixture, log sample, or test name.
|
||||||
|
|
||||||
|
The private deploy binary defines its contexts as typed Rust values. Contexts
|
||||||
|
are compiled into the binary; loading additional context definitions at runtime
|
||||||
|
is deferred until a real requirement justifies that interface.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let contexts = ContextCatalog::new([
|
||||||
|
NamedContext {
|
||||||
|
name: context_name!("e2e"),
|
||||||
|
target: ContextSpec::Remote(RemoteContext {
|
||||||
|
profile: Profile::Prod,
|
||||||
|
registry: oci_registry!("registry.example.com"),
|
||||||
|
registry_project: oci_repository!("harmony/e2e"),
|
||||||
|
domain: domain!("e2e.example.com"),
|
||||||
|
image_pull_secret: k8s_name!("registry-pull"),
|
||||||
|
access: OpenBaoClusterAccess {
|
||||||
|
namespace: openbao_namespace!("example"),
|
||||||
|
url: http_url!("https://secrets.nationtech.io"),
|
||||||
|
role: OpenBaoRoleName::new("example-ci")?,
|
||||||
|
zitadel_url: http_url!("https://identity.example.com"),
|
||||||
|
zitadel_audience: OidcAudience::new("example")?,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
])?;
|
||||||
|
```
|
||||||
|
|
||||||
|
Concrete names and identity details stay private.
|
||||||
|
|
||||||
|
### Context type design
|
||||||
|
|
||||||
|
`ContextSpec` should encode the access-mode choice instead of representing it
|
||||||
|
with unrelated optional fields:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub enum ContextSpec {
|
||||||
|
Local(LocalContext),
|
||||||
|
Remote(RemoteContext),
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`RemoteContext` carries required registry, domain, image-pull, and OpenBao
|
||||||
|
values directly. `LocalContext` carries only its k3d settings. This makes
|
||||||
|
impossible combinations unrepresentable and removes checks such as "exactly one
|
||||||
|
of autoprovision, k3d, or OpenBao must be set."
|
||||||
|
|
||||||
|
Use existing domain types where they are already correct:
|
||||||
|
|
||||||
|
- `Profile` remains an enum. It does not need a string macro.
|
||||||
|
- Kubernetes namespaces, Secret names, and other DNS-label resource names use
|
||||||
|
`K8sName` and a compile-time `k8s_name!` literal.
|
||||||
|
- IP and MAC literals continue using the existing Harmony macros.
|
||||||
|
|
||||||
|
Add types only for distinct grammars or values that are easy to transpose:
|
||||||
|
|
||||||
|
- `ContextName`: CLI selection key. It is not implicitly a Kubernetes name.
|
||||||
|
- `DomainName`: normalized DNS name without scheme, path, port, or wildcard.
|
||||||
|
- `HttpUrl`: HTTP(S)-only remote URL. The existing `Url` also permits local
|
||||||
|
folders and is too broad for OpenBao and Zitadel endpoints.
|
||||||
|
- `OciRegistry`: registry authority with optional port, but no repository path.
|
||||||
|
- `OciRepository`: validated OCI repository path.
|
||||||
|
- `OpenBaoNamespace`: a safe OpenBao path namespace, distinct from `K8sName`.
|
||||||
|
- `OpenBaoRoleName`: auth-role identifier, distinct from the namespace.
|
||||||
|
- `OidcAudience`: non-empty OIDC audience, kept semantically distinct even
|
||||||
|
though the protocol permits several string shapes.
|
||||||
|
|
||||||
|
Do not add a macro for every newtype. Literal macros are justified for the
|
||||||
|
grammars where compile-time diagnostics materially help: DNS names, HTTP URLs,
|
||||||
|
OCI registries/repositories, Kubernetes names, and OpenBao path namespaces.
|
||||||
|
Opaque identifiers can use typed constructors.
|
||||||
|
|
||||||
|
The runtime parser and literal macro must share one validator in
|
||||||
|
`harmony_types`; the macro must not copy the grammar. `harmony_macros` already
|
||||||
|
depends on `harmony_types`, so it can validate the literal during expansion and
|
||||||
|
emit the typed value. Each type needs accepted/rejected unit cases plus
|
||||||
|
compile-fail macro tests.
|
||||||
|
|
||||||
|
### Application integration
|
||||||
|
|
||||||
|
Change the lifecycle entry point from discovering `.harmony/contexts.toml` to
|
||||||
|
receiving the catalog:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
harmony_cli::app::app_main(FleetApp, contexts).await
|
||||||
|
```
|
||||||
|
|
||||||
|
`--context e2e` and `HARMONY_CONTEXT=e2e` select only from contexts compiled
|
||||||
|
into that deploy binary. The public Fleet binary can compile its local context;
|
||||||
|
the private binary compiles the remote E2E context. Both use the same exported
|
||||||
|
`FleetApp` and Scores.
|
||||||
|
|
||||||
|
## CI trust bootstrap
|
||||||
|
|
||||||
|
The CI runner receives one secret:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HARMONY_ZITADEL_KEY_JSON
|
||||||
|
```
|
||||||
|
|
||||||
|
It is the JSON key for a dedicated Zitadel machine identity. Harmony uses that
|
||||||
|
identity to obtain an OpenBao token from `secrets.nationtech.io`. `ConfigClient`
|
||||||
|
then loads the namespace-scoped kubeconfig and deployment secrets. The runner
|
||||||
|
holds no standing kubeconfig, OpenBao token, registry password, or application
|
||||||
|
secret.
|
||||||
|
|
||||||
|
Required server-side setup:
|
||||||
|
|
||||||
|
- [ ] Create a dedicated Zitadel machine identity for E2E CI.
|
||||||
|
- [ ] Grant only the role accepted by the E2E OpenBao auth role.
|
||||||
|
- [ ] Create an OpenBao policy restricted to the E2E config and secret paths.
|
||||||
|
- [ ] Store `ClusterAccess` with the namespace-scoped kubeconfig.
|
||||||
|
- [ ] Store registry credentials and deploy inputs under the E2E namespace.
|
||||||
|
- [ ] Add the machine key JSON as the CI repository's only secret.
|
||||||
|
- [ ] Compile the typed `e2e` context into the private deploy binary.
|
||||||
|
- [ ] Select it with `HARMONY_CONTEXT=e2e`.
|
||||||
|
|
||||||
|
## Known implementation gap
|
||||||
|
|
||||||
|
Cluster access already follows this flow. Image publication currently reads
|
||||||
|
`REGISTRY_USER` and `REGISTRY_TOKEN` directly from the environment. Before the
|
||||||
|
single-secret workflow is complete, registry authentication must be loaded
|
||||||
|
through the resolved `AppContext` and `ConfigClient` like the other secrets.
|
||||||
|
|
||||||
|
The context should resolve once at the start of the combined build-and-publish
|
||||||
|
job. Build itself remains credential-free; publish consumes the registry
|
||||||
|
credential fetched from OpenBao.
|
||||||
|
|
||||||
|
The current `harmony_app::context` TOML parser and `Option<String>` context
|
||||||
|
definition must also be replaced by the typed catalog above. This is a public
|
||||||
|
application-layer change; concrete context values remain private.
|
||||||
|
|
||||||
|
## Phase 1: cluster and identity
|
||||||
|
|
||||||
|
- [ ] Add the context domain types and their shared validators to
|
||||||
|
`harmony_types`.
|
||||||
|
- [ ] Add only the justified literal macros to `harmony_macros`, including
|
||||||
|
compile-fail tests.
|
||||||
|
- [ ] Replace TOML discovery with a caller-supplied `ContextCatalog`.
|
||||||
|
- [ ] Export the canonical `FleetApp` composition so the private deploy binary
|
||||||
|
does not duplicate it.
|
||||||
|
- [ ] Create a dedicated E2E namespace or namespace prefix on the public OKD
|
||||||
|
cluster.
|
||||||
|
- [ ] Configure namespace-scoped RBAC for the OpenBao-brokered kubeconfig.
|
||||||
|
- [ ] Add public DNS and exact-host OKD Routes for E2E services.
|
||||||
|
- [ ] Verify the CI machine identity can read `ClusterAccess` and no unrelated
|
||||||
|
OpenBao path.
|
||||||
|
- [ ] Run a read-only context check from CI with only
|
||||||
|
`HARMONY_ZITADEL_KEY_JSON` present.
|
||||||
|
|
||||||
|
## Phase 2: OCI publication
|
||||||
|
|
||||||
|
- [ ] Move registry credentials behind `ConfigClient`.
|
||||||
|
- [ ] Create a disposable registry project or repository scope for E2E images.
|
||||||
|
- [ ] Run `build` and `publish` in one CI job.
|
||||||
|
- [ ] Verify the pushed manifest digest matches the digest returned by build.
|
||||||
|
- [ ] Pull and run that exact digest in a fresh namespace.
|
||||||
|
- [ ] Apply retention rules so E2E images expire automatically.
|
||||||
|
|
||||||
|
The production publication path remains the same operation against the
|
||||||
|
production registry; only the context and credential policy differ.
|
||||||
|
|
||||||
|
## Phase 3: NATS regression
|
||||||
|
|
||||||
|
- [ ] Deploy the existing certificate and mTLS-capable NATS supercluster Scores
|
||||||
|
unchanged before modifying their APIs.
|
||||||
|
- [ ] Exercise public HTTPS/WebSocket and native TLS/mTLS endpoints through OKD
|
||||||
|
Routes.
|
||||||
|
- [ ] Assert cluster formation, authenticated traffic, SNI routing, reconnect
|
||||||
|
after pod restart, and certificate validation from outside the cluster.
|
||||||
|
- [ ] Record failures as regressions against the existing module before changing
|
||||||
|
Fleet composition.
|
||||||
|
|
||||||
|
## Phase 4: Fleet lifecycle
|
||||||
|
|
||||||
|
- [ ] Build and publish the operator and callout images in one job.
|
||||||
|
- [ ] Pass their digest-pinned references to `deploy`.
|
||||||
|
- [ ] Deploy Fleet through the private `e2e` context using production Scores and
|
||||||
|
typed Refs.
|
||||||
|
- [ ] Enroll an external test agent through the public NATS endpoint.
|
||||||
|
- [ ] Apply, update, and delete a Podman deployment and verify reflected state.
|
||||||
|
- [ ] Exercise Zitadel group changes and OpenBao secret access.
|
||||||
|
- [ ] Verify failed reconciliation recovers after the dependency is restored.
|
||||||
|
- [ ] Remove per-run resources and verify no desired-state or grant residue
|
||||||
|
remains.
|
||||||
|
|
||||||
|
## Guardrails
|
||||||
|
|
||||||
|
- Use a dedicated machine identity, OpenBao role, registry scope, and Kubernetes
|
||||||
|
RBAC grant for E2E.
|
||||||
|
- Keep every concrete private identifier in the private repository or OpenBao.
|
||||||
|
- Put a TTL label on per-run namespaces and clean abandoned runs automatically.
|
||||||
|
- Never print the machine key, OpenBao token, kubeconfig, or registry credential.
|
||||||
|
- Split CI into identity, registry, NATS, and Fleet stages so failures identify
|
||||||
|
one boundary.
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
[Harmony Documentation](./README.md)
|
[Harmony Documentation](./README.md)
|
||||||
|
|
||||||
- [Core Concepts](./concepts.md)
|
- [Core Concepts](./concepts.md)
|
||||||
|
- [Typed Score References](./concepts/score-references.md)
|
||||||
- [Getting Started Guide](./guides/getting-started.md)
|
- [Getting Started Guide](./guides/getting-started.md)
|
||||||
|
|
||||||
## Use Cases
|
## Use Cases
|
||||||
@@ -33,6 +34,10 @@
|
|||||||
|
|
||||||
- [Configuration](./concepts/configuration.md)
|
- [Configuration](./concepts/configuration.md)
|
||||||
|
|
||||||
|
## Reference Designs
|
||||||
|
|
||||||
|
- [Fleet Score References](./reference/fleet-score-references.md)
|
||||||
|
|
||||||
## Architecture Decision Records
|
## Architecture Decision Records
|
||||||
|
|
||||||
- [ADR Overview](./adr/README.md)
|
- [ADR Overview](./adr/README.md)
|
||||||
@@ -57,5 +62,11 @@
|
|||||||
- [019 · Network Bond Setup](./adr/019-Network-bond-setup.md)
|
- [019 · Network Bond Setup](./adr/019-Network-bond-setup.md)
|
||||||
- [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md)
|
- [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md)
|
||||||
- [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md)
|
- [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md)
|
||||||
|
- [022 · Fleet Agent Upgrade Procedure](./adr/022-fleet-agent-upgrade.md)
|
||||||
|
- [023 · Deploy Architecture](./adr/023-deploy-architecture.md)
|
||||||
|
- [024 · Fleet Platform Capability Decomposition](./adr/024-fleet-platform-capability-decomposition.md)
|
||||||
- [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md)
|
- [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md)
|
||||||
- [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md)
|
- [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md)
|
||||||
|
- [026 · Application Lifecycle CLI](./adr/026-application-lifecycle-cli.md)
|
||||||
|
- [027 · Multi-Tenant Cloud Identity](./adr/027-multi-tenant-identity.md)
|
||||||
|
- [028 · Typed Score References](./adr/028-typed-score-references.md)
|
||||||
|
|||||||
176
docs/adr/028-typed-score-references.md
Normal file
176
docs/adr/028-typed-score-references.md
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
# Architecture Decision Record: Typed Score References
|
||||||
|
|
||||||
|
Initial Author: Jean-Gabriel Gill-Couture
|
||||||
|
|
||||||
|
Initial Date: 2026-07-15
|
||||||
|
|
||||||
|
Last Updated Date: 2026-07-15
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Proposed. Fleet is the first application.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
ADR-023 establishes one Score per deployable component and requires deploy
|
||||||
|
crates to compose Scores instead of rebuilding their operations. It does not
|
||||||
|
define how one Score refers to desired state owned by another component.
|
||||||
|
|
||||||
|
Today, deploy code often passes URLs, names, IDs, and credential locations as
|
||||||
|
strings. Consumers then parse those strings or reproduce the producer's naming
|
||||||
|
conventions. Rust cannot distinguish different string contracts, and a refactor
|
||||||
|
can leave valid code that fails only at runtime.
|
||||||
|
|
||||||
|
Earlier composition design attempts treated these relationships as a dependency
|
||||||
|
graph or pipeline. That caused too strong coupling between each component's
|
||||||
|
lifecycle. Harmony needs a smaller contract: a consumer should know what it
|
||||||
|
references without depending on the producer Score or controlling when it runs.
|
||||||
|
|
||||||
|
Infrastructure also does not converge once and remain stable. A referenced
|
||||||
|
component may not exist yet, may still be converging, or may become unavailable
|
||||||
|
later. The reference model must not imply otherwise.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Scores will exchange concrete typed Refs for desired state owned by another
|
||||||
|
component:
|
||||||
|
|
||||||
|
[](../diagrams/score-ref-composition.svg)
|
||||||
|
|
||||||
|
The complete API is small:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct DbRef {
|
||||||
|
// Desired-state properties needed to connect.
|
||||||
|
}
|
||||||
|
|
||||||
|
let db_score = SomeDbScore::new(...);
|
||||||
|
let db_ref = db_score.connection();
|
||||||
|
let app_score = AppScore::new(...).with_db(&db_ref);
|
||||||
|
```
|
||||||
|
|
||||||
|
The producer derives a Ref from its desired state. Creating or passing the Ref
|
||||||
|
performs no I/O and does not require the producer to have run.
|
||||||
|
|
||||||
|
The consumer depends on the Ref type, not the producer Score. The Ref contains
|
||||||
|
the typed identifiers needed to describe the relationship. It does not contain
|
||||||
|
a client, execute provider operations, or transfer ownership of the producer's
|
||||||
|
lifecycle.
|
||||||
|
|
||||||
|
A Ref is eventually consistent. It does not prove that its target exists, has
|
||||||
|
converged, or remains available. The consumer's `Interpret`/`Topology` path must
|
||||||
|
handle that uncertainty. Where Kubernetes already understands the relationship,
|
||||||
|
the consumer should emit the native Kubernetes reference and let Kubernetes
|
||||||
|
reconcile it.
|
||||||
|
|
||||||
|
Refs do not impose execution order. They do not define a dependency graph,
|
||||||
|
pipeline, scheduler, or continuous convergence mechanism.
|
||||||
|
|
||||||
|
### Cyclic relationships
|
||||||
|
|
||||||
|
A cyclic relationship uses the same operation in both directions:
|
||||||
|
|
||||||
|
[](../diagrams/score-ref-cycle.svg)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let broker_score = BrokerScore::new(...);
|
||||||
|
let broker_ref = broker_score.connection();
|
||||||
|
|
||||||
|
let auth_score = AuthScore::new(...).with_broker(&broker_ref);
|
||||||
|
let auth_ref = auth_score.endpoint();
|
||||||
|
|
||||||
|
let broker_score = broker_score.with_auth(&auth_ref);
|
||||||
|
```
|
||||||
|
|
||||||
|
Both Refs derive from desired state before either Score runs. Declaring the
|
||||||
|
cycle therefore needs no graph traversal or scheduler.
|
||||||
|
|
||||||
|
### Values produced later
|
||||||
|
|
||||||
|
A Ref may identify a value that will be generated later, such as an OIDC client
|
||||||
|
ID. Composition can pass the Ref before that value exists. This is the same
|
||||||
|
eventual-consistency rule, not a separate output mechanism.
|
||||||
|
|
||||||
|
### Guarantees
|
||||||
|
|
||||||
|
| Statement | Guarantee |
|
||||||
|
|---|---|
|
||||||
|
| Consumer accepts the intended Ref type | Rust compile time |
|
||||||
|
| Ref construction performs no I/O | Ref API contract |
|
||||||
|
| Target exists or has converged | Not guaranteed |
|
||||||
|
| Runtime systems are compatible or authorized | Not guaranteed(*) |
|
||||||
|
| Target remains available | Not guaranteed |
|
||||||
|
|
||||||
|
> (*) The runtime compatibility or authorization is not guaranteed by the **Ref** itself. But it is usually Harmony's job to provide ref types in a way that makes them compatible. For example, a PostgreSQL score exposing a PostgreSQLRootAccountRef will have the responsibility to provision it.
|
||||||
|
>
|
||||||
|
> In other words, __correct provisioning of a Ref is the score's developer's job, not the ref's job.__
|
||||||
|
|
||||||
|
Changing a Ref's type-level contract produces compiler errors in affected
|
||||||
|
consumers. Rust cannot catch a provider returning incorrect live state, so
|
||||||
|
runtime validation and readiness remain necessary.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
- Typed Refs replace string parsing and copied naming conventions.
|
||||||
|
- Scores share only the desired-state contract needed by the consumer.
|
||||||
|
- Refs can be composed before any component runs.
|
||||||
|
- Consumers remain responsible for infrastructure that is not yet ready or is
|
||||||
|
temporarily unavailable.
|
||||||
|
- Kubernetes keeps reconciling relationships it already understands.
|
||||||
|
- Harmony does not need a graph or pipeline abstraction to express a typed
|
||||||
|
relationship.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
### Positive
|
||||||
|
|
||||||
|
- Refactoring a relationship's typed contract reaches its consumers through the
|
||||||
|
compiler.
|
||||||
|
- Components remain independent of each other's Score implementations and
|
||||||
|
execution order.
|
||||||
|
- Production and E2E can compose the same Score APIs against different
|
||||||
|
Topologies.
|
||||||
|
- Values produced later need not become string dependencies in deploy code.
|
||||||
|
|
||||||
|
### Negative
|
||||||
|
|
||||||
|
- Each real relationship needs a concrete Ref type.
|
||||||
|
- Consumers must handle unresolved and unavailable targets.
|
||||||
|
- Types do not remove runtime readiness, compatibility, or authorization checks.
|
||||||
|
- Existing string-based wiring must be migrated as its owning Scores gain Refs.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
### Continue passing strings
|
||||||
|
|
||||||
|
Rejected. Strings erase domain meaning and force consumers to know producer
|
||||||
|
naming and parsing rules.
|
||||||
|
|
||||||
|
### Pass producer Scores to consumers
|
||||||
|
|
||||||
|
Rejected. This couples component lifecycle and implementation when the consumer
|
||||||
|
only needs a desired-state reference.
|
||||||
|
|
||||||
|
### Make execution return values the composition mechanism
|
||||||
|
|
||||||
|
Rejected for this decision. It requires the producer to run before composition
|
||||||
|
can continue and turns execution order into the dependency model.
|
||||||
|
|
||||||
|
### Introduce a generic graph or output registry
|
||||||
|
|
||||||
|
Rejected. Typed Refs solve the current problem without either abstraction.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
|
||||||
|
ADR-028 does not define continuous convergence, retries, scheduling, graph
|
||||||
|
execution, Ref persistence, or a generic `Ref<T>`. Those decisions require their
|
||||||
|
own evidence and designs.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [Typed Score references](../concepts/score-references.md): concept guide.
|
||||||
|
- [Fleet Score references](../reference/fleet-score-references.md): first application.
|
||||||
|
- [ADR-002](./002-hexagonal-architecture.md): domain and adapter boundaries.
|
||||||
|
- [ADR-015](./015-higher-order-topologies.md): compile-time Topology
|
||||||
|
composition.
|
||||||
|
- [ADR-023](./023-deploy-architecture.md): component Scores and deploy crates.
|
||||||
@@ -59,6 +59,7 @@ Every ADR follows this structure:
|
|||||||
| [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted |
|
| [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted |
|
||||||
| [026](./026-application-lifecycle-cli.md) | Application Lifecycle CLI | Accepted |
|
| [026](./026-application-lifecycle-cli.md) | Application Lifecycle CLI | Accepted |
|
||||||
| [027](./027-multi-tenant-identity.md) | Multi-Tenant Cloud Identity | Accepted |
|
| [027](./027-multi-tenant-identity.md) | Multi-Tenant Cloud Identity | Accepted |
|
||||||
|
| [028](./028-typed-score-references.md) | Typed Score References | Proposed |
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
|
|||||||
@@ -422,9 +422,8 @@ what vocabulary should the new fleet shape stay consistent with?
|
|||||||
apply via typed code.
|
apply via typed code.
|
||||||
- **Implication for fleet:** chart-rendering helpers
|
- **Implication for fleet:** chart-rendering helpers
|
||||||
(`build_operator_deployment` et al.) are template-hydration
|
(`build_operator_deployment` et al.) are template-hydration
|
||||||
edges. They *should* be hidden from domain code. Today they're
|
edges. They *should* be hidden from domain code. Exposing them to
|
||||||
`pub` — visible to consumers like `fleet_staging_install` who
|
deploy callers instead of composing `FleetOperatorScore` is adapter
|
||||||
reach in and grab `operator_secret(opts)`. That's adapter
|
|
||||||
leakage.
|
leakage.
|
||||||
|
|
||||||
### §2.4 — Synthesis: principles for the redesign
|
### §2.4 — Synthesis: principles for the redesign
|
||||||
|
|||||||
125
docs/concepts/score-references.md
Normal file
125
docs/concepts/score-references.md
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
# Typed Score references
|
||||||
|
|
||||||
|
Harmony builds loosely dependent but strongly typed infrastructure components
|
||||||
|
using Refs.
|
||||||
|
|
||||||
|
This means that a `ZitadelScore` does not directly depend on a
|
||||||
|
`PostgreSQLScore`. It can instead accept a `PostgreSQLRootAccountRef` in its
|
||||||
|
builder to know how to connect to PostgreSQL. This is safer than passing strings
|
||||||
|
around: the Ref has a type Rust can check, and consumers do not need to parse or
|
||||||
|
reconstruct the producer's naming conventions.
|
||||||
|
|
||||||
|
The loose dependency matters. Previous design proposals treated composition as a
|
||||||
|
dependency graph or pipeline, which forced components to know too much about one
|
||||||
|
another. A Ref is only a typed reference to declarative desired state. It does
|
||||||
|
not give the consumer control over the producer's lifecycle.
|
||||||
|
|
||||||
|
Most importantly, a Ref does not guarantee that its target exists or has
|
||||||
|
converged when it is passed. Like a Kubernetes reference, it is eventually
|
||||||
|
consistent. Consumers must tolerate the target being absent, starting, or
|
||||||
|
temporarily unavailable.
|
||||||
|
|
||||||
|
Kubernetes gets this right: infrastructure is chaotic. A component can fail
|
||||||
|
after it becomes ready. Drives fail, power goes out, and networks split. The
|
||||||
|
long-term answer is continuous convergence, not an assumption that deployment
|
||||||
|
finishes infrastructure forever.
|
||||||
|
|
||||||
|
The convergence mechanism is outside ADR-028. Choosing eventually consistent
|
||||||
|
Refs now lets Scores declare relationships without committing Harmony to a
|
||||||
|
scheduler, dependency engine, or reconciliation design.
|
||||||
|
|
||||||
|
## The whole pattern
|
||||||
|
|
||||||
|
[](../diagrams/score-ref-composition.svg)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct DbRef {
|
||||||
|
// Desired-state properties needed to connect.
|
||||||
|
}
|
||||||
|
|
||||||
|
let db_score = SomeDbScore::new(...);
|
||||||
|
let some_db_ref = db_score.connection();
|
||||||
|
let app_score = AppScore::new(...).with_db(&some_db_ref);
|
||||||
|
```
|
||||||
|
|
||||||
|
That is the mechanism. `connection()` derives `DbRef` from the database's desired
|
||||||
|
state. It performs no I/O and does not require `SomeDbScore` to have been
|
||||||
|
interpreted.
|
||||||
|
|
||||||
|
`AppScore` knows `DbRef`. It does not need to know `SomeDbScore`, its provider,
|
||||||
|
or its readiness procedure.
|
||||||
|
|
||||||
|
Compare that with copying a connection string:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
app.database_url("postgres://some-db.apps.svc:5432/app");
|
||||||
|
```
|
||||||
|
|
||||||
|
That string duplicates database naming and connection knowledge in the consumer.
|
||||||
|
A rename or layout change can leave valid Rust that fails at runtime. A typed
|
||||||
|
Ref removes that string contract and makes type-level contract changes visible
|
||||||
|
to the compiler.
|
||||||
|
|
||||||
|
## Cyclic relationships
|
||||||
|
|
||||||
|
A cycle is the same pattern in both directions:
|
||||||
|
|
||||||
|
[](../diagrams/score-ref-cycle.svg)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let broker_score = BrokerScore::new(...);
|
||||||
|
let broker_ref = broker_score.connection();
|
||||||
|
|
||||||
|
let auth_score = AuthScore::new(...).with_broker(&broker_ref);
|
||||||
|
let auth_ref = auth_score.endpoint();
|
||||||
|
|
||||||
|
let broker_score = broker_score.with_auth(&auth_ref);
|
||||||
|
```
|
||||||
|
|
||||||
|
No Score has run. Each Ref comes from desired state, so the two relationships
|
||||||
|
can be declared without traversing a graph or resolving either resource.
|
||||||
|
|
||||||
|
## Eventual consistency
|
||||||
|
|
||||||
|
Composition code passes Refs; it does not wait for them to become ready. The
|
||||||
|
consumer's `Interpret`/`Topology` path handles an unresolved target in the way
|
||||||
|
appropriate for that component. It may:
|
||||||
|
|
||||||
|
- emit a native Kubernetes reference and let Kubernetes reconcile it;
|
||||||
|
- wait or retry when the value is needed immediately;
|
||||||
|
- return a non-success outcome for the current interpretation attempt.
|
||||||
|
|
||||||
|
ADR-028 does not choose among those behaviors or define when another attempt
|
||||||
|
runs. It only requires consumers not to treat possession of a Ref as proof of
|
||||||
|
availability.
|
||||||
|
|
||||||
|
Refs may also identify values generated later, such as an OIDC client ID. The
|
||||||
|
same rule applies: passing the Ref does not claim that the value exists yet.
|
||||||
|
|
||||||
|
## What types guarantee
|
||||||
|
|
||||||
|
Concrete types such as `PostgreSQLRootAccountRef`, `OidcApplicationRef`, and
|
||||||
|
`NatsAccountRef` prevent unrelated references from being exchanged by accident.
|
||||||
|
They also remove string parsing and copied naming rules from consumers.
|
||||||
|
|
||||||
|
Types do not prove that:
|
||||||
|
|
||||||
|
- the referenced resource exists;
|
||||||
|
- the producer has converged;
|
||||||
|
- two live systems can communicate;
|
||||||
|
- credentials are authorized;
|
||||||
|
- the infrastructure will remain available.
|
||||||
|
|
||||||
|
Those remain runtime concerns.
|
||||||
|
|
||||||
|
## Authoring rule
|
||||||
|
|
||||||
|
Use a Ref when one Score would otherwise copy a name, URL, ID, credential
|
||||||
|
location, or other desired-state identifier owned by another component. Use an
|
||||||
|
ordinary typed value when there is no such relationship.
|
||||||
|
|
||||||
|
Refs express relationships. They do not impose execution order, create a
|
||||||
|
pipeline, or make Harmony responsible for continuous convergence.
|
||||||
|
|
||||||
|
See [ADR-028](../adr/028-typed-score-references.md) for the decision and the
|
||||||
|
[Fleet reference](../reference/fleet-score-references.md) for the first application.
|
||||||
45
docs/diagrams/fleet-identity-composition.svg
Normal file
45
docs/diagrams/fleet-identity-composition.svg
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="330" viewBox="0 0 1200 330" role="img" aria-labelledby="title desc">
|
||||||
|
<title id="title">PostgreSQL and Zitadel compose identity functionality through a typed Ref</title>
|
||||||
|
<desc id="desc">PostgreSQLScore exports PostgreSQLRootAccountRef to ZitadelScore. Together they provide identity functionality.</desc>
|
||||||
|
<defs>
|
||||||
|
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
|
||||||
|
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
|
||||||
|
</marker>
|
||||||
|
<style>
|
||||||
|
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||||
|
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
|
||||||
|
</style>
|
||||||
|
</defs>
|
||||||
|
<rect width="1200" height="330" fill="#ffffff"/>
|
||||||
|
<g class="sans">
|
||||||
|
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Compose identity</text>
|
||||||
|
<text x="40" y="80" fill="#64748b" font-size="17">A consumer receives a typed contract, not the producer Score.</text>
|
||||||
|
|
||||||
|
<rect x="40" y="120" width="270" height="100" rx="12" fill="#172554"/>
|
||||||
|
<text x="65" y="150" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
|
||||||
|
<text x="65" y="184" fill="#ffffff" font-size="21" font-weight="700" class="mono">PostgreSQLScore</text>
|
||||||
|
<text x="65" y="207" fill="#bfdbfe" font-size="14">provisions PostgreSQL</text>
|
||||||
|
|
||||||
|
<path d="M310 170H385" fill="none" stroke="#0f766e" stroke-width="2.5"/>
|
||||||
|
<rect x="385" y="143" width="290" height="54" rx="27" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
|
||||||
|
<text x="530" y="165" text-anchor="middle" fill="#0f766e" font-size="11" font-weight="750" letter-spacing="1">REF TYPE</text>
|
||||||
|
<text x="530" y="184" text-anchor="middle" fill="#115e59" font-size="14" font-weight="650" class="mono">PostgreSQLRootAccountRef</text>
|
||||||
|
<path d="M675 170H750" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
|
||||||
|
<rect x="750" y="120" width="230" height="100" rx="12" fill="#172554"/>
|
||||||
|
<text x="775" y="150" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
|
||||||
|
<text x="775" y="184" fill="#ffffff" font-size="21" font-weight="700" class="mono">ZitadelScore</text>
|
||||||
|
<text x="775" y="207" fill="#bfdbfe" font-size="14">provisions Zitadel</text>
|
||||||
|
|
||||||
|
<text x="1005" y="181" fill="#94a3b8" font-size="28" font-weight="500">=</text>
|
||||||
|
<rect x="1040" y="120" width="130" height="100" rx="12" fill="#eef2ff" stroke="#6366f1" stroke-width="1.5"/>
|
||||||
|
<text x="1105" y="158" text-anchor="middle" fill="#3730a3" font-size="12" font-weight="750" letter-spacing="1.2">IDENTITY</text>
|
||||||
|
<text x="1105" y="180" text-anchor="middle" fill="#3730a3" font-size="12" font-weight="750" letter-spacing="1.2">FUNCTIONALITY</text>
|
||||||
|
<text x="1105" y="204" text-anchor="middle" fill="#6366f1" font-size="13">composed</text>
|
||||||
|
|
||||||
|
<rect x="40" y="260" width="18" height="18" rx="4" fill="#172554"/>
|
||||||
|
<text x="70" y="275" fill="#64748b" font-size="14">Score owns behavior</text>
|
||||||
|
<rect x="238" y="259" width="54" height="20" rx="10" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="304" y="275" fill="#64748b" font-size="14">Ref carries typed desired state</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 3.2 KiB |
50
docs/diagrams/fleet-messaging-composition.svg
Normal file
50
docs/diagrams/fleet-messaging-composition.svg
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="390" viewBox="0 0 1200 390" role="img" aria-labelledby="title desc">
|
||||||
|
<title id="title">NATS and its authentication callout compose authenticated messaging</title>
|
||||||
|
<desc id="desc">NatsScore exports NatsAccountRef to NatsAuthCalloutScore. NatsAuthCalloutScore exports NatsAuthCalloutRef back to NatsScore. The composed functionality requires OidcProviderRef and exports NatsClientRef.</desc>
|
||||||
|
<defs>
|
||||||
|
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
|
||||||
|
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
|
||||||
|
</marker>
|
||||||
|
<style>
|
||||||
|
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||||
|
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
|
||||||
|
</style>
|
||||||
|
</defs>
|
||||||
|
<rect width="1200" height="390" fill="#ffffff"/>
|
||||||
|
<g class="sans">
|
||||||
|
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Compose authenticated messaging</text>
|
||||||
|
<text x="40" y="80" fill="#64748b" font-size="17">Two Scores can exchange desired-state Refs without imposing execution order.</text>
|
||||||
|
|
||||||
|
<rect x="40" y="108" width="1120" height="220" rx="16" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
|
||||||
|
<text x="70" y="137" fill="#64748b" font-size="12" font-weight="750" letter-spacing="1.3">AUTHENTICATED MESSAGING FUNCTIONALITY</text>
|
||||||
|
|
||||||
|
<text x="70" y="177" fill="#64748b" font-size="12" font-weight="750">REQUIRES</text>
|
||||||
|
<rect x="145" y="155" width="190" height="34" rx="17" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="240" y="177" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">OidcProviderRef</text>
|
||||||
|
|
||||||
|
<rect x="70" y="210" width="235" height="90" rx="12" fill="#172554"/>
|
||||||
|
<text x="94" y="238" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
|
||||||
|
<text x="94" y="270" fill="#ffffff" font-size="21" font-weight="700" class="mono">NatsScore</text>
|
||||||
|
<text x="94" y="291" fill="#bfdbfe" font-size="14">provisions NATS</text>
|
||||||
|
|
||||||
|
<path d="M305 236H650" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
<rect x="365" y="216" width="220" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="475" y="241" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">NatsAccountRef</text>
|
||||||
|
|
||||||
|
<path d="M650 276H305" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
<rect x="350" y="256" width="250" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="475" y="281" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">NatsAuthCalloutRef</text>
|
||||||
|
|
||||||
|
<rect x="650" y="210" width="265" height="90" rx="12" fill="#172554"/>
|
||||||
|
<text x="674" y="238" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
|
||||||
|
<text x="674" y="270" fill="#ffffff" font-size="16" font-weight="700" class="mono">NatsAuthCalloutScore</text>
|
||||||
|
<text x="674" y="291" fill="#bfdbfe" font-size="14">provisions the callout</text>
|
||||||
|
|
||||||
|
<text x="950" y="220" fill="#64748b" font-size="12" font-weight="750">EXPORTS</text>
|
||||||
|
<rect x="950" y="232" width="180" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
|
||||||
|
<text x="1040" y="258" text-anchor="middle" fill="#115e59" font-size="14" font-weight="650" class="mono">NatsClientRef</text>
|
||||||
|
<text x="950" y="299" fill="#475569" font-size="14">One capability for consumers</text>
|
||||||
|
|
||||||
|
<text x="40" y="365" fill="#64748b" font-size="14">The common region is the functionality; the blue boxes remain independent Scores.</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 3.6 KiB |
100
docs/diagrams/fleet-score-references.svg
Normal file
100
docs/diagrams/fleet-score-references.svg
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="790" viewBox="0 0 1400 790" role="img" aria-labelledby="title desc">
|
||||||
|
<title id="title">Fleet reference implementation contract map</title>
|
||||||
|
<desc id="desc">A three-column map lists reusable Fleet functionality, the concrete Ref types it produces, and the Score types that consume each Ref.</desc>
|
||||||
|
<style>
|
||||||
|
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||||
|
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
|
||||||
|
.header { fill: #64748b; font-size: 13px; font-weight: 750; letter-spacing: 1.4px; }
|
||||||
|
.function { fill: #0f172a; font-size: 20px; font-weight: 720; }
|
||||||
|
.detail { fill: #64748b; font-size: 14px; }
|
||||||
|
.ref { fill: #115e59; font-size: 14px; font-weight: 650; }
|
||||||
|
.score { fill: #ffffff; font-size: 13px; font-weight: 650; }
|
||||||
|
</style>
|
||||||
|
<rect width="1400" height="790" fill="#ffffff"/>
|
||||||
|
<g class="sans">
|
||||||
|
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Fleet reference implementation</text>
|
||||||
|
<text x="40" y="80" fill="#64748b" font-size="17">Read each contract left to right. The map shows type compatibility, not execution order.</text>
|
||||||
|
|
||||||
|
<text x="40" y="122" class="header">FUNCTIONALITY</text>
|
||||||
|
<text x="360" y="122" class="header">PRODUCES REF TYPE</text>
|
||||||
|
<text x="740" y="122" class="header">CONSUMED BY SCORE TYPE</text>
|
||||||
|
<line x1="40" y1="135" x2="1360" y2="135" stroke="#94a3b8" stroke-width="1.5"/>
|
||||||
|
|
||||||
|
<!-- Identity contracts -->
|
||||||
|
<rect x="40" y="150" width="280" height="190" rx="12" fill="#eef2ff"/>
|
||||||
|
<rect x="40" y="150" width="6" height="190" rx="3" fill="#6366f1"/>
|
||||||
|
<text x="65" y="190" class="function">Identity</text>
|
||||||
|
<text x="65" y="218" class="detail mono">PostgreSQLScore</text>
|
||||||
|
<text x="65" y="240" class="detail">+</text>
|
||||||
|
<text x="65" y="262" class="detail mono">ZitadelScore</text>
|
||||||
|
|
||||||
|
<rect x="360" y="158" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="184" text-anchor="middle" class="mono ref">OidcProviderRef</text>
|
||||||
|
<rect x="360" y="218" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="244" text-anchor="middle" class="mono ref">OidcApplicationRef</text>
|
||||||
|
<rect x="360" y="278" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="304" text-anchor="middle" class="mono ref">OidcMachineIdentityRef</text>
|
||||||
|
|
||||||
|
<rect x="740" y="158" width="170" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="825" y="184" text-anchor="middle" class="mono score">OpenBaoScore</text>
|
||||||
|
<rect x="920" y="158" width="210" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="1025" y="184" text-anchor="middle" class="mono score" style="font-size:12px">NatsAuthCalloutScore</text>
|
||||||
|
<rect x="1140" y="158" width="220" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="1250" y="184" text-anchor="middle" class="mono score" style="font-size:11px">FleetDeviceEnrollmentScore</text>
|
||||||
|
<rect x="740" y="218" width="220" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="850" y="244" text-anchor="middle" class="mono score">FleetOperatorScore</text>
|
||||||
|
<rect x="740" y="278" width="220" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="850" y="304" text-anchor="middle" class="mono score">FleetOperatorScore</text>
|
||||||
|
|
||||||
|
<line x1="330" y1="210" x2="1360" y2="210" stroke="#e2e8f0"/>
|
||||||
|
<line x1="330" y1="270" x2="1360" y2="270" stroke="#e2e8f0"/>
|
||||||
|
|
||||||
|
<!-- Messaging contract -->
|
||||||
|
<rect x="40" y="360" width="280" height="75" rx="12" fill="#ecfeff"/>
|
||||||
|
<rect x="40" y="360" width="6" height="75" rx="3" fill="#06b6d4"/>
|
||||||
|
<text x="65" y="390" class="function">Messaging</text>
|
||||||
|
<text x="65" y="415" class="detail mono" style="font-size:12px">NatsScore + NatsAuthCalloutScore</text>
|
||||||
|
|
||||||
|
<rect x="360" y="376" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="402" text-anchor="middle" class="mono ref">NatsClientRef</text>
|
||||||
|
<rect x="740" y="376" width="220" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="850" y="402" text-anchor="middle" class="mono score">FleetOperatorScore</text>
|
||||||
|
<rect x="975" y="376" width="205" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="1077" y="402" text-anchor="middle" class="mono score">FleetDeviceScore</text>
|
||||||
|
|
||||||
|
<!-- Secret contracts -->
|
||||||
|
<rect x="40" y="455" width="280" height="130" rx="12" fill="#f0fdf4"/>
|
||||||
|
<rect x="40" y="455" width="6" height="130" rx="3" fill="#22c55e"/>
|
||||||
|
<text x="65" y="495" class="function">Secrets</text>
|
||||||
|
<text x="65" y="523" class="detail mono">OpenBaoScore</text>
|
||||||
|
|
||||||
|
<rect x="360" y="463" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="489" text-anchor="middle" class="mono ref">OpenBaoRef</text>
|
||||||
|
<rect x="360" y="523" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="549" text-anchor="middle" class="mono ref">OpenBaoJwtAuthRef</text>
|
||||||
|
<rect x="740" y="463" width="220" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="850" y="489" text-anchor="middle" class="mono score">FleetOperatorScore</text>
|
||||||
|
<rect x="740" y="523" width="205" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="842" y="549" text-anchor="middle" class="mono score">FleetDeviceScore</text>
|
||||||
|
<line x1="330" y1="515" x2="1360" y2="515" stroke="#e2e8f0"/>
|
||||||
|
|
||||||
|
<!-- Device identity contract -->
|
||||||
|
<rect x="40" y="605" width="280" height="75" rx="12" fill="#fff7ed"/>
|
||||||
|
<rect x="40" y="605" width="6" height="75" rx="3" fill="#f97316"/>
|
||||||
|
<text x="65" y="635" class="function">Device identity</text>
|
||||||
|
<text x="65" y="660" class="detail mono" style="font-size:12px">FleetDeviceEnrollmentScore</text>
|
||||||
|
|
||||||
|
<rect x="360" y="621" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="510" y="647" text-anchor="middle" class="mono ref">DeviceIdentityRef</text>
|
||||||
|
<rect x="740" y="621" width="205" height="42" rx="10" fill="#172554"/>
|
||||||
|
<text x="842" y="647" text-anchor="middle" class="mono score">FleetDeviceScore</text>
|
||||||
|
|
||||||
|
<line x1="40" y1="705" x2="1360" y2="705" stroke="#cbd5e1"/>
|
||||||
|
<rect x="40" y="730" width="120" height="36" rx="9" fill="#172554"/>
|
||||||
|
<text x="100" y="753" text-anchor="middle" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.2">SCORE TYPE</text>
|
||||||
|
<text x="178" y="753" fill="#64748b" font-size="14">owns behavior</text>
|
||||||
|
<rect x="320" y="728" width="145" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="392" y="753" text-anchor="middle" class="mono ref">SomeRef</text>
|
||||||
|
<text x="485" y="753" fill="#64748b" font-size="14">carries typed desired state, not readiness</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 6.5 KiB |
38
docs/diagrams/score-ref-composition.svg
Normal file
38
docs/diagrams/score-ref-composition.svg
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="300" viewBox="0 0 1100 300" role="img" aria-labelledby="title desc">
|
||||||
|
<title id="title">A Score passes a typed Ref to a dependent Score</title>
|
||||||
|
<desc id="desc">SomeDbScore produces DbRef from desired state. AppScore accepts DbRef without depending on SomeDbScore.</desc>
|
||||||
|
<defs>
|
||||||
|
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
|
||||||
|
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
|
||||||
|
</marker>
|
||||||
|
<style>
|
||||||
|
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||||
|
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
|
||||||
|
</style>
|
||||||
|
</defs>
|
||||||
|
<rect width="1100" height="300" fill="#ffffff"/>
|
||||||
|
<g class="sans">
|
||||||
|
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Score → Ref → dependent Score</text>
|
||||||
|
<text x="40" y="80" fill="#64748b" font-size="17">The Ref is the complete relationship between otherwise independent components.</text>
|
||||||
|
|
||||||
|
<text x="55" y="128" fill="#64748b" font-size="12" font-weight="750">PRODUCER</text>
|
||||||
|
<rect x="40" y="140" width="255" height="90" rx="12" fill="#172554"/>
|
||||||
|
<text x="64" y="168" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
|
||||||
|
<text x="64" y="201" fill="#ffffff" font-size="22" font-weight="700" class="mono">SomeDbScore</text>
|
||||||
|
<text x="64" y="221" fill="#bfdbfe" font-size="14">owns database behavior</text>
|
||||||
|
|
||||||
|
<path d="M295 185H390" fill="none" stroke="#0f766e" stroke-width="2.5"/>
|
||||||
|
<rect x="390" y="158" width="250" height="54" rx="27" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
|
||||||
|
<text x="515" y="180" text-anchor="middle" fill="#0f766e" font-size="11" font-weight="750" letter-spacing="1.2">REF TYPE</text>
|
||||||
|
<text x="515" y="200" text-anchor="middle" fill="#115e59" font-size="18" font-weight="700" class="mono">DbRef</text>
|
||||||
|
<path d="M640 185H735" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
|
||||||
|
<text x="750" y="128" fill="#64748b" font-size="12" font-weight="750">CONSUMER</text>
|
||||||
|
<rect x="735" y="140" width="255" height="90" rx="12" fill="#172554"/>
|
||||||
|
<text x="759" y="168" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
|
||||||
|
<text x="759" y="201" fill="#ffffff" font-size="22" font-weight="700" class="mono">AppScore</text>
|
||||||
|
<text x="759" y="221" fill="#bfdbfe" font-size="14">knows DbRef, not SomeDbScore</text>
|
||||||
|
|
||||||
|
<text x="40" y="270" fill="#64748b" font-size="14">Creating and passing DbRef performs no I/O and says nothing about readiness.</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 2.6 KiB |
41
docs/diagrams/score-ref-cycle.svg
Normal file
41
docs/diagrams/score-ref-cycle.svg
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="380" viewBox="0 0 1100 380" role="img" aria-labelledby="title desc">
|
||||||
|
<title id="title">Two Scores exchange typed Refs in a cyclic relationship</title>
|
||||||
|
<desc id="desc">BrokerScore passes BrokerRef to AuthScore. AuthScore passes AuthRef to BrokerScore. Both Refs come from desired state before either Score runs.</desc>
|
||||||
|
<defs>
|
||||||
|
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
|
||||||
|
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
|
||||||
|
</marker>
|
||||||
|
<style>
|
||||||
|
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
|
||||||
|
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
|
||||||
|
</style>
|
||||||
|
</defs>
|
||||||
|
<rect width="1100" height="380" fill="#ffffff"/>
|
||||||
|
<g class="sans">
|
||||||
|
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">A cycle is the same pattern twice</text>
|
||||||
|
<text x="40" y="80" fill="#64748b" font-size="17">Both Refs come from desired state before either Score runs.</text>
|
||||||
|
|
||||||
|
<rect x="40" y="108" width="1020" height="220" rx="16" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
|
||||||
|
<text x="70" y="140" fill="#64748b" font-size="12" font-weight="750" letter-spacing="1.3">COMPOSED FUNCTIONALITY</text>
|
||||||
|
|
||||||
|
<rect x="85" y="178" width="260" height="100" rx="12" fill="#172554"/>
|
||||||
|
<text x="110" y="208" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
|
||||||
|
<text x="110" y="242" fill="#ffffff" font-size="22" font-weight="700" class="mono">BrokerScore</text>
|
||||||
|
<text x="110" y="265" fill="#bfdbfe" font-size="14">owns the broker</text>
|
||||||
|
|
||||||
|
<path d="M345 204H755" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
<rect x="440" y="181" width="220" height="46" rx="23" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="550" y="210" text-anchor="middle" fill="#115e59" font-size="15" font-weight="650" class="mono">BrokerRef</text>
|
||||||
|
|
||||||
|
<path d="M755 252H345" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
|
||||||
|
<rect x="440" y="229" width="220" height="46" rx="23" fill="#f0fdfa" stroke="#0f766e"/>
|
||||||
|
<text x="550" y="258" text-anchor="middle" fill="#115e59" font-size="15" font-weight="650" class="mono">AuthRef</text>
|
||||||
|
|
||||||
|
<rect x="755" y="178" width="260" height="100" rx="12" fill="#172554"/>
|
||||||
|
<text x="780" y="208" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
|
||||||
|
<text x="780" y="242" fill="#ffffff" font-size="22" font-weight="700" class="mono">AuthScore</text>
|
||||||
|
<text x="780" y="265" fill="#bfdbfe" font-size="14">owns authentication</text>
|
||||||
|
|
||||||
|
<text x="40" y="360" fill="#64748b" font-size="14">The cycle describes desired relationships; it does not require a graph scheduler.</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 2.8 KiB |
@@ -1,460 +1,105 @@
|
|||||||
# Fleet staging install on OKD
|
# Deploy Fleet to a remote cluster
|
||||||
|
|
||||||
End-to-end runbook for deploying the fleet stack (Zitadel + NATS +
|
Fleet uses the same lifecycle commands and Scores for local, staging, and
|
||||||
auth callout + operator) on an OKD cluster, with a remote agent
|
production clusters. The selected context changes the target and publication
|
||||||
connecting through the public WSS endpoint. Targets the staging
|
behavior; there is no separate staging installer.
|
||||||
shape — single-instance NATS, public Zitadel + NATS WS Routes with
|
|
||||||
edge-TLS via cert-manager, env-only Secret config (no volume
|
|
||||||
mounts) so default `restricted-v2` SCC is enough.
|
|
||||||
|
|
||||||
Time budget: ~30 min on a warm cluster, ~60 min cold.
|
## Prerequisites
|
||||||
|
|
||||||
## 0. Prereqs
|
- Docker with Buildx, Helm, and kubectl are on `PATH`.
|
||||||
|
- The remote cluster has cert-manager and the CloudNativePG operator.
|
||||||
|
- The image registry contains credentials for `REGISTRY_USER` and
|
||||||
|
`REGISTRY_TOKEN`.
|
||||||
|
- The deploy identity can obtain the cluster kubeconfig through OpenBao.
|
||||||
|
- Public DNS points the context domain at the cluster ingress.
|
||||||
|
|
||||||
- `oc` CLI logged in with cluster-admin (or at least
|
## Define the context
|
||||||
cluster-scoped privileges on the namespaces below — namespace
|
|
||||||
create, CRD apply, ClusterRole create).
|
|
||||||
- `podman` on your laptop, authenticated to the destination registry
|
|
||||||
(default `hub.nationtech.io/harmony` — `podman login` if needed).
|
|
||||||
- `helm` on PATH (used by Harmony's helm chart Scores).
|
|
||||||
- The staging cluster has:
|
|
||||||
- cert-manager installed and a `ClusterIssuer` ready for the
|
|
||||||
cluster's base domain (default name: `letsencrypt-prod` —
|
|
||||||
override with `--cluster-issuer` if yours differs).
|
|
||||||
- CNPG (cloudnative-pg) operator installed (Zitadel relies on it
|
|
||||||
for its Postgres cluster).
|
|
||||||
- DNS: the chosen `--base-domain` resolves to the OKD ingress
|
|
||||||
router. For `cb1.nationtech.io`, that means `*.cb1.nationtech.io`
|
|
||||||
or at least `sso-staging.cb1.nationtech.io` and
|
|
||||||
`nats-fleet-staging.cb1.nationtech.io` must point at the OKD
|
|
||||||
router VIP. If you're using the cluster's apps domain
|
|
||||||
(`apps.cb1.nationtech.io`), set `--base-domain` accordingly.
|
|
||||||
- Access to write a `[credentials]` TOML on whichever machine will
|
|
||||||
run the agent (your laptop is fine for the demo).
|
|
||||||
|
|
||||||
## 1. Build and push images
|
Remote contexts belong in the customer's private repository, not this public
|
||||||
|
repository. Create a context file like this:
|
||||||
|
|
||||||
The staging install pulls operator + auth-callout images from your
|
```toml
|
||||||
registry. The helper script builds both, tags them, and pushes:
|
[contexts.customer-prod]
|
||||||
|
profile = "prod"
|
||||||
|
registry = "hub.example.com"
|
||||||
|
project = "customer"
|
||||||
|
domain = "fleet.example.com"
|
||||||
|
image_pull_secret = "registry-pull"
|
||||||
|
openbao_namespace = "customer-prod"
|
||||||
|
openbao_url = "https://openbao.example.com"
|
||||||
|
zitadel_url = "https://sso.example.com"
|
||||||
|
openbao_role = "customer-prod-cd"
|
||||||
|
zitadel_audience = "openbao"
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `HARMONY_ZITADEL_KEY_JSON` to the deploy service user's Zitadel key JSON.
|
||||||
|
The context uses that identity to obtain its OpenBao-brokered kubeconfig.
|
||||||
|
|
||||||
|
## Provision callout credentials
|
||||||
|
|
||||||
|
Before the first deploy, create the immutable
|
||||||
|
`fleet-system/fleet-callout-credentials` Secret. It must contain
|
||||||
|
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass`.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /path/to/harmony
|
kubectl create namespace fleet-system --dry-run=client -o yaml | kubectl apply -f -
|
||||||
./fleet/scripts/build_and_push_images.sh
|
kubectl -n fleet-system create secret generic fleet-callout-credentials \
|
||||||
|
--from-literal=issuer-nkey-seed="$ISSUER_NKEY_SEED" \
|
||||||
|
--from-literal=issuer-public-key="$ISSUER_PUBLIC_KEY" \
|
||||||
|
--from-literal=nats-auth-pass="$NATS_AUTH_PASS" \
|
||||||
|
--dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
kubectl -n fleet-system patch secret fleet-callout-credentials \
|
||||||
|
--type=merge -p '{"immutable":true}'
|
||||||
```
|
```
|
||||||
|
|
||||||
Defaults: `REGISTRY=hub.nationtech.io/harmony`, `IMAGE_TAG=dev`,
|
The registry pull Secret named by `image_pull_secret` must also exist in each
|
||||||
`PUSH=1`. Override with environment variables. Skip the push (e.g.
|
namespace that pulls private images.
|
||||||
to inspect the images locally first) with `PUSH=0`.
|
|
||||||
|
|
||||||
Output ends with the exact `--operator-image` / `--callout-image`
|
## Deploy
|
||||||
flags to paste into step 4.
|
|
||||||
|
|
||||||
**Verify:**
|
`ship` builds the operator and callout images, pushes their digest-pinned
|
||||||
|
references, then converges the Fleet Scores serially:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
podman images | grep harmony # both refs present locally
|
export REGISTRY_USER=...
|
||||||
podman pull hub.nationtech.io/harmony/harmony-fleet-operator:dev # registry confirmed
|
export REGISTRY_TOKEN=...
|
||||||
|
export HARMONY_ZITADEL_KEY_JSON=...
|
||||||
|
|
||||||
|
cargo run -p harmony-fleet-deploy -- \
|
||||||
|
ship --context customer-prod --config path/to/contexts.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
## 2. Create namespaces
|
For CI, the stages can run independently. Pass the digest references printed by
|
||||||
|
`build` into `publish` and `deploy`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
oc new-project zitadel-staging
|
cargo run -p harmony-fleet-deploy -- build --context customer-prod --config path/to/contexts.toml
|
||||||
oc new-project fleet-staging
|
cargo run -p harmony-fleet-deploy -- publish --context customer-prod --config path/to/contexts.toml \
|
||||||
|
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
|
||||||
|
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
|
||||||
|
cargo run -p harmony-fleet-deploy -- deploy --context customer-prod --config path/to/contexts.toml \
|
||||||
|
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
|
||||||
|
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
|
||||||
```
|
```
|
||||||
|
|
||||||
If `hub.nationtech.io` requires authentication, add the imagePullSecret
|
`deploy` never builds or publishes images. It returns only after each Score's
|
||||||
to both namespaces (each pod that pulls from the registry needs it):
|
readiness check passes, including the operator's authenticated NATS startup.
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# adjust to whatever you have for hub.nationtech.io
|
cargo run -p harmony-fleet-deploy -- status --context customer-prod --config path/to/contexts.toml
|
||||||
oc -n fleet-staging secrets link default <hub-pull-secret> --for=pull
|
cargo run -p harmony-fleet-deploy -- logs --context customer-prod --config path/to/contexts.toml
|
||||||
oc -n zitadel-staging secrets link default <hub-pull-secret> --for=pull
|
kubectl -n fleet-system get deployments,pods
|
||||||
|
kubectl get deployments.fleet.nationtech.io,devices.fleet.nationtech.io
|
||||||
```
|
```
|
||||||
|
|
||||||
(For Zitadel + Postgres the chart pulls from public registries, so
|
For local validation before a remote deploy:
|
||||||
the secret is only strictly required in `fleet-staging` for the
|
|
||||||
operator + callout images. Linking both is safest.)
|
|
||||||
|
|
||||||
## 3. Set KUBECONFIG and verify cluster context
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export KUBECONFIG=$ADMIN_KUBECONFIG
|
cargo run -p harmony-fleet-deploy -- ship --context local
|
||||||
oc whoami
|
HARMONY_FLEET_E2E=1 cargo test -p harmony-fleet-e2e --test ping -- --nocapture
|
||||||
oc config current-context
|
|
||||||
oc get clusterversion # confirm OKD reachable + healthy
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The install runs with this `KUBECONFIG`. **Double-check** before
|
See [`fleet/README.md`](../../fleet/README.md) for component details and the E2E
|
||||||
running step 4 — Harmony's `K8sAnywhereTopology::from_env()` honors
|
workflow.
|
||||||
this and there's no second confirmation prompt.
|
|
||||||
|
|
||||||
## 4. Run `fleet_staging_install`
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cargo run --release -p example_fleet_staging_install -- \
|
|
||||||
--base-domain cb1.nationtech.io \
|
|
||||||
--operator-image hub.nationtech.io/harmony/harmony-fleet-operator:dev \
|
|
||||||
--callout-image hub.nationtech.io/harmony/harmony-nats-callout:dev
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional flags (defaults shown):
|
|
||||||
|
|
||||||
```
|
|
||||||
--cluster-issuer letsencrypt-prod
|
|
||||||
--fleet-namespace fleet-staging
|
|
||||||
--zitadel-namespace zitadel-staging
|
|
||||||
--nats-account FLEET
|
|
||||||
--zitadel-version v4.12.1
|
|
||||||
--project-name fleet
|
|
||||||
--admin-role fleet-admin
|
|
||||||
--device-role device
|
|
||||||
--operator-username fleet-operator
|
|
||||||
--admin-username fleet-ops
|
|
||||||
```
|
|
||||||
|
|
||||||
Step-by-step the binary does:
|
|
||||||
|
|
||||||
1. **Zitadel helm install** — Postgres (CNPG) + Zitadel chart into
|
|
||||||
`--zitadel-namespace`. Edge-TLS Route at `sso-staging.<base>`
|
|
||||||
with cert-manager-driven certificate.
|
|
||||||
2. **Zitadel setup** — project, two roles (`fleet-admin`, `device`),
|
|
||||||
API app `nats`, and two machine users (`fleet-ops` for manual
|
|
||||||
admin work, `fleet-operator` for the operator pod). Both get
|
|
||||||
JSON keys cached at `~/.local/share/harmony/zitadel/client-config.json`.
|
|
||||||
3. **NATS install** — single-instance JetStream, `auth_callout`
|
|
||||||
block referencing the issuer NKey pubkey, WebSocket listener on
|
|
||||||
8080. Edge-TLS Route at `nats-fleet-staging.<base>`.
|
|
||||||
4. **Auth callout deployment** — env-only Secret config (no mounts),
|
|
||||||
wired to the same issuer + Zitadel project audience.
|
|
||||||
5. **Operator deployment** — single Secret holding the credentials
|
|
||||||
TOML (with the operator's JSON keyfile inlined). One env var,
|
|
||||||
`FLEET_OPERATOR_CREDENTIALS_TOML`, no volumes.
|
|
||||||
|
|
||||||
The binary prints the URLs + project_id at the end. Save that block
|
|
||||||
— you'll need the project_id for the agent config.
|
|
||||||
|
|
||||||
**Expected output tail:**
|
|
||||||
|
|
||||||
```
|
|
||||||
=== fleet-staging install complete ===
|
|
||||||
Zitadel: https://sso-staging.cb1.nationtech.io/
|
|
||||||
NATS WS public: wss://nats-fleet-staging.cb1.nationtech.io/
|
|
||||||
NATS in-cluster: nats://fleet-nats.fleet-staging.svc.cluster.local:4222
|
|
||||||
Operator: oc -n fleet-staging get deploy/harmony-fleet-operator
|
|
||||||
Auth callout: oc -n fleet-staging get deploy/fleet-callout
|
|
||||||
Project id: 371xxxxxxxxxxxxxxx
|
|
||||||
Admin user: fleet-ops (machine key in ~/.local/share/harmony/zitadel/client-config.json)
|
|
||||||
Operator user: fleet-operator (machine key embedded in operator's Secret)
|
|
||||||
```
|
|
||||||
|
|
||||||
## 5. Verify each layer
|
|
||||||
|
|
||||||
### 5.1 Zitadel reachable, certificate provisioned
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# pod up
|
|
||||||
oc -n zitadel-staging get pods
|
|
||||||
# expect: zitadel-* Running, zitadel-pg-1/2 Running
|
|
||||||
|
|
||||||
# Route + certificate (cert-manager creates the secret)
|
|
||||||
oc -n zitadel-staging get route
|
|
||||||
oc -n zitadel-staging get certificate
|
|
||||||
|
|
||||||
# OIDC discovery from the public URL
|
|
||||||
curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer
|
|
||||||
# expect: "https://sso-staging.cb1.nationtech.io"
|
|
||||||
```
|
|
||||||
|
|
||||||
If `curl` fails with TLS errors, the cert-manager certificate isn't
|
|
||||||
ready yet. Watch its status:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc -n zitadel-staging describe certificate
|
|
||||||
oc -n cert-manager logs deploy/cert-manager --tail=50
|
|
||||||
```
|
|
||||||
|
|
||||||
A `Ready` condition `True` + `secretName: zitadel-tls` populated
|
|
||||||
means the Route can serve HTTPS.
|
|
||||||
|
|
||||||
### 5.2 NATS pod up, callout connected
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc -n fleet-staging get pods
|
|
||||||
# expect:
|
|
||||||
# fleet-nats-0 2/2 Running (NATS + reloader sidecar)
|
|
||||||
# fleet-callout-... 1/1 Running
|
|
||||||
|
|
||||||
oc -n fleet-staging logs deploy/fleet-callout --tail=30 | grep -E "starting|JWKS|listening"
|
|
||||||
# expect:
|
|
||||||
# starting harmony NATS auth callout
|
|
||||||
# JWKS refreshed count=2
|
|
||||||
# auth callout service listening subject="$SYS.REQ.USER.AUTH"
|
|
||||||
```
|
|
||||||
|
|
||||||
If the callout pod CrashLoopBackOff:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc -n fleet-staging logs deploy/fleet-callout --previous --tail=30
|
|
||||||
```
|
|
||||||
|
|
||||||
Most common: OIDC issuer URL mismatch. The callout's
|
|
||||||
`OIDC_ISSUER_URL` env must byte-equal what Zitadel emits as `iss` in
|
|
||||||
its discovery doc. Check both:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_ISSUER_URL
|
|
||||||
# vs.
|
|
||||||
curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5.3 Operator authenticated and running
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc -n fleet-staging get pods -l app.kubernetes.io/name=harmony-fleet-operator
|
|
||||||
oc -n fleet-staging logs deploy/harmony-fleet-operator --tail=30
|
|
||||||
```
|
|
||||||
|
|
||||||
Look for, in order:
|
|
||||||
|
|
||||||
```
|
|
||||||
minted fresh Zitadel access token audience=<project_id>
|
|
||||||
connected successfully server=4222
|
|
||||||
NATS connected
|
|
||||||
KV bucket ready bucket=desired-state
|
|
||||||
starting Deployment controller
|
|
||||||
device-reconciler: watching device-info KV
|
|
||||||
aggregator: startup complete
|
|
||||||
```
|
|
||||||
|
|
||||||
If you see `Permissions Violation` errors, the callout's
|
|
||||||
`OIDC_AUDIENCE` (project_id at deploy time) doesn't match the
|
|
||||||
project_id in Zitadel today. Re-run step 4 — the live-query fix
|
|
||||||
in the Zitadel setup will refresh.
|
|
||||||
|
|
||||||
### 5.4 NATS WSS reachable from outside the cluster
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -sSI https://nats-fleet-staging.cb1.nationtech.io/ | head -5
|
|
||||||
```
|
|
||||||
|
|
||||||
Expect a 4xx (NATS doesn't speak HTTP, but the TLS handshake should
|
|
||||||
succeed and you'll get back a WebSocket-upgrade-related response).
|
|
||||||
A connection refused or TLS handshake error means the Route or
|
|
||||||
cert-manager is unhappy.
|
|
||||||
|
|
||||||
### 5.5 CRDs registered
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc get crd | grep fleet.nationtech.io
|
|
||||||
# expect:
|
|
||||||
# deployments.fleet.nationtech.io
|
|
||||||
# devices.fleet.nationtech.io
|
|
||||||
```
|
|
||||||
|
|
||||||
## 6. Connect a remote agent
|
|
||||||
|
|
||||||
The fleet agent runs on the device (laptop, Pi, anywhere with
|
|
||||||
outbound HTTPS). It needs:
|
|
||||||
|
|
||||||
- Its own Zitadel machine user with the `device` role grant.
|
|
||||||
- The JSON keyfile from that user.
|
|
||||||
- A `[credentials]` TOML pointing at the public Zitadel + the WSS
|
|
||||||
NATS URL.
|
|
||||||
|
|
||||||
### 6.1 Mint a per-device machine user
|
|
||||||
|
|
||||||
Use `oc port-forward` or a helper to call Zitadel's API. Easier
|
|
||||||
path: drop a quick Score that adds one machine user. For tonight,
|
|
||||||
do it from the Zitadel UI:
|
|
||||||
|
|
||||||
1. Browse to `https://sso-staging.cb1.nationtech.io/ui/console/`,
|
|
||||||
log in as the human admin (password from Zitadel ConfigMap on
|
|
||||||
first install — see `docs/guides/fleet-zitadel-faq.md`).
|
|
||||||
2. Pick the `Default` org → `fleet` project → Roles → confirm
|
|
||||||
`device` exists.
|
|
||||||
3. Org → Users → Service Users → New: name `device-laptop-01`,
|
|
||||||
userName `device-laptop-01`. Save.
|
|
||||||
4. The user's "Personal Information" tab → Authorizations or
|
|
||||||
"Authorization" → "+New" — grant the `fleet` project's `device`
|
|
||||||
role to this user.
|
|
||||||
5. The user's "Keys" tab → "+New", type `JSON`, expiration future
|
|
||||||
date. **Download the keyfile JSON** — Zitadel only shows the
|
|
||||||
private half once. Save as `~/.local/share/harmony/fleet/agents/device-laptop-01.json`.
|
|
||||||
|
|
||||||
### 6.2 Build the agent locally
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cargo build --release -p harmony-fleet-agent
|
|
||||||
ls -la target/release/harmony-fleet-agent
|
|
||||||
```
|
|
||||||
|
|
||||||
### 6.3 Render the agent's config TOML
|
|
||||||
|
|
||||||
```bash
|
|
||||||
PROJECT_ID=$(oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_AUDIENCE)
|
|
||||||
cat > /tmp/fleet-agent-config.toml <<EOF
|
|
||||||
[agent]
|
|
||||||
device_id = "device-laptop-01"
|
|
||||||
|
|
||||||
[nats]
|
|
||||||
urls = ["wss://nats-fleet-staging.cb1.nationtech.io"]
|
|
||||||
|
|
||||||
[credentials]
|
|
||||||
type = "zitadel-jwt"
|
|
||||||
key_path = "/etc/fleet-agent/zitadel-key.json"
|
|
||||||
oidc_issuer_url = "https://sso-staging.cb1.nationtech.io"
|
|
||||||
audience = "$PROJECT_ID"
|
|
||||||
|
|
||||||
[labels]
|
|
||||||
env = "staging"
|
|
||||||
location = "laptop"
|
|
||||||
arch = "$(uname -m)"
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
The agent's username convention is `device-<device_id>`, matching
|
|
||||||
the callout's `DEVICE_ID_PREFIX_STRIP=device-`. The Zitadel machine
|
|
||||||
user must literally be `device-laptop-01` for the JWT-bearer flow
|
|
||||||
to extract the right device id.
|
|
||||||
|
|
||||||
### 6.4 Run the agent
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo mkdir -p /etc/fleet-agent
|
|
||||||
sudo cp ~/.local/share/harmony/fleet/agents/device-laptop-01.json \
|
|
||||||
/etc/fleet-agent/zitadel-key.json
|
|
||||||
sudo chown $(id -u):$(id -g) /etc/fleet-agent/zitadel-key.json
|
|
||||||
sudo chmod 0400 /etc/fleet-agent/zitadel-key.json
|
|
||||||
|
|
||||||
FLEET_AGENT_CONFIG=/tmp/fleet-agent-config.toml \
|
|
||||||
RUST_LOG=info \
|
|
||||||
./target/release/harmony-fleet-agent
|
|
||||||
```
|
|
||||||
|
|
||||||
Watch the log:
|
|
||||||
|
|
||||||
```
|
|
||||||
fleet-agent-v0 starting device_id=device-laptop-01
|
|
||||||
podman socket ready
|
|
||||||
inventory loaded hostname=...
|
|
||||||
connecting to NATS ["wss://nats-fleet-staging.cb1.nationtech.io"]
|
|
||||||
minted fresh Zitadel access token audience=<project_id>
|
|
||||||
connected successfully server=...
|
|
||||||
NATS connected
|
|
||||||
fleet publisher ready
|
|
||||||
watching KV keys filter=device-laptop-01.>
|
|
||||||
```
|
|
||||||
|
|
||||||
If you hit `Permissions Violation` errors after `connected`:
|
|
||||||
- check `oc -n fleet-staging logs deploy/fleet-callout --tail=20` —
|
|
||||||
it'll show why the JWT was rejected (audience, role claim,
|
|
||||||
device_id format).
|
|
||||||
|
|
||||||
### 6.5 Verify the operator created a Device CR
|
|
||||||
|
|
||||||
```bash
|
|
||||||
oc get devices
|
|
||||||
# expect:
|
|
||||||
# NAME AGE
|
|
||||||
# device-laptop-01 Xs
|
|
||||||
oc describe device device-laptop-01
|
|
||||||
# labels block reflects what the agent sent in [labels]
|
|
||||||
```
|
|
||||||
|
|
||||||
## 7. Drive a deployment end to end
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cat > /tmp/hello-web.yaml <<'EOF'
|
|
||||||
apiVersion: fleet.nationtech.io/v1alpha1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: hello-web
|
|
||||||
spec:
|
|
||||||
score:
|
|
||||||
type: PodmanV0
|
|
||||||
data:
|
|
||||||
services:
|
|
||||||
- name: testnginx
|
|
||||||
image: docker.io/nginx:latest
|
|
||||||
ports:
|
|
||||||
- "8080:80"
|
|
||||||
targetSelector:
|
|
||||||
matchLabels:
|
|
||||||
env: staging
|
|
||||||
rollout:
|
|
||||||
strategy: Immediate
|
|
||||||
EOF
|
|
||||||
|
|
||||||
oc apply -f /tmp/hello-web.yaml
|
|
||||||
|
|
||||||
# Status reflect-back from the agent (takes ~5-15s)
|
|
||||||
oc get deployment.fleet.nationtech.io hello-web -o yaml | yq '.status'
|
|
||||||
# expect:
|
|
||||||
# aggregate:
|
|
||||||
# matchedDeviceCount: 1
|
|
||||||
# succeeded: 1
|
|
||||||
# failed: 0
|
|
||||||
# pending: 0
|
|
||||||
|
|
||||||
# On the device:
|
|
||||||
podman ps
|
|
||||||
# expect: testnginx running, port 8080→80
|
|
||||||
curl -sS http://localhost:8080 | head -3
|
|
||||||
```
|
|
||||||
|
|
||||||
## 8. Common failure modes
|
|
||||||
|
|
||||||
| Symptom | Cause / fix |
|
|
||||||
| --- | --- |
|
|
||||||
| `cert-manager` Certificate stuck `False` for 5+ min | DNS for the host doesn't resolve to the OKD router yet. `dig sso-staging.<base> +short` should match the cluster's ingress IP. Or your `letsencrypt-prod` ClusterIssuer is using HTTP01 and the route isn't reachable from Let's Encrypt. |
|
|
||||||
| Operator pod `Error: constructing CredentialSource` | The credentials TOML in the Secret is malformed. `oc -n fleet-staging get secret harmony-fleet-operator-secrets -o jsonpath='{.data.credentials\.toml}' \| base64 -d` and inspect; the `key_json` field must be a valid JSON keyfile string (multi-line triple-quoted in TOML is fine). |
|
|
||||||
| Operator pod `Permissions Violation` after `NATS connected` | Issuer pubkey or project_id mismatch between callout and NATS chart values, or Zitadel was reset and the operator's machine key no longer authenticates. Re-run `cargo run -p example_fleet_staging_install`. |
|
|
||||||
| Agent: `Zitadel token endpoint returned 400: invalid_grant_type` | TOML scope assembly bug or wrong `audience`. Confirm `audience` matches `oc exec deploy/fleet-callout -- printenv OIDC_AUDIENCE`. |
|
|
||||||
| Agent: connects, then `Permissions Violation for Publish to "$KV.device-info..."` | The device's machine user has no `device` role grant. Add via Zitadel UI → user → Authorizations. |
|
|
||||||
| `Deployment.fleet.nationtech.io` CR applied but `matchedDeviceCount: 0` | `targetSelector.matchLabels` doesn't match any Device CR's `metadata.labels`. `oc get devices --show-labels`. |
|
|
||||||
| Container redeploys every 30s on the device | Known FIXME — the agent's `matches_spec` returns false for any spec with env or volumes. For the demo, use trivial specs (the hello-web above is fine). Tracked in `harmony/src/modules/podman/topology.rs`. |
|
|
||||||
|
|
||||||
## 9. Tear down
|
|
||||||
|
|
||||||
The Helm releases own the bulk of the resources, so the cleanest
|
|
||||||
recovery from a broken state is:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
helm -n zitadel-staging uninstall zitadel
|
|
||||||
helm -n fleet-staging uninstall fleet-nats
|
|
||||||
oc -n fleet-staging delete deploy/harmony-fleet-operator deploy/fleet-callout
|
|
||||||
oc -n fleet-staging delete secret harmony-fleet-operator-secrets fleet-callout-secrets
|
|
||||||
oc -n zitadel-staging delete pgcluster zitadel-pg --ignore-not-found
|
|
||||||
oc delete project zitadel-staging fleet-staging
|
|
||||||
|
|
||||||
# CRDs persist (helm.sh/resource-policy: keep). Delete by hand if you
|
|
||||||
# really want a clean slate:
|
|
||||||
oc delete crd deployments.fleet.nationtech.io devices.fleet.nationtech.io
|
|
||||||
```
|
|
||||||
|
|
||||||
The host-side `~/.local/share/harmony/zitadel/client-config.json`
|
|
||||||
caches machine keys + project IDs from this install. Wipe it before
|
|
||||||
re-installing against a freshly reset Zitadel:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
rm -f ~/.local/share/harmony/zitadel/client-config.json
|
|
||||||
```
|
|
||||||
|
|
||||||
(The cache-vs-live drift bug is fixed — `ZitadelSetupScore` now
|
|
||||||
re-queries Zitadel for IDs on every apply — but stale machine-key
|
|
||||||
material from a deleted Zitadel project will fail at JWT-bearer
|
|
||||||
mint until you delete + re-create.)
|
|
||||||
|
|
||||||
## 10. Cross-reference
|
|
||||||
|
|
||||||
- [`fleet-zitadel-faq.md`](./fleet-zitadel-faq.md) — concepts behind
|
|
||||||
Zitadel projects, roles, machine users, audit-trail decisions.
|
|
||||||
- [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) —
|
|
||||||
worked recipe for minting an admin token by hand and using it
|
|
||||||
with `nats kv` commands.
|
|
||||||
- `examples/fleet_staging_install/src/main.rs` — the install code
|
|
||||||
itself; the comments narrate every step.
|
|
||||||
- `harmony/src/modules/fleet/server.rs::FleetServerScore` —
|
|
||||||
composable form of the same install for callers that don't need
|
|
||||||
the intermediate read of `ZitadelClientConfig`.
|
|
||||||
|
|||||||
108
docs/reference/fleet-score-references.md
Normal file
108
docs/reference/fleet-score-references.md
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
# Fleet Score references
|
||||||
|
|
||||||
|
Fleet is the first application of [ADR-028](../adr/028-typed-score-references.md).
|
||||||
|
Its deployment should read like a declaration of what Fleet needs and how those
|
||||||
|
components connect.
|
||||||
|
|
||||||
|
## Target experience
|
||||||
|
|
||||||
|
Deploying a configured Fleet should require one command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./harmony-fleet-deploy ship --context prod
|
||||||
|
```
|
||||||
|
|
||||||
|
The context selects the desired configuration. The command composes the Scores,
|
||||||
|
passes their Refs, and interprets the result. It does not expose the deployment
|
||||||
|
sequence to the operator.
|
||||||
|
|
||||||
|
The corresponding Rust should stay close to the same statement:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let postgres = PostgreSQLScore::new(config.postgres);
|
||||||
|
let zitadel = ZitadelScore::new(config.zitadel)
|
||||||
|
.database(postgres.root_account_ref());
|
||||||
|
|
||||||
|
let nats = NatsScore::new(config.nats);
|
||||||
|
let callout = NatsAuthCalloutScore::new(config.callout)
|
||||||
|
.nats(nats.account_ref());
|
||||||
|
|
||||||
|
let operator = FleetOperatorScore::new(config.operator)
|
||||||
|
.identity(zitadel.application_ref())
|
||||||
|
.messaging(nats.client_ref());
|
||||||
|
```
|
||||||
|
|
||||||
|
The API names are illustrative. The required shape is a short component
|
||||||
|
declaration followed by typed connections to other components.
|
||||||
|
|
||||||
|
## How composition works
|
||||||
|
|
||||||
|
### Identity functionality
|
||||||
|
|
||||||
|
[](../diagrams/fleet-identity-composition.svg)
|
||||||
|
|
||||||
|
### Messaging functionality
|
||||||
|
|
||||||
|
[](../diagrams/fleet-messaging-composition.svg)
|
||||||
|
|
||||||
|
The bidirectional messaging relationship is still declarative: constructing
|
||||||
|
either Ref does not require its target to exist.
|
||||||
|
|
||||||
|
## Complete reference
|
||||||
|
|
||||||
|
[](../diagrams/fleet-score-references.svg)
|
||||||
|
|
||||||
|
The final map lists each functionality, the Ref types it exports, and the Score
|
||||||
|
types that consume them. The names identify intended contracts. They become
|
||||||
|
public APIs only as implementation replaces the corresponding string wiring;
|
||||||
|
the map does not prescribe a generic Ref hierarchy or registry.
|
||||||
|
|
||||||
|
## Composition boundary
|
||||||
|
|
||||||
|
`harmony-fleet-deploy` is composition only. It may:
|
||||||
|
|
||||||
|
- load the selected context;
|
||||||
|
- declare component configuration;
|
||||||
|
- construct Scores;
|
||||||
|
- connect Scores through typed Refs;
|
||||||
|
- interpret the composed deployment.
|
||||||
|
|
||||||
|
It must not:
|
||||||
|
|
||||||
|
- call provider APIs or commands directly;
|
||||||
|
- build Kubernetes or Helm resources;
|
||||||
|
- generate credentials or reconstruct provider outputs;
|
||||||
|
- implement deployment ordering, retries, or readiness checks;
|
||||||
|
- copy provider URLs, IDs, credential locations, or naming conventions.
|
||||||
|
|
||||||
|
All deployment behavior belongs to the Score that owns the component. If a
|
||||||
|
required operation has no Score, the missing work is to add or extend that
|
||||||
|
Score, not to implement the operation in Fleet composition.
|
||||||
|
|
||||||
|
## Ref behavior
|
||||||
|
|
||||||
|
Refs are available from desired state before their resources exist. Passing a
|
||||||
|
Ref is therefore not a readiness check. Each consuming Score handles an absent
|
||||||
|
or unavailable target through its own `Interpret`/`Topology` path, often by
|
||||||
|
emitting a native Kubernetes reference.
|
||||||
|
|
||||||
|
ADR-028 does not require a particular non-success outcome, retry policy, or
|
||||||
|
continuous convergence mechanism.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
|
||||||
|
Fleet validates this design when:
|
||||||
|
|
||||||
|
1. A configured deployment runs as
|
||||||
|
`./harmony-fleet-deploy ship --context prod`.
|
||||||
|
2. Declarations and Ref wiring average at most five non-blank lines per logical
|
||||||
|
component in the canonical Fleet composition. PostgreSQL, Zitadel, each
|
||||||
|
Zitadel application or account, NATS, the auth callout, OpenBao, the Fleet
|
||||||
|
operator, and device enrollment each count as logical components.
|
||||||
|
3. `harmony-fleet-deploy` contains only context loading, configuration,
|
||||||
|
component declarations, Ref wiring, and interpretation.
|
||||||
|
4. Each migrated Ref removes the string or cache contract it replaces.
|
||||||
|
5. Constructing and passing Refs performs no I/O and implies no execution order.
|
||||||
|
6. Production and E2E compose the same Score and Ref APIs.
|
||||||
|
7. No generic graph, scheduler, or convergence mechanism is added as part of
|
||||||
|
ADR-028.
|
||||||
@@ -241,6 +241,7 @@ pub async fn bring_up_stack() -> Result<StackHandles> {
|
|||||||
|
|
||||||
info!("[5/8] provisioning project + roles + machine users in Zitadel");
|
info!("[5/8] provisioning project + roles + machine users in Zitadel");
|
||||||
let setup = ZitadelSetupScore {
|
let setup = ZitadelSetupScore {
|
||||||
|
outputs_namespace: None,
|
||||||
host: ZITADEL_HOST.to_string(),
|
host: ZITADEL_HOST.to_string(),
|
||||||
scheme: Default::default(),
|
scheme: Default::default(),
|
||||||
port: None,
|
port: None,
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ vm-rehearsal = ["harmony/kvm"]
|
|||||||
# `podman` types via the reconciler-contracts shape). `kvm` is the
|
# `podman` types via the reconciler-contracts shape). `kvm` is the
|
||||||
# only feature that pulls libvirt and stays opt-in via `vm-rehearsal`.
|
# only feature that pulls libvirt and stays opt-in via `vm-rehearsal`.
|
||||||
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
|
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
|
||||||
|
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
log.workspace = true
|
log.workspace = true
|
||||||
|
|||||||
@@ -25,11 +25,11 @@ use std::path::PathBuf;
|
|||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::{
|
use harmony::modules::fleet::ensure_fleet_ssh_keypair;
|
||||||
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
|
|
||||||
ensure_fleet_ssh_keypair,
|
|
||||||
};
|
|
||||||
use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials};
|
use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials};
|
||||||
|
use harmony_fleet_deploy::{
|
||||||
|
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
|
||||||
|
};
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
|
|
||||||
// VM-rehearsal-only imports. Hidden behind a feature so `cargo build
|
// VM-rehearsal-only imports. Hidden behind a feature so `cargo build
|
||||||
|
|||||||
@@ -4,46 +4,10 @@ edition = "2024"
|
|||||||
version.workspace = true
|
version.workspace = true
|
||||||
readme.workspace = true
|
readme.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description = "VM-based end-to-end rehearsal: k3d + Zitadel + NATS auth callout + libvirt VM agents + operator → CR → podman → status"
|
description = "VM-based end-to-end rehearsal of the Fleet Scores"
|
||||||
|
|
||||||
[lib]
|
|
||||||
name = "example_fleet_e2e_demo"
|
|
||||||
path = "src/lib.rs"
|
|
||||||
|
|
||||||
[[bin]]
|
|
||||||
name = "fleet-e2e-demo"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|
||||||
[[test]]
|
|
||||||
name = "e2e_walking_skeleton"
|
|
||||||
path = "tests/e2e_walking_skeleton.rs"
|
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", features = ["kvm"] }
|
harmony-fleet-e2e = { path = "../../fleet/harmony-fleet-e2e" }
|
||||||
harmony-k8s = { path = "../../harmony-k8s" }
|
|
||||||
harmony_types = { path = "../../harmony_types" }
|
|
||||||
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
|
|
||||||
harmony-nats-callout = { path = "../../nats/callout" }
|
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
|
||||||
harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" }
|
|
||||||
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
|
||||||
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
|
||||||
k3d-rs = { path = "../../k3d" }
|
|
||||||
async-nats.workspace = true
|
|
||||||
nkeys = "0.4"
|
|
||||||
tokio = { workspace = true, features = ["full"] }
|
|
||||||
tokio-test.workspace = true
|
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
log.workspace = true
|
tokio = { workspace = true, features = ["full"] }
|
||||||
env_logger.workspace = true
|
clap = { workspace = true, features = ["derive"] }
|
||||||
tracing.workspace = true
|
|
||||||
tracing-subscriber.workspace = true
|
|
||||||
futures-util.workspace = true
|
|
||||||
k8s-openapi.workspace = true
|
|
||||||
kube.workspace = true
|
|
||||||
clap = { version = "4", features = ["derive", "env"] }
|
|
||||||
directories = "6.0.0"
|
|
||||||
tempfile = "3"
|
|
||||||
url.workspace = true
|
|
||||||
|
|||||||
@@ -1,312 +0,0 @@
|
|||||||
# Local fleet rehearsal runbook
|
|
||||||
|
|
||||||
End-to-end walkthrough of the IoT fleet platform on your laptop:
|
|
||||||
k3d-hosted control plane (Zitadel + NATS + auth callout) plus two
|
|
||||||
libvirt VMs running the fleet-agent. Mirrors the production topology
|
|
||||||
closely enough that you can watch the auth callout flow, the
|
|
||||||
JetStream KV traffic, and the per-device permission boundary in a
|
|
||||||
real cluster.
|
|
||||||
|
|
||||||
This is not the integration-test harness (that runs unattended). It
|
|
||||||
is a step-by-step sequence with inspection points in between. Run
|
|
||||||
each section, look at what happened, then continue.
|
|
||||||
|
|
||||||
## 0. Prerequisites
|
|
||||||
|
|
||||||
- Linux host with KVM (the user running the commands in `libvirt` /
|
|
||||||
`kvm` group; check with `id`).
|
|
||||||
- `podman`, `qemu-system-x86_64` (and `qemu-system-aarch64` if you
|
|
||||||
pick `--arch aarch64`), `mdbook` (optional), `kubectl`, `nats` CLI
|
|
||||||
(optional, for the manual subscribe step). Most other tooling
|
|
||||||
(k3d, ansible venv, cloud images) is auto-provisioned under
|
|
||||||
`~/.local/share/harmony/`.
|
|
||||||
- `/etc/hosts`: `127.0.0.1 sso.fleet.local` so you can hit Zitadel
|
|
||||||
from your browser through the cluster's HTTP_PORT (see
|
|
||||||
`examples/fleet_auth_callout/src/lib.rs` for the constant).
|
|
||||||
- Free TCP ports `8080` and `30422` on the host.
|
|
||||||
|
|
||||||
Source map for the things you'll inspect:
|
|
||||||
|
|
||||||
| Component | File |
|
|
||||||
| --- | --- |
|
|
||||||
| Bring-up flow | `examples/fleet_e2e_demo/src/lib.rs` |
|
|
||||||
| Per-device Zitadel + agent install | same, `provision_device()` |
|
|
||||||
| NATS Score (auth-callout mode) | `fleet/harmony-fleet-deploy/src/nats.rs::FleetNatsScore::callout` |
|
|
||||||
| Shared agent config schema | `fleet/harmony-fleet-auth/src/agent_config.rs` |
|
|
||||||
| Auth callout deployment Score | `harmony/src/modules/nats_auth_callout/mod.rs` |
|
|
||||||
| Callout decision logic | `nats/callout/src/handler.rs::decide` |
|
|
||||||
| Per-device permissions template | `nats/callout/src/permissions.rs::device_default` |
|
|
||||||
| Agent NATS auth (JWT-bearer mint) | `fleet/harmony-fleet-auth/src/credentials.rs` |
|
|
||||||
| Agent KV publishers + direct pulse | `fleet/harmony-fleet-agent/src/fleet_publisher.rs` |
|
|
||||||
| Walking-skeleton tests | `examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs` |
|
|
||||||
|
|
||||||
The NATS server's helm values are rendered from typed Rust structs
|
|
||||||
via `serde_yaml::to_string` (see `FleetNatsScore::values_yaml`),
|
|
||||||
not by `format!()` string interpolation. Same with the agent's
|
|
||||||
`/etc/fleet-agent/config.toml` — typed `AgentConfig` →
|
|
||||||
`toml::to_string` → ConfigMap. Per ADR-023 principle 2 the e2e
|
|
||||||
demo composes the same `*Score` types the production deploy uses.
|
|
||||||
|
|
||||||
## 1. Provision the VMs
|
|
||||||
|
|
||||||
Each VM is one libvirt domain on the default network
|
|
||||||
(`192.168.122.0/24`). Run `fleet_vm_setup` once per VM. Pass
|
|
||||||
`--only-vm` so it stops at the cloud-init step (the agent install
|
|
||||||
happens later from the e2e bring-up — keeps the two phases legible).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# VM 0
|
|
||||||
cargo run --release -p example-fleet-vm-setup -- \
|
|
||||||
--arch aarch64 \
|
|
||||||
--vm-name vm-device-00 \
|
|
||||||
--only-vm
|
|
||||||
|
|
||||||
# VM 1
|
|
||||||
cargo run --release -p example-fleet-vm-setup -- \
|
|
||||||
--arch aarch64 \
|
|
||||||
--vm-name vm-device-01 \
|
|
||||||
--only-vm
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--arch x86_64` for native KVM speed; `aarch64` runs under
|
|
||||||
qemu-system-aarch64 TCG emulation on x86_64 hosts and is slower but
|
|
||||||
matches Pi targets.
|
|
||||||
|
|
||||||
**Inspect:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
virsh list --all
|
|
||||||
virsh domifaddr vm-device-00
|
|
||||||
virsh domifaddr vm-device-01
|
|
||||||
```
|
|
||||||
|
|
||||||
Note the IPs — you'll pass them in step 2. Confirm SSH works:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
|
|
||||||
fleet-admin@<vm0-ip> uptime
|
|
||||||
```
|
|
||||||
|
|
||||||
The keypair lives under `~/.local/share/harmony/fleet/ssh/`,
|
|
||||||
generated on first run.
|
|
||||||
|
|
||||||
## 2. Bring up the control-plane stack
|
|
||||||
|
|
||||||
This single command does everything: k3d cluster, Zitadel,
|
|
||||||
ZitadelSetupScore (project + roles + 2 device machine users +
|
|
||||||
`fleet-ops` admin), NATS with `auth_callout`, callout image build &
|
|
||||||
sideload, callout Deployment, and finally `FleetDeviceSetupScore`
|
|
||||||
over SSH for each VM (packages, agent binary, JWT keyfile,
|
|
||||||
systemd unit).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
FLEET_E2E_VM_0_IP=<vm0-ip> FLEET_E2E_VM_1_IP=<vm1-ip> \
|
|
||||||
cargo run --release -p example-fleet-e2e-demo -- --num-devices 2
|
|
||||||
```
|
|
||||||
|
|
||||||
The bring-up logs each step as `[e2e-demo X/9]`. Read along with
|
|
||||||
`examples/fleet_e2e_demo/src/lib.rs::bring_up_full_stack` to see
|
|
||||||
what's happening at each line. Stops at `STACK READY` and waits on
|
|
||||||
Ctrl-C (the cluster stays up after Ctrl-C — this is just the
|
|
||||||
foreground holder).
|
|
||||||
|
|
||||||
**Inspect:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export KUBECONFIG=$(k3d kubeconfig write fleet-auth-callout)
|
|
||||||
|
|
||||||
# All workloads up?
|
|
||||||
kubectl get pods -n fleet-system
|
|
||||||
kubectl get pods -n zitadel
|
|
||||||
|
|
||||||
# Callout config the deployment is using:
|
|
||||||
kubectl get deployment -n fleet-system fleet-callout \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].env}' | jq
|
|
||||||
```
|
|
||||||
|
|
||||||
Open Zitadel in the browser: <http://sso.fleet.local:8080/ui/console>
|
|
||||||
(login with `root@zitadel.local` / the bootstrap password printed
|
|
||||||
during step `[e2e-demo 3/9]`). Click into the `fleet` project →
|
|
||||||
`Users` to see the two `device-vm-device-0X` machine users with
|
|
||||||
`device` role grants and the `fleet-ops` admin.
|
|
||||||
|
|
||||||
## 3. Watch the auth callout in action
|
|
||||||
|
|
||||||
The callout is the security boundary: every NATS connect attempt
|
|
||||||
hits `$SYS.REQ.USER.AUTH`, the callout validates the Zitadel JWT
|
|
||||||
in `connect_opts.auth_token`, applies the decision tree in
|
|
||||||
`nats/callout/src/handler.rs::decide`, and signs back a user JWT
|
|
||||||
with role-scoped permissions.
|
|
||||||
|
|
||||||
Tail it while the agents reconnect:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl logs -n fleet-system -l app=fleet-callout -f
|
|
||||||
```
|
|
||||||
|
|
||||||
You'll see one set of lines per (re)connect:
|
|
||||||
|
|
||||||
```
|
|
||||||
received auth callout request user_nkey=U…
|
|
||||||
Zitadel JWT validated, generating user JWT device_id=vm-device-00 role=device
|
|
||||||
sending auth response
|
|
||||||
```
|
|
||||||
|
|
||||||
The `device_id` field is the value AFTER `device_id_prefix_strip`
|
|
||||||
runs (Zitadel emits `client_id=device-vm-device-00`; the callout
|
|
||||||
strips `device-` so permissions are interpolated against the bare
|
|
||||||
device id the agent uses for KV keys). See
|
|
||||||
`nats/callout/src/zitadel.rs::extract_device_id` for the strip.
|
|
||||||
|
|
||||||
**Force a reconnect to make a callout fire on demand:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
|
|
||||||
fleet-admin@<vm0-ip> 'sudo systemctl restart fleet-agent'
|
|
||||||
```
|
|
||||||
|
|
||||||
Watch the callout pod log emit one fresh request/response.
|
|
||||||
|
|
||||||
## 4. Watch the agent
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
|
|
||||||
fleet-admin@<vm0-ip> 'sudo journalctl -u fleet-agent -f'
|
|
||||||
```
|
|
||||||
|
|
||||||
What good looks like, in order:
|
|
||||||
|
|
||||||
| Log line | Where it comes from |
|
|
||||||
| --- | --- |
|
|
||||||
| `minted fresh Zitadel access token audience=…` | `credentials.rs::zitadel_mint` — RFC 7523 JWT-bearer flow, signed with the per-device machine key under `/etc/fleet-agent/zitadel-key.json` |
|
|
||||||
| `connected successfully server=4222` | NATS accepted the JWT minted by the callout |
|
|
||||||
| `fleet publisher ready` | KV buckets opened; `device-info` write succeeded |
|
|
||||||
| `watching KV keys filter=vm-device-00.>` | desired-state subscriber is up |
|
|
||||||
|
|
||||||
Absence of `Permissions Violation` lines is the success signal —
|
|
||||||
those mean the JWT's perms don't match what the agent tried to
|
|
||||||
publish (you'd hit them if `device_id_prefix_strip` were
|
|
||||||
misconfigured, for example).
|
|
||||||
|
|
||||||
## 5. Observe fleet traffic as admin
|
|
||||||
|
|
||||||
The harness mints a `fleet-ops` admin machine user with the
|
|
||||||
`fleet-admin` role; the callout maps that role to
|
|
||||||
`pub/sub allow: [">"]`. The integration test
|
|
||||||
`admin_jwt_reads_any_device_subject` exercises this — easiest path
|
|
||||||
to see it live is to run it with output. The test is
|
|
||||||
`#[ignore]`d on `cargo test` so a developer box doesn't burn a
|
|
||||||
10-minute Zitadel bring-up by accident; `--ignored` opts in:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
FLEET_E2E_VM_0_IP=<vm0-ip> FLEET_E2E_VM_1_IP=<vm1-ip> \
|
|
||||||
cargo test -p example-fleet-e2e-demo \
|
|
||||||
--test e2e_walking_skeleton \
|
|
||||||
admin_jwt_reads_any_device_subject \
|
|
||||||
-- --test-threads=1 --nocapture --ignored
|
|
||||||
```
|
|
||||||
|
|
||||||
It subscribes admin to `device-state.>` (the direct, non-JetStream
|
|
||||||
fan-out subject the agent emits a pulse on every 30s — see
|
|
||||||
`fleet_publisher.rs::publish_state_pulse`) and asserts a message
|
|
||||||
arrives within 30s.
|
|
||||||
|
|
||||||
**Inspect KV state directly** using a bare admin client. The
|
|
||||||
underlying mechanism is in
|
|
||||||
`examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs::admin_nats_client`:
|
|
||||||
mint a JWT-bearer token from `stack.admin_machine_key`, hand it to
|
|
||||||
`async_nats` as `auth_token`. The test
|
|
||||||
`both_devices_heartbeat_within_60s` then reads `device-info` keys
|
|
||||||
directly:
|
|
||||||
|
|
||||||
```rust
|
|
||||||
let js = async_nats::jetstream::new(admin);
|
|
||||||
let bucket = js.get_key_value(BUCKET_DEVICE_INFO).await?;
|
|
||||||
let entry = bucket.entry(&device_info_key("vm-device-00")).await?;
|
|
||||||
```
|
|
||||||
|
|
||||||
To do it from a shell, port-forward NATS and use the `nats` CLI
|
|
||||||
with admin creds — but creds for an auth-callout server take a
|
|
||||||
JWT-bearer token, which the `nats` CLI doesn't speak natively;
|
|
||||||
running the test is the path of least friction.
|
|
||||||
|
|
||||||
## 6. Verify cross-device isolation (currently `#[ignore]`)
|
|
||||||
|
|
||||||
`cross_device_isolation_enforced_in_vm` is an empty test marked
|
|
||||||
`#[ignore = "requires E2eHandles::device_machine_key plumbing"]`
|
|
||||||
in `e2e_walking_skeleton.rs` — the test is a placeholder. The
|
|
||||||
plumbing it's waiting on is straightforward: the existing
|
|
||||||
`DeviceHandle` struct (`examples/fleet_e2e_demo/src/lib.rs:106`)
|
|
||||||
exposes `device_id` + `vm_ip` + `labels` but not the per-device
|
|
||||||
Zitadel machine key the test would need to mint a `device`-role
|
|
||||||
JWT and try cross-device subjects. `provision_device` already
|
|
||||||
creates the key (line ~324, `machine_key_json`) — wiring it through
|
|
||||||
into `DeviceHandle.machine_key` and implementing the test body
|
|
||||||
(mint JWT-bearer for vm-device-00, sub to
|
|
||||||
`device-commands.vm-device-01`, expect `Permissions Violation`)
|
|
||||||
is a single follow-up commit. I haven't touched it because nothing
|
|
||||||
in this branch's scope required it.
|
|
||||||
|
|
||||||
**You can verify the boundary manually right now**, even without
|
|
||||||
the test wired up: tail the callout pod, then SSH onto vm-device-00
|
|
||||||
and run the agent with a tampered config that points it at
|
|
||||||
vm-device-01's keyfile. The callout will issue a JWT for
|
|
||||||
`vm-device-01` (because the JWT-bearer assertion is signed with
|
|
||||||
that user's key); the agent on vm-device-00 will then publish on
|
|
||||||
`$KV.device-info.info.vm-device-00`, which is NOT in the JWT's
|
|
||||||
allow list — NATS rejects with `Permissions Violation`. This is
|
|
||||||
the same gate the test would automate.
|
|
||||||
|
|
||||||
The permissions template is in
|
|
||||||
`nats/callout/src/permissions.rs::device_default` — every allowed
|
|
||||||
subject contains `{device_id}` and is interpolated per-request, so
|
|
||||||
device A's JWT physically cannot publish to device B's subjects.
|
|
||||||
|
|
||||||
## 7. Drive the desired-state loop
|
|
||||||
|
|
||||||
(Not yet covered by a walking-skeleton test, but the agent's
|
|
||||||
reconciler is wired and observable.) From an admin client, write a
|
|
||||||
desired state for vm-device-00:
|
|
||||||
|
|
||||||
```rust
|
|
||||||
// pseudocode — see harmony-reconciler-contracts for the exact types
|
|
||||||
let kv = jetstream.create_key_value(kv::Config {
|
|
||||||
bucket: BUCKET_DESIRED_STATE.into(),
|
|
||||||
history: 1,
|
|
||||||
..Default::default()
|
|
||||||
}).await?;
|
|
||||||
kv.put(
|
|
||||||
&desired_state_key("vm-device-00", &dn("hello-web")),
|
|
||||||
payload.into(),
|
|
||||||
).await?;
|
|
||||||
```
|
|
||||||
|
|
||||||
What happens, observable from the agent's journal:
|
|
||||||
|
|
||||||
1. Agent's KV watcher (filter `vm-device-00.>`) fires.
|
|
||||||
2. Reconciler computes the diff and runs the podman create.
|
|
||||||
3. `write_deployment_state(&state)` fires:
|
|
||||||
- puts `state.vm-device-00.hello-web` into the `device-state`
|
|
||||||
KV bucket (operator-side watch picks it up)
|
|
||||||
- publishes the same payload on direct subject
|
|
||||||
`device-state.vm-device-00` (admin observers see it live)
|
|
||||||
|
|
||||||
You can subscribe to the latter with admin and watch reconcile
|
|
||||||
events stream in real time.
|
|
||||||
|
|
||||||
## 8. Teardown
|
|
||||||
|
|
||||||
The cluster persists across runs (re-running `fleet_e2e_demo`
|
|
||||||
converges drift, doesn't recreate). When you want a clean slate:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
k3d cluster delete fleet-auth-callout
|
|
||||||
|
|
||||||
virsh destroy vm-device-00; virsh undefine vm-device-00 --remove-all-storage
|
|
||||||
virsh destroy vm-device-01; virsh undefine vm-device-01 --remove-all-storage
|
|
||||||
```
|
|
||||||
|
|
||||||
Cached assets (cloud images, k3d binary, ansible venv, SSH key,
|
|
||||||
fleet secrets) live under `~/.local/share/harmony/` and survive
|
|
||||||
cluster/VM destruction by design — first run after a clean reuses
|
|
||||||
them.
|
|
||||||
@@ -1,847 +0,0 @@
|
|||||||
//! VM-based end-to-end rehearsal of the customer demo flow.
|
|
||||||
//!
|
|
||||||
//! Goal: prove the JWT-auth chain works on a real-system agent
|
|
||||||
//! before pointing the demo at OKD. See
|
|
||||||
//! `ROADMAP/fleet_platform/v0_demo_e2e.md` for the full plan.
|
|
||||||
//!
|
|
||||||
//! Bring-up sequence:
|
|
||||||
//! 1. k3d cluster with HTTP + NATS port mappings (re-uses
|
|
||||||
//! fleet_auth_callout's k3d helpers — same cluster name so
|
|
||||||
//! re-runs of either example reuse the same cluster).
|
|
||||||
//! 2. Zitadel + Postgres via ZitadelScore.
|
|
||||||
//! 3. Wait for Zitadel HTTP and the chart-provisioned `iam-admin-pat`
|
|
||||||
//! secret (the chart's setup job is async).
|
|
||||||
//! 4. ZitadelSetupScore for the project + API app + roles + admin
|
|
||||||
//! machine user (no per-device users yet).
|
|
||||||
//! 5. NATS with auth_callout block + the callout pod.
|
|
||||||
//! 6. For each device i:
|
|
||||||
//! - ZitadelSetupScore minting a per-device machine user with
|
|
||||||
//! the `device` role grant. The JSON keyfile is cached in
|
|
||||||
//! `ZitadelClientConfig` and read back here for the agent.
|
|
||||||
//! - libvirt VM via `ProvisionVmScore`.
|
|
||||||
//! - SSH-inject `/etc/hosts` so the VM resolves
|
|
||||||
//! `sso.fleet.local` to the libvirt host.
|
|
||||||
//! - `FleetDeviceSetupScore` with `FleetDeviceAuth::ZitadelJwt`
|
|
||||||
//! pointing at the dropped keyfile.
|
|
||||||
//!
|
|
||||||
//! Tests in `tests/e2e_walking_skeleton.rs` share a single bring-up
|
|
||||||
//! via `OnceCell` and exercise: heartbeats, label-selector targeting,
|
|
||||||
//! status reflect-back, env+volume propagation, admin cross-device
|
|
||||||
//! read, per-device isolation, NATS-pod-restart reconnect.
|
|
||||||
|
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use example_fleet_auth_callout::{
|
|
||||||
ADMIN_ROLE_KEY, API_APP_NAME, CALLOUT_DEPLOYMENT_NAME, CALLOUT_IMAGE_TAG, DEVICE_ROLE_KEY,
|
|
||||||
FLEET_NAMESPACE, HTTP_PORT, NATS_ACCOUNT, NATS_AUTH_PASS, NATS_AUTH_USER, NATS_NAMESPACE,
|
|
||||||
NATS_NODE_PORT, NATS_RELEASE, PROJECT_NAME, ZITADEL_HOST, build_and_load_callout_image,
|
|
||||||
create_k3d, create_topology, deploy_zitadel, ensure_issuer_seed, wait_for_callout_ready,
|
|
||||||
wait_for_zitadel_ready,
|
|
||||||
};
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::fleet::{
|
|
||||||
FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry,
|
|
||||||
ensure_fleet_ssh_keypair,
|
|
||||||
};
|
|
||||||
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
|
|
||||||
use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv};
|
|
||||||
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
|
||||||
use harmony::modules::zitadel::{
|
|
||||||
MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole,
|
|
||||||
ZitadelSetupScore,
|
|
||||||
};
|
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
|
|
||||||
use harmony_fleet_deploy::FleetNatsScore;
|
|
||||||
use harmony_types::id::Id;
|
|
||||||
use log::{info, warn};
|
|
||||||
use nkeys::KeyPair;
|
|
||||||
|
|
||||||
// ---- constants -------------------------------------------------------------
|
|
||||||
|
|
||||||
/// Libvirt's default NAT gateway. The host's IP from inside any VM
|
|
||||||
/// attached to the `default` libvirt network. We bake this in because
|
|
||||||
/// every smoke-a* harness assumes it; if a customer runs their own
|
|
||||||
/// libvirt with a different bridge they can override via env.
|
|
||||||
pub const DEFAULT_LIBVIRT_HOST_IP: &str = "192.168.122.1";
|
|
||||||
|
|
||||||
pub const ADMIN_USERNAME: &str = "fleet-ops";
|
|
||||||
/// Separate machine user for the in-cluster operator. Distinct from
|
|
||||||
/// `fleet-ops` (manual admin tooling) so the audit trail can tell
|
|
||||||
/// operator-driven actions apart from human operator actions. Same
|
|
||||||
/// `fleet-admin` role grant — only the identity differs.
|
|
||||||
pub const OPERATOR_USERNAME: &str = "fleet-operator";
|
|
||||||
pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:dev";
|
|
||||||
|
|
||||||
/// Per-device username convention: `device-${device_id}`. Matches what
|
|
||||||
/// `fleet_rpi_setup` produces, so callout's `device_id_claim =
|
|
||||||
/// "client_id"` extracts the device id verbatim from the `client_id`
|
|
||||||
/// claim Zitadel emits in machine-user access tokens.
|
|
||||||
pub fn device_username(device_id: &str) -> String {
|
|
||||||
format!("device-{device_id}")
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- options + handles -----------------------------------------------------
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct E2eDemoOpts {
|
|
||||||
/// Number of VM-as-device agents to provision.
|
|
||||||
pub num_devices: usize,
|
|
||||||
/// Path to the cross-compiled `fleet-agent` binary uploaded to
|
|
||||||
/// each VM. Defaults to `target/release/fleet-agent` (the same
|
|
||||||
/// path that smoke-a4 produces).
|
|
||||||
pub agent_binary: PathBuf,
|
|
||||||
/// Override for the libvirt host IP (the address VMs see as the
|
|
||||||
/// gateway). Defaults to [`DEFAULT_LIBVIRT_HOST_IP`].
|
|
||||||
pub libvirt_host_ip: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for E2eDemoOpts {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
num_devices: 2,
|
|
||||||
agent_binary: workspace_target_path("release/harmony-fleet-agent"),
|
|
||||||
libvirt_host_ip: DEFAULT_LIBVIRT_HOST_IP.to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct DeviceHandle {
|
|
||||||
pub index: usize,
|
|
||||||
pub device_id: String,
|
|
||||||
pub vm_ip: String,
|
|
||||||
pub labels: std::collections::BTreeMap<String, String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct E2eHandles {
|
|
||||||
pub cluster_name: String,
|
|
||||||
pub nats_url_external: String,
|
|
||||||
pub zitadel_url: String,
|
|
||||||
pub project_id: String,
|
|
||||||
pub issuer_pubkey: String,
|
|
||||||
pub admin_machine_key: String,
|
|
||||||
pub devices: Vec<DeviceHandle>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- bring up --------------------------------------------------------------
|
|
||||||
|
|
||||||
pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result<E2eHandles> {
|
|
||||||
let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
|
||||||
.try_init();
|
|
||||||
|
|
||||||
info!("[e2e-demo 1/9] ensuring k3d cluster");
|
|
||||||
let k3d = create_k3d();
|
|
||||||
k3d.ensure_installed()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("k3d ensure: {e}"))?;
|
|
||||||
let topology = create_topology(&k3d);
|
|
||||||
topology.ensure_ready().await.context("topology init")?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 2/9] deploying Zitadel (cold start: ~5 min)");
|
|
||||||
deploy_zitadel(&topology).await?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 3/9] CoreDNS rewrite + waiting for Zitadel HTTP + iam-admin-pat secret");
|
|
||||||
CoreDNSRewriteScore {
|
|
||||||
rewrites: vec![CoreDNSRewrite {
|
|
||||||
hostname: ZITADEL_HOST.to_string(),
|
|
||||||
target: "zitadel.zitadel.svc.cluster.local".to_string(),
|
|
||||||
}],
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("CoreDNSRewriteScore")?;
|
|
||||||
wait_for_zitadel_ready().await?;
|
|
||||||
wait_for_iam_admin_pat_secret(&topology).await?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 4/9] provisioning project, API app, roles, admin machine user");
|
|
||||||
let admin_setup = ZitadelSetupScore {
|
|
||||||
host: ZITADEL_HOST.to_string(),
|
|
||||||
scheme: Default::default(),
|
|
||||||
port: None,
|
|
||||||
skip_tls: true,
|
|
||||||
endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")),
|
|
||||||
admin_org_id: None,
|
|
||||||
namespace: "zitadel".to_string(),
|
|
||||||
applications: vec![],
|
|
||||||
api_apps: vec![ZitadelApiApp {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
app_name: API_APP_NAME.to_string(),
|
|
||||||
}],
|
|
||||||
roles: vec![
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
key: ADMIN_ROLE_KEY.to_string(),
|
|
||||||
display_name: "Fleet Admin".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
key: DEVICE_ROLE_KEY.to_string(),
|
|
||||||
display_name: "Device".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
groups_claim_action: false,
|
|
||||||
human_users: vec![],
|
|
||||||
org_members: vec![],
|
|
||||||
instance_members: vec![],
|
|
||||||
machine_secrets: vec![],
|
|
||||||
port_forward_service: None,
|
|
||||||
machine_users: vec![
|
|
||||||
ZitadelMachineUser {
|
|
||||||
username: ADMIN_USERNAME.to_string(),
|
|
||||||
name: "Fleet Operations".to_string(),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(PROJECT_NAME.to_string()),
|
|
||||||
grant_roles: vec![ADMIN_ROLE_KEY.to_string()],
|
|
||||||
},
|
|
||||||
// Separate machine user for the in-cluster operator pod.
|
|
||||||
// Same `fleet-admin` role grant as the manual admin
|
|
||||||
// identity, but distinct username so JWT `client_id` lets
|
|
||||||
// log analysis tell operator-driven actions apart from
|
|
||||||
// human operator actions.
|
|
||||||
ZitadelMachineUser {
|
|
||||||
username: OPERATOR_USERNAME.to_string(),
|
|
||||||
name: "Fleet Operator (in-cluster)".to_string(),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(PROJECT_NAME.to_string()),
|
|
||||||
grant_roles: vec![ADMIN_ROLE_KEY.to_string()],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
admin_setup
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("admin ZitadelSetupScore")?;
|
|
||||||
|
|
||||||
let zcfg = ZitadelClientConfig::load()
|
|
||||||
.context("ZitadelSetupScore did not produce a client config cache")?;
|
|
||||||
let project_id = zcfg
|
|
||||||
.project_id_by_name(PROJECT_NAME)
|
|
||||||
.or(zcfg.project_id.as_ref())
|
|
||||||
.context("project_id missing from cache")?
|
|
||||||
.clone();
|
|
||||||
let admin_machine_key = zcfg
|
|
||||||
.machine_key(ADMIN_USERNAME)
|
|
||||||
.context("admin machine key missing from cache")?
|
|
||||||
.clone();
|
|
||||||
|
|
||||||
info!("[e2e-demo 5/9] generating issuer NKey, deploying NATS with auth_callout");
|
|
||||||
let issuer_seed = ensure_issuer_seed(&topology).await?;
|
|
||||||
let issuer_kp = KeyPair::from_seed(&issuer_seed)
|
|
||||||
.map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?;
|
|
||||||
let issuer_pubkey = issuer_kp.public_key();
|
|
||||||
|
|
||||||
// Per ADR-023 principle 2 — e2e uses the same Scores as production.
|
|
||||||
// `FleetNatsScore::callout` renders the auth-callout values block
|
|
||||||
// typed (serde_yaml) rather than the legacy `render_nats_values`
|
|
||||||
// string interpolation. Same upstream chart, same wire format,
|
|
||||||
// schema-checked at compile time.
|
|
||||||
FleetNatsScore::callout(
|
|
||||||
NATS_NAMESPACE,
|
|
||||||
NATS_NODE_PORT as u16,
|
|
||||||
&issuer_pubkey,
|
|
||||||
NATS_ACCOUNT,
|
|
||||||
NATS_AUTH_USER,
|
|
||||||
NATS_AUTH_PASS,
|
|
||||||
)
|
|
||||||
.release_name(NATS_RELEASE)
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("NATS deploy")?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 6/9] building + sideloading callout image into k3d");
|
|
||||||
build_and_load_callout_image(&k3d).await?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 7/9] deploying NatsAuthCalloutScore");
|
|
||||||
let mut callout = NatsAuthCalloutScore::new(
|
|
||||||
CALLOUT_DEPLOYMENT_NAME,
|
|
||||||
FLEET_NAMESPACE,
|
|
||||||
format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"),
|
|
||||||
format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
|
|
||||||
project_id.clone(),
|
|
||||||
NATS_AUTH_USER,
|
|
||||||
NATS_AUTH_PASS,
|
|
||||||
issuer_seed.clone(),
|
|
||||||
)
|
|
||||||
.image(CALLOUT_IMAGE_TAG)
|
|
||||||
.target_account(NATS_ACCOUNT)
|
|
||||||
.admin_role(ADMIN_ROLE_KEY)
|
|
||||||
.device_role(DEVICE_ROLE_KEY)
|
|
||||||
.danger_accept_invalid_certs(true);
|
|
||||||
// Same convention as fleet_auth_callout: the username is in the
|
|
||||||
// access token's `client_id` claim. The role claim path is
|
|
||||||
// project-scoped because the JWT-bearer flow requests project
|
|
||||||
// audience scope.
|
|
||||||
callout.device_id_claim = "client_id".to_string();
|
|
||||||
// Zitadel's `client_id` for a machine user equals its userName, so a
|
|
||||||
// user created as `device-vm-device-00` (the convention shared with
|
|
||||||
// fleet_rpi_setup and fleet_auth_callout) lands in the JWT verbatim.
|
|
||||||
// Strip the `device-` prefix so the callout interpolates permissions
|
|
||||||
// against the bare device id (`vm-device-00`) the agent uses for KV
|
|
||||||
// keys + direct subjects.
|
|
||||||
callout.device_id_prefix_strip = "device-".to_string();
|
|
||||||
callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles");
|
|
||||||
callout
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("callout deploy")?;
|
|
||||||
wait_for_callout_ready(&topology).await?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 8/10] building + sideloading operator image into k3d");
|
|
||||||
build_and_load_operator_image(&k3d).await?;
|
|
||||||
|
|
||||||
info!("[e2e-demo 9/10] deploying fleet operator with Zitadel JWT auth");
|
|
||||||
let operator_machine_key = zcfg
|
|
||||||
.machine_key(OPERATOR_USERNAME)
|
|
||||||
.with_context(|| format!("machine key for {OPERATOR_USERNAME} missing from cache"))?
|
|
||||||
.clone();
|
|
||||||
deploy_operator(&topology, &project_id, &operator_machine_key).await?;
|
|
||||||
wait_for_operator_ready(&topology).await?;
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"[e2e-demo 10/10] provisioning {} VM(s) and onboarding agent(s)",
|
|
||||||
opts.num_devices
|
|
||||||
);
|
|
||||||
let mut devices = Vec::with_capacity(opts.num_devices);
|
|
||||||
for i in 0..opts.num_devices {
|
|
||||||
let handle = provision_device(i, &opts, &topology, &project_id).await?;
|
|
||||||
devices.push(handle);
|
|
||||||
}
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"full stack ready: {} device(s), operator + admin role configured",
|
|
||||||
devices.len()
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(E2eHandles {
|
|
||||||
cluster_name: example_fleet_auth_callout::CLUSTER_NAME.to_string(),
|
|
||||||
nats_url_external: format!("nats://127.0.0.1:{NATS_NODE_PORT}"),
|
|
||||||
zitadel_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
|
|
||||||
project_id,
|
|
||||||
issuer_pubkey,
|
|
||||||
admin_machine_key,
|
|
||||||
devices,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- per-device provisioning ----------------------------------------------
|
|
||||||
|
|
||||||
async fn provision_device(
|
|
||||||
index: usize,
|
|
||||||
opts: &E2eDemoOpts,
|
|
||||||
topology: &K8sAnywhereTopology,
|
|
||||||
project_id: &str,
|
|
||||||
) -> Result<DeviceHandle> {
|
|
||||||
let device_id = format!("vm-device-{index:02}");
|
|
||||||
let username = device_username(&device_id);
|
|
||||||
info!("[device {index}] minting Zitadel machine user {username}");
|
|
||||||
|
|
||||||
// Per-device ZitadelSetupScore (search-then-create — running this
|
|
||||||
// for an existing user is a NOOP that just refreshes the cache
|
|
||||||
// entry pointing at the persisted machine key). The keyfile is
|
|
||||||
// re-minted because Zitadel doesn't expose the private half of
|
|
||||||
// an existing key — accept that any prior key drifts to "stale
|
|
||||||
// until expiry" on the previous device installation.
|
|
||||||
let device_setup = ZitadelSetupScore {
|
|
||||||
host: ZITADEL_HOST.to_string(),
|
|
||||||
scheme: Default::default(),
|
|
||||||
port: None,
|
|
||||||
skip_tls: true,
|
|
||||||
endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")),
|
|
||||||
admin_org_id: None,
|
|
||||||
namespace: "zitadel".to_string(),
|
|
||||||
applications: vec![],
|
|
||||||
api_apps: vec![],
|
|
||||||
roles: vec![],
|
|
||||||
groups_claim_action: false,
|
|
||||||
human_users: vec![],
|
|
||||||
org_members: vec![],
|
|
||||||
instance_members: vec![],
|
|
||||||
machine_secrets: vec![],
|
|
||||||
port_forward_service: None,
|
|
||||||
machine_users: vec![ZitadelMachineUser {
|
|
||||||
username: username.clone(),
|
|
||||||
name: format!("Fleet Device {device_id}"),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(PROJECT_NAME.to_string()),
|
|
||||||
grant_roles: vec![DEVICE_ROLE_KEY.to_string()],
|
|
||||||
}],
|
|
||||||
};
|
|
||||||
device_setup
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.with_context(|| format!("ZitadelSetupScore for {username}"))?;
|
|
||||||
|
|
||||||
let zcfg = ZitadelClientConfig::load()
|
|
||||||
.context("ZitadelClientConfig disappeared between admin and device setup")?;
|
|
||||||
let machine_key_json = zcfg
|
|
||||||
.machine_key(&username)
|
|
||||||
.with_context(|| format!("machine key for {username} missing from cache"))?
|
|
||||||
.clone();
|
|
||||||
|
|
||||||
// -- VM provisioning would go here. Deferred to keep the harness
|
|
||||||
// cold-start observable in pieces — the kvm bits (ProvisionVmScore)
|
|
||||||
// require root + libvirtd + the cloud image. Today the harness
|
|
||||||
// expects the operator to have provisioned VMs out-of-band (e.g.
|
|
||||||
// via fleet_vm_setup, or a pre-existing libvirt domain). We read
|
|
||||||
// the IP from a convention path (see `discover_vm_ip`) so the
|
|
||||||
// test driver can iterate on the agent path without re-paying VM
|
|
||||||
// boot every test cycle.
|
|
||||||
//
|
|
||||||
// Follow-up: fold ProvisionVmScore::ensure_vm here once the
|
|
||||||
// bring-up has been demonstrated end-to-end at least once.
|
|
||||||
let vm_ip = discover_vm_ip(index)
|
|
||||||
.with_context(|| format!("could not resolve IP for device {index}"))?;
|
|
||||||
|
|
||||||
info!("[device {index}] {device_id} at {vm_ip} — installing agent with Zitadel JWT auth");
|
|
||||||
let labels = build_device_labels(&device_id, index);
|
|
||||||
let agent_score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig {
|
|
||||||
device_id: Id::from(device_id.clone()),
|
|
||||||
labels: labels.clone(),
|
|
||||||
// Agent connects to NATS at the libvirt host's IP via the
|
|
||||||
// NodePort. The libvirt default network NATs the VM through
|
|
||||||
// the host so the host's port mapping is reachable.
|
|
||||||
nats_urls: vec![format!("nats://{}:{NATS_NODE_PORT}", opts.libvirt_host_ip)],
|
|
||||||
auth: FleetDeviceAuth::ZitadelJwt {
|
|
||||||
machine_key_json,
|
|
||||||
// Issuer URL the agent uses MUST match the issuer
|
|
||||||
// string Zitadel returns — Zitadel derives that from
|
|
||||||
// the request's Host header. We hit Zitadel via the
|
|
||||||
// host's port mapping, so the agent's URL is
|
|
||||||
// `http://sso.fleet.local:<host-port>`. The /etc/hosts
|
|
||||||
// entry below points sso.fleet.local at the libvirt
|
|
||||||
// host so the VM resolves it.
|
|
||||||
oidc_issuer_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
|
|
||||||
audience: project_id.to_string(),
|
|
||||||
// Local rehearsal hits Zitadel over plain HTTP through
|
|
||||||
// the cluster ingress; no TLS validation needed.
|
|
||||||
danger_accept_invalid_certs: true,
|
|
||||||
},
|
|
||||||
agent_binary_path: opts.agent_binary.clone(),
|
|
||||||
hosts_entries: vec![HostsEntry {
|
|
||||||
ip: opts.libvirt_host_ip.clone(),
|
|
||||||
hostname: ZITADEL_HOST.to_string(),
|
|
||||||
}],
|
|
||||||
openbao: None,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Apply the score over SSH against the VM. Same pattern as
|
|
||||||
// fleet_rpi_setup, but synthesized inline so the harness can drive
|
|
||||||
// multiple VMs in sequence without copying the CLI plumbing.
|
|
||||||
apply_fleet_setup_to_vm(index, &vm_ip, agent_score).await?;
|
|
||||||
|
|
||||||
Ok(DeviceHandle {
|
|
||||||
index,
|
|
||||||
device_id,
|
|
||||||
vm_ip,
|
|
||||||
labels,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn apply_fleet_setup_to_vm(
|
|
||||||
index: usize,
|
|
||||||
vm_ip: &str,
|
|
||||||
score: FleetDeviceSetupScore,
|
|
||||||
) -> Result<()> {
|
|
||||||
ensure_ansible_venv()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("ansible venv: {e}"))?;
|
|
||||||
let ssh = ensure_fleet_ssh_keypair()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?;
|
|
||||||
let ip = vm_ip
|
|
||||||
.parse()
|
|
||||||
.with_context(|| format!("VM IP '{vm_ip}' is not a valid IP address"))?;
|
|
||||||
let creds = SshCredentials {
|
|
||||||
// Matches the cloud-init admin user that fleet_vm_setup +
|
|
||||||
// smoke-a4 create. If the operator overrode that during
|
|
||||||
// out-of-band VM provisioning, follow-up: thread the
|
|
||||||
// username through E2eDemoOpts.
|
|
||||||
user: "fleet-admin".to_string(),
|
|
||||||
private_key_path: ssh.private_key.clone(),
|
|
||||||
remote_python: Some("/usr/bin/python3".to_string()),
|
|
||||||
sudo_password: None,
|
|
||||||
};
|
|
||||||
let topology = LinuxHostTopology::new(format!("vm-device-{index:02}"), ip, creds);
|
|
||||||
use harmony::score::Score;
|
|
||||||
score
|
|
||||||
.create_interpret()
|
|
||||||
.execute(&Inventory::empty(), &topology)
|
|
||||||
.await
|
|
||||||
.with_context(|| format!("FleetDeviceSetupScore against VM {index} ({vm_ip})"))?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_device_labels(
|
|
||||||
device_id: &str,
|
|
||||||
index: usize,
|
|
||||||
) -> std::collections::BTreeMap<String, String> {
|
|
||||||
// Two devices, two distinct group labels by default — lets
|
|
||||||
// selector tests target "exactly one device". Label scheme
|
|
||||||
// matches the demo runbook.
|
|
||||||
let mut labels = std::collections::BTreeMap::new();
|
|
||||||
labels.insert(
|
|
||||||
"group".to_string(),
|
|
||||||
if index == 0 {
|
|
||||||
"group-a".to_string()
|
|
||||||
} else {
|
|
||||||
"group-b".to_string()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
labels.insert("arch".to_string(), std::env::consts::ARCH.to_string());
|
|
||||||
labels.insert("role".to_string(), "rehearsal".to_string());
|
|
||||||
labels.insert("device-id".to_string(), device_id.to_string());
|
|
||||||
labels
|
|
||||||
}
|
|
||||||
|
|
||||||
fn discover_vm_ip(index: usize) -> Result<String> {
|
|
||||||
// Convention: a `FLEET_E2E_VM_<i>_IP` env var points at the
|
|
||||||
// pre-provisioned VM's IP. This keeps the harness usable on a
|
|
||||||
// workstation where the operator runs `fleet_vm_setup` once per
|
|
||||||
// device out-of-band, then re-runs the e2e harness against the
|
|
||||||
// already-booted VMs.
|
|
||||||
let key = format!("FLEET_E2E_VM_{index}_IP");
|
|
||||||
std::env::var(&key)
|
|
||||||
.with_context(|| format!("set {key} to the libvirt VM's IP (default network)"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- iam-admin-pat readiness ----------------------------------------------
|
|
||||||
|
|
||||||
/// Wait for the Zitadel chart's setup job to write the `iam-admin-pat`
|
|
||||||
/// secret. The Helm release reports Ready before the job completes,
|
|
||||||
/// so calling ZitadelSetupScore immediately after Zitadel deploy
|
|
||||||
/// races. ZitadelSetupScore itself reads this secret to authenticate
|
|
||||||
/// to the management API.
|
|
||||||
async fn wait_for_iam_admin_pat_secret(topology: &K8sAnywhereTopology) -> Result<()> {
|
|
||||||
use k8s_openapi::api::core::v1::Secret;
|
|
||||||
let k8s = topology
|
|
||||||
.k8s_client()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
|
|
||||||
for attempt in 1..=120 {
|
|
||||||
if let Some(secret) = k8s
|
|
||||||
.get_resource::<Secret>("iam-admin-pat", Some("zitadel"))
|
|
||||||
.await?
|
|
||||||
&& let Some(data) = secret.data
|
|
||||||
&& data.contains_key("pat")
|
|
||||||
{
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if attempt % 10 == 0 {
|
|
||||||
warn!("iam-admin-pat secret not yet present in zitadel ns ({attempt}/120)");
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
}
|
|
||||||
anyhow::bail!(
|
|
||||||
"timed out waiting for iam-admin-pat secret in 'zitadel' namespace — \
|
|
||||||
is FirstInstance.Org.Machine.Pat configured in ZitadelScore Helm values?"
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- operator deploy -------------------------------------------------------
|
|
||||||
|
|
||||||
const OPERATOR_NAMESPACE: &str = FLEET_NAMESPACE;
|
|
||||||
|
|
||||||
/// k3d's data directory under `$XDG_DATA_HOME`. Mirrors
|
|
||||||
/// `example_fleet_auth_callout::data_dir` (the latter is private —
|
|
||||||
/// duplicated here rather than re-exported so the operator wiring is
|
|
||||||
/// self-contained).
|
|
||||||
fn k3d_data_dir() -> PathBuf {
|
|
||||||
directories::BaseDirs::new()
|
|
||||||
.map(|dirs| dirs.data_dir().join("harmony").join("k3d"))
|
|
||||||
.unwrap_or_else(|| PathBuf::from("/tmp/harmony"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build the operator's release binary, package it into an OCI image,
|
|
||||||
/// and sideload into the k3d cluster. Mirrors
|
|
||||||
/// `build_and_load_callout_image`. The Dockerfile lives in the
|
|
||||||
/// operator crate.
|
|
||||||
async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> {
|
|
||||||
use std::process::Stdio;
|
|
||||||
|
|
||||||
let workspace_root = std::env::var("CARGO_MANIFEST_DIR")
|
|
||||||
.map(|d| PathBuf::from(d).join("..").join(".."))
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("."));
|
|
||||||
let workspace_root = workspace_root.canonicalize().unwrap_or(workspace_root);
|
|
||||||
|
|
||||||
info!("cargo build --release -p harmony-fleet-operator");
|
|
||||||
let status = tokio::process::Command::new("cargo")
|
|
||||||
.args(["build", "--release", "-p", "harmony-fleet-operator"])
|
|
||||||
.current_dir(&workspace_root)
|
|
||||||
.status()
|
|
||||||
.await?;
|
|
||||||
if !status.success() {
|
|
||||||
anyhow::bail!("cargo build for fleet operator failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stage the binary + Dockerfile into a clean temp dir so podman
|
|
||||||
// build doesn't drag the whole target/ tree across.
|
|
||||||
let ctx = tempfile::tempdir()?;
|
|
||||||
let bin_dst = ctx.path().join("target/release");
|
|
||||||
std::fs::create_dir_all(&bin_dst)?;
|
|
||||||
std::fs::copy(
|
|
||||||
workspace_root.join("target/release/harmony-fleet-operator"),
|
|
||||||
bin_dst.join("harmony-fleet-operator"),
|
|
||||||
)
|
|
||||||
.context("staging operator binary into build context")?;
|
|
||||||
let dockerfile_src = workspace_root.join("fleet/harmony-fleet-operator/Dockerfile");
|
|
||||||
if !dockerfile_src.exists() {
|
|
||||||
anyhow::bail!(
|
|
||||||
"missing fleet/harmony-fleet-operator/Dockerfile — operator image staging \
|
|
||||||
expects it next to Cargo.toml; either add it or update the bring-up."
|
|
||||||
);
|
|
||||||
}
|
|
||||||
std::fs::copy(&dockerfile_src, ctx.path().join("Dockerfile"))?;
|
|
||||||
|
|
||||||
info!("podman build → {OPERATOR_IMAGE_TAG}");
|
|
||||||
let status = tokio::process::Command::new("podman")
|
|
||||||
.args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."])
|
|
||||||
.current_dir(ctx.path())
|
|
||||||
.stderr(Stdio::inherit())
|
|
||||||
.status()
|
|
||||||
.await?;
|
|
||||||
if !status.success() {
|
|
||||||
anyhow::bail!("podman build for operator failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
let tar_path =
|
|
||||||
std::env::temp_dir().join(format!("harmony-operator-image-{}.tar", std::process::id()));
|
|
||||||
let _ = std::fs::remove_file(&tar_path);
|
|
||||||
let status = tokio::process::Command::new("podman")
|
|
||||||
.args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG])
|
|
||||||
.status()
|
|
||||||
.await?;
|
|
||||||
if !status.success() {
|
|
||||||
anyhow::bail!("podman save for operator failed");
|
|
||||||
}
|
|
||||||
info!("k3d image import {OPERATOR_IMAGE_TAG}");
|
|
||||||
let cluster_name = k3d
|
|
||||||
.cluster_name()
|
|
||||||
.unwrap_or(example_fleet_auth_callout::CLUSTER_NAME)
|
|
||||||
.to_string();
|
|
||||||
let tar_path_str = tar_path.to_str().unwrap().to_string();
|
|
||||||
let cluster_for_blocking = cluster_name.clone();
|
|
||||||
let data_dir = k3d_data_dir();
|
|
||||||
tokio::task::spawn_blocking(move || {
|
|
||||||
k3d_rs::K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([
|
|
||||||
"image",
|
|
||||||
"import",
|
|
||||||
tar_path_str.as_str(),
|
|
||||||
"-c",
|
|
||||||
cluster_for_blocking.as_str(),
|
|
||||||
])
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
.map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?;
|
|
||||||
let _ = std::fs::remove_file(&tar_path);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
|
|
||||||
/// ClusterRoleBinding + Secret + Deployment via Harmony's
|
|
||||||
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
|
|
||||||
/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and
|
|
||||||
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
|
|
||||||
async fn deploy_operator(
|
|
||||||
topology: &K8sAnywhereTopology,
|
|
||||||
project_id: &str,
|
|
||||||
operator_machine_key: &str,
|
|
||||||
) -> Result<()> {
|
|
||||||
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
|
|
||||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
|
||||||
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
|
||||||
use harmony_fleet_deploy::operator::chart::{
|
|
||||||
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
|
|
||||||
build_cluster_role_binding, build_operator_deployment, build_service_account,
|
|
||||||
operator_secret,
|
|
||||||
};
|
|
||||||
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
|
||||||
use kube::CustomResourceExt;
|
|
||||||
|
|
||||||
// Render the [credentials] TOML the operator pod consumes via the
|
|
||||||
// OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret
|
|
||||||
// key). The Zitadel JSON keyfile is embedded inline under
|
|
||||||
// `key_json`; the operator never sees a file. Triple-quoted TOML
|
|
||||||
// string keeps the JSON's `"`s untouched.
|
|
||||||
let credentials_toml = format!(
|
|
||||||
r#"type = "zitadel-jwt"
|
|
||||||
oidc_issuer_url = "http://{host}:{port}"
|
|
||||||
audience = "{project_id}"
|
|
||||||
danger_accept_invalid_certs = true
|
|
||||||
key_json = """
|
|
||||||
{key_json}
|
|
||||||
"""
|
|
||||||
"#,
|
|
||||||
host = ZITADEL_HOST,
|
|
||||||
port = HTTP_PORT,
|
|
||||||
key_json = operator_machine_key,
|
|
||||||
);
|
|
||||||
|
|
||||||
let opts = ChartOptions {
|
|
||||||
output_dir: PathBuf::new(), // unused on this code path
|
|
||||||
image: OPERATOR_IMAGE_TAG.to_string(),
|
|
||||||
image_pull_policy: "IfNotPresent".to_string(),
|
|
||||||
nats_url: format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"),
|
|
||||||
log_level: "info,kube_runtime=warn".to_string(),
|
|
||||||
credentials: Some(OperatorCredentials { credentials_toml }),
|
|
||||||
chart_version: None,
|
|
||||||
web_auth_config_json: None,
|
|
||||||
web_cookie_key_json: None,
|
|
||||||
};
|
|
||||||
|
|
||||||
// CRDs first — the operator watches them on startup.
|
|
||||||
let crds: Vec<CustomResourceDefinition> = vec![FleetDeployment::crd(), Device::crd()];
|
|
||||||
K8sResourceScore::<CustomResourceDefinition> {
|
|
||||||
resource: crds,
|
|
||||||
namespace: None,
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator CRD apply")?;
|
|
||||||
|
|
||||||
// RBAC.
|
|
||||||
K8sResourceScore::single(
|
|
||||||
build_service_account(),
|
|
||||||
Some(OPERATOR_NAMESPACE.to_string()),
|
|
||||||
)
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator ServiceAccount apply")?;
|
|
||||||
|
|
||||||
K8sResourceScore::single(build_cluster_role(), None)
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator ClusterRole apply")?;
|
|
||||||
|
|
||||||
K8sResourceScore::single(build_cluster_role_binding(OPERATOR_NAMESPACE), None)
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator ClusterRoleBinding apply")?;
|
|
||||||
|
|
||||||
// Secret holding both the credentials TOML and the keyfile.
|
|
||||||
let secret = operator_secret(&opts).expect("credentials present in opts");
|
|
||||||
K8sResourceScore::single(secret, Some(OPERATOR_NAMESPACE.to_string()))
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator Secret apply")?;
|
|
||||||
|
|
||||||
// Deployment last so it pulls the up-to-date Secret.
|
|
||||||
K8sResourceScore::single(
|
|
||||||
build_operator_deployment(&opts),
|
|
||||||
Some(OPERATOR_NAMESPACE.to_string()),
|
|
||||||
)
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("operator Deployment apply")?;
|
|
||||||
|
|
||||||
info!("operator deployment {OPERATOR_NAMESPACE}/{RELEASE_NAME} applied");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for_operator_ready(topology: &K8sAnywhereTopology) -> Result<()> {
|
|
||||||
use harmony_fleet_deploy::operator::chart::RELEASE_NAME;
|
|
||||||
use k8s_openapi::api::apps::v1::Deployment as K8sDeployment;
|
|
||||||
let k8s = topology
|
|
||||||
.k8s_client()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
|
|
||||||
for attempt in 1..=120 {
|
|
||||||
if let Some(d) = k8s
|
|
||||||
.get_resource::<K8sDeployment>(RELEASE_NAME, Some(OPERATOR_NAMESPACE))
|
|
||||||
.await?
|
|
||||||
&& let Some(status) = d.status
|
|
||||||
&& status.ready_replicas.unwrap_or(0) >= 1
|
|
||||||
{
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if attempt % 10 == 0 {
|
|
||||||
warn!("operator Deployment not yet Ready ({attempt}/120)");
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
}
|
|
||||||
anyhow::bail!("timed out waiting for operator Deployment to become Ready")
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- helpers ---------------------------------------------------------------
|
|
||||||
|
|
||||||
fn workspace_target_path(rel: &str) -> PathBuf {
|
|
||||||
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("."));
|
|
||||||
manifest_dir.join("..").join("..").join("target").join(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- next-steps panel ------------------------------------------------------
|
|
||||||
|
|
||||||
impl E2eHandles {
|
|
||||||
pub fn print_next_steps(&self) {
|
|
||||||
println!();
|
|
||||||
println!("============================================================");
|
|
||||||
println!(" E2E DEMO REHEARSAL — STACK READY");
|
|
||||||
println!("============================================================");
|
|
||||||
println!(" k3d cluster: {}", self.cluster_name);
|
|
||||||
println!(" Zitadel: {}", self.zitadel_url);
|
|
||||||
println!(" NATS (host): {}", self.nats_url_external);
|
|
||||||
println!(" Project ID: {}", self.project_id);
|
|
||||||
println!(" Issuer pubkey: {}", self.issuer_pubkey);
|
|
||||||
println!();
|
|
||||||
println!(" Devices ({}):", self.devices.len());
|
|
||||||
for d in &self.devices {
|
|
||||||
let labels: Vec<String> = d.labels.iter().map(|(k, v)| format!("{k}={v}")).collect();
|
|
||||||
println!(
|
|
||||||
" [{}] {} @ {} ({})",
|
|
||||||
d.index,
|
|
||||||
d.device_id,
|
|
||||||
d.vm_ip,
|
|
||||||
labels.join(",")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
println!();
|
|
||||||
println!(" Run the test suite:");
|
|
||||||
println!();
|
|
||||||
println!(" cargo test -p example-fleet-e2e-demo \\");
|
|
||||||
println!(" --test e2e_walking_skeleton -- --test-threads=1 --nocapture");
|
|
||||||
println!();
|
|
||||||
println!(" Ctrl-C exits without tearing the cluster down — re-run");
|
|
||||||
println!(" the bring-up to converge any drift.");
|
|
||||||
println!("============================================================");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod unit_tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn device_username_matches_callout_convention() {
|
|
||||||
// Callout's device_id_claim is `client_id`, which Zitadel
|
|
||||||
// populates from the machine user's username. The test we
|
|
||||||
// run later asserts the agent's per-device subjects match
|
|
||||||
// its device_id, which therefore must equal the username
|
|
||||||
// minus the "device-" prefix the callout knows about.
|
|
||||||
assert_eq!(device_username("vm-device-00"), "device-vm-device-00");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn device_labels_split_into_distinct_groups() {
|
|
||||||
let l0 = build_device_labels("vm-device-00", 0);
|
|
||||||
let l1 = build_device_labels("vm-device-01", 1);
|
|
||||||
assert_eq!(l0.get("group").unwrap(), "group-a");
|
|
||||||
assert_eq!(l1.get("group").unwrap(), "group-b");
|
|
||||||
assert_ne!(l0.get("group"), l1.get("group"));
|
|
||||||
// Ubiquitous labels: device-id + arch + role on both.
|
|
||||||
for l in [&l0, &l1] {
|
|
||||||
assert!(l.contains_key("device-id"));
|
|
||||||
assert!(l.contains_key("arch"));
|
|
||||||
assert_eq!(l.get("role").unwrap(), "rehearsal");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,51 +1,22 @@
|
|||||||
//! `cargo run -p example-fleet-e2e-demo -- --num-devices 2 ...`
|
use anyhow::Result;
|
||||||
//!
|
|
||||||
//! Brings up the full E2E rehearsal stack: k3d + Zitadel + NATS auth
|
|
||||||
//! callout + per-device Zitadel machine users + (out-of-band)
|
|
||||||
//! libvirt VMs + agents authenticating via JWT-bearer.
|
|
||||||
//!
|
|
||||||
//! See `src/lib.rs` and `ROADMAP/fleet_platform/v0_demo_e2e.md`.
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use example_fleet_e2e_demo::{DEFAULT_LIBVIRT_HOST_IP, E2eDemoOpts, bring_up_full_stack};
|
use harmony_fleet_e2e::{AuthMode, VmStack, VmStackOptions};
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser)]
|
||||||
#[command(
|
|
||||||
name = "fleet-e2e-demo",
|
|
||||||
about = "VM-based end-to-end rehearsal of the fleet platform demo flow"
|
|
||||||
)]
|
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Number of VM-as-device agents to bring up. Each one needs its
|
#[arg(long, default_value_t = 1)]
|
||||||
/// own libvirt domain (provisioned out-of-band today via
|
|
||||||
/// `fleet_vm_setup` — see `FLEET_E2E_VM_<i>_IP` env vars below).
|
|
||||||
#[arg(long, default_value_t = 2)]
|
|
||||||
num_devices: usize,
|
num_devices: usize,
|
||||||
/// Path to the cross-compiled `fleet-agent` binary uploaded to
|
|
||||||
/// each VM. Same binary that smoke-a4 produces.
|
|
||||||
#[arg(long, default_value = "target/release/harmony-fleet-agent")]
|
|
||||||
agent_binary: PathBuf,
|
|
||||||
/// Override for the libvirt host IP (the address VMs see as the
|
|
||||||
/// gateway). Defaults to the libvirt default network's gateway.
|
|
||||||
#[arg(long, default_value = DEFAULT_LIBVIRT_HOST_IP)]
|
|
||||||
libvirt_host_ip: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
let handles = bring_up_full_stack(E2eDemoOpts {
|
let _stack = VmStack::bring_up(VmStackOptions {
|
||||||
num_devices: cli.num_devices,
|
num_devices: cli.num_devices,
|
||||||
agent_binary: cli.agent_binary,
|
auth_mode: AuthMode::Callout,
|
||||||
libvirt_host_ip: cli.libvirt_host_ip,
|
..Default::default()
|
||||||
})
|
})
|
||||||
.await
|
.await?;
|
||||||
.context("bring_up_full_stack")?;
|
|
||||||
handles.print_next_steps();
|
|
||||||
|
|
||||||
println!();
|
|
||||||
println!(" Press Ctrl-C to exit (cluster keeps running).");
|
|
||||||
tokio::signal::ctrl_c().await?;
|
tokio::signal::ctrl_c().await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,161 +0,0 @@
|
|||||||
//! End-to-end walking-skeleton tests for the VM-based demo rehearsal.
|
|
||||||
//!
|
|
||||||
//! Shares one bring-up across the whole suite via `OnceCell`. Run
|
|
||||||
//! sequentially — they touch shared k3d + libvirt VM state.
|
|
||||||
//!
|
|
||||||
//! Pre-flight (manual, before `cargo test`):
|
|
||||||
//!
|
|
||||||
//! - libvirt + qemu installed; default network active.
|
|
||||||
//! - Two cloud-init Ubuntu VMs provisioned (e.g. via
|
|
||||||
//! `cargo run -p example_fleet_vm_setup`). Their IPs exported as
|
|
||||||
//! `FLEET_E2E_VM_0_IP` and `FLEET_E2E_VM_1_IP`.
|
|
||||||
//! - SSH keypair the VMs trust at `~/.ssh/id_ed25519` (or
|
|
||||||
//! override path; harness reads the standard pair).
|
|
||||||
//!
|
|
||||||
//! Run:
|
|
||||||
//!
|
|
||||||
//! ```bash
|
|
||||||
//! FLEET_E2E_VM_0_IP=192.168.122.42 \
|
|
||||||
//! FLEET_E2E_VM_1_IP=192.168.122.43 \
|
|
||||||
//! cargo test -p example-fleet-e2e-demo --test e2e_walking_skeleton \
|
|
||||||
//! -- --test-threads=1 --nocapture
|
|
||||||
//! ```
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use async_nats::ConnectOptions;
|
|
||||||
use example_fleet_auth_callout::{mint_access_token, scopes_for_project};
|
|
||||||
use example_fleet_e2e_demo::{E2eDemoOpts, E2eHandles, bring_up_full_stack};
|
|
||||||
use futures_util::StreamExt;
|
|
||||||
use tokio::sync::OnceCell;
|
|
||||||
|
|
||||||
static STACK: OnceCell<Arc<E2eHandles>> = OnceCell::const_new();
|
|
||||||
|
|
||||||
async fn shared_stack() -> Result<Arc<E2eHandles>> {
|
|
||||||
let cell = STACK
|
|
||||||
.get_or_try_init(|| async {
|
|
||||||
let h = bring_up_full_stack(E2eDemoOpts::default()).await?;
|
|
||||||
anyhow::Ok(Arc::new(h))
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
Ok(cell.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn admin_nats_client(stack: &E2eHandles) -> Result<async_nats::Client> {
|
|
||||||
let token = mint_access_token(
|
|
||||||
&stack.zitadel_url,
|
|
||||||
&stack.admin_machine_key,
|
|
||||||
&scopes_for_project(&stack.project_id),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.context("mint admin Zitadel token")?;
|
|
||||||
ConnectOptions::with_token(token)
|
|
||||||
.connection_timeout(Duration::from_secs(5))
|
|
||||||
.connect(&stack.nats_url_external)
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("admin connect: {e}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- Test 1 -------------------------------------------------------------
|
|
||||||
|
|
||||||
/// Each provisioned VM publishes a DeviceInfo within the heartbeat
|
|
||||||
/// window. Reads from the `device-info` KV bucket via the admin
|
|
||||||
/// client (admin role can subscribe to anything).
|
|
||||||
#[tokio::test]
|
|
||||||
#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"]
|
|
||||||
async fn both_devices_heartbeat_within_60s() -> Result<()> {
|
|
||||||
let _ = tracing_subscriber::fmt().with_env_filter("info").try_init();
|
|
||||||
let stack = shared_stack().await?;
|
|
||||||
let admin = admin_nats_client(&stack).await?;
|
|
||||||
|
|
||||||
let js = async_nats::jetstream::new(admin);
|
|
||||||
let bucket = js
|
|
||||||
.get_key_value(harmony_reconciler_contracts::BUCKET_DEVICE_INFO)
|
|
||||||
.await
|
|
||||||
.context("device-info bucket")?;
|
|
||||||
|
|
||||||
let deadline = std::time::Instant::now() + Duration::from_secs(60);
|
|
||||||
let expected: std::collections::HashSet<String> =
|
|
||||||
stack.devices.iter().map(|d| d.device_id.clone()).collect();
|
|
||||||
let mut seen = std::collections::HashSet::new();
|
|
||||||
|
|
||||||
while std::time::Instant::now() < deadline && seen != expected {
|
|
||||||
for d in &stack.devices {
|
|
||||||
let key = harmony_reconciler_contracts::device_info_key(&d.device_id);
|
|
||||||
if let Some(_e) = bucket.entry(&key).await? {
|
|
||||||
seen.insert(d.device_id.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
seen, expected,
|
|
||||||
"each provisioned device must publish DeviceInfo within 60s; saw {seen:?}"
|
|
||||||
);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- Test 5 (admin cross-device read) -----------------------------------
|
|
||||||
|
|
||||||
/// The admin's Zitadel JWT carries `fleet-admin` role. Callout maps
|
|
||||||
/// that to `pub/sub allow: [">"]`, so subscribing to `device-state.>`
|
|
||||||
/// is admitted and observes every device's traffic.
|
|
||||||
#[tokio::test]
|
|
||||||
#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"]
|
|
||||||
async fn admin_jwt_reads_any_device_subject() -> Result<()> {
|
|
||||||
let _ = tracing_subscriber::fmt().with_env_filter("info").try_init();
|
|
||||||
let stack = shared_stack().await?;
|
|
||||||
let admin = admin_nats_client(&stack).await?;
|
|
||||||
|
|
||||||
let mut sub = admin.subscribe("device-state.>").await?;
|
|
||||||
admin.flush().await?;
|
|
||||||
|
|
||||||
// Hold the subscription open long enough that any device's
|
|
||||||
// periodic state publication should land. We don't pump traffic
|
|
||||||
// ourselves — the agents themselves publish per-deployment state
|
|
||||||
// on every reconcile tick. If no traffic arrives in 30s it means
|
|
||||||
// either the agents aren't connected or they're not publishing,
|
|
||||||
// both of which are fatal for the demo.
|
|
||||||
let result = tokio::time::timeout(Duration::from_secs(30), sub.next()).await;
|
|
||||||
assert!(
|
|
||||||
result.is_ok() && result.as_ref().unwrap().is_some(),
|
|
||||||
"admin must observe at least one device-state.* message in 30s"
|
|
||||||
);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- Test 6 (per-device isolation) ---------------------------------------
|
|
||||||
|
|
||||||
/// A per-device JWT has subject permissions scoped to its own
|
|
||||||
/// `device-state.{device_id}` and `device-commands.{device_id}`. The
|
|
||||||
/// callout enforces this; subscribing to a sibling device's commands
|
|
||||||
/// must fail at NATS connect-time or at SUB-time.
|
|
||||||
///
|
|
||||||
/// Skipped here because the per-device JWT minting helper (analogous
|
|
||||||
/// to `mint_access_token` but for a `device` role user) needs the
|
|
||||||
/// per-device machine key to be plumbed back from `bring_up_full_stack`
|
|
||||||
/// through `E2eHandles`. Follow-up commit adds
|
|
||||||
/// `E2eHandles::device_machine_key(idx)` so this test can be
|
|
||||||
/// implemented without re-running `ZitadelSetupScore` from the test
|
|
||||||
/// body.
|
|
||||||
#[tokio::test]
|
|
||||||
#[ignore = "requires E2eHandles::device_machine_key plumbing"]
|
|
||||||
async fn cross_device_isolation_enforced_in_vm() {}
|
|
||||||
|
|
||||||
// -- Test 7 (load-bearing reconnect) -------------------------------------
|
|
||||||
|
|
||||||
/// Kill the NATS pod, wait for the new one to come up, verify both
|
|
||||||
/// agents reconnect with fresh JWTs and resume publishing within
|
|
||||||
/// 30 seconds. This is the test that validates the "never lose
|
|
||||||
/// connectivity to a device" guarantee under realistic disturbance.
|
|
||||||
///
|
|
||||||
/// Skipped pending operator install in the harness — without the
|
|
||||||
/// operator the agents have no `desired-state` to publish status
|
|
||||||
/// against, so verifying "publishing resumed" needs a separate
|
|
||||||
/// signal. Follow-up commit observes the agents' periodic
|
|
||||||
/// heartbeat publication directly via the device-heartbeat KV.
|
|
||||||
#[tokio::test]
|
|
||||||
#[ignore = "requires NATS-pod-restart driver and heartbeat-presence assertion"]
|
|
||||||
async fn agent_recovers_from_nats_pod_restart() {}
|
|
||||||
@@ -9,7 +9,7 @@ name = "fleet_load_test"
|
|||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", default-features = false }
|
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
async-nats = { workspace = true }
|
async-nats = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
|
|||||||
@@ -28,13 +28,13 @@ use anyhow::{Context, Result};
|
|||||||
use async_nats::jetstream::{self, kv};
|
use async_nats::jetstream::{self, kv};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::modules::fleet::operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
|
use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
|
||||||
use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
|
|
||||||
use harmony_reconciler_contracts::{
|
use harmony_reconciler_contracts::{
|
||||||
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
|
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
|
||||||
DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key,
|
DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key,
|
||||||
device_info_key, device_state_key,
|
device_info_key, device_state_key,
|
||||||
};
|
};
|
||||||
|
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
|
||||||
use k8s_openapi::api::core::v1::Namespace;
|
use k8s_openapi::api::core::v1::Namespace;
|
||||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! Install a single-node NATS server into the cluster `KUBECONFIG`
|
//! Install a single-node NATS server into the cluster `KUBECONFIG`
|
||||||
//! points at, using harmony's `NatsBasicScore` + `K8sBareTopology`.
|
//! points at, using harmony's `NatsScore` + `K8sBareTopology`.
|
||||||
//!
|
//!
|
||||||
//! This binary is the glue between the smoke harness (`smoke-a4.sh`)
|
//! This binary is the glue between the smoke harness (`smoke-a4.sh`)
|
||||||
//! and the framework Score. Typical usage from a demo script:
|
//! and the framework Score. Typical usage from a demo script:
|
||||||
@@ -20,13 +20,13 @@ use anyhow::{Context, Result};
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::k8s::K8sBareTopology;
|
use harmony::modules::k8s::K8sBareTopology;
|
||||||
use harmony::modules::nats::NatsBasicScore;
|
use harmony::modules::nats::NatsScore;
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(
|
#[command(
|
||||||
name = "fleet_nats_install",
|
name = "fleet_nats_install",
|
||||||
about = "Install single-node NATS (JetStream) via NatsBasicScore"
|
about = "Install single-node NATS (JetStream) via NatsScore"
|
||||||
)]
|
)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Target namespace. Created if missing.
|
/// Target namespace. Created if missing.
|
||||||
@@ -45,7 +45,7 @@ struct Cli {
|
|||||||
/// NodePort when `--expose=node-port`. Must be in the cluster's
|
/// NodePort when `--expose=node-port`. Must be in the cluster's
|
||||||
/// nodeport range (default 30000-32767). Ignored otherwise.
|
/// nodeport range (default 30000-32767). Ignored otherwise.
|
||||||
#[arg(long, default_value_t = 30422)]
|
#[arg(long, default_value_t = 30422)]
|
||||||
node_port: i32,
|
node_port: u16,
|
||||||
/// Override the NATS container image.
|
/// Override the NATS container image.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
image: Option<String>,
|
image: Option<String>,
|
||||||
@@ -67,7 +67,7 @@ async fn main() -> Result<()> {
|
|||||||
.map_err(|e| anyhow::anyhow!(e))
|
.map_err(|e| anyhow::anyhow!(e))
|
||||||
.context("building K8sBareTopology from KUBECONFIG")?;
|
.context("building K8sBareTopology from KUBECONFIG")?;
|
||||||
|
|
||||||
let mut score = NatsBasicScore::new(&cli.name, &cli.namespace);
|
let mut score = NatsScore::new(&cli.name, &cli.namespace);
|
||||||
match cli.expose {
|
match cli.expose {
|
||||||
ExposeMode::ClusterIp => {}
|
ExposeMode::ClusterIp => {}
|
||||||
ExposeMode::NodePort => score = score.node_port(cli.node_port),
|
ExposeMode::NodePort => score = score.node_port(cli.node_port),
|
||||||
@@ -81,7 +81,7 @@ async fn main() -> Result<()> {
|
|||||||
let outcome = interpret
|
let outcome = interpret
|
||||||
.execute(&Inventory::empty(), &topology)
|
.execute(&Inventory::empty(), &topology)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("execute NatsBasicScore: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("execute NatsScore: {e}"))?;
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
"NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}",
|
"NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}",
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ path = "src/main.rs"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony" }
|
harmony = { path = "../../harmony" }
|
||||||
|
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
||||||
harmony_cli = { path = "../../harmony_cli" }
|
harmony_cli = { path = "../../harmony_cli" }
|
||||||
harmony_secret = { path = "../../harmony_secret" }
|
harmony_secret = { path = "../../harmony_secret" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ use anyhow::{Context, Result};
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::config::secret::SudoPassword;
|
use harmony::config::secret::SudoPassword;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
|
|
||||||
use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec};
|
use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec};
|
||||||
|
use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
|
||||||
use harmony_secret::SecretManager;
|
use harmony_secret::SecretManager;
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
use log::info;
|
use log::info;
|
||||||
|
|||||||
@@ -21,7 +21,7 @@
|
|||||||
//!
|
//!
|
||||||
//! Behaviour:
|
//! Behaviour:
|
||||||
//! - Installs single-node NATS (JetStream) into `--nats-namespace`
|
//! - Installs single-node NATS (JetStream) into `--nats-namespace`
|
||||||
//! using `NatsBasicScore`, exposed per `--nats-expose`.
|
//! using `NatsScore`, exposed per `--nats-expose`.
|
||||||
//! - Installs the operator chart into `--operator-namespace` via
|
//! - Installs the operator chart into `--operator-namespace` via
|
||||||
//! `FleetOperatorScore` (which renders the chart in a tempdir
|
//! `FleetOperatorScore` (which renders the chart in a tempdir
|
||||||
//! and helm-installs it).
|
//! and helm-installs it).
|
||||||
@@ -47,7 +47,7 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::nats::NatsBasicScore;
|
use harmony::modules::nats::NatsScore;
|
||||||
use harmony::modules::zitadel::ZitadelScore;
|
use harmony::modules::zitadel::ZitadelScore;
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::K8sAnywhereTopology;
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
@@ -72,7 +72,7 @@ struct Cli {
|
|||||||
nats_expose: NatsExpose,
|
nats_expose: NatsExpose,
|
||||||
/// NodePort when `--nats-expose=node-port`. Ignored otherwise.
|
/// NodePort when `--nats-expose=node-port`. Ignored otherwise.
|
||||||
#[arg(long, default_value_t = 30422)]
|
#[arg(long, default_value_t = 30422)]
|
||||||
nats_node_port: i32,
|
nats_node_port: u16,
|
||||||
/// Optional NATS image override (`repository:tag`).
|
/// Optional NATS image override (`repository:tag`).
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
nats_image: Option<String>,
|
nats_image: Option<String>,
|
||||||
@@ -124,7 +124,7 @@ async fn main() -> Result<()> {
|
|||||||
|
|
||||||
let topology = K8sAnywhereTopology::from_env();
|
let topology = K8sAnywhereTopology::from_env();
|
||||||
|
|
||||||
let mut nats = NatsBasicScore::new(&cli.nats_name, &cli.nats_namespace);
|
let mut nats = NatsScore::new(&cli.nats_name, &cli.nats_namespace);
|
||||||
match cli.nats_expose {
|
match cli.nats_expose {
|
||||||
NatsExpose::ClusterIp => {}
|
NatsExpose::ClusterIp => {}
|
||||||
NatsExpose::NodePort => nats = nats.node_port(cli.nats_node_port),
|
NatsExpose::NodePort => nats = nats.node_port(cli.nats_node_port),
|
||||||
@@ -135,17 +135,14 @@ async fn main() -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Point the operator at NATS via the in-cluster service DNS the
|
// Point the operator at NATS via the in-cluster service DNS the
|
||||||
// NatsBasicScore install creates. ClusterIP and LoadBalancer both
|
// NatsScore install creates. ClusterIP and LoadBalancer both
|
||||||
// expose the same `<release>.<namespace>:4222` for in-cluster
|
// expose the same `<release>.<namespace>:4222` for in-cluster
|
||||||
// callers.
|
// callers.
|
||||||
let nats_url = format!("nats://{}.{}:4222", cli.nats_name, cli.nats_namespace);
|
let operator = FleetOperatorScore::new(&cli.operator_image)
|
||||||
|
|
||||||
let operator = FleetOperatorScore::new()
|
|
||||||
.namespace(&cli.operator_namespace)
|
.namespace(&cli.operator_namespace)
|
||||||
.release_name(&cli.operator_release)
|
.release_name(&cli.operator_release)
|
||||||
.image(&cli.operator_image)
|
|
||||||
.image_pull_policy(&cli.operator_image_pull_policy)
|
.image_pull_policy(&cli.operator_image_pull_policy)
|
||||||
.nats_url(&nats_url)
|
.messaging(&nats.client_ref())
|
||||||
.log_level(&cli.log_level);
|
.log_level(&cli.log_level);
|
||||||
|
|
||||||
// FleetServerScore now takes NatsK8sScore (auth-callout-aware,
|
// FleetServerScore now takes NatsK8sScore (auth-callout-aware,
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "example-fleet-staging-deploy"
|
|
||||||
edition = "2024"
|
|
||||||
version.workspace = true
|
|
||||||
readme.workspace = true
|
|
||||||
license.workspace = true
|
|
||||||
description = "Deploy the fleet platform stack (Zitadel + NATS + auth callout) onto an OKD/Kubernetes cluster. Operator-side, run-once-per-customer."
|
|
||||||
|
|
||||||
[lib]
|
|
||||||
name = "example_fleet_staging_deploy"
|
|
||||||
path = "src/lib.rs"
|
|
||||||
|
|
||||||
[[bin]]
|
|
||||||
name = "fleet-staging-deploy"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
harmony = { path = "../../harmony" }
|
|
||||||
harmony-k8s = { path = "../../harmony-k8s" }
|
|
||||||
harmony_types = { path = "../../harmony_types" }
|
|
||||||
harmony-nats-callout = { path = "../../nats/callout" }
|
|
||||||
nkeys = "0.4"
|
|
||||||
async-nats.workspace = true
|
|
||||||
reqwest = { workspace = true }
|
|
||||||
tokio = { workspace = true, features = ["full"] }
|
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
|
||||||
anyhow.workspace = true
|
|
||||||
log.workspace = true
|
|
||||||
env_logger.workspace = true
|
|
||||||
tracing.workspace = true
|
|
||||||
tracing-subscriber.workspace = true
|
|
||||||
clap = { version = "4", features = ["derive", "env"] }
|
|
||||||
k8s-openapi.workspace = true
|
|
||||||
kube.workspace = true
|
|
||||||
url.workspace = true
|
|
||||||
@@ -1,579 +0,0 @@
|
|||||||
//! Operator-side staging deploy harness.
|
|
||||||
//!
|
|
||||||
//! Runs once per customer instance against an OKD / Kubernetes cluster
|
|
||||||
//! to bring up the fleet platform's central services:
|
|
||||||
//!
|
|
||||||
//! 1. Zitadel + Postgres (HTTPS via OKD HAProxy ingress, edge TLS).
|
|
||||||
//! 2. The fleet project + roles (`fleet-admin`, `device`) + an API app
|
|
||||||
//! (so the project ID can be the JWT-bearer audience).
|
|
||||||
//! 3. NATS with `auth_callout` and a WSS ingress (so Pis on a customer
|
|
||||||
//! LAN connect through `wss://nats.<base>/`).
|
|
||||||
//! 4. The auth callout Deployment, configured to validate Zitadel JWTs
|
|
||||||
//! and emit per-device permissions on user JWTs to NATS.
|
|
||||||
//!
|
|
||||||
//! Everything keys off [`FleetDomainConfig::base_domain`] —
|
|
||||||
//! `zitadel.<base>`, `nats.<base>`, `api.<base>` are the only
|
|
||||||
//! customer-visible hostnames. Pi-side onboarding (see
|
|
||||||
//! `examples/fleet_rpi_setup/`) consumes the Zitadel admin PAT plus
|
|
||||||
//! the project ID this harness prints, so the operator's flow is:
|
|
||||||
//!
|
|
||||||
//! ```text
|
|
||||||
//! cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io
|
|
||||||
//! ↓ prints PROJECT_ID, NATS WSS URL, instructions to extract iam-admin-pat
|
|
||||||
//! HARMONY_ZITADEL_ADMIN_PAT=$(kubectl -n zitadel get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d) \
|
|
||||||
//! cargo run -p example-fleet-rpi-setup -- \
|
|
||||||
//! --pi-host 192.168.1.42 \
|
|
||||||
//! --bootstrap-token "$HARMONY_ZITADEL_ADMIN_PAT" \
|
|
||||||
//! --zitadel-issuer-url https://zitadel.customer1.nationtech.io \
|
|
||||||
//! --zitadel-project-id <PROJECT_ID printed above> \
|
|
||||||
//! --nats-url wss://nats.customer1.nationtech.io/ \
|
|
||||||
//! --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent
|
|
||||||
//! ```
|
|
||||||
//!
|
|
||||||
//! The harness is **idempotent** by design — re-running picks up
|
|
||||||
//! existing resources via the new helm-upgrade-by-default behavior +
|
|
||||||
//! ZitadelSetupScore's search-then-create flow + a persisted issuer
|
|
||||||
//! NKey in a K8s secret so user JWTs survive restarts.
|
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::nats::NatsHelmChartScore;
|
|
||||||
use harmony::modules::nats_auth_callout::{NatsAuthCalloutScore, render_auth_callout_block};
|
|
||||||
use harmony::modules::zitadel::{
|
|
||||||
ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelRole,
|
|
||||||
ZitadelScore, ZitadelSetupScore,
|
|
||||||
};
|
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
|
|
||||||
use log::info;
|
|
||||||
use nkeys::KeyPair;
|
|
||||||
|
|
||||||
// ---- domain config ---------------------------------------------------------
|
|
||||||
|
|
||||||
/// Single source of truth for all customer-visible hostnames. Every
|
|
||||||
/// `<app>.<customer>.<base>` URL the staging deploy emits derives from
|
|
||||||
/// the one base domain — no hostnames are hardcoded so the same code
|
|
||||||
/// runs across customers / staging / canary instances.
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct FleetDomainConfig {
|
|
||||||
/// e.g. `customer1.nationtech.io`. The deploy emits
|
|
||||||
/// `zitadel.<base>`, `nats.<base>`, `api.<base>` against it.
|
|
||||||
pub base_domain: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FleetDomainConfig {
|
|
||||||
pub fn new(base_domain: impl Into<String>) -> Self {
|
|
||||||
Self {
|
|
||||||
base_domain: base_domain.into(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn zitadel_host(&self) -> String {
|
|
||||||
format!("zitadel.{}", self.base_domain)
|
|
||||||
}
|
|
||||||
pub fn nats_wss_host(&self) -> String {
|
|
||||||
format!("nats.{}", self.base_domain)
|
|
||||||
}
|
|
||||||
pub fn zitadel_issuer_url(&self) -> String {
|
|
||||||
format!("https://{}", self.zitadel_host())
|
|
||||||
}
|
|
||||||
pub fn nats_wss_url(&self) -> String {
|
|
||||||
format!("wss://{}/", self.nats_wss_host())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- naming + constants ----------------------------------------------------
|
|
||||||
|
|
||||||
pub const FLEET_NAMESPACE: &str = "fleet-staging";
|
|
||||||
pub const ZITADEL_NAMESPACE: &str = "zitadel-staging";
|
|
||||||
pub const NATS_RELEASE: &str = "fleet-nats";
|
|
||||||
pub const CALLOUT_DEPLOYMENT_NAME: &str = "fleet-callout";
|
|
||||||
pub const PROJECT_NAME: &str = "fleet";
|
|
||||||
pub const API_APP_NAME: &str = "nats";
|
|
||||||
pub const CLI_APP_NAME: &str = "harmony-cli";
|
|
||||||
pub const ADMIN_ROLE_KEY: &str = "fleet-admin";
|
|
||||||
pub const DEVICE_ROLE_KEY: &str = "device";
|
|
||||||
pub const NATS_AUTH_USER: &str = "auth";
|
|
||||||
pub const NATS_ACCOUNT: &str = "DEVICES";
|
|
||||||
pub const NATS_SYSTEM_USER: &str = "sys-admin";
|
|
||||||
pub const ISSUER_SEED_SECRET: &str = "callout-issuer-seed";
|
|
||||||
|
|
||||||
// ---- handles ---------------------------------------------------------------
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct StagingHandles {
|
|
||||||
pub domain: FleetDomainConfig,
|
|
||||||
pub project_id: String,
|
|
||||||
pub issuer_pubkey: String,
|
|
||||||
/// Tag of the callout image expected to exist in a registry the
|
|
||||||
/// cluster pulls from. The operator pushes it before running the
|
|
||||||
/// deploy; this field is just the name we put on the Deployment
|
|
||||||
/// for traceability.
|
|
||||||
pub callout_image: String,
|
|
||||||
/// OIDC client_id of the `harmony-cli` Device Code app — what the
|
|
||||||
/// `fleet_sso_login` CLI sends in its device-authorization request.
|
|
||||||
/// `None` if the app pre-existed without the cache picking it up
|
|
||||||
/// (re-running the staging deploy after `rm -rf
|
|
||||||
/// ~/.local/share/harmony/zitadel/`).
|
|
||||||
pub cli_client_id: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- bring up --------------------------------------------------------------
|
|
||||||
|
|
||||||
pub struct StagingDeployOpts {
|
|
||||||
pub domain: FleetDomainConfig,
|
|
||||||
pub kubeconfig_context: Option<String>,
|
|
||||||
/// Image reference the cluster will pull. Operator must have
|
|
||||||
/// pushed this beforehand (e.g. `quay.io/customer/harmony-nats-callout:demo`).
|
|
||||||
pub callout_image: String,
|
|
||||||
/// Per-NATS-account password for the callout's own NATS connection.
|
|
||||||
/// Stored in a K8s secret + listed in the chart's
|
|
||||||
/// `accounts.<account>.users` so the callout bypasses callout to
|
|
||||||
/// connect (otherwise it'd deadlock authenticating itself).
|
|
||||||
pub nats_auth_pass: String,
|
|
||||||
/// SYS account password (for `kubectl exec nats-box` debugging).
|
|
||||||
pub nats_system_pass: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn bring_up_staging(opts: StagingDeployOpts) -> Result<StagingHandles> {
|
|
||||||
let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
|
||||||
.try_init();
|
|
||||||
|
|
||||||
if let Some(ctx) = &opts.kubeconfig_context {
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("HARMONY_K8S_CONTEXT", ctx);
|
|
||||||
std::env::set_var("HARMONY_USE_LOCAL_K3D", "false");
|
|
||||||
std::env::set_var("HARMONY_AUTOINSTALL", "false");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let topology = K8sAnywhereTopology::from_env();
|
|
||||||
topology.ensure_ready().await.context("topology init")?;
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"[1/5] deploying Zitadel at https://{}",
|
|
||||||
opts.domain.zitadel_host()
|
|
||||||
);
|
|
||||||
deploy_zitadel(&opts.domain, &topology).await?;
|
|
||||||
|
|
||||||
info!("[2/5] waiting for Zitadel HTTPS to respond");
|
|
||||||
wait_for_zitadel_ready(&opts.domain).await?;
|
|
||||||
|
|
||||||
info!("[3/5] provisioning project '{PROJECT_NAME}', api app, CLI device-code app, and roles");
|
|
||||||
provision_zitadel_project(&opts.domain, &topology).await?;
|
|
||||||
let project_id = read_project_id()?;
|
|
||||||
let cli_client_id = read_cli_client_id();
|
|
||||||
info!(" → project_id = {project_id}");
|
|
||||||
if let Some(cid) = &cli_client_id {
|
|
||||||
info!(" → cli_client_id = {cid}");
|
|
||||||
} else {
|
|
||||||
log::warn!(
|
|
||||||
" → cli_client_id missing from cache; CLI login won't work until you reset the local zitadel cache"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("[4/5] generating issuer NKey + deploying NATS with auth_callout + WSS ingress");
|
|
||||||
let issuer_seed = ensure_issuer_seed(&topology).await?;
|
|
||||||
let issuer_kp = KeyPair::from_seed(&issuer_seed)
|
|
||||||
.map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?;
|
|
||||||
let issuer_pubkey = issuer_kp.public_key();
|
|
||||||
|
|
||||||
NatsHelmChartScore::new(
|
|
||||||
NATS_RELEASE.to_string(),
|
|
||||||
FLEET_NAMESPACE.to_string(),
|
|
||||||
render_nats_values(
|
|
||||||
&opts.domain,
|
|
||||||
&issuer_pubkey,
|
|
||||||
&opts.nats_auth_pass,
|
|
||||||
&opts.nats_system_pass,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("NATS deploy")?;
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"[5/5] deploying NatsAuthCalloutScore (image: {})",
|
|
||||||
opts.callout_image
|
|
||||||
);
|
|
||||||
NatsAuthCalloutScore::new(
|
|
||||||
CALLOUT_DEPLOYMENT_NAME,
|
|
||||||
FLEET_NAMESPACE,
|
|
||||||
format!("nats://{NATS_RELEASE}.{FLEET_NAMESPACE}.svc.cluster.local:4222"),
|
|
||||||
opts.domain.zitadel_issuer_url(),
|
|
||||||
// The aud the callout validates against is the project ID —
|
|
||||||
// Zitadel emits it in access tokens minted via the
|
|
||||||
// project-id-audience scope.
|
|
||||||
project_id.clone(),
|
|
||||||
NATS_AUTH_USER,
|
|
||||||
opts.nats_auth_pass.clone(),
|
|
||||||
issuer_seed,
|
|
||||||
)
|
|
||||||
.image(&opts.callout_image)
|
|
||||||
.target_account(NATS_ACCOUNT)
|
|
||||||
.admin_role(ADMIN_ROLE_KEY)
|
|
||||||
.device_role(DEVICE_ROLE_KEY)
|
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
|
||||||
.await
|
|
||||||
.context("callout deploy")?;
|
|
||||||
|
|
||||||
Ok(StagingHandles {
|
|
||||||
domain: opts.domain,
|
|
||||||
project_id,
|
|
||||||
issuer_pubkey,
|
|
||||||
callout_image: opts.callout_image,
|
|
||||||
cli_client_id,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_cli_client_id() -> Option<String> {
|
|
||||||
ZitadelClientConfig::load()?
|
|
||||||
.client_id(CLI_APP_NAME)
|
|
||||||
.cloned()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn deploy_zitadel(domain: &FleetDomainConfig, topology: &K8sAnywhereTopology) -> Result<()> {
|
|
||||||
let z = ZitadelScore {
|
|
||||||
host: domain.zitadel_host(),
|
|
||||||
zitadel_version: "v4.12.1".to_string(),
|
|
||||||
// OKD HAProxy edge-terminates TLS for us, so the issuer URL
|
|
||||||
// is `https://zitadel.<base>` (port 443 implied) — leave
|
|
||||||
// external_port at None so Zitadel's emitted issuer omits the
|
|
||||||
// port, matching what clients reach.
|
|
||||||
external_secure: true,
|
|
||||||
external_port: None,
|
|
||||||
namespace: ZITADEL_NAMESPACE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
z.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("ZitadelScore")?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn provision_zitadel_project(
|
|
||||||
domain: &FleetDomainConfig,
|
|
||||||
topology: &K8sAnywhereTopology,
|
|
||||||
) -> Result<()> {
|
|
||||||
let setup = ZitadelSetupScore {
|
|
||||||
host: domain.zitadel_host(),
|
|
||||||
// Direct HTTPS through OKD's HAProxy ingress — operator runs
|
|
||||||
// anywhere with kubeconfig + DNS access. Defaults give
|
|
||||||
// `https://<host>` (port 443).
|
|
||||||
scheme: Default::default(),
|
|
||||||
port: None,
|
|
||||||
skip_tls: false,
|
|
||||||
endpoint: None,
|
|
||||||
admin_org_id: None,
|
|
||||||
namespace: ZITADEL_NAMESPACE.to_string(),
|
|
||||||
applications: vec![ZitadelApplication {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
app_name: CLI_APP_NAME.to_string(),
|
|
||||||
// Device Code grant — the only browser-driven OIDC flow
|
|
||||||
// that fits a CLI tool: prints a verification URL + user
|
|
||||||
// code, polls for a token, no embedded web server / open
|
|
||||||
// listener required.
|
|
||||||
app_type: ZitadelAppType::DeviceCode,
|
|
||||||
}],
|
|
||||||
api_apps: vec![ZitadelApiApp {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
app_name: API_APP_NAME.to_string(),
|
|
||||||
}],
|
|
||||||
roles: vec![
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
key: ADMIN_ROLE_KEY.to_string(),
|
|
||||||
display_name: "Fleet Admin".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
key: DEVICE_ROLE_KEY.to_string(),
|
|
||||||
display_name: "Device".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
// No machine users provisioned here — `fleet_rpi_setup` mints
|
|
||||||
// them on demand per device, so the staging deploy stays
|
|
||||||
// device-count-agnostic.
|
|
||||||
groups_claim_action: false,
|
|
||||||
machine_users: vec![],
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
setup
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.context("ZitadelSetupScore")?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_project_id() -> Result<String> {
|
|
||||||
let cfg = ZitadelClientConfig::load()
|
|
||||||
.context("ZitadelSetupScore did not produce a client config cache")?;
|
|
||||||
cfg.project_id_by_name(PROJECT_NAME)
|
|
||||||
.or(cfg.project_id.as_ref())
|
|
||||||
.context("project_id missing from ZitadelClientConfig cache")
|
|
||||||
.cloned()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Persist the callout's issuer NKey seed in a K8s secret so re-runs
|
|
||||||
/// of the staging deploy don't invalidate previously-issued user JWTs
|
|
||||||
/// already in flight on customer Pis.
|
|
||||||
async fn ensure_issuer_seed(topology: &K8sAnywhereTopology) -> Result<String> {
|
|
||||||
use k8s_openapi::ByteString;
|
|
||||||
use k8s_openapi::api::core::v1::{Namespace, Secret};
|
|
||||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
|
|
||||||
use std::collections::BTreeMap;
|
|
||||||
|
|
||||||
let k8s = topology
|
|
||||||
.k8s_client()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
|
|
||||||
|
|
||||||
if k8s
|
|
||||||
.get_resource::<Namespace>(FLEET_NAMESPACE, None)
|
|
||||||
.await?
|
|
||||||
.is_none()
|
|
||||||
{
|
|
||||||
let ns = Namespace {
|
|
||||||
metadata: ObjectMeta {
|
|
||||||
name: Some(FLEET_NAMESPACE.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
k8s.create(&ns, None).await.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(existing) = k8s
|
|
||||||
.get_resource::<Secret>(ISSUER_SEED_SECRET, Some(FLEET_NAMESPACE))
|
|
||||||
.await?
|
|
||||||
&& let Some(data) = existing.data
|
|
||||||
&& let Some(seed_bytes) = data.get("seed")
|
|
||||||
{
|
|
||||||
let seed = String::from_utf8(seed_bytes.0.clone())?;
|
|
||||||
return Ok(seed.trim().to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
let seed = KeyPair::new_account()
|
|
||||||
.seed()
|
|
||||||
.map_err(|e| anyhow::anyhow!("nkey seed: {e}"))?;
|
|
||||||
let mut data = BTreeMap::new();
|
|
||||||
data.insert("seed".to_string(), ByteString(seed.as_bytes().to_vec()));
|
|
||||||
let secret = Secret {
|
|
||||||
metadata: ObjectMeta {
|
|
||||||
name: Some(ISSUER_SEED_SECRET.to_string()),
|
|
||||||
namespace: Some(FLEET_NAMESPACE.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
data: Some(data),
|
|
||||||
type_: Some("Opaque".to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
k8s.create(&secret, Some(FLEET_NAMESPACE)).await.ok();
|
|
||||||
Ok(seed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- NATS values -----------------------------------------------------------
|
|
||||||
|
|
||||||
/// Render NATS Helm values for an OKD-flavored deployment with WSS
|
|
||||||
/// ingress + auth callout + JetStream.
|
|
||||||
///
|
|
||||||
/// **Why WSS rather than plain NATS-on-TLS:** OKD's default ingress
|
|
||||||
/// controller (HAProxy) is HTTP-aware and edge-terminates TLS. NATS
|
|
||||||
/// over WebSocket goes through that ingress unchanged; native NATS
|
|
||||||
/// TCP would require a TCP loadbalancer service or a passthrough
|
|
||||||
/// Route, both of which are extra infra the customer's cluster may
|
|
||||||
/// not have. WSS is also the default async-nats client transport on
|
|
||||||
/// `wss://...` URLs — no special agent code needed.
|
|
||||||
pub fn render_nats_values(
|
|
||||||
domain: &FleetDomainConfig,
|
|
||||||
issuer_pubkey: &str,
|
|
||||||
nats_auth_pass: &str,
|
|
||||||
nats_system_pass: &str,
|
|
||||||
) -> String {
|
|
||||||
let auth_callout = render_auth_callout_block(issuer_pubkey, NATS_AUTH_USER, NATS_ACCOUNT);
|
|
||||||
let auth_callout_indented = auth_callout
|
|
||||||
.lines()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, l)| {
|
|
||||||
if i == 0 {
|
|
||||||
l.to_string()
|
|
||||||
} else {
|
|
||||||
format!(" {l}")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("\n");
|
|
||||||
format!(
|
|
||||||
r#"fullnameOverride: {nats_release}
|
|
||||||
config:
|
|
||||||
cluster:
|
|
||||||
enabled: false
|
|
||||||
jetstream:
|
|
||||||
enabled: true
|
|
||||||
fileStorage:
|
|
||||||
enabled: true
|
|
||||||
size: 5Gi
|
|
||||||
websocket:
|
|
||||||
enabled: true
|
|
||||||
port: 8443
|
|
||||||
ingress:
|
|
||||||
enabled: true
|
|
||||||
className: openshift-default
|
|
||||||
pathType: Prefix
|
|
||||||
hosts:
|
|
||||||
- {nats_wss_host}
|
|
||||||
annotations:
|
|
||||||
# OKD HAProxy edge-terminates TLS — the chart's default Route
|
|
||||||
# generation needs `route.openshift.io/termination: edge` so
|
|
||||||
# the Route's TLS block is "edge", matching the cluster's wildcard
|
|
||||||
# cert behavior. Switch to `reencrypt` if you need TLS all the
|
|
||||||
# way to the NATS pod.
|
|
||||||
route.openshift.io/termination: edge
|
|
||||||
haproxy.router.openshift.io/timeout: "1h"
|
|
||||||
merge:
|
|
||||||
{auth_callout_indented}
|
|
||||||
accounts:
|
|
||||||
{nats_account}:
|
|
||||||
jetstream: enabled
|
|
||||||
users:
|
|
||||||
- user: "{auth_user}"
|
|
||||||
password: "{auth_pass}"
|
|
||||||
SYS:
|
|
||||||
users:
|
|
||||||
- user: "{sys_user}"
|
|
||||||
password: "{sys_pass}"
|
|
||||||
system_account: SYS
|
|
||||||
service:
|
|
||||||
ports:
|
|
||||||
nats:
|
|
||||||
enabled: true
|
|
||||||
"#,
|
|
||||||
nats_release = NATS_RELEASE,
|
|
||||||
nats_wss_host = domain.nats_wss_host(),
|
|
||||||
nats_account = NATS_ACCOUNT,
|
|
||||||
auth_user = NATS_AUTH_USER,
|
|
||||||
auth_pass = nats_auth_pass,
|
|
||||||
sys_user = NATS_SYSTEM_USER,
|
|
||||||
sys_pass = nats_system_pass,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- readiness -------------------------------------------------------------
|
|
||||||
|
|
||||||
async fn wait_for_zitadel_ready(domain: &FleetDomainConfig) -> Result<()> {
|
|
||||||
let issuer = domain.zitadel_issuer_url();
|
|
||||||
let well_known = format!("{issuer}/.well-known/openid-configuration");
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(Duration::from_secs(5))
|
|
||||||
.build()?;
|
|
||||||
for attempt in 1..=180 {
|
|
||||||
match client.get(&well_known).send().await {
|
|
||||||
Ok(r) if r.status().is_success() => return Ok(()),
|
|
||||||
Ok(r) if attempt % 30 == 0 => {
|
|
||||||
info!("Zitadel HTTPS {} (attempt {attempt}/180)", r.status());
|
|
||||||
}
|
|
||||||
Err(e) if attempt % 30 == 0 => {
|
|
||||||
info!("Zitadel unreachable: {e} (attempt {attempt}/180)");
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
||||||
}
|
|
||||||
anyhow::bail!("timed out waiting for Zitadel at {well_known}")
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- helpful printout ------------------------------------------------------
|
|
||||||
|
|
||||||
impl StagingHandles {
|
|
||||||
/// Print the operator's "what to do next" panel after a successful
|
|
||||||
/// staging deploy. Pasted at the end of the binary's run.
|
|
||||||
pub fn print_next_steps(&self) {
|
|
||||||
let zitadel = self.domain.zitadel_issuer_url();
|
|
||||||
let nats = self.domain.nats_wss_url();
|
|
||||||
println!();
|
|
||||||
println!("============================================================");
|
|
||||||
println!(" STAGING DEPLOY COMPLETE");
|
|
||||||
println!("============================================================");
|
|
||||||
println!(" Base domain: {}", self.domain.base_domain);
|
|
||||||
println!(" Zitadel: {zitadel}");
|
|
||||||
println!(" NATS (WSS): {nats}");
|
|
||||||
println!(" Project ID: {}", self.project_id);
|
|
||||||
println!(" Callout image: {}", self.callout_image);
|
|
||||||
println!(" Issuer pubkey: {}", self.issuer_pubkey);
|
|
||||||
if let Some(cid) = &self.cli_client_id {
|
|
||||||
println!(" CLI client_id: {cid}");
|
|
||||||
println!();
|
|
||||||
println!(" CLI SSO login (developer-side):");
|
|
||||||
println!();
|
|
||||||
println!(" cargo run -p example-fleet-sso-login -- \\");
|
|
||||||
println!(" --base-domain {} \\", self.domain.base_domain);
|
|
||||||
println!(" --client-id {cid}");
|
|
||||||
}
|
|
||||||
println!();
|
|
||||||
println!(" Onboard a Pi:");
|
|
||||||
println!();
|
|
||||||
println!(" PAT=$(kubectl -n zitadel get secret iam-admin-pat \\");
|
|
||||||
println!(" -o jsonpath='{{.data.pat}}' | base64 -d)");
|
|
||||||
println!();
|
|
||||||
println!(" cargo run -p example-fleet-rpi-setup -- \\");
|
|
||||||
println!(" --pi-host <PI_IP> \\");
|
|
||||||
println!(" --bootstrap-token \"$PAT\" \\");
|
|
||||||
println!(" --zitadel-issuer-url {zitadel} \\");
|
|
||||||
println!(" --zitadel-project-id {} \\", self.project_id);
|
|
||||||
println!(" --nats-url {nats} \\");
|
|
||||||
println!(" --agent-binary <path-to-aarch64-fleet-agent>");
|
|
||||||
println!();
|
|
||||||
println!("============================================================");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn domain_config_derives_hostnames() {
|
|
||||||
let d = FleetDomainConfig::new("customer1.nationtech.io");
|
|
||||||
assert_eq!(d.zitadel_host(), "zitadel.customer1.nationtech.io");
|
|
||||||
assert_eq!(d.nats_wss_host(), "nats.customer1.nationtech.io");
|
|
||||||
assert_eq!(
|
|
||||||
d.zitadel_issuer_url(),
|
|
||||||
"https://zitadel.customer1.nationtech.io"
|
|
||||||
);
|
|
||||||
assert_eq!(d.nats_wss_url(), "wss://nats.customer1.nationtech.io/");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nats_values_render_includes_wss_ingress_and_auth_callout() {
|
|
||||||
let d = FleetDomainConfig::new("acme.io");
|
|
||||||
let yaml = render_nats_values(&d, "ABCDEF", "auth-pass", "sys-pass");
|
|
||||||
// WSS plumbing.
|
|
||||||
assert!(yaml.contains("websocket:"));
|
|
||||||
assert!(yaml.contains("port: 8443"));
|
|
||||||
assert!(yaml.contains("nats.acme.io"));
|
|
||||||
// OKD edge-TLS annotations.
|
|
||||||
assert!(yaml.contains("openshift-default"));
|
|
||||||
assert!(yaml.contains("route.openshift.io/termination: edge"));
|
|
||||||
// Auth callout wired through with the issuer pubkey.
|
|
||||||
assert!(yaml.contains("auth_callout"));
|
|
||||||
assert!(yaml.contains("issuer: ABCDEF"));
|
|
||||||
assert!(yaml.contains("auth_users: [ auth ]"));
|
|
||||||
assert!(yaml.contains("system_account: SYS"));
|
|
||||||
// Account user.
|
|
||||||
assert!(yaml.contains("password: \"auth-pass\""));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nats_values_inline_account_block_under_merge() {
|
|
||||||
// Prevent regressions where the auth_callout block leaks
|
|
||||||
// outside the `merge:` indentation level — chart expects it
|
|
||||||
// under config.merge.
|
|
||||||
let d = FleetDomainConfig::new("x.io");
|
|
||||||
let yaml = render_nats_values(&d, "K", "p", "s");
|
|
||||||
let idx_merge = yaml.find("\n merge:\n").expect("merge block present");
|
|
||||||
let idx_callout = yaml.find("auth_callout:").expect("auth_callout present");
|
|
||||||
assert!(idx_callout > idx_merge, "auth_callout must follow merge:");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
//! `cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io ...`
|
|
||||||
//!
|
|
||||||
//! Operator-side, run-once-per-customer-instance harness. Brings up
|
|
||||||
//! the central fleet platform services (Zitadel + NATS + auth callout)
|
|
||||||
//! against an OKD/K8s cluster pointed to by `KUBECONFIG`. Prints the
|
|
||||||
//! exact follow-up command the operator runs against a Pi to onboard
|
|
||||||
//! the first device.
|
|
||||||
//!
|
|
||||||
//! See `src/lib.rs` for the architectural notes.
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use clap::Parser;
|
|
||||||
use example_fleet_staging_deploy::{FleetDomainConfig, StagingDeployOpts, bring_up_staging};
|
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
|
||||||
#[command(
|
|
||||||
name = "fleet-staging-deploy",
|
|
||||||
about = "Deploy Zitadel + NATS + auth callout onto an OKD cluster"
|
|
||||||
)]
|
|
||||||
struct Cli {
|
|
||||||
/// Base DNS domain. All cluster-visible services derive from this:
|
|
||||||
/// `zitadel.<base>`, `nats.<base>`. The customer's wildcard cert /
|
|
||||||
/// CoreDNS / DNS provider must already point this at the cluster.
|
|
||||||
#[arg(long, env = "FLEET_BASE_DOMAIN")]
|
|
||||||
base_domain: String,
|
|
||||||
/// kubeconfig context to deploy against. Defaults to the
|
|
||||||
/// kubeconfig's current-context. Set this when your kubeconfig
|
|
||||||
/// has multiple contexts and you don't want to rely on the
|
|
||||||
/// global current.
|
|
||||||
#[arg(long, env = "FLEET_KUBE_CONTEXT")]
|
|
||||||
kube_context: Option<String>,
|
|
||||||
/// Container image reference for the harmony-nats-callout binary.
|
|
||||||
/// The cluster pulls this; operator must have pushed it before
|
|
||||||
/// running the deploy. Defaults to a quay.io path that the
|
|
||||||
/// customer should override per their registry.
|
|
||||||
#[arg(
|
|
||||||
long,
|
|
||||||
env = "FLEET_CALLOUT_IMAGE",
|
|
||||||
default_value = "quay.io/nationtech/harmony-nats-callout:demo"
|
|
||||||
)]
|
|
||||||
callout_image: String,
|
|
||||||
/// Password for the NATS service-account user the callout uses on
|
|
||||||
/// its own NATS connection. Stored in a K8s secret + listed in
|
|
||||||
/// the chart's `accounts.DEVICES.users` (which bypass callout —
|
|
||||||
/// otherwise the callout would deadlock authenticating itself).
|
|
||||||
#[arg(long, env = "FLEET_NATS_AUTH_PASS")]
|
|
||||||
nats_auth_pass: String,
|
|
||||||
/// Password for the NATS SYS account (used for nats-box debugging
|
|
||||||
/// inside the cluster).
|
|
||||||
#[arg(long, env = "FLEET_NATS_SYSTEM_PASS")]
|
|
||||||
nats_system_pass: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::main]
|
|
||||||
async fn main() -> Result<()> {
|
|
||||||
let cli = Cli::parse();
|
|
||||||
let domain = FleetDomainConfig::new(cli.base_domain);
|
|
||||||
|
|
||||||
let handles = bring_up_staging(StagingDeployOpts {
|
|
||||||
domain,
|
|
||||||
kubeconfig_context: cli.kube_context,
|
|
||||||
callout_image: cli.callout_image,
|
|
||||||
nats_auth_pass: cli.nats_auth_pass,
|
|
||||||
nats_system_pass: cli.nats_system_pass,
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.context("staging deploy")?;
|
|
||||||
|
|
||||||
handles.print_next_steps();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "example_fleet_staging_install"
|
|
||||||
edition = "2024"
|
|
||||||
version.workspace = true
|
|
||||||
readme.workspace = true
|
|
||||||
license.workspace = true
|
|
||||||
description = "Production-shape fleet install: Zitadel + NATS + auth callout + operator on OKD"
|
|
||||||
|
|
||||||
[[bin]]
|
|
||||||
name = "fleet_staging_install"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
harmony = { path = "../../harmony" }
|
|
||||||
harmony_cli = { path = "../../harmony_cli" }
|
|
||||||
harmony_config = { path = "../../harmony_config" }
|
|
||||||
harmony_secret = { path = "../../harmony_secret" }
|
|
||||||
harmony-k8s = { path = "../../harmony-k8s" }
|
|
||||||
harmony-nats-callout = { path = "../../nats/callout" }
|
|
||||||
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
|
||||||
nkeys = "0.4"
|
|
||||||
rand = "0.9"
|
|
||||||
base64.workspace = true
|
|
||||||
anyhow.workspace = true
|
|
||||||
clap = { version = "4", features = ["derive", "env"] }
|
|
||||||
tokio.workspace = true
|
|
||||||
tracing = { workspace = true }
|
|
||||||
serde = { workspace = true }
|
|
||||||
schemars = "0.8"
|
|
||||||
@@ -1,417 +0,0 @@
|
|||||||
//! Production-shape fleet install for OKD (or any cluster with the same
|
|
||||||
//! capabilities): Zitadel SSO + NATS (auth-callout) + operator + OpenBao,
|
|
||||||
//! composed from Scores.
|
|
||||||
//!
|
|
||||||
//! Tunables come from [`ConfigClient`] (`HARMONY_CONFIG_FleetStagingConfig`
|
|
||||||
//! env JSON → OpenBao → interactive prompt), not a bespoke CLI. The only
|
|
||||||
//! flags are `harmony_cli`'s: `--filter`/`--list`/`-y` select which workload
|
|
||||||
//! Scores to (re)deploy — e.g. `--filter FleetOperatorScore` bumps the
|
|
||||||
//! operator without touching NATS or the callout.
|
|
||||||
//!
|
|
||||||
//! Zitadel + OpenBao are an idempotent bootstrap: ZitadelSetupScore mints the
|
|
||||||
//! `project_id` + `fleet-operator` machine key that the callout and operator
|
|
||||||
//! Scores consume, so it must converge (and cache to disk) before they're
|
|
||||||
//! built. That data flow is why those two can't sit in the filterable batch.
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use clap::Parser;
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::nats::capability::NatsCluster;
|
|
||||||
use harmony::modules::nats::score_nats_k8s::{AuthCalloutCfg, NatsK8sScore, WebSocketRouteCfg};
|
|
||||||
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
|
||||||
use harmony::modules::openbao::{
|
|
||||||
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore,
|
|
||||||
cached_root_token,
|
|
||||||
};
|
|
||||||
use harmony::modules::zitadel::{
|
|
||||||
MachineKeyType, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig,
|
|
||||||
ZitadelMachineUser, ZitadelRole, ZitadelScore, ZitadelSetupScore,
|
|
||||||
};
|
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
|
|
||||||
use harmony_config::{Config, ConfigClient, StoreSource};
|
|
||||||
use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, OperatorCredentials};
|
|
||||||
use harmony_k8s::KubernetesDistribution;
|
|
||||||
use harmony_secret::{OpenbaoSecretStore, OpenbaoStoreOptions};
|
|
||||||
use nkeys::KeyPair;
|
|
||||||
use schemars::JsonSchema;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use tracing::info;
|
|
||||||
|
|
||||||
/// Non-secret install tunables. `base_domain` drives every public hostname;
|
|
||||||
/// the image refs and `*-stg.<base>` hosts have no safe default, so an empty
|
|
||||||
/// value is rejected at startup. Everything else defaults to the staging
|
|
||||||
/// conventions.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
|
|
||||||
#[serde(default)]
|
|
||||||
struct FleetStagingConfig {
|
|
||||||
base_domain: String,
|
|
||||||
operator_image: String,
|
|
||||||
callout_image: String,
|
|
||||||
cluster_issuer: String,
|
|
||||||
fleet_namespace: String,
|
|
||||||
zitadel_namespace: String,
|
|
||||||
nats_account: String,
|
|
||||||
zitadel_version: String,
|
|
||||||
project_name: String,
|
|
||||||
admin_role: String,
|
|
||||||
device_role: String,
|
|
||||||
operator_username: String,
|
|
||||||
admin_username: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for FleetStagingConfig {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
base_domain: String::new(),
|
|
||||||
operator_image: String::new(),
|
|
||||||
callout_image: String::new(),
|
|
||||||
cluster_issuer: "letsencrypt-prod".to_string(),
|
|
||||||
fleet_namespace: "fleet-staging".to_string(),
|
|
||||||
zitadel_namespace: "zitadel-staging".to_string(),
|
|
||||||
nats_account: "FLEET".to_string(),
|
|
||||||
zitadel_version: "v4.12.1".to_string(),
|
|
||||||
project_name: "fleet".to_string(),
|
|
||||||
admin_role: "fleet-admin".to_string(),
|
|
||||||
device_role: "device".to_string(),
|
|
||||||
operator_username: "fleet-operator".to_string(),
|
|
||||||
admin_username: "fleet-ops".to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::main]
|
|
||||||
async fn main() -> Result<()> {
|
|
||||||
harmony_cli::cli_logger::init();
|
|
||||||
let args = harmony_cli::Args::parse();
|
|
||||||
|
|
||||||
let cfg: FleetStagingConfig = ConfigClient::for_namespace("harmony")
|
|
||||||
.await
|
|
||||||
.get_or_prompt()
|
|
||||||
.await
|
|
||||||
.context("loading FleetStagingConfig")?;
|
|
||||||
anyhow::ensure!(
|
|
||||||
!cfg.base_domain.is_empty()
|
|
||||||
&& !cfg.operator_image.is_empty()
|
|
||||||
&& !cfg.callout_image.is_empty(),
|
|
||||||
"base_domain, operator_image and callout_image must be set"
|
|
||||||
);
|
|
||||||
|
|
||||||
let topology = K8sAnywhereTopology::from_env();
|
|
||||||
topology.ensure_ready().await?;
|
|
||||||
|
|
||||||
let zitadel_host = format!("sso-stg.{}", cfg.base_domain);
|
|
||||||
let nats_ws_host = format!("nats-fleet-stg.{}", cfg.base_domain);
|
|
||||||
let secrets_host = format!("secrets-stg.{}", cfg.base_domain);
|
|
||||||
let nats_release = "fleet-nats";
|
|
||||||
let cli_app_name = "harmony-cli";
|
|
||||||
|
|
||||||
// ---- Bootstrap (idempotent): Zitadel mints the project + operator key ---
|
|
||||||
ZitadelScore {
|
|
||||||
host: zitadel_host.clone(),
|
|
||||||
zitadel_version: cfg.zitadel_version.clone(),
|
|
||||||
external_secure: true,
|
|
||||||
external_port: None,
|
|
||||||
namespace: cfg.zitadel_namespace.clone(),
|
|
||||||
cluster_issuer: cfg.cluster_issuer.clone(),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::empty(), &topology)
|
|
||||||
.await
|
|
||||||
.context("Zitadel helm install")?;
|
|
||||||
|
|
||||||
ZitadelSetupScore {
|
|
||||||
host: zitadel_host.clone(),
|
|
||||||
scheme: Default::default(),
|
|
||||||
port: None,
|
|
||||||
skip_tls: false,
|
|
||||||
endpoint: None,
|
|
||||||
admin_org_id: None,
|
|
||||||
namespace: cfg.zitadel_namespace.clone(),
|
|
||||||
// Devices read per-deployment secrets gated by group membership;
|
|
||||||
// the Action surfaces role keys as the `groups` claim OpenBao
|
|
||||||
// binds against (ADR-025).
|
|
||||||
groups_claim_action: true,
|
|
||||||
human_users: vec![],
|
|
||||||
org_members: vec![],
|
|
||||||
instance_members: vec![],
|
|
||||||
machine_secrets: vec![],
|
|
||||||
port_forward_service: None,
|
|
||||||
// Device-code OIDC app for human admin login from
|
|
||||||
// `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel
|
|
||||||
// generates is read back below and printed for `--admin-oidc-client-id`.
|
|
||||||
applications: vec![ZitadelApplication {
|
|
||||||
project_name: cfg.project_name.clone(),
|
|
||||||
app_name: cli_app_name.to_string(),
|
|
||||||
app_type: ZitadelAppType::DeviceCode,
|
|
||||||
}],
|
|
||||||
api_apps: vec![ZitadelApiApp {
|
|
||||||
project_name: cfg.project_name.clone(),
|
|
||||||
app_name: "nats".to_string(),
|
|
||||||
}],
|
|
||||||
roles: vec![
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: cfg.project_name.clone(),
|
|
||||||
key: cfg.admin_role.clone(),
|
|
||||||
display_name: "Fleet Admin".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
ZitadelRole {
|
|
||||||
project_name: cfg.project_name.clone(),
|
|
||||||
key: cfg.device_role.clone(),
|
|
||||||
display_name: "Device".to_string(),
|
|
||||||
group: None,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
machine_users: vec![
|
|
||||||
ZitadelMachineUser {
|
|
||||||
username: cfg.admin_username.clone(),
|
|
||||||
name: "Fleet Operations".to_string(),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(cfg.project_name.clone()),
|
|
||||||
grant_roles: vec![cfg.admin_role.clone()],
|
|
||||||
},
|
|
||||||
ZitadelMachineUser {
|
|
||||||
username: cfg.operator_username.clone(),
|
|
||||||
name: "Fleet Operator (in-cluster)".to_string(),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(cfg.project_name.clone()),
|
|
||||||
grant_roles: vec![cfg.admin_role.clone()],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::empty(), &topology)
|
|
||||||
.await
|
|
||||||
.context("Zitadel setup (project + roles + machine users)")?;
|
|
||||||
|
|
||||||
// Read back the project_id + operator key + device-code client_id.
|
|
||||||
let zcfg = ZitadelClientConfig::load()
|
|
||||||
.context("ZitadelSetupScore did not produce a client config cache")?;
|
|
||||||
let project_id = zcfg
|
|
||||||
.project_id_by_name(&cfg.project_name)
|
|
||||||
.or(zcfg.project_id.as_ref())
|
|
||||||
.context("project_id missing from cache after setup")?
|
|
||||||
.clone();
|
|
||||||
let operator_machine_key = zcfg
|
|
||||||
.machine_key(&cfg.operator_username)
|
|
||||||
.with_context(|| {
|
|
||||||
format!(
|
|
||||||
"machine key for {} missing from cache",
|
|
||||||
cfg.operator_username
|
|
||||||
)
|
|
||||||
})?
|
|
||||||
.clone();
|
|
||||||
let cli_client_id = zcfg
|
|
||||||
.client_id(cli_app_name)
|
|
||||||
.with_context(|| format!("OIDC client_id for app '{cli_app_name}' missing from cache"))?
|
|
||||||
.clone();
|
|
||||||
|
|
||||||
// ---- OpenBao: deploy + policy, co-located in the fleet namespace --------
|
|
||||||
// The operator's credentials are seeded here so a later
|
|
||||||
// `harmony-fleet-deploy --from-tag <tag>` upgrades the operator alone.
|
|
||||||
let openbao = OpenbaoInstance {
|
|
||||||
namespace: cfg.fleet_namespace.clone(),
|
|
||||||
release: "openbao".to_string(),
|
|
||||||
};
|
|
||||||
OpenbaoScore {
|
|
||||||
instance: openbao.clone(),
|
|
||||||
host: secrets_host.clone(),
|
|
||||||
openshift: true,
|
|
||||||
tls_issuer: Some(cfg.cluster_issuer.clone()),
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::empty(), &topology)
|
|
||||||
.await
|
|
||||||
.context("OpenBao deploy")?;
|
|
||||||
OpenbaoSetupScore {
|
|
||||||
instance: openbao.clone(),
|
|
||||||
policies: vec![OpenbaoPolicy {
|
|
||||||
name: "fleet-deployer".to_string(),
|
|
||||||
hcl: r#"path "secret/data/harmony/*" { capabilities = ["read"] }
|
|
||||||
path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
|
|
||||||
.to_string(),
|
|
||||||
}],
|
|
||||||
// Device login (ADR-025): batch tokens so re-login is free, 1h
|
|
||||||
// TTL as the group-membership convergence bound, groups bound
|
|
||||||
// from the Action-flattened `groups` claim.
|
|
||||||
jwt_auth: Some(OpenbaoJwtAuth {
|
|
||||||
oidc_discovery_url: format!("https://{zitadel_host}"),
|
|
||||||
jwt_validation_pubkeys: String::new(),
|
|
||||||
bound_issuer: format!("https://{zitadel_host}"),
|
|
||||||
role_name: "fleet-device".to_string(),
|
|
||||||
bound_audiences: project_id.clone(),
|
|
||||||
user_claim: "sub".to_string(),
|
|
||||||
groups_claim: "groups".to_string(),
|
|
||||||
token_type: "batch".to_string(),
|
|
||||||
ttl: "1h".to_string(),
|
|
||||||
max_ttl: "1h".to_string(),
|
|
||||||
}),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::empty(), &topology)
|
|
||||||
.await
|
|
||||||
.context("OpenBao setup")?;
|
|
||||||
|
|
||||||
// ---- Workload Scores: filterable via `harmony_cli::Args` ----------------
|
|
||||||
// The callout signs user JWTs with this account NKey; NATS is configured
|
|
||||||
// with the matching pubkey via the auth_callout block in its helm values.
|
|
||||||
let issuer_kp = KeyPair::new_account();
|
|
||||||
let issuer_seed = issuer_kp
|
|
||||||
.seed()
|
|
||||||
.map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?;
|
|
||||||
let nats_auth_user = "auth";
|
|
||||||
let nats_auth_pass = generate_alphanum(24);
|
|
||||||
let nats_url = format!(
|
|
||||||
"nats://{nats_release}.{}.svc.cluster.local:4222",
|
|
||||||
cfg.fleet_namespace
|
|
||||||
);
|
|
||||||
|
|
||||||
let nats = NatsK8sScore {
|
|
||||||
distribution: KubernetesDistribution::OpenshiftFamily,
|
|
||||||
cluster: NatsCluster {
|
|
||||||
namespace: cfg.fleet_namespace.clone(),
|
|
||||||
// `domain` and the static-string fields below are only read in the
|
|
||||||
// supercluster path (gateway Some), which staging doesn't take.
|
|
||||||
domain: cfg.base_domain.clone(),
|
|
||||||
replicas: 1,
|
|
||||||
name: nats_release.to_string(),
|
|
||||||
gateway_advertise: String::new(),
|
|
||||||
dns_name: nats_ws_host.clone(),
|
|
||||||
supercluster_ca_secret_name: "fleet-nats-supercluster-ca",
|
|
||||||
tls_cert_name: "fleet-nats-tls",
|
|
||||||
jetstream_enabled: "true",
|
|
||||||
},
|
|
||||||
peers: None,
|
|
||||||
ca_bundle: None,
|
|
||||||
gateway: None,
|
|
||||||
auth_callout: Some(AuthCalloutCfg {
|
|
||||||
issuer_pubkey: issuer_kp.public_key(),
|
|
||||||
auth_user: nats_auth_user.to_string(),
|
|
||||||
auth_pass: nats_auth_pass.clone(),
|
|
||||||
account: cfg.nats_account.clone(),
|
|
||||||
}),
|
|
||||||
websocket: Some(WebSocketRouteCfg {
|
|
||||||
host: nats_ws_host.clone(),
|
|
||||||
cluster_issuer: cfg.cluster_issuer.clone(),
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut callout = NatsAuthCalloutScore::new(
|
|
||||||
"fleet-callout",
|
|
||||||
&cfg.fleet_namespace,
|
|
||||||
nats_url.clone(),
|
|
||||||
format!("https://{zitadel_host}"),
|
|
||||||
project_id.clone(),
|
|
||||||
nats_auth_user,
|
|
||||||
&nats_auth_pass,
|
|
||||||
&issuer_seed,
|
|
||||||
)
|
|
||||||
.image(&cfg.callout_image)
|
|
||||||
.target_account(&cfg.nats_account)
|
|
||||||
.admin_role(&cfg.admin_role)
|
|
||||||
.device_role(&cfg.device_role)
|
|
||||||
.danger_accept_invalid_certs(false);
|
|
||||||
callout.device_id_claim = "client_id".to_string();
|
|
||||||
callout.device_id_prefix_strip = "device-".to_string();
|
|
||||||
callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles");
|
|
||||||
|
|
||||||
let credentials = OperatorCredentials::zitadel_jwt(
|
|
||||||
&format!("https://{zitadel_host}"),
|
|
||||||
&project_id,
|
|
||||||
&operator_machine_key,
|
|
||||||
);
|
|
||||||
let mut operator = FleetOperatorScore::new()
|
|
||||||
.namespace(&cfg.fleet_namespace)
|
|
||||||
.release_name("harmony-fleet-operator")
|
|
||||||
.image(&cfg.operator_image)
|
|
||||||
.image_pull_policy("Always")
|
|
||||||
.nats_url(nats_url.clone())
|
|
||||||
.log_level("info,kube_runtime=warn");
|
|
||||||
operator.credentials = Some(credentials.clone());
|
|
||||||
|
|
||||||
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> =
|
|
||||||
vec![Box::new(nats), Box::new(callout), Box::new(operator)];
|
|
||||||
harmony_cli::run(Inventory::empty(), topology.clone(), scores, Some(args))
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
||||||
|
|
||||||
// ---- Seed operator credentials as FleetDeploySecrets --------------------
|
|
||||||
// Reached via port-forward with the cached root token, so it doesn't wait
|
|
||||||
// on the public route/cert. No kubeconfig — CD callers use their own context.
|
|
||||||
let k8s = topology
|
|
||||||
.k8s_client()
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!(e))?;
|
|
||||||
let pf = k8s
|
|
||||||
.port_forward(&openbao.pod(), &openbao.namespace, 8200, 8200)
|
|
||||||
.await
|
|
||||||
.context("port-forward to OpenBao")?;
|
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
|
||||||
let store = OpenbaoSecretStore::new(OpenbaoStoreOptions {
|
|
||||||
base_url: format!("http://127.0.0.1:{}", pf.port()),
|
|
||||||
kv_mount: "secret".to_string(),
|
|
||||||
auth_mount: "token".to_string(),
|
|
||||||
skip_tls: true,
|
|
||||||
token: Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?),
|
|
||||||
username: None,
|
|
||||||
password: None,
|
|
||||||
zitadel_sso_url: None,
|
|
||||||
zitadel_client_id: None,
|
|
||||||
jwt_role: None,
|
|
||||||
jwt_auth_mount: None,
|
|
||||||
zitadel_jwt_bearer: None,
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.context("OpenBao client")?;
|
|
||||||
ConfigClient::new(vec![
|
|
||||||
Arc::new(StoreSource::new("harmony".to_string(), store))
|
|
||||||
as Arc<dyn harmony_config::ConfigSource>,
|
|
||||||
])
|
|
||||||
.set(&FleetDeploySecrets {
|
|
||||||
operator_credentials_toml: credentials.credentials_toml.clone(),
|
|
||||||
kubeconfig: None,
|
|
||||||
operator_cookie_key_b64: generate_cookie_key_b64(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.context("seed FleetDeploySecrets")?;
|
|
||||||
|
|
||||||
info!("=== fleet-staging install complete ===");
|
|
||||||
info!(
|
|
||||||
"Zitadel: https://{zitadel_host}/ (admin user {})",
|
|
||||||
cfg.admin_username
|
|
||||||
);
|
|
||||||
info!("NATS WS public: wss://{nats_ws_host}/");
|
|
||||||
info!("OpenBao: https://{secrets_host}/");
|
|
||||||
info!("Project id: {project_id}");
|
|
||||||
info!("SSO client_id: {cli_client_id} (app '{cli_app_name}', device-code grant)");
|
|
||||||
info!(
|
|
||||||
"Enroll a device: fleet_device_enroll --target ssh://<user>@<device> \
|
|
||||||
--issuer-url https://{zitadel_host} --audience {project_id} \
|
|
||||||
--nats-url wss://{nats_ws_host} --admin-oidc-client-id {cli_client_id} \
|
|
||||||
--agent-binary <path>"
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn generate_alphanum(len: usize) -> String {
|
|
||||||
use rand::Rng;
|
|
||||||
const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
||||||
let mut rng = rand::rng();
|
|
||||||
(0..len)
|
|
||||||
.map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char)
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Standard-base64 of 64 random bytes — the operator's session-cookie
|
|
||||||
/// signing key. Generated per install and seeded as a secret.
|
|
||||||
fn generate_cookie_key_b64() -> String {
|
|
||||||
use base64::Engine;
|
|
||||||
use rand::Rng;
|
|
||||||
let mut bytes = [0u8; 64];
|
|
||||||
rand::rng().fill(&mut bytes);
|
|
||||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
|
||||||
}
|
|
||||||
@@ -10,6 +10,7 @@ path = "src/main.rs"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", features = ["kvm"] }
|
harmony = { path = "../../harmony", features = ["kvm"] }
|
||||||
|
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
log.workspace = true
|
log.workspace = true
|
||||||
|
|||||||
@@ -12,13 +12,13 @@ use anyhow::{Context, Result};
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::{
|
use harmony::modules::fleet::{
|
||||||
FleetDeviceSetupConfig, FleetDeviceSetupScore, ProvisionVmScore,
|
ProvisionVmScore, check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair,
|
||||||
check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair,
|
|
||||||
};
|
};
|
||||||
use harmony::modules::kvm::KvmVirtualMachineHost;
|
use harmony::modules::kvm::KvmVirtualMachineHost;
|
||||||
use harmony::modules::kvm::config::init_executor;
|
use harmony::modules::kvm::config::init_executor;
|
||||||
use harmony::modules::linux::{LinuxHostTopology, SshCredentials};
|
use harmony::modules::linux::{LinuxHostTopology, SshCredentials};
|
||||||
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
|
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
|
||||||
|
use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
@@ -213,7 +213,7 @@ async fn main() -> Result<()> {
|
|||||||
nats_urls: vec![cli.nats_url.clone()],
|
nats_urls: vec![cli.nats_url.clone()],
|
||||||
// VM smoke harness keeps shared-creds for v0; the customer-
|
// VM smoke harness keeps shared-creds for v0; the customer-
|
||||||
// facing Pi flow uses Zitadel JWT (see fleet_rpi_setup).
|
// facing Pi flow uses Zitadel JWT (see fleet_rpi_setup).
|
||||||
auth: harmony::modules::fleet::FleetDeviceAuth::TomlShared {
|
auth: FleetDeviceAuth::TomlShared {
|
||||||
nats_user: cli.nats_user.clone(),
|
nats_user: cli.nats_user.clone(),
|
||||||
nats_pass: cli.nats_pass.clone(),
|
nats_pass: cli.nats_pass.clone(),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -9,7 +9,8 @@ name = "harmony_apply_deployment"
|
|||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
|
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
||||||
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
kube = { workspace = true, features = ["runtime", "derive"] }
|
kube = { workspace = true, features = ["runtime", "derive"] }
|
||||||
k8s-openapi = { workspace = true }
|
k8s-openapi = { workspace = true }
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
|
|||||||
@@ -12,8 +12,8 @@
|
|||||||
//! (not `iot_`-anything), in line with the review call to position
|
//! (not `iot_`-anything), in line with the review call to position
|
||||||
//! the operator as a generic fleet/reconcile tool.
|
//! the operator as a generic fleet/reconcile tool.
|
||||||
//!
|
//!
|
||||||
//! The CRD types live in `harmony::modules::fleet::operator`; the score types
|
//! The CRD types live in `harmony_fleet_operator`; the score types
|
||||||
//! live in `harmony::modules::podman` (PodmanV0 being the first
|
//! live in `harmony_reconciler_contracts` (PodmanV0 being the first
|
||||||
//! reconciler variant — future variants drop in alongside).
|
//! reconciler variant — future variants drop in alongside).
|
||||||
//!
|
//!
|
||||||
//! Typical demo-driver usage:
|
//! Typical demo-driver usage:
|
||||||
@@ -38,11 +38,10 @@
|
|||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use harmony::modules::fleet::operator::crd::{
|
use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
|
||||||
Deployment, DeploymentSpec, Rollout, RolloutStrategy,
|
use harmony_reconciler_contracts::{
|
||||||
|
EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, VolumeMount,
|
||||||
};
|
};
|
||||||
use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
|
|
||||||
use harmony::topology::{EnvVar, RestartPolicy, VolumeMount};
|
|
||||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
||||||
@@ -143,7 +142,7 @@ async fn main() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mirrors `harmony::topology::RestartPolicy` so we can keep the CLI
|
/// Mirrors the contract `RestartPolicy` so we can keep the CLI
|
||||||
/// schema stable even if the underlying enum gains variants.
|
/// schema stable even if the underlying enum gains variants.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
|
||||||
enum CliRestart {
|
enum CliRestart {
|
||||||
|
|||||||
@@ -249,6 +249,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
host: OPENBAO_HOST.to_string(),
|
host: OPENBAO_HOST.to_string(),
|
||||||
openshift: false,
|
openshift: false,
|
||||||
tls_issuer: None,
|
tls_issuer: None,
|
||||||
|
node_port: None,
|
||||||
}
|
}
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
.interpret(&Inventory::autoload(), &topology)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ async fn main() -> Result<()> {
|
|||||||
host: cfg.host.clone(),
|
host: cfg.host.clone(),
|
||||||
openshift: cfg.openshift,
|
openshift: cfg.openshift,
|
||||||
tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()),
|
tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()),
|
||||||
|
node_port: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
// JWT auth composes in only when both issuer and audience are set; it
|
// JWT auth composes in only when both issuer and audience are set; it
|
||||||
@@ -128,6 +129,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
|
|||||||
policies,
|
policies,
|
||||||
users: vec![],
|
users: vec![],
|
||||||
jwt_auth,
|
jwt_auth,
|
||||||
|
oidc_application: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)];
|
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)];
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ IoT / decentralized-edge orchestration for harmony. A fleet stack is:
|
|||||||
| **Operator** | [`harmony-fleet-operator`](harmony-fleet-operator/) | Watches `Deployment` CRs, writes desired state into NATS JetStream KV, aggregates device state back into CR status. Runtime binary; no `harmony` dep. |
|
| **Operator** | [`harmony-fleet-operator`](harmony-fleet-operator/) | Watches `Deployment` CRs, writes desired state into NATS JetStream KV, aggregates device state back into CR status. Runtime binary; no `harmony` dep. |
|
||||||
| **Agent** | [`harmony-fleet-agent`](harmony-fleet-agent/) | One per device. Watches the desired-state KV, drives the local runtime (podman today), publishes heartbeats + per-deployment state, answers `device-commands.*` request/reply. |
|
| **Agent** | [`harmony-fleet-agent`](harmony-fleet-agent/) | One per device. Watches the desired-state KV, drives the local runtime (podman today), publishes heartbeats + per-deployment state, answers `device-commands.*` request/reply. |
|
||||||
| **Auth** | [`harmony-fleet-auth`](harmony-fleet-auth/) | Shared NATS credential plumbing — `TomlShared` (dev) and `ZitadelJwt` (prod with auth-callout). |
|
| **Auth** | [`harmony-fleet-auth`](harmony-fleet-auth/) | Shared NATS credential plumbing — `TomlShared` (dev) and `ZitadelJwt` (prod with auth-callout). |
|
||||||
| **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. Imports `harmony` and exposes one `*Score` per component (`FleetOperatorScore`, `FleetAgentScore`, `FleetNatsScore`, `FleetServerScore`). Both the production CLI and the e2e harness compose these — see [ADR-023](../docs/adr/023-deploy-architecture.md). |
|
| **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. It composes independent component Scores through typed references; there is no Fleet-wide aggregate Score. See [ADR-023](../docs/adr/023-deploy-architecture.md) and [ADR-028](../docs/adr/028-typed-score-references.md). |
|
||||||
| **E2E harness** | [`harmony-fleet-e2e`](harmony-fleet-e2e/) | Brings the stack up in a fresh k3d namespace and runs integration tests against it. |
|
| **E2E harness** | [`harmony-fleet-e2e`](harmony-fleet-e2e/) | Brings the stack up in a fresh k3d namespace and runs integration tests against it. |
|
||||||
|
|
||||||
The on-the-wire types both ends agree on (KV bucket names, key formats, command-protocol payloads) live in [`../harmony-reconciler-contracts`](../harmony-reconciler-contracts/).
|
The on-the-wire types both ends agree on (KV bucket names, key formats, command-protocol payloads) live in [`../harmony-reconciler-contracts`](../harmony-reconciler-contracts/).
|
||||||
@@ -16,6 +16,20 @@ The on-the-wire types both ends agree on (KV bucket names, key formats, command-
|
|||||||
|
|
||||||
`FleetOperatorScore`, `FleetAgentScore`, etc. are real Rust types with capability-bound `Topology` parameters. Production deploys, the e2e harness, and any future control-plane tool all compose the **same** Scores; the only thing that changes is the `Topology` instance. **No handrolled YAML or imperative manifest factories anywhere.** Read [ADR-023](../docs/adr/023-deploy-architecture.md) before adding deploy logic.
|
`FleetOperatorScore`, `FleetAgentScore`, etc. are real Rust types with capability-bound `Topology` parameters. Production deploys, the e2e harness, and any future control-plane tool all compose the **same** Scores; the only thing that changes is the `Topology` instance. **No handrolled YAML or imperative manifest factories anywhere.** Read [ADR-023](../docs/adr/023-deploy-architecture.md) before adding deploy logic.
|
||||||
|
|
||||||
|
## Local Fleet
|
||||||
|
|
||||||
|
The local context creates or reuses the `harmony` k3d cluster, builds and imports
|
||||||
|
the Fleet images, and deploys PostgreSQL, Zitadel, NATS with auth callout,
|
||||||
|
OpenBao, and the Fleet operator:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo run -p harmony-fleet-deploy -- ship --context local
|
||||||
|
```
|
||||||
|
|
||||||
|
The command returns only after the operator has authenticated through Zitadel
|
||||||
|
and initialized its NATS JetStream state. The credentials Score creates an
|
||||||
|
immutable local Kubernetes Secret on the first run and reuses it thereafter.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Quickstart — run the e2e ping test
|
## Quickstart — run the e2e ping test
|
||||||
@@ -31,10 +45,10 @@ What it does, in order:
|
|||||||
1. Ensures a `fleet-e2e` k3d cluster exists (creates one if not). NodePort `30423` on the host forwards to NATS inside the cluster.
|
1. Ensures a `fleet-e2e` k3d cluster exists (creates one if not). NodePort `30423` on the host forwards to NATS inside the cluster.
|
||||||
2. Builds `harmony-fleet-agent` in release mode, packages it into `localhost/harmony-fleet-agent:e2e`, and sideloads the image into the k3d cluster's containerd store.
|
2. Builds `harmony-fleet-agent` in release mode, packages it into `localhost/harmony-fleet-agent:e2e`, and sideloads the image into the k3d cluster's containerd store.
|
||||||
3. Mints a per-bring-up namespace `e2e-<uuid8>` and prunes any leftover `e2e-*` namespaces from prior runs (NodePort `30423` is cluster-scoped, so a stuck `Terminating` namespace would block the new bring-up — the prune waits up to 90 s for full cleanup before proceeding).
|
3. Mints a per-bring-up namespace `e2e-<uuid8>` and prunes any leftover `e2e-*` namespaces from prior runs (NodePort `30423` is cluster-scoped, so a stuck `Terminating` namespace would block the new bring-up — the prune waits up to 90 s for full cleanup before proceeding).
|
||||||
4. Deploys NATS via `FleetNatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service).
|
4. Deploys NATS via `NatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service).
|
||||||
5. Waits for NATS to be reachable from the host on `nats://localhost:30423` (admin/e2e-admin).
|
5. Waits for NATS to be reachable from the host on `nats://localhost:30423` (admin/e2e-admin).
|
||||||
6. Deploys one `FleetAgentScore { target: Pod }` — runs with `runtime_enabled = false` so it skips podman and only runs the command-server + heartbeat loop.
|
6. Deploys one `FleetAgentScore { target: Pod }` — runs with `runtime_enabled = false` so it skips podman and only runs the command-server + heartbeat loop.
|
||||||
7. Waits for the agent Deployment to be Ready.
|
7. `FleetAgentScore` waits for the agent Deployment to be Ready.
|
||||||
8. The test publishes `device-commands.<device_id>.ping` via `FleetCommandsClient::ping` and asserts the agent replies with `{ device_id, agent_version, uptime_s }`.
|
8. The test publishes `device-commands.<device_id>.ping` via `FleetCommandsClient::ping` and asserts the agent replies with `{ device_id, agent_version, uptime_s }`.
|
||||||
|
|
||||||
Cold first run: ~80 s (release build of the agent dominates). Warm: ~25 s.
|
Cold first run: ~80 s (release build of the agent dominates). Warm: ~25 s.
|
||||||
@@ -102,14 +116,17 @@ k3d cluster delete fleet-e2e
|
|||||||
|
|
||||||
## Production deploys
|
## Production deploys
|
||||||
|
|
||||||
`harmony-fleet-deploy` puts the published operator chart on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach) — the `harmony apply` / CD path. It loads `FleetDeploySecrets` from config (Env → OpenBao) and runs one `FleetOperatorScore`; auth is Zitadel-SSO-only. The full bring-up stack (`FleetNatsScore` + `FleetAgentScore` + …) is composed by the e2e harness directly over the same lib Scores, not by this binary.
|
Production contexts live in the customer's private repository. They provide a
|
||||||
|
registry, project, domain, image-pull Secret, and OpenBao-brokered kubeconfig.
|
||||||
|
The cluster must also contain the immutable
|
||||||
|
`fleet-system/fleet-callout-credentials` Secret with
|
||||||
|
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass` keys. The deploy
|
||||||
|
validates this Secret before configuring NATS.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Deploy a released tag (version parsed from it in Rust):
|
|
||||||
cargo run -p harmony-fleet-deploy -- \
|
cargo run -p harmony-fleet-deploy -- \
|
||||||
--filter FleetOperatorScore \
|
ship --context customer-prod \
|
||||||
--from-tag harmony-fleet-operator-v0.0.2 \
|
--config path/to/contexts.toml
|
||||||
--namespace fleet-system --yes
|
|
||||||
```
|
```
|
||||||
|
|
||||||
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
|
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
|
||||||
@@ -132,22 +149,12 @@ cargo run -p harmony-fleet-operator --features web-frontend -- serve-web --mock
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Existing manual rehearsal — `examples/fleet_e2e_demo`
|
## VM rehearsal
|
||||||
|
|
||||||
`examples/fleet_e2e_demo` brings up a *fuller* stack than the e2e harness — real Zitadel, the auth-callout, libvirt VM agents over SSH — at the cost of a 5-min cold start. It's the manual rehearsal flow; not what you want during the dev loop. See the example's [`RUNBOOK.md`](../examples/fleet_e2e_demo/RUNBOOK.md).
|
`examples/fleet_e2e_demo` is a small executable over
|
||||||
|
`harmony-fleet-e2e::VmStack`. It runs the same callout, OpenBao, operator, and
|
||||||
|
device enrollment Scores as the maintained VM tests:
|
||||||
|
|
||||||
The harness and the rehearsal will converge: the [follow-up PR](#whats-next) lifts `FleetCalloutScore` + a mock-OIDC fixture into `harmony-fleet-deploy`, at which point the harness can run the full production auth path in ~30 s instead of 5 min, and `fleet_e2e_demo` thins down to a caller over the same Scores.
|
```bash
|
||||||
|
cargo run -p example-fleet-e2e-demo -- --num-devices 2
|
||||||
---
|
```
|
||||||
|
|
||||||
## What's next
|
|
||||||
|
|
||||||
This branch lands the deploy-architecture cleanup (ADR-023), the per-component Scores, and the ping path. Slated immediately after:
|
|
||||||
|
|
||||||
1. **Zitadel + auth callout in `harmony-fleet-deploy`.** New `FleetCalloutScore` (preset over `NatsAuthCalloutScore`) plus an in-cluster mock-OIDC fixture so the e2e harness can exercise the real auth-callout code path without paying Zitadel's 5-min cold-start cost. The harness's `AuthMode::Callout` variant is already on the public API for this.
|
|
||||||
2. **Operator pod in the e2e harness.** `FleetOperatorScore` is already in the deploy crate; wiring it into the harness gives integration tests against the actual `Deployment` / `Device` reconcile loops.
|
|
||||||
3. **`Verb::Logs` and `Verb::Exec`** — the next two verbs on the `device-commands.*` protocol. Same harness, same TDD shape as `ping`.
|
|
||||||
4. **CRD types out of `harmony` core.** `harmony::modules::fleet::operator::crd` is the last fleet-deploy thing still living in `harmony`. The `ReconcileScore` payload coupling is the only blocker.
|
|
||||||
5. **Smoke-test contract.** ADR-023 principle 4 — every Score blocks on a smoke test before `deploy` returns success. Today the e2e suite plays that role; the trait/companion shape lands once it's been validated in practice.
|
|
||||||
|
|
||||||
See [`PLAN_requests_over_nats.md`](PLAN_requests_over_nats.md) for the full TDD-style plan this branch implements.
|
|
||||||
|
|||||||
@@ -8,13 +8,13 @@ rust-version = "1.85"
|
|||||||
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
||||||
harmony_secret = { path = "../../harmony_secret" }
|
harmony_secret = { path = "../../harmony_secret" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
|
|
||||||
async-nats = { workspace = true }
|
async-nats = { workspace = true }
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
futures-util = { workspace = true }
|
futures-util = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
sha2.workspace = true
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
@@ -22,3 +22,4 @@ anyhow = { workspace = true }
|
|||||||
clap = { workspace = true }
|
clap = { workspace = true }
|
||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
|
podman-api = "0.9"
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
mod command_server;
|
mod command_server;
|
||||||
mod config;
|
mod config;
|
||||||
mod fleet_publisher;
|
mod fleet_publisher;
|
||||||
|
mod podman;
|
||||||
mod reconciler;
|
mod reconciler;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -20,12 +21,9 @@ use harmony_reconciler_contracts::{
|
|||||||
BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter,
|
BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter,
|
||||||
};
|
};
|
||||||
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::podman::PodmanTopology;
|
|
||||||
use harmony::topology::Topology;
|
|
||||||
|
|
||||||
use crate::command_server::CommandServer;
|
use crate::command_server::CommandServer;
|
||||||
use crate::fleet_publisher::FleetPublisher;
|
use crate::fleet_publisher::FleetPublisher;
|
||||||
|
use crate::podman::PodmanRuntime;
|
||||||
use crate::reconciler::Reconciler;
|
use crate::reconciler::Reconciler;
|
||||||
|
|
||||||
/// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
|
/// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
|
||||||
@@ -146,9 +144,11 @@ async fn publish_heartbeat_loop(fleet: Arc<FleetPublisher>) {
|
|||||||
/// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise.
|
/// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise.
|
||||||
/// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state
|
/// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state
|
||||||
/// on heartbeat but not always send it over the wire
|
/// on heartbeat but not always send it over the wire
|
||||||
fn local_inventory(inventory: &Inventory) -> InventorySnapshot {
|
fn local_inventory() -> InventorySnapshot {
|
||||||
InventorySnapshot {
|
InventorySnapshot {
|
||||||
hostname: inventory.location.name.clone(),
|
hostname: std::fs::read_to_string("/proc/sys/kernel/hostname")
|
||||||
|
.map(|hostname| hostname.trim().to_string())
|
||||||
|
.unwrap_or_else(|_| "localhost".to_string()),
|
||||||
arch: std::env::consts::ARCH.to_string(),
|
arch: std::env::consts::ARCH.to_string(),
|
||||||
os: std::env::consts::OS.to_string(),
|
os: std::env::consts::OS.to_string(),
|
||||||
kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
||||||
@@ -206,7 +206,7 @@ async fn main() -> Result<()> {
|
|||||||
// reconciler depends on the topology.
|
// reconciler depends on the topology.
|
||||||
let topology = if cfg.agent.runtime_enabled {
|
let topology = if cfg.agent.runtime_enabled {
|
||||||
let t = Arc::new(
|
let t = Arc::new(
|
||||||
PodmanTopology::from_default_socket()
|
PodmanRuntime::from_default_socket()
|
||||||
.map_err(|e| anyhow::anyhow!("failed to open podman socket: {e}"))?,
|
.map_err(|e| anyhow::anyhow!("failed to open podman socket: {e}"))?,
|
||||||
);
|
);
|
||||||
t.ensure_ready().await.context("podman socket not ready")?;
|
t.ensure_ready().await.context("podman socket not ready")?;
|
||||||
@@ -220,9 +220,8 @@ async fn main() -> Result<()> {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let inventory = Arc::new(Inventory::from_localhost());
|
let inventory_snapshot = local_inventory();
|
||||||
tracing::info!(hostname = %inventory.location.name, "inventory loaded");
|
tracing::info!(hostname = %inventory_snapshot.hostname, "inventory loaded");
|
||||||
let inventory_snapshot = local_inventory(&inventory);
|
|
||||||
|
|
||||||
let creds = credential_source_from_config(&cfg.credentials)
|
let creds = credential_source_from_config(&cfg.credentials)
|
||||||
.context("building NATS credential source")?;
|
.context("building NATS credential source")?;
|
||||||
@@ -291,7 +290,6 @@ async fn main() -> Result<()> {
|
|||||||
Arc::new(Reconciler::new(
|
Arc::new(Reconciler::new(
|
||||||
device_id.clone(),
|
device_id.clone(),
|
||||||
t.clone(),
|
t.clone(),
|
||||||
inventory.clone(),
|
|
||||||
Some(fleet.clone()),
|
Some(fleet.clone()),
|
||||||
secrets,
|
secrets,
|
||||||
))
|
))
|
||||||
|
|||||||
302
fleet/harmony-fleet-agent/src/podman.rs
Normal file
302
fleet/harmony-fleet-agent/src/podman.rs
Normal file
@@ -0,0 +1,302 @@
|
|||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow, bail};
|
||||||
|
use futures_util::StreamExt;
|
||||||
|
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy, VolumeMount};
|
||||||
|
use podman_api::Podman;
|
||||||
|
use podman_api::models::{ContainerMount, PortMapping};
|
||||||
|
use podman_api::opts::{
|
||||||
|
ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts,
|
||||||
|
ContainerRestartPolicy, ContainerStopOpts, PullOpts,
|
||||||
|
};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment";
|
||||||
|
const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256";
|
||||||
|
const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by";
|
||||||
|
const MANAGED_BY_VALUE: &str = "harmony";
|
||||||
|
const STOP_TIMEOUT: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
pub struct PodmanRuntime {
|
||||||
|
podman: Podman,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PodmanRuntime {
|
||||||
|
pub fn from_default_socket() -> Result<Self> {
|
||||||
|
Self::from_unix_socket(default_user_socket())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_unix_socket(socket: PathBuf) -> Result<Self> {
|
||||||
|
Ok(Self {
|
||||||
|
podman: Podman::unix(&socket),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn ensure_ready(&self) -> Result<()> {
|
||||||
|
self.podman
|
||||||
|
.info()
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow!("podman socket unreachable: {error}"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> {
|
||||||
|
for service in &score.services {
|
||||||
|
self.ensure_service_running(service, deployment).await?;
|
||||||
|
}
|
||||||
|
let desired = score
|
||||||
|
.services
|
||||||
|
.iter()
|
||||||
|
.map(|service| service.name.as_str())
|
||||||
|
.collect::<HashSet<_>>();
|
||||||
|
for container in self.deployment_containers(deployment).await? {
|
||||||
|
let name = container_name(&container);
|
||||||
|
if !desired.contains(name.as_str()) {
|
||||||
|
self.remove_service(&name).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn remove_deployment(&self, deployment: &str) -> Result<()> {
|
||||||
|
for container in self.deployment_containers(deployment).await? {
|
||||||
|
self.remove_service(&container_name(&container)).await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn remove_service(&self, name: &str) -> Result<()> {
|
||||||
|
let opts = ContainerStopOpts::builder()
|
||||||
|
.timeout(STOP_TIMEOUT.as_secs() as usize)
|
||||||
|
.build();
|
||||||
|
let container = self.podman.containers().get(name);
|
||||||
|
if container.exists().await.unwrap_or(false) {
|
||||||
|
let _ = container.stop(&opts).await;
|
||||||
|
}
|
||||||
|
self.remove_container(name).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_service_running(
|
||||||
|
&self,
|
||||||
|
service: &PodmanService,
|
||||||
|
deployment: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
let existing = self.get_by_name(&service.name).await?;
|
||||||
|
if let Some(existing) = existing.as_ref() {
|
||||||
|
if matches_spec(existing, service) {
|
||||||
|
if existing.state.as_deref() == Some("running") {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let id = existing.id.clone().unwrap_or_else(|| service.name.clone());
|
||||||
|
self.podman.containers().get(id).start(None).await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
self.remove_container(&service.name).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.ensure_image_present(&service.image).await?;
|
||||||
|
let labels = HashMap::from([
|
||||||
|
(MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
|
||||||
|
(DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
|
||||||
|
(SPEC_LABEL.to_string(), spec_hash(service)?),
|
||||||
|
]);
|
||||||
|
let ports = service
|
||||||
|
.ports
|
||||||
|
.iter()
|
||||||
|
.map(|port| parse_port_mapping(port))
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
let env: HashMap<String, String> = service
|
||||||
|
.env
|
||||||
|
.iter()
|
||||||
|
.map(|env| (env.name.clone(), env.value.clone()))
|
||||||
|
.collect();
|
||||||
|
let mounts = service
|
||||||
|
.volumes
|
||||||
|
.iter()
|
||||||
|
.map(volume_to_mount)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let mut builder = ContainerCreateOpts::builder()
|
||||||
|
.name(&service.name)
|
||||||
|
.image(&service.image)
|
||||||
|
.labels(labels)
|
||||||
|
.portmappings(ports)
|
||||||
|
.env(env)
|
||||||
|
.restart_policy(map_restart_policy(service.restart_policy));
|
||||||
|
if !mounts.is_empty() {
|
||||||
|
builder = builder.mounts(mounts);
|
||||||
|
}
|
||||||
|
let created = self.podman.containers().create(&builder.build()).await?;
|
||||||
|
self.podman.containers().get(created.id).start(None).await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_by_name(&self, name: &str) -> Result<Option<podman_api::models::ListContainer>> {
|
||||||
|
let opts = ContainerListOpts::builder()
|
||||||
|
.all(true)
|
||||||
|
.filter([ContainerListFilter::Name(name.to_string())])
|
||||||
|
.build();
|
||||||
|
Ok(self
|
||||||
|
.podman
|
||||||
|
.containers()
|
||||||
|
.list(&opts)
|
||||||
|
.await?
|
||||||
|
.into_iter()
|
||||||
|
.next())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deployment_containers(
|
||||||
|
&self,
|
||||||
|
deployment: &str,
|
||||||
|
) -> Result<Vec<podman_api::models::ListContainer>> {
|
||||||
|
let opts = ContainerListOpts::builder()
|
||||||
|
.all(true)
|
||||||
|
.filter([ContainerListFilter::LabelKeyVal(
|
||||||
|
DEPLOYMENT_LABEL.to_string(),
|
||||||
|
deployment.to_string(),
|
||||||
|
)])
|
||||||
|
.build();
|
||||||
|
Ok(self.podman.containers().list(&opts).await?)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_image_present(&self, image: &str) -> Result<()> {
|
||||||
|
let images = self.podman.images();
|
||||||
|
if images.get(image).exists().await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut pull = images.pull(&PullOpts::builder().reference(image).build());
|
||||||
|
while let Some(event) = pull.next().await {
|
||||||
|
if let Some(error) = event?.error {
|
||||||
|
bail!("podman pull {image} failed: {error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_container(&self, name: &str) -> Result<()> {
|
||||||
|
let opts = ContainerDeleteOpts::builder().force(true).build();
|
||||||
|
match self.podman.containers().get(name).delete(&opts).await {
|
||||||
|
Ok(()) => Ok(()),
|
||||||
|
Err(error) if is_not_found(&error) => Ok(()),
|
||||||
|
Err(error) => Err(error.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_user_socket() -> PathBuf {
|
||||||
|
if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") {
|
||||||
|
return PathBuf::from(format!("{dir}/podman/podman.sock"));
|
||||||
|
}
|
||||||
|
let uid = std::fs::read_to_string("/proc/self/status")
|
||||||
|
.ok()
|
||||||
|
.and_then(|status| {
|
||||||
|
status
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.strip_prefix("Uid:"))
|
||||||
|
.and_then(|value| value.split_whitespace().next())
|
||||||
|
.and_then(|value| value.parse::<u32>().ok())
|
||||||
|
})
|
||||||
|
.unwrap_or(0);
|
||||||
|
PathBuf::from(format!("/run/user/{uid}/podman/podman.sock"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_port_mapping(raw: &str) -> Result<PortMapping> {
|
||||||
|
let (host, container) = raw
|
||||||
|
.split_once(':')
|
||||||
|
.ok_or_else(|| anyhow!("invalid port mapping '{raw}' - expected 'HOST:CONTAINER'"))?;
|
||||||
|
Ok(PortMapping {
|
||||||
|
host_port: Some(
|
||||||
|
host.parse()
|
||||||
|
.map_err(|_| anyhow!("invalid host port '{host}' in mapping '{raw}'"))?,
|
||||||
|
),
|
||||||
|
container_port: Some(
|
||||||
|
container
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| anyhow!("invalid container port '{container}' in mapping '{raw}'"))?,
|
||||||
|
),
|
||||||
|
host_ip: None,
|
||||||
|
protocol: None,
|
||||||
|
range: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanService) -> bool {
|
||||||
|
observed
|
||||||
|
.labels
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|labels| labels.get(SPEC_LABEL))
|
||||||
|
.is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spec_hash(service: &PodmanService) -> Result<String> {
|
||||||
|
Ok(format!(
|
||||||
|
"{:x}",
|
||||||
|
Sha256::digest(serde_json::to_vec(service)?)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn container_name(container: &podman_api::models::ListContainer) -> String {
|
||||||
|
container
|
||||||
|
.names
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|names| names.first())
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| container.id.clone())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn volume_to_mount(volume: &VolumeMount) -> ContainerMount {
|
||||||
|
ContainerMount {
|
||||||
|
_type: Some("bind".to_string()),
|
||||||
|
source: Some(volume.host_path.clone()),
|
||||||
|
destination: Some(volume.container_path.clone()),
|
||||||
|
options: Some(vec![if volume.read_only { "ro" } else { "rw" }.to_string()]),
|
||||||
|
uid_mappings: None,
|
||||||
|
gid_mappings: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_restart_policy(policy: RestartPolicy) -> ContainerRestartPolicy {
|
||||||
|
match policy {
|
||||||
|
RestartPolicy::No => ContainerRestartPolicy::No,
|
||||||
|
RestartPolicy::UnlessStopped => ContainerRestartPolicy::UnlessStopped,
|
||||||
|
RestartPolicy::OnFailure => ContainerRestartPolicy::OnFailure,
|
||||||
|
RestartPolicy::Always => ContainerRestartPolicy::Always,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_not_found(error: &podman_api::Error) -> bool {
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.to_lowercase()
|
||||||
|
.contains("no such container")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use harmony_reconciler_contracts::EnvVar;
|
||||||
|
|
||||||
|
fn service() -> PodmanService {
|
||||||
|
PodmanService {
|
||||||
|
name: "web".into(),
|
||||||
|
image: "nginx:latest".into(),
|
||||||
|
ports: vec!["8080:80".into()],
|
||||||
|
env: vec![EnvVar::new("MODE", "prod")],
|
||||||
|
secret_env: vec![],
|
||||||
|
volumes: vec![],
|
||||||
|
restart_policy: RestartPolicy::Always,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn spec_hash_is_stable_and_covers_desired_state() {
|
||||||
|
let original = service();
|
||||||
|
assert_eq!(spec_hash(&original).unwrap(), spec_hash(&original).unwrap());
|
||||||
|
|
||||||
|
let mut changed = original;
|
||||||
|
changed.env[0].value = "dev".into();
|
||||||
|
assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,15 +4,15 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase};
|
use harmony_reconciler_contracts::{
|
||||||
|
DeploymentName, DeploymentState, EnvVar, Id, Phase, PodmanV0Score, ReconcileScore,
|
||||||
|
};
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::podman::{EnvVar, PodmanTopology, PodmanV0Score, ReconcileScore};
|
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony_secret::SecretStore;
|
use harmony_secret::SecretStore;
|
||||||
|
|
||||||
use crate::fleet_publisher::FleetPublisher;
|
use crate::fleet_publisher::FleetPublisher;
|
||||||
|
use crate::podman::PodmanRuntime;
|
||||||
|
|
||||||
/// Where deployment secrets come from (ADR-025). `prefix` scopes the
|
/// Where deployment secrets come from (ADR-025). `prefix` scopes the
|
||||||
/// KV reads to `<prefix>/<deployment>/<secret>` — it must match the
|
/// KV reads to `<prefix>/<deployment>/<secret>` — it must match the
|
||||||
@@ -25,19 +25,21 @@ pub struct SecretSource {
|
|||||||
|
|
||||||
/// Cache key → last-seen state, populated by `apply` and consulted by the
|
/// Cache key → last-seen state, populated by `apply` and consulted by the
|
||||||
/// 30-second periodic tick and the delete path.
|
/// 30-second periodic tick and the delete path.
|
||||||
struct CachedEntry {
|
#[derive(Clone)]
|
||||||
/// Serialized score JSON. Used for string-compare idempotency per
|
enum CachedEntry {
|
||||||
/// ROADMAP §5.5 — cheaper and more deterministic than a hash.
|
Apply {
|
||||||
serialized: String,
|
serialized: String,
|
||||||
/// Parsed score. Cached so the periodic reconcile tick and delete
|
score: PodmanV0Score,
|
||||||
/// handlers don't have to re-parse the JSON.
|
resolved: bool,
|
||||||
score: PodmanV0Score,
|
},
|
||||||
|
Remove {
|
||||||
|
score: Option<PodmanV0Score>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Reconciler {
|
pub struct Reconciler {
|
||||||
device_id: Id,
|
device_id: Id,
|
||||||
topology: Arc<PodmanTopology>,
|
runtime: Arc<PodmanRuntime>,
|
||||||
inventory: Arc<Inventory>,
|
|
||||||
/// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per
|
/// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per
|
||||||
/// KV key — in v0 there is no fan-out from one key to many scores.
|
/// KV key — in v0 there is no fan-out from one key to many scores.
|
||||||
state: Mutex<HashMap<String, CachedEntry>>,
|
state: Mutex<HashMap<String, CachedEntry>>,
|
||||||
@@ -58,15 +60,13 @@ pub struct Reconciler {
|
|||||||
impl Reconciler {
|
impl Reconciler {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
device_id: Id,
|
device_id: Id,
|
||||||
topology: Arc<PodmanTopology>,
|
runtime: Arc<PodmanRuntime>,
|
||||||
inventory: Arc<Inventory>,
|
|
||||||
fleet: Option<Arc<FleetPublisher>>,
|
fleet: Option<Arc<FleetPublisher>>,
|
||||||
secrets: Option<SecretSource>,
|
secrets: Option<SecretSource>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
device_id,
|
device_id,
|
||||||
topology,
|
runtime,
|
||||||
inventory,
|
|
||||||
state: Mutex::new(HashMap::new()),
|
state: Mutex::new(HashMap::new()),
|
||||||
deployments: Mutex::new(HashMap::new()),
|
deployments: Mutex::new(HashMap::new()),
|
||||||
fleet,
|
fleet,
|
||||||
@@ -75,10 +75,9 @@ impl Reconciler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Materialize every `secret_env` reference into a plain env var.
|
/// Materialize every `secret_env` reference into a plain env var.
|
||||||
/// Resolution happens once per spec change, never on the periodic
|
/// Resolution is retried until it succeeds, then the resolved score is
|
||||||
/// tick: secret values bind at apply time (rotation = spec bump or
|
/// reused until the spec changes. This keeps the secret store off the
|
||||||
/// re-apply), which keeps the secret store off the 30s hot path —
|
/// periodic hot path for converged deployments.
|
||||||
/// fleet-scale login load is the ADR-025 constraint.
|
|
||||||
async fn resolve_secrets(
|
async fn resolve_secrets(
|
||||||
&self,
|
&self,
|
||||||
deployment: Option<&DeploymentName>,
|
deployment: Option<&DeploymentName>,
|
||||||
@@ -150,13 +149,7 @@ impl Reconciler {
|
|||||||
/// entry. Idempotent: a delete for a never-applied deployment is
|
/// entry. Idempotent: a delete for a never-applied deployment is
|
||||||
/// a no-op in memory and a harmless tombstone write on the wire.
|
/// a no-op in memory and a harmless tombstone write on the wire.
|
||||||
async fn drop_phase(&self, deployment: &DeploymentName) {
|
async fn drop_phase(&self, deployment: &DeploymentName) {
|
||||||
let was_known = {
|
self.deployments.lock().await.remove(deployment);
|
||||||
let mut phases = self.deployments.lock().await;
|
|
||||||
phases.remove(deployment).is_some()
|
|
||||||
};
|
|
||||||
if !was_known {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Some(publisher) = &self.fleet {
|
if let Some(publisher) = &self.fleet {
|
||||||
publisher.delete_deployment_state(deployment).await;
|
publisher.delete_deployment_state(deployment).await;
|
||||||
}
|
}
|
||||||
@@ -182,30 +175,32 @@ impl Reconciler {
|
|||||||
|
|
||||||
{
|
{
|
||||||
let state = self.state.lock().await;
|
let state = self.state.lock().await;
|
||||||
if let Some(existing) = state.get(key) {
|
if let Some(CachedEntry::Apply {
|
||||||
if existing.serialized == serialized {
|
serialized: existing,
|
||||||
|
..
|
||||||
|
}) = state.get(key)
|
||||||
|
{
|
||||||
|
if existing == &serialized {
|
||||||
tracing::debug!(key, "score unchanged — noop");
|
tracing::debug!(key, "score unchanged — noop");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.state.lock().await.insert(
|
||||||
|
key.to_string(),
|
||||||
|
CachedEntry::Apply {
|
||||||
|
serialized,
|
||||||
|
score: incoming,
|
||||||
|
resolved: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
if let Some(name) = &deployment {
|
if let Some(name) = &deployment {
|
||||||
self.apply_phase(name, Phase::Pending, None).await;
|
self.apply_phase(name, Phase::Pending, None).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let resolved = match self.resolve_secrets(deployment.as_ref(), incoming).await {
|
match self.reconcile_cached(key).await {
|
||||||
Ok(score) => score,
|
|
||||||
Err(e) => {
|
|
||||||
if let Some(name) = &deployment {
|
|
||||||
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
match self.run_score(key, &resolved).await {
|
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
if let Some(name) = &deployment {
|
if let Some(name) = &deployment {
|
||||||
self.apply_phase(name, Phase::Running, None).await;
|
self.apply_phase(name, Phase::Running, None).await;
|
||||||
@@ -219,82 +214,56 @@ impl Reconciler {
|
|||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut state = self.state.lock().await;
|
|
||||||
// Cache the *resolved* score: the periodic tick re-runs it
|
|
||||||
// as-is, so converged deployments never re-hit the secret
|
|
||||||
// store (see `resolve_secrets`).
|
|
||||||
state.insert(
|
|
||||||
key.to_string(),
|
|
||||||
CachedEntry {
|
|
||||||
serialized,
|
|
||||||
score: resolved,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handle a Delete/Purge event. Stops and removes every container
|
/// Handle a Delete/Purge event. The tombstone remains cached until all
|
||||||
/// referenced by the last cached score for this key. Idempotent: if we
|
/// deployment containers are removed, including when no score survived a
|
||||||
/// never saw a Put for this key (agent restart after delete), logs and
|
/// restart.
|
||||||
/// returns ok.
|
|
||||||
pub async fn remove(&self, key: &str) -> Result<()> {
|
pub async fn remove(&self, key: &str) -> Result<()> {
|
||||||
let deployment = deployment_from_key(key);
|
|
||||||
let mut state = self.state.lock().await;
|
let mut state = self.state.lock().await;
|
||||||
let Some(entry) = state.remove(key) else {
|
let score = match state.get(key) {
|
||||||
tracing::info!(key, "delete for unknown key — nothing to remove");
|
Some(CachedEntry::Apply { score, .. }) => Some(score.clone()),
|
||||||
if let Some(name) = &deployment {
|
Some(CachedEntry::Remove { score }) => score.clone(),
|
||||||
self.drop_phase(name).await;
|
None => None,
|
||||||
}
|
|
||||||
return Ok(());
|
|
||||||
};
|
};
|
||||||
|
state.insert(key.to_string(), CachedEntry::Remove { score });
|
||||||
drop(state);
|
drop(state);
|
||||||
|
self.remove_cached(key).await
|
||||||
use harmony::topology::ContainerRuntime;
|
|
||||||
for service in &entry.score.services {
|
|
||||||
if let Err(e) = self.topology.remove_service(&service.name).await {
|
|
||||||
tracing::warn!(
|
|
||||||
key,
|
|
||||||
service = %service.name,
|
|
||||||
error = %e,
|
|
||||||
"failed to remove container"
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
tracing::info!(key, service = %service.name, "removed container");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(name) = &deployment {
|
|
||||||
self.drop_phase(name).await;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of
|
/// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of
|
||||||
/// event-driven PLEG. Agent polls podman every 30s as ground truth;
|
/// event-driven PLEG. Agent polls podman every 30s as ground truth;
|
||||||
/// KV watch events are accelerators." Re-runs each cached score against
|
/// KV watch events are accelerators." Retries cached applies and removals
|
||||||
/// podman-api; the underlying `ensure_service_running` is idempotent
|
/// against podman-api.
|
||||||
/// so a converged state produces no log noise.
|
|
||||||
pub async fn tick(&self) -> Result<()> {
|
pub async fn tick(&self) -> Result<()> {
|
||||||
let snapshot: Vec<(String, PodmanV0Score)> = {
|
let keys: Vec<String> = self.state.lock().await.keys().cloned().collect();
|
||||||
let state = self.state.lock().await;
|
for key in keys {
|
||||||
state
|
|
||||||
.iter()
|
|
||||||
.map(|(k, v)| (k.clone(), v.score.clone()))
|
|
||||||
.collect()
|
|
||||||
};
|
|
||||||
for (key, score) in snapshot {
|
|
||||||
let deployment = deployment_from_key(&key);
|
let deployment = deployment_from_key(&key);
|
||||||
match self.run_score(&key, &score).await {
|
let removing = matches!(
|
||||||
|
self.state.lock().await.get(&key),
|
||||||
|
Some(CachedEntry::Remove { .. })
|
||||||
|
);
|
||||||
|
let result = if removing {
|
||||||
|
self.remove_cached(&key).await
|
||||||
|
} else {
|
||||||
|
self.reconcile_cached(&key).await
|
||||||
|
};
|
||||||
|
match result {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
if let Some(name) = &deployment {
|
if !removing {
|
||||||
self.apply_phase(name, Phase::Running, None).await;
|
if let Some(name) = &deployment {
|
||||||
|
self.apply_phase(name, Phase::Running, None).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(key, error = %e, "periodic reconcile failed");
|
tracing::warn!(key, error = %e, "periodic reconcile failed");
|
||||||
if let Some(name) = &deployment {
|
if !removing {
|
||||||
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
|
if let Some(name) = &deployment {
|
||||||
.await;
|
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
|
||||||
|
.await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -313,13 +282,78 @@ impl Reconciler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn reconcile_cached(&self, key: &str) -> Result<()> {
|
||||||
|
let Some(CachedEntry::Apply {
|
||||||
|
serialized,
|
||||||
|
mut score,
|
||||||
|
resolved,
|
||||||
|
}) = self.state.lock().await.get(key).cloned()
|
||||||
|
else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
|
||||||
|
if !resolved {
|
||||||
|
score = self
|
||||||
|
.resolve_secrets(deployment_from_key(key).as_ref(), score)
|
||||||
|
.await?;
|
||||||
|
let mut state = self.state.lock().await;
|
||||||
|
let Some(CachedEntry::Apply {
|
||||||
|
serialized: current,
|
||||||
|
score: cached_score,
|
||||||
|
resolved,
|
||||||
|
}) = state.get_mut(key)
|
||||||
|
else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if current != &serialized {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
*cached_score = score.clone();
|
||||||
|
*resolved = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.run_score(key, &score).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_cached(&self, key: &str) -> Result<()> {
|
||||||
|
let Some(CachedEntry::Remove { score }) = self.state.lock().await.get(key).cloned() else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let deployment = deployment_from_key(key);
|
||||||
|
if let Some(name) = &deployment {
|
||||||
|
self.runtime.remove_deployment(name.as_str()).await?;
|
||||||
|
} else if let Some(score) = score {
|
||||||
|
for service in &score.services {
|
||||||
|
self.runtime.remove_service(&service.name).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let removed = {
|
||||||
|
let mut state = self.state.lock().await;
|
||||||
|
if matches!(state.get(key), Some(CachedEntry::Remove { .. })) {
|
||||||
|
state.remove(key);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if removed {
|
||||||
|
if let Some(name) = &deployment {
|
||||||
|
self.drop_phase(name).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> {
|
async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> {
|
||||||
let interpret = Score::<PodmanTopology>::create_interpret(score);
|
let deployment = deployment_from_key(key)
|
||||||
let outcome = interpret
|
.map(|name| name.to_string())
|
||||||
.execute(&self.inventory, &self.topology)
|
.unwrap_or_else(|| key.to_string());
|
||||||
|
self.runtime
|
||||||
|
.reconcile(&deployment, score)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("PodmanV0Score interpret failed for {key}: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("PodmanV0Score reconcile failed for {key}: {e}"))?;
|
||||||
tracing::info!(key, outcome = ?outcome, "reconciled");
|
tracing::info!(key, services = score.services.len(), "reconciled");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -350,26 +384,19 @@ mod tests {
|
|||||||
//! `drop_phase` directly with an inert topology (no real podman
|
//! `drop_phase` directly with an inert topology (no real podman
|
||||||
//! socket) and a `None` FleetPublisher.
|
//! socket) and a `None` FleetPublisher.
|
||||||
use super::*;
|
use super::*;
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::podman::PodmanTopology;
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Mutex as StdMutex;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
fn reconciler() -> Reconciler {
|
fn reconciler() -> Reconciler {
|
||||||
reconciler_with_secrets(None)
|
reconciler_with_secrets(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reconciler_with_secrets(secrets: Option<SecretSource>) -> Reconciler {
|
fn reconciler_with_secrets(secrets: Option<SecretSource>) -> Reconciler {
|
||||||
let topology = Arc::new(
|
let runtime = Arc::new(
|
||||||
PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(),
|
PodmanRuntime::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(),
|
||||||
);
|
);
|
||||||
let inventory = Arc::new(Inventory::empty());
|
Reconciler::new(Id::from("test-device".to_string()), runtime, None, secrets)
|
||||||
Reconciler::new(
|
|
||||||
Id::from("test-device".to_string()),
|
|
||||||
topology,
|
|
||||||
inventory,
|
|
||||||
None,
|
|
||||||
secrets,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn dn(s: &str) -> DeploymentName {
|
fn dn(s: &str) -> DeploymentName {
|
||||||
@@ -403,8 +430,40 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct FlakyStore {
|
||||||
|
calls: AtomicUsize,
|
||||||
|
value: StdMutex<Option<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl SecretStore for FlakyStore {
|
||||||
|
async fn get_raw(
|
||||||
|
&self,
|
||||||
|
_namespace: &str,
|
||||||
|
_key: &str,
|
||||||
|
) -> Result<Vec<u8>, harmony_secret::SecretStoreError> {
|
||||||
|
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.value
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.map(|value| value.as_bytes().to_vec())
|
||||||
|
.ok_or_else(|| harmony_secret::SecretStoreError::Store("temporarily denied".into()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_raw(
|
||||||
|
&self,
|
||||||
|
_namespace: &str,
|
||||||
|
_key: &str,
|
||||||
|
_value: &[u8],
|
||||||
|
) -> Result<(), harmony_secret::SecretStoreError> {
|
||||||
|
unimplemented!("tests never write")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score {
|
fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score {
|
||||||
use harmony::modules::podman::{PodmanService, SecretEnvVar};
|
use harmony_reconciler_contracts::{PodmanService, SecretEnvVar};
|
||||||
PodmanV0Score {
|
PodmanV0Score {
|
||||||
services: vec![PodmanService {
|
services: vec![PodmanService {
|
||||||
name: "web-svc".to_string(),
|
name: "web-svc".to_string(),
|
||||||
@@ -479,6 +538,73 @@ mod tests {
|
|||||||
assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed));
|
assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tick_retries_secret_resolution_then_reuses_resolved_score() {
|
||||||
|
let store = Arc::new(FlakyStore::default());
|
||||||
|
let r = reconciler_with_secrets(Some(SecretSource {
|
||||||
|
store: store.clone(),
|
||||||
|
prefix: "fleet-test".to_string(),
|
||||||
|
}));
|
||||||
|
let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score(
|
||||||
|
"DB_PASSWORD",
|
||||||
|
"db_password",
|
||||||
|
)))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
r.apply("test-device.web", &payload)
|
||||||
|
.await
|
||||||
|
.expect_err("first secret read fails");
|
||||||
|
*store.value.lock().unwrap() = Some("hunter2".to_string());
|
||||||
|
|
||||||
|
r.tick().await.unwrap();
|
||||||
|
r.tick().await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(store.calls.load(Ordering::Relaxed), 2);
|
||||||
|
let state = r.state.lock().await;
|
||||||
|
let Some(CachedEntry::Apply {
|
||||||
|
score, resolved, ..
|
||||||
|
}) = state.get("test-device.web")
|
||||||
|
else {
|
||||||
|
panic!("apply retry state missing");
|
||||||
|
};
|
||||||
|
assert!(*resolved);
|
||||||
|
assert_eq!(
|
||||||
|
score.services[0].env[0],
|
||||||
|
EnvVar::new("DB_PASSWORD", "hunter2")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_removal_is_retried_and_later_put_cancels_it() {
|
||||||
|
let r = reconciler();
|
||||||
|
|
||||||
|
r.remove("test-device.web")
|
||||||
|
.await
|
||||||
|
.expect_err("unreachable Podman keeps the tombstone pending");
|
||||||
|
assert!(matches!(
|
||||||
|
r.state.lock().await.get("test-device.web"),
|
||||||
|
Some(CachedEntry::Remove { score: None })
|
||||||
|
));
|
||||||
|
|
||||||
|
r.tick().await.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
r.state.lock().await.get("test-device.web"),
|
||||||
|
Some(CachedEntry::Remove { .. })
|
||||||
|
));
|
||||||
|
|
||||||
|
let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(PodmanV0Score {
|
||||||
|
services: vec![],
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
r.apply("test-device.web", &payload)
|
||||||
|
.await
|
||||||
|
.expect_err("unreachable Podman keeps the new apply pending");
|
||||||
|
assert!(matches!(
|
||||||
|
r.state.lock().await.get("test-device.web"),
|
||||||
|
Some(CachedEntry::Apply { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn plain_scores_resolve_without_store() {
|
async fn plain_scores_resolve_without_store() {
|
||||||
// No secret_env → no store needed; resolution is a no-op.
|
// No secret_env → no store needed; resolution is a no-op.
|
||||||
|
|||||||
@@ -24,9 +24,11 @@
|
|||||||
mod agent_config;
|
mod agent_config;
|
||||||
mod config;
|
mod config;
|
||||||
mod credentials;
|
mod credentials;
|
||||||
|
mod observation;
|
||||||
|
|
||||||
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
|
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
|
||||||
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
|
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
|
||||||
|
pub use observation::AgentObservation;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
//! Per ADR-023 P7 this is a companion struct, not a new method on
|
//! Per ADR-023 P7 this is a companion struct, not a new method on
|
||||||
//! `Score` / `Interpret`. The base trait surface stays unchanged.
|
//! `Score` / `Interpret`. The base trait surface stays unchanged.
|
||||||
|
|
||||||
use harmony_fleet_auth::AgentConfig;
|
use crate::AgentConfig;
|
||||||
use harmony_reconciler_contracts::{
|
use harmony_reconciler_contracts::{
|
||||||
BUCKET_DESIRED_STATE, DeploymentName, Id, desired_state_key, desired_state_watch_filter,
|
BUCKET_DESIRED_STATE, DeploymentName, Id, desired_state_key, desired_state_watch_filter,
|
||||||
};
|
};
|
||||||
@@ -104,7 +104,7 @@ impl AgentObservation {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use harmony_fleet_auth::{AgentSection, CredentialsSection, NatsSection};
|
use crate::{AgentSection, CredentialsSection, NatsSection};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
fn cfg(device_id: &str, runtime_enabled: bool) -> AgentConfig {
|
fn cfg(device_id: &str, runtime_enabled: bool) -> AgentConfig {
|
||||||
@@ -5,6 +5,7 @@ version.workspace = true
|
|||||||
readme.workspace = true
|
readme.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, callout. Imports harmony for the Score-Topology-Interpret machinery; consumed by the production CLI, the e2e harness, and future control-plane tooling."
|
description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, callout. Imports harmony for the Score-Topology-Interpret machinery; consumed by the production CLI, the e2e harness, and future control-plane tooling."
|
||||||
|
default-run = "harmony-fleet-deploy"
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
@@ -15,20 +16,15 @@ path = "src/lib.rs"
|
|||||||
name = "harmony-fleet-deploy"
|
name = "harmony-fleet-deploy"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
# `harmony-fleet-publish --from-tag <tag>` builds + publishes the
|
|
||||||
# operator's image + chart for a release.
|
|
||||||
[[bin]]
|
|
||||||
name = "harmony-fleet-publish"
|
|
||||||
path = "src/bin/harmony-fleet-publish.rs"
|
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", features = ["podman"] }
|
harmony = { path = "../../harmony", features = ["podman"] }
|
||||||
harmony_cli = { path = "../../harmony_cli" }
|
harmony_cli = { path = "../../harmony_cli" }
|
||||||
|
harmony_app = { path = "../../harmony_app" }
|
||||||
harmony_config = { path = "../../harmony_config" }
|
harmony_config = { path = "../../harmony_config" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
harmony_macros = { path = "../../harmony_macros" }
|
harmony_macros = { path = "../../harmony_macros" }
|
||||||
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-fleet-operator = { path = "../harmony-fleet-operator" }
|
||||||
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
|
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
|
||||||
|
|
||||||
anyhow = { workspace = true }
|
anyhow = { workspace = true }
|
||||||
@@ -40,12 +36,15 @@ kube = { workspace = true, features = ["runtime", "derive"] }
|
|||||||
log = { workspace = true }
|
log = { workspace = true }
|
||||||
env_logger = { workspace = true }
|
env_logger = { workspace = true }
|
||||||
non-blank-string-rs = "1"
|
non-blank-string-rs = "1"
|
||||||
|
inquire.workspace = true
|
||||||
schemars = "0.8"
|
schemars = "0.8"
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
serde_yaml = { workspace = true }
|
serde_yaml = { workspace = true }
|
||||||
|
similar.workspace = true
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["full"] }
|
tokio = { workspace = true, features = ["full"] }
|
||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
url.workspace = true
|
||||||
|
|||||||
@@ -2,9 +2,8 @@
|
|||||||
//! instance.
|
//! instance.
|
||||||
//!
|
//!
|
||||||
//! v1 supports the in-cluster Pod form factor (the e2e harness and
|
//! v1 supports the in-cluster Pod form factor (the e2e harness and
|
||||||
//! same-cluster production deploys). The VM/SSH/RPi path stays in
|
//! same-cluster production deploys). [`crate::FleetDeviceSetupScore`]
|
||||||
//! `harmony::modules::fleet::FleetDeviceSetupScore` until a follow-up
|
//! owns the VM/SSH/RPi form factor.
|
||||||
//! migrates it into this crate.
|
|
||||||
//!
|
//!
|
||||||
//! Composition: the Pod target Score interprets to two
|
//! Composition: the Pod target Score interprets to two
|
||||||
//! [`K8sResourceScore`] applies in sequence — a `ConfigMap` carrying
|
//! [`K8sResourceScore`] applies in sequence — a `ConfigMap` carrying
|
||||||
@@ -16,6 +15,7 @@
|
|||||||
//! compile error — that's the point.
|
//! compile error — that's the point.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use harmony::data::Version;
|
use harmony::data::Version;
|
||||||
@@ -217,6 +217,24 @@ impl<T: Topology + K8sclient> Interpret<T> for FleetAgentInterpret {
|
|||||||
source: e,
|
source: e,
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
topology
|
||||||
|
.k8s_client()
|
||||||
|
.await
|
||||||
|
.map_err(InterpretError::new)?
|
||||||
|
.wait_until_deployment_ready(
|
||||||
|
&FleetAgentScore::resource_name(&device_id),
|
||||||
|
Some(&self.score.namespace),
|
||||||
|
Some(Duration::from_secs(120)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
InterpretError::new(format!(
|
||||||
|
"agent deployment {}/{} not ready: {error}",
|
||||||
|
self.score.namespace,
|
||||||
|
FleetAgentScore::resource_name(&device_id)
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
Ok(Outcome::success_with_details(
|
Ok(Outcome::success_with_details(
|
||||||
outcome.message,
|
outcome.message,
|
||||||
vec![
|
vec![
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
//! `harmony-fleet-publish` — build + publish the operator image + chart
|
|
||||||
//! at a version. `docker` / `helm` must be on PATH and logged in to the
|
|
||||||
//! registry (CI's login actions; dev's manual login).
|
|
||||||
|
|
||||||
use anyhow::{Result, bail};
|
|
||||||
use clap::Parser;
|
|
||||||
use harmony_fleet_deploy::release::{release_operator, version_from_tag};
|
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
|
||||||
#[command(
|
|
||||||
name = "harmony-fleet-publish",
|
|
||||||
about = "Build + publish the operator image + chart at a version"
|
|
||||||
)]
|
|
||||||
struct Cli {
|
|
||||||
/// The image + chart version, e.g. `0.1.0` or `0.0.0-dev.1730000000`
|
|
||||||
/// (must be valid semver — helm rejects anything else). Wins over
|
|
||||||
/// `--from-tag`; the laptop/dev path.
|
|
||||||
#[arg(long)]
|
|
||||||
version: Option<String>,
|
|
||||||
|
|
||||||
/// Parse the version out of a release tag like
|
|
||||||
/// `harmony-fleet-operator-v0.1.0`. Defaults to `$GITHUB_REF_NAME`
|
|
||||||
/// so the release workflow passes nothing. Used when `--version`
|
|
||||||
/// is unset.
|
|
||||||
#[arg(long, env = "GITHUB_REF_NAME")]
|
|
||||||
from_tag: Option<String>,
|
|
||||||
|
|
||||||
#[arg(long, default_value = "hub.nationtech.io")]
|
|
||||||
registry: String,
|
|
||||||
|
|
||||||
#[arg(long, default_value = "harmony")]
|
|
||||||
project: String,
|
|
||||||
|
|
||||||
/// Build + package only; skip both pushes (local k3d smoke-test).
|
|
||||||
#[arg(long)]
|
|
||||||
no_push: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Cli {
|
|
||||||
fn version(&self) -> Result<String> {
|
|
||||||
match (&self.version, &self.from_tag) {
|
|
||||||
(Some(v), _) => Ok(v.clone()),
|
|
||||||
(None, Some(tag)) => version_from_tag(tag),
|
|
||||||
(None, None) => bail!("set --version (e.g. 0.1.0) or --from-tag"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() -> Result<()> {
|
|
||||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
|
|
||||||
let cli = Cli::parse();
|
|
||||||
let version = cli.version()?;
|
|
||||||
release_operator(&version, &cli.registry, &cli.project, !cli.no_push)
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
//! Score companions — observe deployed-Score behavior without
|
|
||||||
//! widening the `Score` / `Interpret` trait surface (ADR-023 P7).
|
|
||||||
//!
|
|
||||||
//! A companion takes a Score (or, when the runtime contract lives
|
|
||||||
//! elsewhere, the typed config the Score emits) and exposes a
|
|
||||||
//! derived view a caller cares about — what the agent will watch on
|
|
||||||
//! NATS, which CRDs the operator owns, what the readiness probe
|
|
||||||
//! checks. Companions never call into the framework or topology;
|
|
||||||
//! they're pure functions over data the Score already exposes.
|
|
||||||
//!
|
|
||||||
//! Today: one companion, [`AgentObservation`], used by the e2e
|
|
||||||
//! harness to derive the device-scoped KV watch filter from a typed
|
|
||||||
//! [`harmony_fleet_auth::AgentConfig`]. Future companions for the
|
|
||||||
//! operator's CRD ownership + the readiness/smoke-test contract
|
|
||||||
//! land alongside this module.
|
|
||||||
|
|
||||||
pub mod agent;
|
|
||||||
|
|
||||||
pub use agent::AgentObservation;
|
|
||||||
@@ -11,19 +11,17 @@ use harmony_types::id::Id;
|
|||||||
use log::{debug, info, warn};
|
use log::{debug, info, warn};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::data::Version;
|
use harmony::data::Version;
|
||||||
use crate::domain::interpret::{
|
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
|
||||||
Interpret, InterpretError, InterpretName, InterpretStatus, Outcome,
|
use harmony::inventory::Inventory;
|
||||||
};
|
use harmony::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig};
|
||||||
use crate::domain::inventory::Inventory;
|
use harmony::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials};
|
||||||
use crate::domain::topology::{
|
use harmony::score::Score;
|
||||||
|
use harmony::topology::{
|
||||||
FileDelivery, FileFetcher, FileSource, FileSpec, HostReachable, LinuxHostConfiguration,
|
FileDelivery, FileFetcher, FileSource, FileSpec, HostReachable, LinuxHostConfiguration,
|
||||||
PackageInstaller, SystemdManager, SystemdScope, SystemdUnitSpec, Topology, UnixUserManager,
|
PackageInstaller, SystemdManager, SystemdScope, SystemdUnitSpec, Topology, UnixUserManager,
|
||||||
UserSpec,
|
UserSpec,
|
||||||
};
|
};
|
||||||
use crate::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig};
|
|
||||||
use crate::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials};
|
|
||||||
use crate::score::Score;
|
|
||||||
|
|
||||||
/// User-visible configuration for the setup Score. Everything a customer
|
/// User-visible configuration for the setup Score. Everything a customer
|
||||||
/// needs to tell us to bring a device into the fleet.
|
/// needs to tell us to bring a device into the fleet.
|
||||||
@@ -760,6 +758,23 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
|
|||||||
"✅ running unchanged"
|
"✅ running unchanged"
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut active = false;
|
||||||
|
for _ in 0..30 {
|
||||||
|
if SystemdManager::systemd_unit_active(topology, "fleet-agent", SystemdScope::System)
|
||||||
|
.await
|
||||||
|
.map_err(wrap)?
|
||||||
|
{
|
||||||
|
active = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
if !active {
|
||||||
|
return Err(InterpretError::new(
|
||||||
|
"fleet-agent.service did not become active within 30s".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
// Build human-readable recap. Style matches OKDAddNodeScore:
|
// Build human-readable recap. Style matches OKDAddNodeScore:
|
||||||
// one fact per line, "Key: value" prefix, all the operationally
|
// one fact per line, "Key: value" prefix, all the operationally
|
||||||
// useful state in a single Outcome details block so a CLI run
|
// useful state in a single Outcome details block so a CLI run
|
||||||
@@ -811,7 +826,7 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn wrap(e: crate::executors::ExecutorError) -> InterpretError {
|
fn wrap(e: harmony::executors::ExecutorError) -> InterpretError {
|
||||||
InterpretError::new(e.to_string())
|
InterpretError::new(e.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7,38 +7,16 @@
|
|||||||
//! depend on this crate — they're the targets being deployed, not
|
//! depend on this crate — they're the targets being deployed, not
|
||||||
//! the deployer.
|
//! the deployer.
|
||||||
//!
|
//!
|
||||||
//! Coverage today:
|
//! The binary composes independent component Scores through typed references;
|
||||||
//!
|
//! it does not own provider operations or a Fleet-wide aggregate Score.
|
||||||
//! - [`operator`] — `FleetOperatorScore` (moved out of
|
|
||||||
//! `harmony::modules::fleet::operator`), helm chart generator,
|
|
||||||
//! CRD apply via [`operator::install_crds`].
|
|
||||||
//! - [`nats`] — `FleetNatsScore` (preset over `NatsHelmChartScore`
|
|
||||||
//! with the values the fleet stack needs).
|
|
||||||
//! - [`agent`] — `FleetAgentScore` (Pod-target only in v1; the
|
|
||||||
//! VM/SSH target is `FleetDeviceSetupScore` in `harmony` core,
|
|
||||||
//! slated to move here in a follow-up).
|
|
||||||
//!
|
|
||||||
//! Out of scope for now:
|
|
||||||
//!
|
|
||||||
//! - **Callout** (mock-OIDC fixture + `NatsAuthCalloutScore` wiring) —
|
|
||||||
//! PR 1.5.
|
|
||||||
//! - **Smoke-test contract**. ADR-023 principle 4 says `deploy`
|
|
||||||
//! returns only after a smoke test passes; the trait/struct shape
|
|
||||||
//! for that contract is open. The e2e harness's `ping` test plays
|
|
||||||
//! the role of the smoke test for now.
|
|
||||||
|
|
||||||
pub mod agent;
|
pub mod agent;
|
||||||
pub mod companion;
|
mod device_setup;
|
||||||
pub mod nats;
|
|
||||||
pub mod operator;
|
pub mod operator;
|
||||||
pub mod release;
|
|
||||||
pub mod secrets;
|
|
||||||
pub mod server;
|
|
||||||
|
|
||||||
pub use agent::{FleetAgentScore, PodTarget};
|
pub use agent::{FleetAgentScore, PodTarget};
|
||||||
pub use companion::AgentObservation;
|
pub use device_setup::{
|
||||||
pub use nats::{FleetNatsScore, UserPassCredentials};
|
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
|
||||||
pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart};
|
HostsEntry, merge_hosts_file,
|
||||||
pub use release::{release_operator, version_from_tag};
|
};
|
||||||
pub use secrets::{FleetDeployConfig, FleetDeploySecrets};
|
pub use operator::{FleetOperatorScore, OperatorCredentials};
|
||||||
pub use server::FleetServerScore;
|
|
||||||
|
|||||||
@@ -1,149 +1,153 @@
|
|||||||
//! `harmony-fleet-deploy` — deploy the published fleet operator chart
|
use anyhow::Result;
|
||||||
//! (`harmony apply`). Loads its secrets from config and runs one
|
use async_trait::async_trait;
|
||||||
//! [`FleetOperatorScore`] against [`K8sAnywhereTopology`].
|
use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService};
|
||||||
//!
|
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
||||||
//! The full dev/e2e stack (NATS + agent, user/pass) is composed by the
|
use harmony::modules::openbao::{OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore};
|
||||||
//! e2e harness directly over the `harmony_fleet_deploy` Scores — not
|
use harmony::modules::postgresql::K8sPostgreSQLScore;
|
||||||
//! here. This binary is the prod/CD path: Zitadel-SSO-only, no
|
use harmony::modules::zitadel::{ZitadelAppType, ZitadelScore, ZitadelSetupScore};
|
||||||
//! handrolled manifests.
|
use harmony::score::Score;
|
||||||
|
|
||||||
use std::io::Write;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
|
||||||
use clap::Parser;
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::topology::K8sAnywhereTopology;
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
use harmony_cli::Args as HarmonyCliArgs;
|
use harmony_app::{AppContext, AppError, AppIdentity, HarmonyApp, ImageRefs, ImageSpec, Profile};
|
||||||
use harmony_config::ConfigClient;
|
use harmony_fleet_deploy::FleetOperatorScore;
|
||||||
use harmony_fleet_deploy::{
|
|
||||||
FleetDeployConfig, FleetDeploySecrets, FleetOperatorScore, version_from_tag,
|
|
||||||
};
|
|
||||||
use harmony_zitadel_auth::{OperatorCookieKey, ZitadelAuthConfig};
|
|
||||||
use tracing::info;
|
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
const PROJECT: &str = "fleet";
|
||||||
#[command(
|
const ADMIN_ROLE: &str = "fleet-admin";
|
||||||
name = "harmony-fleet-deploy",
|
const DEVICE_ROLE: &str = "device";
|
||||||
about = "Deploy the published harmony fleet operator chart"
|
const OPERATOR_APP: &str = "harmony-fleet-operator";
|
||||||
)]
|
const OPERATOR_USER: &str = "fleet-operator";
|
||||||
struct CliConfig {
|
const NATS_ACCOUNT: &str = "FLEET";
|
||||||
/// Override the k8s namespace from config (e.g. `fleet-staging`).
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_NAMESPACE")]
|
|
||||||
namespace: Option<String>,
|
|
||||||
|
|
||||||
/// Release tag to deploy (e.g. `harmony-fleet-operator-v0.0.2`); the
|
struct FleetApp;
|
||||||
/// version is parsed from it in Rust so the workflow passes a tag and
|
|
||||||
/// never parses one in YAML. Wins over `--operator-chart-version`.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_TAG")]
|
|
||||||
from_tag: Option<String>,
|
|
||||||
|
|
||||||
/// Bare chart version, for the laptop path.
|
#[async_trait]
|
||||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")]
|
impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
|
||||||
operator_chart_version: Option<String>,
|
fn identity(&self) -> AppIdentity {
|
||||||
|
AppIdentity {
|
||||||
/// Override the OCI chart registry from config.
|
name: "fleet".to_string(),
|
||||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_REGISTRY")]
|
namespace: "fleet-system".to_string(),
|
||||||
operator_chart_registry: Option<String>,
|
|
||||||
|
|
||||||
/// Override the OCI chart project from config.
|
|
||||||
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_PROJECT")]
|
|
||||||
operator_chart_project: Option<String>,
|
|
||||||
|
|
||||||
/// Config namespace `FleetDeploySecrets` and `FleetDeployConfig` resolve under (Env → OpenBao).
|
|
||||||
#[arg(
|
|
||||||
long,
|
|
||||||
env = "HARMONY_CONFIG_NAMESPACE",
|
|
||||||
default_value = "fleet-staging"
|
|
||||||
)]
|
|
||||||
config_namespace: String,
|
|
||||||
|
|
||||||
#[command(flatten)]
|
|
||||||
harmony_cli: HarmonyCliArgs,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CliConfig {
|
|
||||||
fn chart_version(&self) -> Result<String> {
|
|
||||||
match (&self.from_tag, &self.operator_chart_version) {
|
|
||||||
(Some(tag), _) => version_from_tag(tag),
|
|
||||||
(None, Some(v)) => Ok(v.clone()),
|
|
||||||
(None, None) => bail!("set --from-tag or --operator-chart-version"),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
|
||||||
|
Ok(vec![
|
||||||
|
ImageSpec {
|
||||||
|
name: "operator".to_string(),
|
||||||
|
image: ctx.image("harmony-fleet-operator")?,
|
||||||
|
context: ".".into(),
|
||||||
|
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
|
||||||
|
build_args: Vec::new(),
|
||||||
|
},
|
||||||
|
ImageSpec {
|
||||||
|
name: "callout".to_string(),
|
||||||
|
image: ctx.image("harmony-nats-callout")?,
|
||||||
|
context: ".".into(),
|
||||||
|
dockerfile: "nats/callout/Dockerfile".into(),
|
||||||
|
build_args: Vec::new(),
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn scores(
|
||||||
|
&self,
|
||||||
|
ctx: &AppContext,
|
||||||
|
images: &ImageRefs,
|
||||||
|
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
|
||||||
|
let namespace = "fleet-system";
|
||||||
|
let zitadel_namespace = "zitadel";
|
||||||
|
let openbao_namespace = "openbao";
|
||||||
|
let image_pull_secret = ctx.image_pull_secret()?;
|
||||||
|
|
||||||
|
let postgres = K8sPostgreSQLScore::new(zitadel_namespace).cluster_name("zitadel-pg");
|
||||||
|
let database = postgres.root_account_ref();
|
||||||
|
|
||||||
|
let mut zitadel = ZitadelScore::new(
|
||||||
|
ctx.service_host("zitadel", zitadel_namespace)?,
|
||||||
|
zitadel_namespace,
|
||||||
|
)
|
||||||
|
.database(database);
|
||||||
|
if ctx.profile() == Profile::Local {
|
||||||
|
// Q : should that be handled by harmony internally?
|
||||||
|
zitadel = zitadel.http(Some(8080));
|
||||||
|
}
|
||||||
|
let provider = zitadel.provider_ref();
|
||||||
|
|
||||||
|
let identity = ZitadelSetupScore::for_provider(&provider, zitadel_namespace, namespace)
|
||||||
|
.application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
|
||||||
|
.api_application(PROJECT, "nats")
|
||||||
|
.role(PROJECT, ADMIN_ROLE, "Fleet Admin")
|
||||||
|
.role(PROJECT, DEVICE_ROLE, "Device")
|
||||||
|
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
|
||||||
|
.port_forward("zitadel")
|
||||||
|
.groups_claim();
|
||||||
|
let application = identity.application_ref(OPERATOR_APP);
|
||||||
|
let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
|
||||||
|
|
||||||
|
let credentials = match ctx.profile() {
|
||||||
|
// Q: this looks weird. Why do we need to generate creds locally but not in prod? I
|
||||||
|
// would think that in both cases creds should be "ensured". Created if not existing.
|
||||||
|
// Maybe in prod it should use harmony config with a remote openbao and store them there
|
||||||
|
// (as we plan on using external secrets asap)
|
||||||
|
Profile::Local => NatsAuthCalloutCredentialsScore::generated,
|
||||||
|
Profile::Prod => NatsAuthCalloutCredentialsScore::existing,
|
||||||
|
}(namespace, "fleet-callout-credentials");
|
||||||
|
let service = match ctx.profile() {
|
||||||
|
// FIXME this should definitely not surface up to the end user deployment crate. This is
|
||||||
|
// topology dependent and is the topology's job (local k3d vs regular cluster) to decide
|
||||||
|
// how to expose a port and then the score ref should be appropriate
|
||||||
|
Profile::Local => NatsService::NodePort(30422),
|
||||||
|
Profile::Prod => NatsService::LoadBalancer,
|
||||||
|
};
|
||||||
|
// I feel like this should not be a standalone nats score but rather be configuration passed
|
||||||
|
// to the main nats score that is installing the nats cluster
|
||||||
|
let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
|
||||||
|
.with_jetstream_size("2Gi");
|
||||||
|
let account = nats.account_ref();
|
||||||
|
let callout =
|
||||||
|
NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
|
||||||
|
.credentials(&credentials.credentials_ref())
|
||||||
|
.with_oidc(&provider, &application)
|
||||||
|
.image(images.require("callout")?)
|
||||||
|
.image_pull_secret(image_pull_secret)
|
||||||
|
.admin_role(ADMIN_ROLE)
|
||||||
|
.device_role(DEVICE_ROLE)
|
||||||
|
.device_id_claim("client_id");
|
||||||
|
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
|
||||||
|
|
||||||
|
let mut openbao = OpenbaoScore::new(
|
||||||
|
openbao_namespace,
|
||||||
|
"openbao",
|
||||||
|
ctx.service_host("openbao", openbao_namespace)?,
|
||||||
|
);
|
||||||
|
if ctx.profile() == Profile::Prod {
|
||||||
|
openbao = openbao.tls("letsencrypt-prod");
|
||||||
|
}
|
||||||
|
let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
|
||||||
|
&provider,
|
||||||
|
&application,
|
||||||
|
OpenbaoJwtAuth::oidc("fleet-device"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let operator = FleetOperatorScore::new(images.require("operator")?)
|
||||||
|
.namespace(namespace)
|
||||||
|
.image_pull_secret(image_pull_secret)
|
||||||
|
.messaging(&nats.client_ref())
|
||||||
|
.identity(&provider, &application, &operator_identity);
|
||||||
|
|
||||||
|
Ok(vec![
|
||||||
|
Box::new(postgres),
|
||||||
|
Box::new(zitadel),
|
||||||
|
Box::new(credentials),
|
||||||
|
Box::new(nats),
|
||||||
|
Box::new(identity),
|
||||||
|
Box::new(callout),
|
||||||
|
Box::new(openbao),
|
||||||
|
Box::new(openbao_setup),
|
||||||
|
Box::new(operator),
|
||||||
|
])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
harmony_cli::cli_logger::init();
|
harmony_cli::app::app_main(FleetApp).await
|
||||||
let cli = CliConfig::parse();
|
|
||||||
let version = cli.chart_version()?;
|
|
||||||
|
|
||||||
let config_client = ConfigClient::for_namespace(&cli.config_namespace).await;
|
|
||||||
|
|
||||||
let secrets: FleetDeploySecrets = config_client
|
|
||||||
.get()
|
|
||||||
.await
|
|
||||||
.context("loading FleetDeploySecrets (set HARMONY_CONFIG_FleetDeploySecrets or OpenBao)")?;
|
|
||||||
|
|
||||||
let config: FleetDeployConfig = config_client
|
|
||||||
.get_or_prompt()
|
|
||||||
.await
|
|
||||||
.context("loading FleetDeployConfig")?;
|
|
||||||
|
|
||||||
// Point KUBECONFIG at the scoped deployer credential before the
|
|
||||||
// topology reads it, so the runner pod needs no standing permissions.
|
|
||||||
// Held to end of scope so the tempfile outlives the deploy.
|
|
||||||
let _kubeconfig = match &secrets.kubeconfig {
|
|
||||||
Some(kubeconfig) => {
|
|
||||||
let mut f = tempfile::NamedTempFile::new().context("kubeconfig tempfile")?;
|
|
||||||
f.write_all(kubeconfig.as_bytes())
|
|
||||||
.context("writing kubeconfig")?;
|
|
||||||
// SAFETY: single-threaded startup, before any topology reads it.
|
|
||||||
unsafe { std::env::set_var("KUBECONFIG", f.path()) };
|
|
||||||
Some(f)
|
|
||||||
}
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let namespace = cli.namespace.unwrap_or(config.namespace);
|
|
||||||
let registry = cli
|
|
||||||
.operator_chart_registry
|
|
||||||
.unwrap_or(config.operator_chart_registry);
|
|
||||||
let project = cli
|
|
||||||
.operator_chart_project
|
|
||||||
.unwrap_or(config.operator_chart_project);
|
|
||||||
|
|
||||||
// Coherent with the other staging hosts (sso-stg., secrets-stg.).
|
|
||||||
let ui_host = format!("fleet-stg.{}", config.base_domain);
|
|
||||||
|
|
||||||
// Dashboard SSO config: hosts derive from base_domain; client_id +
|
|
||||||
// audiences come from config, cookie key from secrets. Baked into the
|
|
||||||
// operator Secret for the pod's ConfigClient to read.
|
|
||||||
let web_auth = ZitadelAuthConfig {
|
|
||||||
zitadel_base: format!("https://sso-stg.{}", config.base_domain),
|
|
||||||
base_url: format!("https://{ui_host}"),
|
|
||||||
client_id: config.operator_oidc_client_id,
|
|
||||||
scope: "openid profile email".to_string(),
|
|
||||||
trusted_audiences: config.operator_trusted_audiences,
|
|
||||||
logout_redirect_uri: format!("https://{ui_host}/"),
|
|
||||||
};
|
|
||||||
let cookie = OperatorCookieKey {
|
|
||||||
cookie_key_b64: secrets.operator_cookie_key_b64,
|
|
||||||
};
|
|
||||||
|
|
||||||
let operator = FleetOperatorScore::new()
|
|
||||||
.namespace(namespace)
|
|
||||||
.nats_url(config.nats_url)
|
|
||||||
.credentials(secrets.operator_credentials_toml)
|
|
||||||
.published_chart(registry, project, version)
|
|
||||||
.ingress(ui_host, Some(config.cluster_issuer))
|
|
||||||
.web_auth(web_auth, cookie);
|
|
||||||
|
|
||||||
harmony_cli::run(
|
|
||||||
Inventory::autoload(),
|
|
||||||
K8sAnywhereTopology::from_env(),
|
|
||||||
vec![Box::new(operator)],
|
|
||||||
Some(cli.harmony_cli),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
//! operator Deployment into a helm chart the cluster runs itself.
|
//! operator Deployment into a helm chart the cluster runs itself.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::hash::{DefaultHasher, Hash, Hasher};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
@@ -25,7 +26,8 @@ use k8s_openapi::api::apps::v1::{
|
|||||||
};
|
};
|
||||||
use k8s_openapi::api::core::v1::{
|
use k8s_openapi::api::core::v1::{
|
||||||
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile,
|
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile,
|
||||||
Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec,
|
Secret, SecretKeySelector, SecretVolumeSource, SecurityContext, Service, ServiceAccount,
|
||||||
|
ServicePort, ServiceSpec, Volume, VolumeMount,
|
||||||
};
|
};
|
||||||
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
|
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
|
||||||
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
||||||
@@ -36,8 +38,9 @@ use kube::api::ObjectMeta;
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
|
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
|
||||||
use harmony::modules::fleet::operator::crd::{Deployment, Device};
|
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
|
||||||
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
||||||
|
use harmony_fleet_operator::{Deployment, Device};
|
||||||
|
|
||||||
/// Inputs for chart generation. Default values are aimed at a
|
/// Inputs for chart generation. Default values are aimed at a
|
||||||
/// local-dev k3d install; override via the `chart` subcommand flags.
|
/// local-dev k3d install; override via the `chart` subcommand flags.
|
||||||
@@ -77,56 +80,24 @@ pub struct ChartOptions {
|
|||||||
/// JSON of the dashboard's `OperatorCookieKey`, stored under
|
/// JSON of the dashboard's `OperatorCookieKey`, stored under
|
||||||
/// [`ENV_WEB_COOKIE_KEY`].
|
/// [`ENV_WEB_COOKIE_KEY`].
|
||||||
pub web_cookie_key_json: Option<String>,
|
pub web_cookie_key_json: Option<String>,
|
||||||
|
pub identity: Option<OperatorIdentityRefs>,
|
||||||
|
pub identity_version: Option<String>,
|
||||||
|
pub image_pull_secret: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What the operator pod needs to authenticate to NATS via the auth
|
#[derive(Debug, Clone, Serialize)]
|
||||||
/// callout: a TOML snippet matching the agent's `[credentials]`
|
pub struct OperatorIdentityRefs {
|
||||||
/// table.
|
pub provider: OidcProviderRef,
|
||||||
///
|
pub application: OidcApplicationRef,
|
||||||
/// **Single Secret key**: the credentials TOML is the only payload.
|
pub machine: OidcMachineIdentityRef,
|
||||||
/// The Zitadel JSON keyfile content is embedded inline inside the
|
}
|
||||||
/// TOML as the `key_json` field (multi-line triple-quoted string).
|
|
||||||
/// One env-var-from-Secret on the operator Pod
|
/// Static NATS credentials for deployments without typed OIDC identity refs.
|
||||||
/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume
|
|
||||||
/// mounts, OKD restricted-v2 SCC compatible.
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct OperatorCredentials {
|
pub struct OperatorCredentials {
|
||||||
/// TOML payload, e.g.
|
|
||||||
/// ```text
|
|
||||||
/// type = "zitadel-jwt"
|
|
||||||
/// oidc_issuer_url = "http://sso.fleet.local:8080"
|
|
||||||
/// audience = "<project_id>"
|
|
||||||
/// key_json = '''{"type":"serviceaccount","keyId":"...","key":"-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----\n","userId":"..."}'''
|
|
||||||
/// ```
|
|
||||||
///
|
|
||||||
/// `key_json` **must** use TOML *literal* multi-line strings
|
|
||||||
/// (`'''...'''`), NOT *basic* multi-line strings (`"""..."""`).
|
|
||||||
/// Basic strings interpret backslash escapes — every `\n` in the
|
|
||||||
/// embedded RSA private key would expand to a literal 0x0A before
|
|
||||||
/// JSON parsing, and JSON rejects raw control chars inside strings
|
|
||||||
/// ("control character … found while parsing a string"). Literal
|
|
||||||
/// strings preserve `\n` as the two chars `\\` + `n`, the JSON
|
|
||||||
/// parser then interprets that as a string escape and decodes the
|
|
||||||
/// multi-line PEM correctly.
|
|
||||||
pub credentials_toml: String,
|
pub credentials_toml: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl OperatorCredentials {
|
|
||||||
/// Build the auth-callout credentials from a Zitadel machine key.
|
|
||||||
/// `key_json` goes in a TOML *literal* string — see the field docs
|
|
||||||
/// for why basic strings corrupt the embedded PEM.
|
|
||||||
pub fn zitadel_jwt(oidc_issuer_url: &str, audience: &str, key_json: &str) -> Self {
|
|
||||||
Self {
|
|
||||||
credentials_toml: format!(
|
|
||||||
"type = \"zitadel-jwt\"\n\
|
|
||||||
oidc_issuer_url = \"{oidc_issuer_url}\"\n\
|
|
||||||
audience = \"{audience}\"\n\
|
|
||||||
key_json = '''{key_json}'''\n"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for ChartOptions {
|
impl Default for ChartOptions {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -143,6 +114,9 @@ impl Default for ChartOptions {
|
|||||||
chart_version: None,
|
chart_version: None,
|
||||||
web_auth_config_json: None,
|
web_auth_config_json: None,
|
||||||
web_cookie_key_json: None,
|
web_cookie_key_json: None,
|
||||||
|
identity: None,
|
||||||
|
identity_version: None,
|
||||||
|
image_pull_secret: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -401,12 +375,32 @@ fn operator_service() -> Service {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn config_hash(opts: &ChartOptions) -> String {
|
||||||
|
let mut secret_hash = DefaultHasher::new();
|
||||||
|
opts.credentials
|
||||||
|
.as_ref()
|
||||||
|
.map(|credentials| &credentials.credentials_toml)
|
||||||
|
.hash(&mut secret_hash);
|
||||||
|
opts.web_auth_config_json.hash(&mut secret_hash);
|
||||||
|
opts.web_cookie_key_json.hash(&mut secret_hash);
|
||||||
|
opts.identity
|
||||||
|
.as_ref()
|
||||||
|
.map(|identity| identity.machine.secret_name())
|
||||||
|
.hash(&mut secret_hash);
|
||||||
|
opts.identity_version.hash(&mut secret_hash);
|
||||||
|
format!("{:x}", secret_hash.finish())
|
||||||
|
}
|
||||||
|
|
||||||
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
||||||
let mut match_labels = BTreeMap::new();
|
let mut match_labels = BTreeMap::new();
|
||||||
match_labels.insert(
|
match_labels.insert(
|
||||||
"app.kubernetes.io/name".to_string(),
|
"app.kubernetes.io/name".to_string(),
|
||||||
RELEASE_NAME.to_string(),
|
RELEASE_NAME.to_string(),
|
||||||
);
|
);
|
||||||
|
let pod_annotations = BTreeMap::from([(
|
||||||
|
"harmony.nationtech.io/config-hash".to_string(),
|
||||||
|
config_hash(opts),
|
||||||
|
)]);
|
||||||
|
|
||||||
let mut env = vec![
|
let mut env = vec![
|
||||||
EnvVar {
|
EnvVar {
|
||||||
@@ -420,6 +414,32 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
if let Some(identity) = &opts.identity {
|
||||||
|
env.extend([
|
||||||
|
EnvVar {
|
||||||
|
name: "FLEET_OPERATOR_OIDC_ISSUER".to_string(),
|
||||||
|
value: Some(identity.provider.issuer().to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
EnvVar {
|
||||||
|
name: "FLEET_OPERATOR_OIDC_AUDIENCE".to_string(),
|
||||||
|
value_from: Some(EnvVarSource {
|
||||||
|
config_map_key_ref: Some(k8s_openapi::api::core::v1::ConfigMapKeySelector {
|
||||||
|
name: identity.application.config_map_name().to_string(),
|
||||||
|
key: identity.application.project_id_key().to_string(),
|
||||||
|
optional: Some(false),
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
EnvVar {
|
||||||
|
name: "FLEET_OPERATOR_OIDC_KEY_PATH".to_string(),
|
||||||
|
value: Some("/var/run/secrets/fleet/key.json".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
// The whole `[credentials]` TOML payload — including the JSON
|
// The whole `[credentials]` TOML payload — including the JSON
|
||||||
// keyfile inlined under `key_json` — travels as a single env
|
// keyfile inlined under `key_json` — travels as a single env
|
||||||
@@ -490,18 +510,47 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
|||||||
template: PodTemplateSpec {
|
template: PodTemplateSpec {
|
||||||
metadata: Some(ObjectMeta {
|
metadata: Some(ObjectMeta {
|
||||||
labels: Some(match_labels),
|
labels: Some(match_labels),
|
||||||
|
annotations: Some(pod_annotations),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
spec: Some(PodSpec {
|
spec: Some(PodSpec {
|
||||||
service_account_name: Some(SERVICE_ACCOUNT.to_string()),
|
service_account_name: Some(SERVICE_ACCOUNT.to_string()),
|
||||||
|
image_pull_secrets: opts.image_pull_secret.as_ref().map(|name| {
|
||||||
|
vec![k8s_openapi::api::core::v1::LocalObjectReference {
|
||||||
|
name: name.clone(),
|
||||||
|
}]
|
||||||
|
}),
|
||||||
containers: vec![Container {
|
containers: vec![Container {
|
||||||
name: "operator".to_string(),
|
name: "operator".to_string(),
|
||||||
image: Some(opts.image.clone()),
|
image: Some(opts.image.clone()),
|
||||||
image_pull_policy: Some(opts.image_pull_policy.clone()),
|
image_pull_policy: Some(opts.image_pull_policy.clone()),
|
||||||
env: Some(env),
|
env: Some(env),
|
||||||
|
volume_mounts: opts.identity.as_ref().map(|_| {
|
||||||
|
vec![VolumeMount {
|
||||||
|
name: "operator-identity".to_string(),
|
||||||
|
mount_path: "/var/run/secrets/fleet".to_string(),
|
||||||
|
read_only: Some(true),
|
||||||
|
..Default::default()
|
||||||
|
}]
|
||||||
|
}),
|
||||||
security_context: Some(container_security_context()),
|
security_context: Some(container_security_context()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}],
|
}],
|
||||||
|
volumes: opts.identity.as_ref().map(|identity| {
|
||||||
|
vec![Volume {
|
||||||
|
name: "operator-identity".to_string(),
|
||||||
|
secret: Some(SecretVolumeSource {
|
||||||
|
secret_name: Some(identity.machine.secret_name().to_string()),
|
||||||
|
items: Some(vec![k8s_openapi::api::core::v1::KeyToPath {
|
||||||
|
key: identity.machine.key_json_key().to_string(),
|
||||||
|
path: "key.json".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}]),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}]
|
||||||
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
@@ -511,27 +560,6 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-export the manifest builders so the e2e bring-up can apply the
|
|
||||||
// operator inline (Score-style) without re-implementing the manifests.
|
|
||||||
//
|
|
||||||
// The SA + Deployment helpers return namespace-neutral manifests;
|
|
||||||
// callers inject the target namespace through `K8sResourceScore::single`.
|
|
||||||
// The CRB takes the SA's namespace as an explicit argument because the
|
|
||||||
// CRB subject must reference a concrete namespace — there's no
|
|
||||||
// kube-side "current namespace" for cluster-scoped resources.
|
|
||||||
pub fn build_service_account() -> ServiceAccount {
|
|
||||||
service_account()
|
|
||||||
}
|
|
||||||
pub fn build_cluster_role() -> ClusterRole {
|
|
||||||
cluster_role()
|
|
||||||
}
|
|
||||||
pub fn build_cluster_role_binding(subject_namespace: &str) -> ClusterRoleBinding {
|
|
||||||
cluster_role_binding(subject_namespace)
|
|
||||||
}
|
|
||||||
pub fn build_operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
|
||||||
operator_deployment(opts)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Minimum-privilege container security context.
|
/// Minimum-privilege container security context.
|
||||||
///
|
///
|
||||||
/// - `runAsNonRoot: true` — a compromised operator pod with
|
/// - `runAsNonRoot: true` — a compromised operator pod with
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
//! Install the operator's CRDs into a target Kubernetes cluster via
|
|
||||||
//! a harmony Score.
|
|
||||||
//!
|
|
||||||
//! Moved from `fleet/harmony-fleet-operator/src/install.rs` per
|
|
||||||
//! ADR-023: this is deploy-side logic, so it lives in the deploy
|
|
||||||
//! crate, not in the operator runtime binary. The runtime binary's
|
|
||||||
//! `Install` subcommand is gone; users invoke `harmony-fleet-deploy
|
|
||||||
//! operator-install-crds` (or compose [`install_crds`] from another
|
|
||||||
//! deploy flow) instead.
|
|
||||||
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::fleet::operator::crd::{Deployment, Device};
|
|
||||||
use harmony::modules::k8s::K8sBareTopology;
|
|
||||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
|
||||||
use harmony::score::Score;
|
|
||||||
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
|
||||||
use kube::CustomResourceExt;
|
|
||||||
use thiserror::Error;
|
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum InstallCrdsError {
|
|
||||||
#[error("building K8sBareTopology from KUBECONFIG: {0}")]
|
|
||||||
Topology(String),
|
|
||||||
#[error("executing K8sResourceScore for Deployment + Device CRDs: {0}")]
|
|
||||||
Interpret(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Apply the operator's CRDs to whatever cluster `KUBECONFIG` points
|
|
||||||
/// at. Returns once the apply call completes — does **not** wait for
|
|
||||||
/// the apiserver to mark each CRD `Established`; the caller does that
|
|
||||||
/// (e.g. with `kubectl wait --for=condition=Established`) if it
|
|
||||||
/// cares.
|
|
||||||
pub async fn install_crds() -> Result<(), InstallCrdsError> {
|
|
||||||
let topology = K8sBareTopology::from_kubeconfig("harmony-fleet-operator-install")
|
|
||||||
.await
|
|
||||||
.map_err(InstallCrdsError::Topology)?;
|
|
||||||
let inventory = Inventory::empty();
|
|
||||||
|
|
||||||
let crds: Vec<CustomResourceDefinition> = vec![Deployment::crd(), Device::crd()];
|
|
||||||
let score = K8sResourceScore::<CustomResourceDefinition> {
|
|
||||||
resource: crds,
|
|
||||||
namespace: None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let interpret = Score::<K8sBareTopology>::create_interpret(&score);
|
|
||||||
let outcome = interpret
|
|
||||||
.execute(&inventory, &topology)
|
|
||||||
.await
|
|
||||||
.map_err(|e| InstallCrdsError::Interpret(e.to_string()))?;
|
|
||||||
|
|
||||||
tracing::info!(?outcome, "CRDs installed");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@@ -3,17 +3,12 @@
|
|||||||
//!
|
//!
|
||||||
//! Lives in `harmony-fleet-deploy` per ADR-023: deploy logic for any
|
//! Lives in `harmony-fleet-deploy` per ADR-023: deploy logic for any
|
||||||
//! fleet component sits in this crate, not in `harmony` core. The
|
//! fleet component sits in this crate, not in `harmony` core. The
|
||||||
//! operator's CRD types ([`harmony::modules::fleet::operator::crd`])
|
//! operator's CRD types ([`harmony_fleet_operator::crd`])
|
||||||
//! stay in `harmony` for the moment — a follow-up PR finishes lifting
|
//! stay in `harmony` for the moment — a follow-up PR finishes lifting
|
||||||
//! them out — but the *Score* + chart generator are home here.
|
//! them out — but the *Score* + chart generator are home here.
|
||||||
|
|
||||||
pub mod chart;
|
pub(crate) mod chart;
|
||||||
pub mod install;
|
|
||||||
pub mod score;
|
pub mod score;
|
||||||
|
|
||||||
pub use chart::{
|
pub use chart::OperatorCredentials;
|
||||||
ChartOptions, OperatorCredentials, RELEASE_NAME, SECRET_KEY_CREDENTIALS_TOML, SECRET_NAME,
|
pub use score::FleetOperatorScore;
|
||||||
build_chart, operator_secret,
|
|
||||||
};
|
|
||||||
pub use install::install_crds;
|
|
||||||
pub use score::{FleetOperatorScore, PublishedChart};
|
|
||||||
|
|||||||
@@ -18,12 +18,12 @@
|
|||||||
//! use harmony::inventory::Inventory;
|
//! use harmony::inventory::Inventory;
|
||||||
//!
|
//!
|
||||||
//! let topology = K8sBareTopology::from_kubeconfig("fleet-operator").await?;
|
//! let topology = K8sBareTopology::from_kubeconfig("fleet-operator").await?;
|
||||||
//! let score = FleetOperatorScore::new()
|
//! let score = FleetOperatorScore::new("hub.nationtech.io/harmony/harmony-fleet-operator:dev")
|
||||||
//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev");
|
//! .messaging(&nats.client_ref());
|
||||||
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
|
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
use std::str::FromStr;
|
use std::{str::FromStr, time::Duration};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
@@ -36,22 +36,17 @@ use harmony::inventory::Inventory;
|
|||||||
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
|
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
|
||||||
use harmony::modules::k8s::ingress::K8sIngressScore;
|
use harmony::modules::k8s::ingress::K8sIngressScore;
|
||||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
use harmony::modules::k8s::resource::K8sResourceScore;
|
||||||
|
use harmony::modules::nats::NatsClientRef;
|
||||||
|
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::{HelmCommand, K8sclient, Topology};
|
use harmony::topology::{HelmCommand, K8sclient, Topology};
|
||||||
|
use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret};
|
||||||
|
use kube::api::ListParams;
|
||||||
|
|
||||||
use crate::operator::chart;
|
use crate::operator::chart;
|
||||||
use crate::operator::chart::{ChartOptions, OperatorCredentials, build_chart, operator_secret};
|
use crate::operator::chart::{
|
||||||
|
ChartOptions, OperatorCredentials, OperatorIdentityRefs, build_chart, operator_secret,
|
||||||
/// The already-published OCI chart to install (the CD `harmony apply`
|
};
|
||||||
/// path). When set, the operator installs
|
|
||||||
/// `oci://{registry}/{project}/harmony-fleet-operator-chart:{version}` and the
|
|
||||||
/// score's `image` is ignored (the image is baked into the chart).
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub struct PublishedChart {
|
|
||||||
pub registry: String,
|
|
||||||
pub project: String,
|
|
||||||
pub version: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Declarative install of the harmony fleet operator. Construct via
|
/// Declarative install of the harmony fleet operator. Construct via
|
||||||
/// [`new`](Self::new), tune with the builder-style methods, hand to
|
/// [`new`](Self::new), tune with the builder-style methods, hand to
|
||||||
@@ -66,12 +61,9 @@ pub struct FleetOperatorScore {
|
|||||||
pub release_name: String,
|
pub release_name: String,
|
||||||
pub image: String,
|
pub image: String,
|
||||||
pub image_pull_policy: String,
|
pub image_pull_policy: String,
|
||||||
pub nats_url: String,
|
nats: Option<NatsClientRef>,
|
||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
pub credentials: Option<OperatorCredentials>,
|
pub credentials: Option<OperatorCredentials>,
|
||||||
/// `None` renders + installs the chart from local source (dev/e2e);
|
|
||||||
/// `Some` installs the published OCI chart at a pinned version (CD).
|
|
||||||
pub published_chart: Option<PublishedChart>,
|
|
||||||
/// `Some(host)` exposes the UI via an Ingress after install (CD);
|
/// `Some(host)` exposes the UI via an Ingress after install (CD);
|
||||||
/// `None` leaves it cluster-internal (dev/e2e harnesses).
|
/// `None` leaves it cluster-internal (dev/e2e harnesses).
|
||||||
pub operator_ui_host: Option<String>,
|
pub operator_ui_host: Option<String>,
|
||||||
@@ -82,6 +74,8 @@ pub struct FleetOperatorScore {
|
|||||||
/// for the pod's `ConfigClient` to read. `None` leaves the dashboard
|
/// for the pod's `ConfigClient` to read. `None` leaves the dashboard
|
||||||
/// unauthenticated (dev/e2e).
|
/// unauthenticated (dev/e2e).
|
||||||
pub web_auth: Option<WebAuth>,
|
pub web_auth: Option<WebAuth>,
|
||||||
|
pub identity: Option<OperatorIdentityRefs>,
|
||||||
|
pub image_pull_secret: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The dashboard's auth inputs the operator reads via `ConfigClient`.
|
/// The dashboard's auth inputs the operator reads via `ConfigClient`.
|
||||||
@@ -94,22 +88,23 @@ pub struct WebAuth {
|
|||||||
impl FleetOperatorScore {
|
impl FleetOperatorScore {
|
||||||
/// Build a score targeting the `fleet-system` namespace with the
|
/// Build a score targeting the `fleet-system` namespace with the
|
||||||
/// dev-default image and NATS URL. Use the builders to override.
|
/// dev-default image and NATS URL. Use the builders to override.
|
||||||
pub fn new() -> Self {
|
pub fn new(image: impl Into<String>) -> Self {
|
||||||
let defaults = ChartOptions::default();
|
let defaults = ChartOptions::default();
|
||||||
Self {
|
Self {
|
||||||
// FleetOperatorScore's own default; the chart itself is
|
// FleetOperatorScore's own default; the chart itself is
|
||||||
// namespace-neutral. Callers override via `.namespace(..)`.
|
// namespace-neutral. Callers override via `.namespace(..)`.
|
||||||
namespace: "fleet-system".to_string(),
|
namespace: "fleet-system".to_string(),
|
||||||
release_name: "harmony-fleet-operator".to_string(),
|
release_name: "harmony-fleet-operator".to_string(),
|
||||||
image: defaults.image,
|
image: image.into(),
|
||||||
image_pull_policy: defaults.image_pull_policy,
|
image_pull_policy: defaults.image_pull_policy,
|
||||||
nats_url: defaults.nats_url,
|
nats: None,
|
||||||
log_level: defaults.log_level,
|
log_level: defaults.log_level,
|
||||||
credentials: defaults.credentials,
|
credentials: defaults.credentials,
|
||||||
published_chart: None,
|
|
||||||
operator_ui_host: None,
|
operator_ui_host: None,
|
||||||
cluster_issuer: None,
|
cluster_issuer: None,
|
||||||
web_auth: None,
|
web_auth: None,
|
||||||
|
identity: None,
|
||||||
|
image_pull_secret: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,22 +128,6 @@ impl FleetOperatorScore {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Install the published OCI chart at `version` instead of rendering
|
|
||||||
/// one from local source (the CD `harmony apply` path).
|
|
||||||
pub fn published_chart(
|
|
||||||
mut self,
|
|
||||||
registry: impl Into<String>,
|
|
||||||
project: impl Into<String>,
|
|
||||||
version: impl Into<String>,
|
|
||||||
) -> Self {
|
|
||||||
self.published_chart = Some(PublishedChart {
|
|
||||||
registry: registry.into(),
|
|
||||||
project: project.into(),
|
|
||||||
version: version.into(),
|
|
||||||
});
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn namespace(mut self, ns: impl Into<String>) -> Self {
|
pub fn namespace(mut self, ns: impl Into<String>) -> Self {
|
||||||
self.namespace = ns.into();
|
self.namespace = ns.into();
|
||||||
self
|
self
|
||||||
@@ -159,18 +138,32 @@ impl FleetOperatorScore {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn image(mut self, image: impl Into<String>) -> Self {
|
|
||||||
self.image = image.into();
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn image_pull_policy(mut self, policy: impl Into<String>) -> Self {
|
pub fn image_pull_policy(mut self, policy: impl Into<String>) -> Self {
|
||||||
self.image_pull_policy = policy.into();
|
self.image_pull_policy = policy.into();
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn nats_url(mut self, url: impl Into<String>) -> Self {
|
pub fn messaging(mut self, nats: &NatsClientRef) -> Self {
|
||||||
self.nats_url = url.into();
|
self.nats = Some(nats.clone());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn identity(
|
||||||
|
mut self,
|
||||||
|
provider: &OidcProviderRef,
|
||||||
|
application: &OidcApplicationRef,
|
||||||
|
machine: &OidcMachineIdentityRef,
|
||||||
|
) -> Self {
|
||||||
|
self.identity = Some(OperatorIdentityRefs {
|
||||||
|
provider: provider.clone(),
|
||||||
|
application: application.clone(),
|
||||||
|
machine: machine.clone(),
|
||||||
|
});
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self {
|
||||||
|
self.image_pull_secret = secret.map(str::to_string);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -190,12 +183,6 @@ impl FleetOperatorScore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for FleetOperatorScore {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Topology + HelmCommand + K8sclient> Score<T> for FleetOperatorScore {
|
impl<T: Topology + HelmCommand + K8sclient> Score<T> for FleetOperatorScore {
|
||||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||||
Box::new(FleetOperatorInterpret {
|
Box::new(FleetOperatorInterpret {
|
||||||
@@ -213,6 +200,51 @@ pub struct FleetOperatorInterpret {
|
|||||||
score: FleetOperatorScore,
|
score: FleetOperatorScore,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn smoke_test_operator<T: K8sclient>(
|
||||||
|
namespace: &str,
|
||||||
|
expected_config_hash: &str,
|
||||||
|
topology: &T,
|
||||||
|
) -> Result<(), InterpretError> {
|
||||||
|
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
|
||||||
|
tokio::time::timeout(Duration::from_secs(180), async {
|
||||||
|
loop {
|
||||||
|
let pods = k8s
|
||||||
|
.list_resources::<Pod>(
|
||||||
|
Some(namespace),
|
||||||
|
Some(
|
||||||
|
ListParams::default()
|
||||||
|
.labels(&format!("app.kubernetes.io/name={}", chart::RELEASE_NAME)),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if let Ok(pods) = pods {
|
||||||
|
for pod in pods.items {
|
||||||
|
let has_expected_config = pod
|
||||||
|
.metadata
|
||||||
|
.annotations
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|annotations| {
|
||||||
|
annotations.get("harmony.nationtech.io/config-hash")
|
||||||
|
})
|
||||||
|
.is_some_and(|hash| hash == expected_config_hash);
|
||||||
|
if let Some(name) = pod.metadata.name
|
||||||
|
&& has_expected_config
|
||||||
|
&& k8s
|
||||||
|
.pod_logs(namespace, &name, Some(100))
|
||||||
|
.await
|
||||||
|
.is_ok_and(|logs| logs.contains("KV bucket ready"))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterpret {
|
impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterpret {
|
||||||
async fn execute(
|
async fn execute(
|
||||||
@@ -220,6 +252,74 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
|
|||||||
inventory: &Inventory,
|
inventory: &Inventory,
|
||||||
topology: &T,
|
topology: &T,
|
||||||
) -> Result<Outcome, InterpretError> {
|
) -> Result<Outcome, InterpretError> {
|
||||||
|
let nats_url = self
|
||||||
|
.score
|
||||||
|
.nats
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| InterpretError::new("NatsClientRef is required".to_string()))?
|
||||||
|
.url();
|
||||||
|
if let Some(identity) = &self.score.identity
|
||||||
|
&& (identity.application.namespace() != self.score.namespace
|
||||||
|
|| identity.machine.namespace() != self.score.namespace)
|
||||||
|
{
|
||||||
|
return Err(InterpretError::new(format!(
|
||||||
|
"operator identity outputs must be in namespace '{}'",
|
||||||
|
self.score.namespace
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
|
||||||
|
k8s.ensure_namespace(&self.score.namespace)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
InterpretError::new(format!(
|
||||||
|
"failed to ensure operator namespace '{}': {e}",
|
||||||
|
self.score.namespace
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let identity_version = if let Some(identity) = &self.score.identity {
|
||||||
|
Some(
|
||||||
|
tokio::time::timeout(Duration::from_secs(180), async {
|
||||||
|
loop {
|
||||||
|
let application = k8s
|
||||||
|
.get_resource::<ConfigMap>(
|
||||||
|
identity.application.config_map_name(),
|
||||||
|
Some(identity.application.namespace()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
|
let machine = k8s
|
||||||
|
.get_resource::<Secret>(
|
||||||
|
identity.machine.secret_name(),
|
||||||
|
Some(identity.machine.namespace()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
|
if let (Some(application), Some(machine)) = (application, machine)
|
||||||
|
&& let Some(project_id) = application
|
||||||
|
.data
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|data| data.get(identity.application.project_id_key()))
|
||||||
|
{
|
||||||
|
return format!(
|
||||||
|
"{}:{}",
|
||||||
|
project_id,
|
||||||
|
machine.metadata.resource_version.unwrap_or_default()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
InterpretError::new("timed out waiting for operator identity refs".to_string())
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let credentials = self.score.credentials.clone();
|
||||||
// Apply the credentials Secret BEFORE the helm install (the
|
// Apply the credentials Secret BEFORE the helm install (the
|
||||||
// chart's Deployment references it via secretKeyRef). Applied
|
// chart's Deployment references it via secretKeyRef). Applied
|
||||||
// directly, not via the chart — it's environment-specific. The
|
// directly, not via the chart — it's environment-specific. The
|
||||||
@@ -236,11 +336,23 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
|
|||||||
),
|
),
|
||||||
None => (None, None),
|
None => (None, None),
|
||||||
};
|
};
|
||||||
if let Some(creds) = &self.score.credentials
|
let expected_config_hash = chart::config_hash(&ChartOptions {
|
||||||
|
credentials: credentials.clone(),
|
||||||
|
web_auth_config_json: web_auth_config_json.clone(),
|
||||||
|
web_cookie_key_json: web_cookie_key_json.clone(),
|
||||||
|
identity: self.score.identity.clone(),
|
||||||
|
identity_version: identity_version.clone(),
|
||||||
|
image_pull_secret: self.score.image_pull_secret.clone(),
|
||||||
|
..ChartOptions::default()
|
||||||
|
});
|
||||||
|
if let Some(creds) = &credentials
|
||||||
&& let Some(secret) = operator_secret(&ChartOptions {
|
&& let Some(secret) = operator_secret(&ChartOptions {
|
||||||
credentials: Some(creds.clone()),
|
credentials: Some(creds.clone()),
|
||||||
web_auth_config_json,
|
web_auth_config_json: web_auth_config_json.clone(),
|
||||||
web_cookie_key_json,
|
web_cookie_key_json: web_cookie_key_json.clone(),
|
||||||
|
identity: self.score.identity.clone(),
|
||||||
|
identity_version: identity_version.clone(),
|
||||||
|
image_pull_secret: self.score.image_pull_secret.clone(),
|
||||||
..ChartOptions::default()
|
..ChartOptions::default()
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
@@ -254,69 +366,57 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Install through HelmChartScore from either the published OCI
|
let tmp = tempfile::tempdir()
|
||||||
// chart (CD) or a freshly rendered local chart (dev/e2e). Each
|
.map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?;
|
||||||
// branch runs its own install so the local tempdir stays alive
|
let chart_path = build_chart(&ChartOptions {
|
||||||
// across it.
|
output_dir: tmp.path().to_path_buf(),
|
||||||
let helm_outcome = if let Some(p) = &self.score.published_chart {
|
image: self.score.image.clone(),
|
||||||
let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, chart::CHART_NAME);
|
image_pull_policy: self.score.image_pull_policy.clone(),
|
||||||
info!(
|
nats_url: nats_url.to_string(),
|
||||||
"Installing helm release '{}' from published chart {chart_ref} version {}",
|
log_level: self.score.log_level.clone(),
|
||||||
self.score.release_name, p.version
|
credentials: credentials.clone(),
|
||||||
);
|
chart_version: None,
|
||||||
HelmChartScore {
|
web_auth_config_json: web_auth_config_json.clone(),
|
||||||
namespace: Some(non_blank(&self.score.namespace, "namespace")?),
|
web_cookie_key_json: web_cookie_key_json.clone(),
|
||||||
release_name: non_blank(&self.score.release_name, "release_name")?,
|
identity: self.score.identity.clone(),
|
||||||
chart_name: non_blank(&chart_ref, "chart_name")?,
|
identity_version,
|
||||||
chart_version: Some(non_blank(&p.version, "chart_version")?),
|
image_pull_secret: self.score.image_pull_secret.clone(),
|
||||||
values_overrides: None,
|
})
|
||||||
values_yaml: None,
|
.map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
|
||||||
create_namespace: true,
|
let chart_path_str = chart_path
|
||||||
install_only: false,
|
.to_str()
|
||||||
force_conflicts: false,
|
.ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?;
|
||||||
repository: None,
|
let helm_outcome = HelmChartScore {
|
||||||
}
|
namespace: Some(non_blank(&self.score.namespace, "namespace")?),
|
||||||
.interpret(inventory, topology)
|
release_name: non_blank(&self.score.release_name, "release_name")?,
|
||||||
.await?
|
chart_name: non_blank(chart_path_str, "chart_name")?,
|
||||||
} else {
|
chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?),
|
||||||
let tmp = tempfile::tempdir()
|
values_overrides: None,
|
||||||
.map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?;
|
values_yaml: None,
|
||||||
let chart_path = build_chart(&ChartOptions {
|
create_namespace: false,
|
||||||
output_dir: tmp.path().to_path_buf(),
|
install_only: false,
|
||||||
image: self.score.image.clone(),
|
force_conflicts: false,
|
||||||
image_pull_policy: self.score.image_pull_policy.clone(),
|
repository: None,
|
||||||
nats_url: self.score.nats_url.clone(),
|
}
|
||||||
log_level: self.score.log_level.clone(),
|
.interpret(inventory, topology)
|
||||||
credentials: self.score.credentials.clone(),
|
.await?;
|
||||||
chart_version: None,
|
|
||||||
// The auth Secret is applied separately above; the
|
k8s.wait_until_deployment_ready(
|
||||||
// rendered chart only needs the Deployment env wiring.
|
&self.score.release_name,
|
||||||
web_auth_config_json: None,
|
Some(&self.score.namespace),
|
||||||
web_cookie_key_json: None,
|
Some(Duration::from_secs(180)),
|
||||||
})
|
)
|
||||||
.map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
|
.await
|
||||||
let chart_path_str = chart_path.to_str().ok_or_else(|| {
|
.map_err(|e| {
|
||||||
InterpretError::new("operator chart path is not utf-8".to_string())
|
InterpretError::new(format!(
|
||||||
})?;
|
"operator deployment {}/{} not ready: {e}",
|
||||||
info!(
|
self.score.namespace, self.score.release_name
|
||||||
"Installing helm release '{}' from rendered chart {}",
|
))
|
||||||
self.score.release_name, chart_path_str
|
})?;
|
||||||
);
|
|
||||||
HelmChartScore {
|
if credentials.is_some() || self.score.identity.is_some() {
|
||||||
namespace: Some(non_blank(&self.score.namespace, "namespace")?),
|
smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?;
|
||||||
release_name: non_blank(&self.score.release_name, "release_name")?,
|
}
|
||||||
chart_name: non_blank(chart_path_str, "chart_name")?,
|
|
||||||
chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?),
|
|
||||||
values_overrides: None,
|
|
||||||
values_yaml: None,
|
|
||||||
create_namespace: true,
|
|
||||||
install_only: false,
|
|
||||||
force_conflicts: false,
|
|
||||||
repository: None,
|
|
||||||
}
|
|
||||||
.interpret(inventory, topology)
|
|
||||||
.await?
|
|
||||||
};
|
|
||||||
|
|
||||||
// Expose the UI. Applied after the chart so the backing Service
|
// Expose the UI. Applied after the chart so the backing Service
|
||||||
// (shipped in the chart) exists. Skipped when no host is set —
|
// (shipped in the chart) exists. Skipped when no host is set —
|
||||||
@@ -324,7 +424,7 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
|
|||||||
let mut details = vec![
|
let mut details = vec![
|
||||||
format!("operator namespace: {}", self.score.namespace),
|
format!("operator namespace: {}", self.score.namespace),
|
||||||
format!("operator release: {}", self.score.release_name),
|
format!("operator release: {}", self.score.release_name),
|
||||||
format!("operator NATS URL: {}", self.score.nats_url),
|
format!("operator NATS URL: {nats_url}"),
|
||||||
];
|
];
|
||||||
if let Some(host) = &self.score.operator_ui_host {
|
if let Some(host) = &self.score.operator_ui_host {
|
||||||
let to_fqdn = |s: &str| {
|
let to_fqdn = |s: &str| {
|
||||||
@@ -383,7 +483,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn defaults_target_fleet_system() {
|
fn defaults_target_fleet_system() {
|
||||||
let s = FleetOperatorScore::new();
|
let s = FleetOperatorScore::new("operator:dev");
|
||||||
assert_eq!(s.namespace, "fleet-system");
|
assert_eq!(s.namespace, "fleet-system");
|
||||||
assert_eq!(s.release_name, "harmony-fleet-operator");
|
assert_eq!(s.release_name, "harmony-fleet-operator");
|
||||||
assert_eq!(s.image_pull_policy, "IfNotPresent");
|
assert_eq!(s.image_pull_policy, "IfNotPresent");
|
||||||
@@ -391,27 +491,19 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn builders_compose() {
|
fn builders_compose() {
|
||||||
let s = FleetOperatorScore::new()
|
let nats = harmony::modules::nats::NatsScore::new("nats", "default").client_ref();
|
||||||
|
let s = FleetOperatorScore::new("hub.example/op:v1")
|
||||||
.namespace("custom-ns")
|
.namespace("custom-ns")
|
||||||
.image("hub.example/op:v1")
|
|
||||||
.image_pull_policy("Always")
|
.image_pull_policy("Always")
|
||||||
.nats_url("nats://nats:4222")
|
.messaging(&nats)
|
||||||
.log_level("debug");
|
.log_level("debug");
|
||||||
assert_eq!(s.namespace, "custom-ns");
|
assert_eq!(s.namespace, "custom-ns");
|
||||||
assert_eq!(s.image, "hub.example/op:v1");
|
assert_eq!(s.image, "hub.example/op:v1");
|
||||||
assert_eq!(s.image_pull_policy, "Always");
|
assert_eq!(s.image_pull_policy, "Always");
|
||||||
assert_eq!(s.nats_url, "nats://nats:4222");
|
assert_eq!(
|
||||||
|
s.nats.unwrap().url(),
|
||||||
|
"nats://nats.default.svc.cluster.local:4222"
|
||||||
|
);
|
||||||
assert_eq!(s.log_level, "debug");
|
assert_eq!(s.log_level, "debug");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn credentials_set_on_published_chart_path() {
|
|
||||||
let s = FleetOperatorScore::new()
|
|
||||||
.credentials("type = \"zitadel-jwt\"")
|
|
||||||
.published_chart("hub.example", "proj", "1.2.3");
|
|
||||||
let creds = s.credentials.expect("credentials wired");
|
|
||||||
assert_eq!(creds.credentials_toml, "type = \"zitadel-jwt\"");
|
|
||||||
// The published-chart interpret applies this Secret before install.
|
|
||||||
assert!(s.published_chart.is_some());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,119 +0,0 @@
|
|||||||
//! Build + publish the operator's image + chart for a tagged release.
|
|
||||||
//!
|
|
||||||
//! A release is five commands — docker build/push, hydrate the chart,
|
|
||||||
//! helm package/push — plus parsing the version out of the git tag.
|
|
||||||
//! Kept as a plain function with `anyhow` (this is binary glue, not
|
|
||||||
//! library API). When a second component needs releasing, lift the
|
|
||||||
//! per-component constants below into a small spec then — not before.
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::process::Command;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
|
||||||
|
|
||||||
use crate::operator::chart::{CHART_NAME, ChartOptions, build_chart};
|
|
||||||
|
|
||||||
const TAG_PREFIX: &str = "harmony-fleet-operator-";
|
|
||||||
const IMAGE_NAME: &str = "harmony-fleet-operator";
|
|
||||||
const DOCKERFILE: &str = "fleet/harmony-fleet-operator/Dockerfile";
|
|
||||||
|
|
||||||
/// Version from a release tag: `harmony-fleet-operator-v0.1.0` → `0.1.0`.
|
|
||||||
/// One version drives both the image tag and the chart version.
|
|
||||||
pub fn version_from_tag(tag: &str) -> Result<String> {
|
|
||||||
let rest = tag
|
|
||||||
.strip_prefix(TAG_PREFIX)
|
|
||||||
.with_context(|| format!("tag '{tag}' is not a '{TAG_PREFIX}*' release tag"))?;
|
|
||||||
let version = rest.strip_prefix('v').unwrap_or(rest);
|
|
||||||
if version.is_empty() {
|
|
||||||
bail!("tag '{tag}' has no version after the prefix");
|
|
||||||
}
|
|
||||||
Ok(version.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build + push the operator image and helm chart at `version`. With
|
|
||||||
/// `push = false`, builds + packages only (a local k3d smoke-test).
|
|
||||||
/// `docker` and `helm` must be on PATH and logged in to the registry.
|
|
||||||
pub fn release_operator(version: &str, registry: &str, project: &str, push: bool) -> Result<()> {
|
|
||||||
let image = format!("{registry}/{project}/{IMAGE_NAME}:{version}");
|
|
||||||
let oci_repo = format!("oci://{registry}/{project}");
|
|
||||||
|
|
||||||
log::info!("docker build {image}");
|
|
||||||
run("docker", &["build", "-f", DOCKERFILE, "-t", &image, "."])?;
|
|
||||||
if push {
|
|
||||||
log::info!("docker push {image}");
|
|
||||||
run("docker", &["push", &image])?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let tmp = tempfile::tempdir().context("chart tempdir")?;
|
|
||||||
let chart_dir = build_chart(&ChartOptions {
|
|
||||||
output_dir: tmp.path().to_path_buf(),
|
|
||||||
image: image.clone(),
|
|
||||||
chart_version: Some(version.to_string()),
|
|
||||||
..ChartOptions::default()
|
|
||||||
})
|
|
||||||
.map_err(|e| anyhow::anyhow!("hydrate operator chart: {e}"))?;
|
|
||||||
|
|
||||||
let tgz = helm_package(&chart_dir, tmp.path())?;
|
|
||||||
if push {
|
|
||||||
log::info!("helm push {} {oci_repo}", tgz.display());
|
|
||||||
run("helm", &["push", path_str(&tgz)?, &oci_repo])?;
|
|
||||||
}
|
|
||||||
|
|
||||||
log::info!("released image={image} chart={oci_repo}/{CHART_NAME}:{version} pushed={push}");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn run(bin: &str, args: &[&str]) -> Result<()> {
|
|
||||||
let status = Command::new(bin)
|
|
||||||
.args(args)
|
|
||||||
.status()
|
|
||||||
.with_context(|| format!("spawn {bin} (is it installed and in PATH?)"))?;
|
|
||||||
if !status.success() {
|
|
||||||
bail!("`{bin} {}` failed ({status})", args.join(" "));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result<PathBuf> {
|
|
||||||
let output = Command::new("helm")
|
|
||||||
.args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?])
|
|
||||||
.output()
|
|
||||||
.context("spawn helm package")?;
|
|
||||||
if !output.status.success() {
|
|
||||||
bail!("helm package failed ({})", output.status);
|
|
||||||
}
|
|
||||||
// helm prints "…saved it to: <path>"; the last token is the path.
|
|
||||||
String::from_utf8_lossy(&output.stdout)
|
|
||||||
.split_whitespace()
|
|
||||||
.last()
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.context("helm package printed no path")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn path_str(p: &Path) -> Result<&str> {
|
|
||||||
p.to_str()
|
|
||||||
.with_context(|| format!("path not utf-8: {}", p.display()))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn version_strips_prefix_and_leading_v() {
|
|
||||||
assert_eq!(
|
|
||||||
version_from_tag("harmony-fleet-operator-v0.1.0").unwrap(),
|
|
||||||
"0.1.0"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
version_from_tag("harmony-fleet-operator-2.3.4").unwrap(),
|
|
||||||
"2.3.4"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn rejects_foreign_or_empty_tags() {
|
|
||||||
assert!(version_from_tag("harmony-fleet-agent-v0.1.0").is_err());
|
|
||||||
assert!(version_from_tag("harmony-fleet-operator-").is_err());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
//! Secrets and config for the published-chart (CD) operator deploy, via
|
|
||||||
//! [`harmony_config::ConfigClient`]. SSO-only by construction: no
|
|
||||||
//! user/pass field exists, so dev-only user/pass auth can't reach a prod
|
|
||||||
//! deploy. Resolved EnvSource → OpenBao, so the in-cluster runner pulls
|
|
||||||
//! them at job time with no plaintext on the pod.
|
|
||||||
|
|
||||||
use harmony_config::Config;
|
|
||||||
use schemars::JsonSchema;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
/// `#[config(secret)]` keeps every field out of cleartext SQLite and
|
|
||||||
/// masks it in logs.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
|
|
||||||
#[config(secret)]
|
|
||||||
pub struct FleetDeploySecrets {
|
|
||||||
/// Operator's zitadel-jwt `[credentials]` TOML — its NATS auth. The
|
|
||||||
/// CD path's only credential; there is no user/pass fallback.
|
|
||||||
pub operator_credentials_toml: String,
|
|
||||||
|
|
||||||
/// Scoped fleet-deployer credential, fetched at job time so the
|
|
||||||
/// runner pod holds no standing permissions. `None` uses the ambient
|
|
||||||
/// `KUBECONFIG`.
|
|
||||||
#[serde(default)]
|
|
||||||
pub kubeconfig: Option<String>,
|
|
||||||
|
|
||||||
/// Operator dashboard session-cookie signing key (standard-base64 of
|
|
||||||
/// ≥64 bytes). `#[serde(default)]` so older seeds without it still
|
|
||||||
/// load — but the dashboard won't authenticate until it's set.
|
|
||||||
#[serde(default)]
|
|
||||||
pub operator_cookie_key_b64: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Non-secret deploy config: k8s namespaces + chart coords. Loaded via
|
|
||||||
/// `ConfigClient::for_namespace("fleet-staging")` alongside `FleetDeploySecrets`.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
|
|
||||||
pub struct FleetDeployConfig {
|
|
||||||
/// K8s namespace where the operator, NATS, and callout live.
|
|
||||||
pub namespace: String,
|
|
||||||
|
|
||||||
/// Full NATS URL the operator connects to (e.g. `nats://fleet-nats.fleet-staging:4222`).
|
|
||||||
pub nats_url: String,
|
|
||||||
|
|
||||||
/// K8s namespace where Zitadel lives (for operator UI SSO).
|
|
||||||
pub zitadel_namespace: String,
|
|
||||||
|
|
||||||
/// K8s namespace where OpenBao lives (for operator secret fetching).
|
|
||||||
pub openbao_namespace: String,
|
|
||||||
|
|
||||||
/// OCI chart registry (e.g. `hub.nationtech.io`).
|
|
||||||
pub operator_chart_registry: String,
|
|
||||||
|
|
||||||
/// OCI chart project (e.g. `harmony`).
|
|
||||||
pub operator_chart_project: String,
|
|
||||||
|
|
||||||
/// Public base domain. The operator UI host is derived as
|
|
||||||
/// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`.
|
|
||||||
pub base_domain: String,
|
|
||||||
|
|
||||||
/// cert-manager `ClusterIssuer` for the operator UI's TLS cert.
|
|
||||||
pub cluster_issuer: String,
|
|
||||||
|
|
||||||
/// Zitadel OIDC app the operator dashboard authenticates against.
|
|
||||||
#[serde(default)]
|
|
||||||
pub operator_oidc_client_id: String,
|
|
||||||
|
|
||||||
/// Token audiences the dashboard accepts (Zitadel project/app ids).
|
|
||||||
#[serde(default)]
|
|
||||||
pub operator_trusted_audiences: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for FleetDeployConfig {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
namespace: "fleet-staging".to_string(),
|
|
||||||
nats_url: "nats://fleet-nats.fleet-staging:4222".to_string(),
|
|
||||||
zitadel_namespace: "zitadel-staging".to_string(),
|
|
||||||
openbao_namespace: "openbao-staging".to_string(),
|
|
||||||
operator_chart_registry: "hub.nationtech.io".to_string(),
|
|
||||||
operator_chart_project: "harmony".to_string(),
|
|
||||||
base_domain: "cb1.nationtech.io".to_string(),
|
|
||||||
cluster_issuer: "letsencrypt-prod".to_string(),
|
|
||||||
operator_oidc_client_id: String::new(),
|
|
||||||
operator_trusted_audiences: Vec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use harmony_config::ConfigClass;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn secret_class_and_optional_kubeconfig() {
|
|
||||||
let s: FleetDeploySecrets =
|
|
||||||
serde_json::from_str(r#"{ "operator_credentials_toml": "x" }"#).unwrap();
|
|
||||||
assert!(s.kubeconfig.is_none());
|
|
||||||
// Secrets must never land in cleartext SQLite.
|
|
||||||
assert_eq!(FleetDeploySecrets::CLASS, ConfigClass::Secret);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn config_class_is_not_secret() {
|
|
||||||
assert_eq!(FleetDeployConfig::CLASS, ConfigClass::Standard);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn config_defaults() {
|
|
||||||
let c = FleetDeployConfig::default();
|
|
||||||
assert_eq!(c.namespace, "fleet-staging");
|
|
||||||
assert_eq!(c.zitadel_namespace, "zitadel-staging");
|
|
||||||
assert_eq!(c.openbao_namespace, "openbao-staging");
|
|
||||||
assert_eq!(c.base_domain, "cb1.nationtech.io");
|
|
||||||
assert_eq!(c.cluster_issuer, "letsencrypt-prod");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,234 +0,0 @@
|
|||||||
//! [`FleetServerScore`] — one-shot install of harmony's fleet
|
|
||||||
//! server-side stack into a Kubernetes cluster.
|
|
||||||
//!
|
|
||||||
//! Composes:
|
|
||||||
//! 1. [`NatsBasicScore`](crate::modules::nats::NatsBasicScore) — single-node
|
|
||||||
//! NATS with JetStream, in-cluster.
|
|
||||||
//! 2. [`FleetOperatorScore`](super::FleetOperatorScore) — operator
|
|
||||||
//! Deployment + CRDs (Deployment / Device) + RBAC.
|
|
||||||
//! 3. (Optional) [`ZitadelScore`] — central OIDC identity provider
|
|
||||||
//! (Postgres + Zitadel helm chart). Set `identity = Some(...)` to
|
|
||||||
//! install; `None` skips it. Adds the `K8sclient + PostgreSQL`
|
|
||||||
//! capability requirements to the topology either way (they're
|
|
||||||
//! static trait bounds; pick a topology that supplies them, e.g.
|
|
||||||
//! [`K8sAnywhereTopology`](crate::topology::K8sAnywhereTopology)).
|
|
||||||
//!
|
|
||||||
//! Argo is the next follow-up — same pattern, optional field, hangs
|
|
||||||
//! off the same sequential chain in [`Interpret::execute`].
|
|
||||||
//!
|
|
||||||
//! NATS goes first deliberately: the operator pod's `NATS_URL`
|
|
||||||
//! environment points at the in-cluster NATS service, so installing
|
|
||||||
//! the operator first leads to a few minutes of `connect_with_retry`
|
|
||||||
//! noise in the operator log until NATS comes up.
|
|
||||||
//!
|
|
||||||
//! Idempotency: each inner Score is itself idempotent (NatsBasicScore
|
|
||||||
//! delegates to the upstream `nats/nats` chart via HelmChartScore,
|
|
||||||
//! FleetOperatorScore pins `chart_version`, ZitadelScore relies on
|
|
||||||
//! HelmChartScore + CNPG idempotency). Running this Score twice is
|
|
||||||
//! safe and produces no churn.
|
|
||||||
//!
|
|
||||||
//! Typical usage:
|
|
||||||
//!
|
|
||||||
//! ```ignore
|
|
||||||
//! use harmony::modules::fleet::{FleetOperatorScore, FleetServerScore};
|
|
||||||
//! use harmony::modules::nats::NatsBasicScore;
|
|
||||||
//! use harmony::modules::zitadel::ZitadelScore;
|
|
||||||
//! use harmony::score::Score;
|
|
||||||
//! use harmony::inventory::Inventory;
|
|
||||||
//! use harmony::topology::K8sAnywhereTopology;
|
|
||||||
//!
|
|
||||||
//! let topology = K8sAnywhereTopology::from_env();
|
|
||||||
//! let nats = NatsBasicScore::new("fleet-nats", "fleet-system").load_balancer();
|
|
||||||
//! let operator = FleetOperatorScore::new()
|
|
||||||
//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev");
|
|
||||||
//! let mut score = FleetServerScore::new(nats, operator);
|
|
||||||
//! score.identity = Some(ZitadelScore {
|
|
||||||
//! host: "zitadel.localhost".into(),
|
|
||||||
//! external_secure: false,
|
|
||||||
//! ..Default::default()
|
|
||||||
//! });
|
|
||||||
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
|
|
||||||
//! ```
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use harmony::data::Version;
|
|
||||||
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
|
|
||||||
use harmony::inventory::Inventory;
|
|
||||||
use harmony::modules::nats::capability::Nats;
|
|
||||||
use harmony::modules::nats::score_nats_k8s::NatsK8sScore;
|
|
||||||
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
|
||||||
use harmony::modules::postgresql::capability::PostgreSQL;
|
|
||||||
use harmony::modules::zitadel::{ZitadelScore, ZitadelSetupScore};
|
|
||||||
use harmony::score::Score;
|
|
||||||
use harmony::topology::{HelmCommand, K8sclient, TlsRouter, Topology};
|
|
||||||
use harmony_types::id::Id;
|
|
||||||
use serde::Serialize;
|
|
||||||
|
|
||||||
use crate::operator::FleetOperatorScore;
|
|
||||||
|
|
||||||
/// Install the fleet server-side stack on Kubernetes: NATS (via
|
|
||||||
/// `NatsK8sScore`, OKD-aware Routes), operator, optionally Zitadel
|
|
||||||
/// (helm + post-install identity setup), optionally the auth callout.
|
|
||||||
///
|
|
||||||
/// Execution order is deliberate:
|
|
||||||
///
|
|
||||||
/// 1. `identity` (Zitadel helm chart + cert-manager Route) FIRST
|
|
||||||
/// so the Zitadel API is reachable when `identity_setup` runs.
|
|
||||||
/// 2. `identity_setup` provisions project + roles + machine users
|
|
||||||
/// via the Zitadel management API. Produces the keys the
|
|
||||||
/// operator needs to authenticate.
|
|
||||||
/// 3. `nats` brings up the NATS pod. When the score's
|
|
||||||
/// `auth_callout` field is `Some`, the rendered helm values
|
|
||||||
/// include the matching `authorization.auth_callout` block
|
|
||||||
/// pointing at the (still-missing) callout pod's issuer.
|
|
||||||
/// 4. `auth_callout` deploys the callout pod itself. Once it's
|
|
||||||
/// Ready, NATS auth handshakes start succeeding.
|
|
||||||
/// 5. `operator` last. By this point Zitadel, NATS, and the
|
|
||||||
/// callout are all up; the operator's pod boots, mints its
|
|
||||||
/// Zitadel token, connects through the callout, watches its
|
|
||||||
/// CRDs.
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub struct FleetServerScore {
|
|
||||||
pub nats: NatsK8sScore,
|
|
||||||
pub operator: FleetOperatorScore,
|
|
||||||
/// Helm install of Zitadel + Postgres. Required when the rest
|
|
||||||
/// of the chain wants Zitadel-backed auth.
|
|
||||||
pub identity: Option<ZitadelScore>,
|
|
||||||
/// Post-install Zitadel API calls — project, roles, machine
|
|
||||||
/// users, JWT keys. Without this the operator has no machine
|
|
||||||
/// user to authenticate as.
|
|
||||||
pub identity_setup: Option<ZitadelSetupScore>,
|
|
||||||
/// Auth callout deployment. Pair with
|
|
||||||
/// `nats.auth_callout = Some(...)` so the rendered NATS helm
|
|
||||||
/// values delegate to this callout's issuer pubkey.
|
|
||||||
pub auth_callout: Option<NatsAuthCalloutScore>,
|
|
||||||
// TODO(argo): pub gitops: Option<ArgoScore>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FleetServerScore {
|
|
||||||
pub fn new(nats: NatsK8sScore, operator: FleetOperatorScore) -> Self {
|
|
||||||
Self {
|
|
||||||
nats,
|
|
||||||
operator,
|
|
||||||
identity: None,
|
|
||||||
identity_setup: None,
|
|
||||||
auth_callout: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Topology + HelmCommand + Nats + K8sclient + PostgreSQL + TlsRouter + 'static> Score<T>
|
|
||||||
for FleetServerScore
|
|
||||||
{
|
|
||||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
|
||||||
Box::new(FleetServerInterpret {
|
|
||||||
score: self.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn name(&self) -> String {
|
|
||||||
"FleetServerScore".to_string()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct FleetServerInterpret {
|
|
||||||
score: FleetServerScore,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl<T: Topology + HelmCommand + Nats + K8sclient + PostgreSQL + TlsRouter + 'static> Interpret<T>
|
|
||||||
for FleetServerInterpret
|
|
||||||
{
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
// Use Score::interpret (not create_interpret().execute()) so each
|
|
||||||
// inner score fires its own InterpretExecutionStarted/Finished
|
|
||||||
// events — that's what gives the cli_logger the per-score
|
|
||||||
// progress lines and what surfaces the inner outcomes' details
|
|
||||||
// in the cli_reporter end-of-run summary.
|
|
||||||
|
|
||||||
// 1. Identity: Zitadel helm chart needs to be up before the
|
|
||||||
// setup score can talk to its API.
|
|
||||||
if let Some(identity) = &self.score.identity {
|
|
||||||
identity.interpret(inventory, topology).await?;
|
|
||||||
}
|
|
||||||
// 2. Identity setup: project + roles + machine users via the
|
|
||||||
// Zitadel management API. Output (machine keys) feeds the
|
|
||||||
// operator's credentials at apply time. The operator
|
|
||||||
// Score has those bytes already (callers wire them in via
|
|
||||||
// `OperatorCredentials.credentials_toml` before passing
|
|
||||||
// `FleetServerScore` to the framework).
|
|
||||||
if let Some(setup) = &self.score.identity_setup {
|
|
||||||
setup.interpret(inventory, topology).await?;
|
|
||||||
}
|
|
||||||
// 3. NATS: helm-install NATS with whatever auth_callout
|
|
||||||
// config was baked into `nats` already.
|
|
||||||
self.score.nats.interpret(inventory, topology).await?;
|
|
||||||
// 4. Auth callout deployment.
|
|
||||||
if let Some(callout) = &self.score.auth_callout {
|
|
||||||
callout.interpret(inventory, topology).await?;
|
|
||||||
}
|
|
||||||
// 5. Operator last so it doesn't waste reconnect attempts
|
|
||||||
// while NATS / callout are still coming up. The pod's own
|
|
||||||
// `connect_with_retry` handles the small remaining race.
|
|
||||||
self.score.operator.interpret(inventory, topology).await?;
|
|
||||||
// TODO(argo): chain GitOps install here.
|
|
||||||
|
|
||||||
// Hand the cli_reporter a few next-step pointers so the
|
|
||||||
// post-install summary at the bottom of the run is genuinely
|
|
||||||
// useful.
|
|
||||||
let mut details = vec![
|
|
||||||
format!(
|
|
||||||
"NATS in-cluster URL: nats://{}.{}:4222",
|
|
||||||
self.score.nats.cluster.name, self.score.nats.cluster.namespace
|
|
||||||
),
|
|
||||||
format!(
|
|
||||||
"Operator deployment: kubectl -n {} get deploy",
|
|
||||||
self.score.operator.namespace
|
|
||||||
),
|
|
||||||
"CRDs: kubectl get crd | grep fleet.nationtech.io".to_string(),
|
|
||||||
];
|
|
||||||
if let Some(ws) = &self.score.nats.websocket {
|
|
||||||
details.push(format!("NATS WS public URL: wss://{}/", ws.host));
|
|
||||||
}
|
|
||||||
if let Some(identity) = &self.score.identity {
|
|
||||||
let scheme = if identity.external_secure {
|
|
||||||
"https"
|
|
||||||
} else {
|
|
||||||
"http"
|
|
||||||
};
|
|
||||||
details.push(format!(
|
|
||||||
"Zitadel: {scheme}://{}/ (kubectl -n zitadel get all)",
|
|
||||||
identity.host
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if self.score.auth_callout.is_some() {
|
|
||||||
details
|
|
||||||
.push("Auth callout: kubectl -n fleet-system get deploy/fleet-callout".to_string());
|
|
||||||
}
|
|
||||||
Ok(Outcome::success_with_details(
|
|
||||||
"Fleet server stack installed".to_string(),
|
|
||||||
details,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_name(&self) -> InterpretName {
|
|
||||||
InterpretName::Custom("FleetServerInterpret")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_version(&self) -> Version {
|
|
||||||
Version::from("0.1.0").expect("static version literal")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_status(&self) -> InterpretStatus {
|
|
||||||
InterpretStatus::QUEUED
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_children(&self) -> Vec<Id> {
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -51,6 +51,7 @@ harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
|||||||
harmony-fleet-deploy = { path = "../harmony-fleet-deploy" }
|
harmony-fleet-deploy = { path = "../harmony-fleet-deploy" }
|
||||||
harmony-fleet-operator = { path = "../harmony-fleet-operator" }
|
harmony-fleet-operator = { path = "../harmony-fleet-operator" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
|
harmony_secret = { path = "../../harmony_secret" }
|
||||||
harmony-k8s = { path = "../../harmony-k8s" }
|
harmony-k8s = { path = "../../harmony-k8s" }
|
||||||
harmony_types = { path = "../../harmony_types" }
|
harmony_types = { path = "../../harmony_types" }
|
||||||
k3d-rs = { path = "../../k3d" }
|
k3d-rs = { path = "../../k3d" }
|
||||||
|
|||||||
@@ -24,22 +24,20 @@ use std::time::Duration;
|
|||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
|
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
|
||||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService};
|
||||||
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
|
||||||
use harmony::modules::openbao::{
|
use harmony::modules::openbao::{
|
||||||
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
|
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
|
||||||
};
|
};
|
||||||
|
use harmony::modules::postgresql::K8sPostgreSQLScore;
|
||||||
use harmony::modules::zitadel::{
|
use harmony::modules::zitadel::{
|
||||||
MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole,
|
OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef, ZitadelAppType,
|
||||||
ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials,
|
ZitadelClientConfig, ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials,
|
||||||
};
|
};
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::K8sAnywhereTopology;
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
use harmony_fleet_deploy::FleetNatsScore;
|
|
||||||
use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode};
|
use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode};
|
||||||
use k8s_openapi::api::core::v1::{Secret, Service, ServicePort, ServiceSpec};
|
use k8s_openapi::api::core::v1::Secret;
|
||||||
use kube::api::ObjectMeta;
|
|
||||||
use nkeys::KeyPair;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
use crate::images::CALLOUT_IMAGE_TAG;
|
use crate::images::CALLOUT_IMAGE_TAG;
|
||||||
@@ -70,9 +68,9 @@ pub const ADMIN_USERNAME: &str = "e2e-admin";
|
|||||||
|
|
||||||
const NATS_ACCOUNT: &str = "FLEET";
|
const NATS_ACCOUNT: &str = "FLEET";
|
||||||
const NATS_AUTH_USER: &str = "auth";
|
const NATS_AUTH_USER: &str = "auth";
|
||||||
|
const OPERATOR_APP: &str = "harmony-fleet-operator";
|
||||||
// Dev-only static credential for the callout's own NATS user, same
|
// Dev-only static credential for the callout's own NATS user, same
|
||||||
// posture as the user-pass mode creds in `stack.rs`.
|
// posture as the user-pass mode creds in `stack.rs`.
|
||||||
const NATS_AUTH_PASS: &str = "e2e-callout-auth";
|
|
||||||
const CALLOUT_DEPLOYMENT: &str = "fleet-callout";
|
const CALLOUT_DEPLOYMENT: &str = "fleet-callout";
|
||||||
|
|
||||||
/// Handles the callout bring-up leaves behind for tests.
|
/// Handles the callout bring-up leaves behind for tests.
|
||||||
@@ -87,6 +85,9 @@ pub struct CalloutHandles {
|
|||||||
/// Zitadel IAM admin PAT (from the chart's `iam-admin-pat`
|
/// Zitadel IAM admin PAT (from the chart's `iam-admin-pat`
|
||||||
/// Secret) — used to enroll device machine users.
|
/// Secret) — used to enroll device machine users.
|
||||||
pub admin_pat: String,
|
pub admin_pat: String,
|
||||||
|
pub provider: OidcProviderRef,
|
||||||
|
pub application: OidcApplicationRef,
|
||||||
|
pub operator_identity: OidcMachineIdentityRef,
|
||||||
pub openbao: OpenbaoInstance,
|
pub openbao: OpenbaoInstance,
|
||||||
pub openbao_root_token: String,
|
pub openbao_root_token: String,
|
||||||
}
|
}
|
||||||
@@ -95,72 +96,34 @@ pub fn issuer_url() -> String {
|
|||||||
format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}")
|
format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// NodePort Service in front of Zitadel with in-cluster port ==
|
|
||||||
/// nodePort == [`ZITADEL_PORT`] (targetPort = the chart's 8080), so
|
|
||||||
/// the issuer URL's port number is valid both inside and outside the
|
|
||||||
/// cluster.
|
|
||||||
fn zitadel_nodeport_service() -> Service {
|
|
||||||
let selector = std::collections::BTreeMap::from([
|
|
||||||
("app.kubernetes.io/name".to_string(), "zitadel".to_string()),
|
|
||||||
(
|
|
||||||
"app.kubernetes.io/instance".to_string(),
|
|
||||||
"zitadel".to_string(),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
Service {
|
|
||||||
metadata: ObjectMeta {
|
|
||||||
name: Some("zitadel-e2e".to_string()),
|
|
||||||
namespace: Some(ZITADEL_NAMESPACE.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
spec: Some(ServiceSpec {
|
|
||||||
type_: Some("NodePort".to_string()),
|
|
||||||
selector: Some(selector),
|
|
||||||
ports: Some(vec![ServicePort {
|
|
||||||
name: Some("http".to_string()),
|
|
||||||
port: ZITADEL_PORT as i32,
|
|
||||||
target_port: Some(
|
|
||||||
k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(8080),
|
|
||||||
),
|
|
||||||
node_port: Some(ZITADEL_PORT as i32),
|
|
||||||
..Default::default()
|
|
||||||
}]),
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deploy Zitadel + project/roles/admin-user, returning everything
|
/// Deploy Zitadel + project/roles/admin-user, returning everything
|
||||||
/// except the OpenBao handles (filled by [`deploy_openbao`]).
|
/// except the OpenBao handles (filled by [`deploy_openbao`]).
|
||||||
pub async fn bring_up_zitadel(
|
pub async fn bring_up_zitadel(
|
||||||
topology: &K8sAnywhereTopology,
|
topology: &K8sAnywhereTopology,
|
||||||
kube_client: &kube::Client,
|
kube_client: &kube::Client,
|
||||||
|
outputs_namespace: &str,
|
||||||
) -> Result<CalloutHandles> {
|
) -> Result<CalloutHandles> {
|
||||||
ZitadelScore {
|
let postgres = K8sPostgreSQLScore::new(ZITADEL_NAMESPACE).cluster_name("zitadel-pg");
|
||||||
host: ZITADEL_HOST.to_string(),
|
let database = postgres.root_account_ref();
|
||||||
zitadel_version: "v4.12.1".to_string(),
|
postgres
|
||||||
external_secure: false,
|
.interpret(&Inventory::autoload(), topology)
|
||||||
external_port: Some(ZITADEL_PORT as u32),
|
.await
|
||||||
namespace: ZITADEL_NAMESPACE.to_string(),
|
.map_err(|e| anyhow::anyhow!("Zitadel PostgreSQL deploy: {e}"))?;
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?;
|
|
||||||
|
|
||||||
K8sResourceScore::single(
|
let zitadel = ZitadelScore::new(ZITADEL_HOST, ZITADEL_NAMESPACE)
|
||||||
zitadel_nodeport_service(),
|
.http(Some(ZITADEL_PORT.into()))
|
||||||
Some(ZITADEL_NAMESPACE.to_string()),
|
.database(database)
|
||||||
)
|
.node_port(ZITADEL_PORT);
|
||||||
.interpret(&Inventory::autoload(), topology)
|
let provider = zitadel.provider_ref();
|
||||||
.await
|
zitadel
|
||||||
.map_err(|e| anyhow::anyhow!("Zitadel NodePort service: {e}"))?;
|
.interpret(&Inventory::autoload(), topology)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?;
|
||||||
|
|
||||||
CoreDNSRewriteScore {
|
CoreDNSRewriteScore {
|
||||||
rewrites: vec![CoreDNSRewrite {
|
rewrites: vec![CoreDNSRewrite {
|
||||||
hostname: ZITADEL_HOST.to_string(),
|
hostname: ZITADEL_HOST.to_string(),
|
||||||
target: format!("zitadel-e2e.{ZITADEL_NAMESPACE}.svc.cluster.local"),
|
target: format!("zitadel-nodeport.{ZITADEL_NAMESPACE}.svc.cluster.local"),
|
||||||
}],
|
}],
|
||||||
}
|
}
|
||||||
.interpret(&Inventory::autoload(), topology)
|
.interpret(&Inventory::autoload(), topology)
|
||||||
@@ -169,46 +132,26 @@ pub async fn bring_up_zitadel(
|
|||||||
|
|
||||||
wait_for_zitadel_ready().await?;
|
wait_for_zitadel_ready().await?;
|
||||||
|
|
||||||
let roles = [ADMIN_ROLE, DEVICE_ROLE]
|
let mut identity =
|
||||||
|
ZitadelSetupScore::for_provider(&provider, ZITADEL_NAMESPACE, outputs_namespace)
|
||||||
|
.application(PROJECT_NAME, OPERATOR_APP, ZitadelAppType::DeviceCode)
|
||||||
|
.api_application(PROJECT_NAME, "nats")
|
||||||
|
.machine_identity(PROJECT_NAME, ADMIN_USERNAME, "E2E Admin", [ADMIN_ROLE])
|
||||||
|
.groups_claim();
|
||||||
|
identity.endpoint = Some(format!("http://127.0.0.1:{ZITADEL_PORT}"));
|
||||||
|
identity.skip_tls = true;
|
||||||
|
for role in [ADMIN_ROLE, DEVICE_ROLE]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.chain(GROUP_ROLES.iter().copied())
|
.chain(GROUP_ROLES.iter().copied())
|
||||||
.map(|key| ZitadelRole {
|
{
|
||||||
project_name: PROJECT_NAME.to_string(),
|
identity = identity.role(PROJECT_NAME, role, role);
|
||||||
key: key.to_string(),
|
|
||||||
display_name: key.to_string(),
|
|
||||||
group: None,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
ZitadelSetupScore {
|
|
||||||
machine_users: vec![ZitadelMachineUser {
|
|
||||||
username: ADMIN_USERNAME.to_string(),
|
|
||||||
name: "E2E Admin".to_string(),
|
|
||||||
create_pat: false,
|
|
||||||
machine_key: Some(MachineKeyType::Json),
|
|
||||||
project_name: Some(PROJECT_NAME.to_string()),
|
|
||||||
grant_roles: vec![ADMIN_ROLE.to_string()],
|
|
||||||
}],
|
|
||||||
roles,
|
|
||||||
// The flattening Action turns project role keys into the
|
|
||||||
// `groups` array claim OpenBao binds against (ADR-025) — this
|
|
||||||
// bring-up is its live verification.
|
|
||||||
groups_claim_action: true,
|
|
||||||
human_users: vec![],
|
|
||||||
org_members: vec![],
|
|
||||||
instance_members: vec![],
|
|
||||||
machine_secrets: vec![],
|
|
||||||
port_forward_service: None,
|
|
||||||
applications: vec![],
|
|
||||||
api_apps: vec![ZitadelApiApp {
|
|
||||||
project_name: PROJECT_NAME.to_string(),
|
|
||||||
app_name: "nats".to_string(),
|
|
||||||
}],
|
|
||||||
..zitadel_connection()
|
|
||||||
}
|
}
|
||||||
.interpret(&Inventory::autoload(), topology)
|
let application = identity.application_ref(OPERATOR_APP);
|
||||||
.await
|
let operator_identity = identity.machine_identity_ref(ADMIN_USERNAME);
|
||||||
.map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?;
|
identity
|
||||||
|
.interpret(&Inventory::autoload(), topology)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?;
|
||||||
|
|
||||||
let zcfg = ZitadelClientConfig::load()
|
let zcfg = ZitadelClientConfig::load()
|
||||||
.context("ZitadelSetupScore did not produce a client config cache")?;
|
.context("ZitadelSetupScore did not produce a client config cache")?;
|
||||||
@@ -228,6 +171,9 @@ pub async fn bring_up_zitadel(
|
|||||||
project_id,
|
project_id,
|
||||||
admin_machine_key,
|
admin_machine_key,
|
||||||
admin_pat,
|
admin_pat,
|
||||||
|
provider,
|
||||||
|
application,
|
||||||
|
operator_identity,
|
||||||
// Placeholder until deploy_openbao fills them in.
|
// Placeholder until deploy_openbao fills them in.
|
||||||
openbao: OpenbaoInstance {
|
openbao: OpenbaoInstance {
|
||||||
namespace: String::new(),
|
namespace: String::new(),
|
||||||
@@ -281,7 +227,7 @@ pub async fn enroll_device(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Deploy NATS in callout mode + the callout service itself.
|
/// Deploy NATS in callout mode + the callout service itself.
|
||||||
/// Returns the `FleetNatsScore` so the caller can derive the
|
/// Returns the `NatsScore` so the caller can derive the
|
||||||
/// in-cluster URL the same way the user-pass path does.
|
/// in-cluster URL the same way the user-pass path does.
|
||||||
pub async fn deploy_nats_callout(
|
pub async fn deploy_nats_callout(
|
||||||
topology: &K8sAnywhereTopology,
|
topology: &K8sAnywhereTopology,
|
||||||
@@ -289,44 +235,38 @@ pub async fn deploy_nats_callout(
|
|||||||
release: &str,
|
release: &str,
|
||||||
node_port: u16,
|
node_port: u16,
|
||||||
handles: &CalloutHandles,
|
handles: &CalloutHandles,
|
||||||
) -> Result<FleetNatsScore> {
|
) -> Result<NatsScore> {
|
||||||
let issuer_kp = KeyPair::new_account();
|
let credentials = NatsAuthCalloutCredentialsScore::generated(
|
||||||
let issuer_seed = issuer_kp
|
namespace,
|
||||||
.seed()
|
format!("{CALLOUT_DEPLOYMENT}-credentials"),
|
||||||
.map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?;
|
);
|
||||||
|
let credentials_ref = credentials.credentials_ref();
|
||||||
let nats = FleetNatsScore::callout(
|
credentials
|
||||||
namespace.to_string(),
|
.interpret(&Inventory::autoload(), topology)
|
||||||
node_port,
|
.await
|
||||||
issuer_kp.public_key(),
|
.map_err(|e| anyhow::anyhow!("callout credentials deploy: {e}"))?;
|
||||||
|
let nats = NatsScore::callout_account(
|
||||||
|
release,
|
||||||
|
namespace,
|
||||||
|
NatsService::NodePort(node_port),
|
||||||
NATS_ACCOUNT,
|
NATS_ACCOUNT,
|
||||||
|
);
|
||||||
|
let callout = NatsAuthCalloutScore::for_account(
|
||||||
|
CALLOUT_DEPLOYMENT,
|
||||||
|
namespace,
|
||||||
|
&nats.account_ref(),
|
||||||
NATS_AUTH_USER,
|
NATS_AUTH_USER,
|
||||||
NATS_AUTH_PASS,
|
|
||||||
)
|
)
|
||||||
.release_name(release);
|
.credentials(&credentials_ref)
|
||||||
|
.with_oidc(&handles.provider, &handles.application)
|
||||||
|
.image(CALLOUT_IMAGE_TAG)
|
||||||
|
.admin_role(ADMIN_ROLE)
|
||||||
|
.device_role(DEVICE_ROLE)
|
||||||
|
.device_id_claim("client_id");
|
||||||
|
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
|
||||||
nats.interpret(&Inventory::autoload(), topology)
|
nats.interpret(&Inventory::autoload(), topology)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?;
|
||||||
|
|
||||||
let mut callout = NatsAuthCalloutScore::new(
|
|
||||||
CALLOUT_DEPLOYMENT,
|
|
||||||
namespace,
|
|
||||||
nats.in_cluster_url(),
|
|
||||||
handles.issuer_url.clone(),
|
|
||||||
handles.project_id.clone(),
|
|
||||||
NATS_AUTH_USER,
|
|
||||||
NATS_AUTH_PASS,
|
|
||||||
issuer_seed,
|
|
||||||
)
|
|
||||||
.image(CALLOUT_IMAGE_TAG)
|
|
||||||
.target_account(NATS_ACCOUNT)
|
|
||||||
.admin_role(ADMIN_ROLE)
|
|
||||||
.device_role(DEVICE_ROLE);
|
|
||||||
// Machine-user access tokens carry the userName in `client_id`;
|
|
||||||
// usernames ARE device ids here (see `enroll_device`).
|
|
||||||
callout.device_id_claim = "client_id".to_string();
|
|
||||||
callout.device_id_prefix_strip = String::new();
|
|
||||||
callout.roles_claim = format!("urn:zitadel:iam:org:project:{}:roles", handles.project_id);
|
|
||||||
callout
|
callout
|
||||||
.interpret(&Inventory::autoload(), topology)
|
.interpret(&Inventory::autoload(), topology)
|
||||||
.await
|
.await
|
||||||
@@ -335,10 +275,6 @@ pub async fn deploy_nats_callout(
|
|||||||
Ok(nats)
|
Ok(nats)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn callout_deployment_name() -> &'static str {
|
|
||||||
CALLOUT_DEPLOYMENT
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Deploy OpenBao into the bring-up namespace: JWT auth bound to the
|
/// Deploy OpenBao into the bring-up namespace: JWT auth bound to the
|
||||||
/// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging
|
/// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging
|
||||||
/// shape) plus a NodePort Service so VMs reach it through the libvirt
|
/// shape) plus a NodePort Service so VMs reach it through the libvirt
|
||||||
@@ -352,70 +288,21 @@ pub async fn deploy_openbao(
|
|||||||
namespace: namespace.to_string(),
|
namespace: namespace.to_string(),
|
||||||
release: OPENBAO_RELEASE.to_string(),
|
release: OPENBAO_RELEASE.to_string(),
|
||||||
};
|
};
|
||||||
OpenbaoScore {
|
OpenbaoScore::new(namespace, OPENBAO_RELEASE, "openbao-e2e.local")
|
||||||
instance: instance.clone(),
|
.node_port(OPENBAO_NODE_PORT)
|
||||||
host: "openbao-e2e.local".to_string(),
|
|
||||||
openshift: false,
|
|
||||||
tls_issuer: None,
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?;
|
|
||||||
|
|
||||||
OpenbaoSetupScore {
|
|
||||||
instance: instance.clone(),
|
|
||||||
kv_mount: KV_MOUNT.to_string(),
|
|
||||||
jwt_auth: Some(OpenbaoJwtAuth {
|
|
||||||
oidc_discovery_url: handles.issuer_url.clone(),
|
|
||||||
jwt_validation_pubkeys: String::new(),
|
|
||||||
bound_issuer: handles.issuer_url.clone(),
|
|
||||||
role_name: "fleet-device".to_string(),
|
|
||||||
bound_audiences: handles.project_id.clone(),
|
|
||||||
user_claim: "sub".to_string(),
|
|
||||||
groups_claim: "groups".to_string(),
|
|
||||||
token_type: "batch".to_string(),
|
|
||||||
ttl: "1h".to_string(),
|
|
||||||
max_ttl: "1h".to_string(),
|
|
||||||
}),
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?;
|
|
||||||
|
|
||||||
// NodePort for VM access. Same selector shape as the chart's own
|
|
||||||
// ClusterIP Service.
|
|
||||||
let selector = std::collections::BTreeMap::from([
|
|
||||||
("app.kubernetes.io/name".to_string(), "openbao".to_string()),
|
|
||||||
(
|
|
||||||
"app.kubernetes.io/instance".to_string(),
|
|
||||||
OPENBAO_RELEASE.to_string(),
|
|
||||||
),
|
|
||||||
("component".to_string(), "server".to_string()),
|
|
||||||
]);
|
|
||||||
let svc = Service {
|
|
||||||
metadata: ObjectMeta {
|
|
||||||
name: Some("openbao-nodeport".to_string()),
|
|
||||||
namespace: Some(namespace.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
spec: Some(ServiceSpec {
|
|
||||||
type_: Some("NodePort".to_string()),
|
|
||||||
selector: Some(selector),
|
|
||||||
ports: Some(vec![ServicePort {
|
|
||||||
name: Some("http".to_string()),
|
|
||||||
port: 8200,
|
|
||||||
node_port: Some(OPENBAO_NODE_PORT as i32),
|
|
||||||
..Default::default()
|
|
||||||
}]),
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
K8sResourceScore::single(svc, Some(namespace.to_string()))
|
|
||||||
.interpret(&Inventory::autoload(), topology)
|
.interpret(&Inventory::autoload(), topology)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("OpenBao NodePort service: {e}"))?;
|
.map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?;
|
||||||
|
|
||||||
|
OpenbaoSetupScore::new(instance.clone())
|
||||||
|
.with_oidc_application(
|
||||||
|
&handles.provider,
|
||||||
|
&handles.application,
|
||||||
|
OpenbaoJwtAuth::oidc("fleet-device"),
|
||||||
|
)
|
||||||
|
.interpret(&Inventory::autoload(), topology)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?;
|
||||||
|
|
||||||
handles.openbao = instance.clone();
|
handles.openbao = instance.clone();
|
||||||
handles.openbao_root_token =
|
handles.openbao_root_token =
|
||||||
|
|||||||
@@ -17,11 +17,11 @@ use std::time::{Duration, Instant};
|
|||||||
|
|
||||||
use async_nats::Client;
|
use async_nats::Client;
|
||||||
use async_nats::jetstream::{self, kv};
|
use async_nats::jetstream::{self, kv};
|
||||||
use harmony::modules::podman::{PodmanV0Score, ReconcileScore};
|
|
||||||
use harmony_reconciler_contracts::{
|
use harmony_reconciler_contracts::{
|
||||||
BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase,
|
BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase,
|
||||||
desired_state_key, device_state_key,
|
desired_state_key, device_state_key,
|
||||||
};
|
};
|
||||||
|
use harmony_reconciler_contracts::{PodmanV0Score, ReconcileScore};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
/// One handle per test, opens the buckets the test reads/writes.
|
/// One handle per test, opens the buckets the test reads/writes.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
//! In-cluster end-to-end harness for the harmony fleet stack.
|
//! In-cluster end-to-end harness for the harmony fleet stack.
|
||||||
//!
|
//!
|
||||||
//! Brings up the fleet stack ([`FleetNatsScore`] +
|
//! Brings up the fleet stack ([`NatsScore`] +
|
||||||
//! [`FleetAgentScore`]) in a fresh per-bring-up k3d namespace and
|
//! [`FleetAgentScore`]) in a fresh per-bring-up k3d namespace and
|
||||||
//! hands the test a connected NATS admin client + the device ids the
|
//! hands the test a connected NATS admin client + the device ids the
|
||||||
//! agents are running under. Bring-up is shared across tests in the
|
//! agents are running under. Bring-up is shared across tests in the
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
//! Per ADR-023, the harness composes the same `*Score` types the
|
//! Per ADR-023, the harness composes the same `*Score` types the
|
||||||
//! production CLI uses. It does **not** build manifests itself.
|
//! production CLI uses. It does **not** build manifests itself.
|
||||||
//!
|
//!
|
||||||
//! [`FleetNatsScore`]: harmony_fleet_deploy::FleetNatsScore
|
//! [`NatsScore`]: harmony::modules::nats::NatsScore
|
||||||
//! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore
|
//! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore
|
||||||
|
|
||||||
pub mod callout;
|
pub mod callout;
|
||||||
|
|||||||
@@ -14,9 +14,13 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
|
use harmony::inventory::Inventory;
|
||||||
|
use harmony::modules::k8s::namespace::K8sNamespaceScore;
|
||||||
|
use harmony::score::Score;
|
||||||
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
use k8s_openapi::api::core::v1::Namespace;
|
use k8s_openapi::api::core::v1::Namespace;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
use kube::api::{Api, ObjectMeta, PostParams};
|
use kube::api::Api;
|
||||||
|
|
||||||
/// Env var that, when set to `"1"`, prevents the guard from deleting
|
/// Env var that, when set to `"1"`, prevents the guard from deleting
|
||||||
/// the namespace on drop. Convenience for debugging hung tests.
|
/// the namespace on drop. Convenience for debugging hung tests.
|
||||||
@@ -43,24 +47,15 @@ impl std::fmt::Debug for NamespaceGuard {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl NamespaceGuard {
|
impl NamespaceGuard {
|
||||||
pub(crate) async fn create(client: Arc<Client>, namespace: String) -> Result<Self> {
|
pub(crate) async fn create(
|
||||||
let api: Api<Namespace> = Api::all((*client).clone());
|
client: Arc<Client>,
|
||||||
let ns = Namespace {
|
topology: &K8sAnywhereTopology,
|
||||||
metadata: ObjectMeta {
|
namespace: String,
|
||||||
name: Some(namespace.clone()),
|
) -> Result<Self> {
|
||||||
labels: Some(
|
K8sNamespaceScore::new(&namespace)
|
||||||
[
|
.label("harmony.io/managed-by", "fleet-e2e")
|
||||||
("harmony.io/managed-by".to_string(), "fleet-e2e".to_string()),
|
.label("harmony.io/scope", "ephemeral")
|
||||||
("harmony.io/scope".to_string(), "ephemeral".to_string()),
|
.interpret(&Inventory::autoload(), topology)
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
api.create(&PostParams::default(), &ns)
|
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("creating namespace {namespace}"))?;
|
.with_context(|| format!("creating namespace {namespace}"))?;
|
||||||
let keep = matches!(
|
let keep = matches!(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
//! same `*Score` types production uses. The only thing this module
|
//! same `*Score` types production uses. The only thing this module
|
||||||
//! owns is the *plumbing* around them: ensure a k3d cluster exists,
|
//! owns is the *plumbing* around them: ensure a k3d cluster exists,
|
||||||
//! sideload the agent image into it, mint a per-bring-up namespace,
|
//! sideload the agent image into it, mint a per-bring-up namespace,
|
||||||
//! compose [`FleetNatsScore`] + [`FleetAgentScore`] against the test
|
//! compose [`NatsScore`] + [`FleetAgentScore`] against the test
|
||||||
//! Topology, wait for pods to be Ready, and hand the test an admin
|
//! Topology, wait for pods to be Ready, and hand the test an admin
|
||||||
//! NATS client.
|
//! NATS client.
|
||||||
//!
|
//!
|
||||||
@@ -24,15 +24,14 @@ use std::sync::Arc;
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::operator::crd::Deployment as FleetDeployment;
|
use harmony::modules::nats::{NatsScore, NatsService, UserPassCredentials};
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::K8sAnywhereTopology;
|
use harmony::topology::K8sAnywhereTopology;
|
||||||
use harmony_fleet_deploy::agent::PodTarget;
|
use harmony_fleet_deploy::agent::PodTarget;
|
||||||
use harmony_fleet_deploy::nats::UserPassCredentials;
|
|
||||||
use harmony_fleet_deploy::operator::OperatorCredentials;
|
use harmony_fleet_deploy::operator::OperatorCredentials;
|
||||||
use harmony_fleet_deploy::{FleetAgentScore, FleetNatsScore, FleetOperatorScore};
|
use harmony_fleet_deploy::{FleetAgentScore, FleetOperatorScore};
|
||||||
|
use harmony_fleet_operator::Deployment as FleetDeployment;
|
||||||
use k3d_rs::{K3d, PortMapping};
|
use k3d_rs::{K3d, PortMapping};
|
||||||
use k8s_openapi::api::apps::v1::Deployment;
|
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
@@ -182,14 +181,10 @@ impl Stack {
|
|||||||
/// use [`shared_stack`].
|
/// use [`shared_stack`].
|
||||||
pub async fn bring_up(opts: StackOptions) -> Result<Self, BringUpError> {
|
pub async fn bring_up(opts: StackOptions) -> Result<Self, BringUpError> {
|
||||||
let callout_mode = opts.auth_mode == AuthMode::Callout;
|
let callout_mode = opts.auth_mode == AuthMode::Callout;
|
||||||
// Static user-pass material is what the Pod agent + operator
|
// Pod agents still use static credentials; callout-mode devices are VMs.
|
||||||
// paths inject today; behind the callout they'd be rejected
|
if callout_mode && opts.num_devices > 0 {
|
||||||
// at connect. The VM harness brings callout-authenticated
|
|
||||||
// agents instead.
|
|
||||||
if callout_mode && (opts.num_devices > 0 || opts.deploy_operator) {
|
|
||||||
return Err(BringUpError::Callout(anyhow::anyhow!(
|
return Err(BringUpError::Callout(anyhow::anyhow!(
|
||||||
"callout mode supports infra-only bring-up (VM agents); \
|
"callout mode uses VM agents; Pod agents still authenticate with static creds"
|
||||||
Pod agents / operator still authenticate with static creds"
|
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,12 +231,18 @@ impl Stack {
|
|||||||
prune_stale_operator_cluster_resources(&kube_client).await;
|
prune_stale_operator_cluster_resources(&kube_client).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let namespace = unique_namespace();
|
||||||
|
let guard = NamespaceGuard::create(kube_client.clone(), &topology, namespace.clone())
|
||||||
|
.await
|
||||||
|
.map_err(BringUpError::Namespace)?;
|
||||||
|
tracing::info!(namespace = %namespace, "namespace created");
|
||||||
|
|
||||||
// Zitadel + project/roles/admin identity. Sits outside the
|
// Zitadel + project/roles/admin identity. Sits outside the
|
||||||
// per-bring-up namespace (stable `zitadel` ns) so the
|
// per-bring-up namespace (stable `zitadel` ns) so the
|
||||||
// multi-minute first install amortizes across runs.
|
// multi-minute first install amortizes across runs.
|
||||||
let mut callout_handles = if callout_mode {
|
let mut callout_handles = if callout_mode {
|
||||||
Some(
|
Some(
|
||||||
crate::callout::bring_up_zitadel(&topology, &kube_client)
|
crate::callout::bring_up_zitadel(&topology, &kube_client, &namespace)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::Callout)?,
|
.map_err(BringUpError::Callout)?,
|
||||||
)
|
)
|
||||||
@@ -249,15 +250,9 @@ impl Stack {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let namespace = unique_namespace();
|
|
||||||
let guard = NamespaceGuard::create(kube_client.clone(), namespace.clone())
|
|
||||||
.await
|
|
||||||
.map_err(BringUpError::Namespace)?;
|
|
||||||
tracing::info!(namespace = %namespace, "namespace created");
|
|
||||||
|
|
||||||
// Deploy NATS via the production Score. The harness owns no
|
// Deploy NATS via the production Score. The harness owns no
|
||||||
// bespoke nats-server.conf, no NodePort Service — that's all
|
// bespoke nats-server.conf, no NodePort Service — that's all
|
||||||
// inside `FleetNatsScore`. NATS readiness is gated by the
|
// inside `NatsScore`. NATS readiness is gated by the
|
||||||
// host-side `connect_nats_admin` retry loop further down (the
|
// host-side `connect_nats_admin` retry loop further down (the
|
||||||
// upstream NATS chart deploys a StatefulSet, not a Deployment,
|
// upstream NATS chart deploys a StatefulSet, not a Deployment,
|
||||||
// so a generic Deployment-poll wouldn't match; the agent's
|
// so a generic Deployment-poll wouldn't match; the agent's
|
||||||
@@ -267,19 +262,20 @@ impl Stack {
|
|||||||
// deploy crate — keeping them out of `harmony-fleet-deploy`
|
// deploy crate — keeping them out of `harmony-fleet-deploy`
|
||||||
// means production builds can't accidentally ship them.
|
// means production builds can't accidentally ship them.
|
||||||
let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}");
|
let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}");
|
||||||
let (nats_client, nats_in_cluster_url) = match callout_handles.as_mut() {
|
let (nats_client, nats_ref) = match callout_handles.as_mut() {
|
||||||
None => {
|
None => {
|
||||||
let nats = FleetNatsScore::user_pass(
|
let nats = NatsScore::user_pass(
|
||||||
|
NATS_RELEASE,
|
||||||
namespace.clone(),
|
namespace.clone(),
|
||||||
NATS_NODE_PORT,
|
NatsService::NodePort(NATS_NODE_PORT),
|
||||||
|
"APP",
|
||||||
UserPassCredentials {
|
UserPassCredentials {
|
||||||
admin_user: NATS_ADMIN_USER.to_string(),
|
admin_user: NATS_ADMIN_USER.to_string(),
|
||||||
admin_pass: NATS_ADMIN_PASS.to_string(),
|
admin_pass: NATS_ADMIN_PASS.to_string(),
|
||||||
device_user: NATS_DEVICE_USER.to_string(),
|
device_user: NATS_DEVICE_USER.to_string(),
|
||||||
device_pass: NATS_DEVICE_PASS.to_string(),
|
device_pass: NATS_DEVICE_PASS.to_string(),
|
||||||
},
|
},
|
||||||
)
|
);
|
||||||
.release_name(NATS_RELEASE);
|
|
||||||
nats.interpret(&Inventory::autoload(), &topology)
|
nats.interpret(&Inventory::autoload(), &topology)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::NatsDeploy)?;
|
.map_err(BringUpError::NatsDeploy)?;
|
||||||
@@ -288,7 +284,7 @@ impl Stack {
|
|||||||
// up, can't connect, hits CrashLoopBackOff for a
|
// up, can't connect, hits CrashLoopBackOff for a
|
||||||
// while, and the agent Deployment-Ready wait
|
// while, and the agent Deployment-Ready wait
|
||||||
// spuriously times out.
|
// spuriously times out.
|
||||||
(connect_nats_admin(&nats_url).await?, nats.in_cluster_url())
|
(connect_nats_admin(&nats_url).await?, nats.client_ref())
|
||||||
}
|
}
|
||||||
Some(handles) => {
|
Some(handles) => {
|
||||||
let nats = crate::callout::deploy_nats_callout(
|
let nats = crate::callout::deploy_nats_callout(
|
||||||
@@ -300,47 +296,46 @@ impl Stack {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::Callout)?;
|
.map_err(BringUpError::Callout)?;
|
||||||
wait_for_deployment(
|
|
||||||
&kube_client,
|
|
||||||
&namespace,
|
|
||||||
crate::callout::callout_deployment_name(),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let client = crate::callout::connect_nats_admin_sso(&nats_url, handles)
|
let client = crate::callout::connect_nats_admin_sso(&nats_url, handles)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::Callout)?;
|
.map_err(BringUpError::Callout)?;
|
||||||
crate::callout::deploy_openbao(&topology, &namespace, handles)
|
crate::callout::deploy_openbao(&topology, &namespace, handles)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::Callout)?;
|
.map_err(BringUpError::Callout)?;
|
||||||
(client, nats.in_cluster_url())
|
(client, nats.client_ref())
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if opts.deploy_operator {
|
if opts.deploy_operator {
|
||||||
let mut operator = FleetOperatorScore::new()
|
let mut operator = FleetOperatorScore::new(opts.operator_image.clone())
|
||||||
.namespace(namespace.clone())
|
.namespace(namespace.clone())
|
||||||
.image(opts.operator_image.clone())
|
.messaging(&nats_ref)
|
||||||
.nats_url(nats_in_cluster_url.clone())
|
|
||||||
.log_level(opts.log_level.clone());
|
.log_level(opts.log_level.clone());
|
||||||
|
|
||||||
operator.credentials = Some(OperatorCredentials {
|
if let Some(handles) = callout_handles.as_ref() {
|
||||||
credentials_toml: format!(
|
operator = operator.identity(
|
||||||
r#"type = "toml-shared"
|
&handles.provider,
|
||||||
nats_user = "{NATS_ADMIN_USER}"
|
&handles.application,
|
||||||
nats_pass = "{NATS_ADMIN_PASS}"
|
&handles.operator_identity,
|
||||||
"#
|
);
|
||||||
),
|
} else {
|
||||||
});
|
operator.credentials = Some(OperatorCredentials {
|
||||||
|
credentials_toml: format!(
|
||||||
|
r#"type = "toml-shared"
|
||||||
|
nats_user = "{NATS_ADMIN_USER}"
|
||||||
|
nats_pass = "{NATS_ADMIN_PASS}"
|
||||||
|
"#
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
operator
|
operator
|
||||||
.interpret(&Inventory::autoload(), &topology)
|
.interpret(&Inventory::autoload(), &topology)
|
||||||
.await
|
.await
|
||||||
.map_err(BringUpError::OperatorDeploy)?;
|
.map_err(BringUpError::OperatorDeploy)?;
|
||||||
|
|
||||||
wait_for_deployment(&kube_client, &namespace, "harmony-fleet-operator").await?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deploy one agent Pod per requested device, all sharing the
|
// Deploy one agent Pod per requested device, all sharing the
|
||||||
// in-cluster NATS DNS the FleetNatsScore exposes.
|
// in-cluster NATS DNS the NatsScore exposes.
|
||||||
let device_ids: Vec<String> = if opts.deploy_agent {
|
let device_ids: Vec<String> = if opts.deploy_agent {
|
||||||
(0..opts.num_devices)
|
(0..opts.num_devices)
|
||||||
.map(|i| format!("vm-device-{i:02}-{}", &namespace["e2e-".len()..]))
|
.map(|i| format!("vm-device-{i:02}-{}", &namespace["e2e-".len()..]))
|
||||||
@@ -356,7 +351,7 @@ impl Stack {
|
|||||||
PodTarget::user_pass(
|
PodTarget::user_pass(
|
||||||
device_id.clone(),
|
device_id.clone(),
|
||||||
opts.agent_image.clone(),
|
opts.agent_image.clone(),
|
||||||
nats_in_cluster_url.clone(),
|
nats_ref.url().to_string(),
|
||||||
NATS_DEVICE_USER,
|
NATS_DEVICE_USER,
|
||||||
NATS_DEVICE_PASS,
|
NATS_DEVICE_PASS,
|
||||||
)
|
)
|
||||||
@@ -367,14 +362,6 @@ impl Stack {
|
|||||||
.await
|
.await
|
||||||
.map_err(BringUpError::AgentDeploy)?;
|
.map_err(BringUpError::AgentDeploy)?;
|
||||||
}
|
}
|
||||||
for device_id in &device_ids {
|
|
||||||
wait_for_deployment(
|
|
||||||
&kube_client,
|
|
||||||
&namespace,
|
|
||||||
&FleetAgentScore::resource_name(device_id),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@@ -591,46 +578,6 @@ async fn prune_stale_namespaces(client: &Client) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_deployment(
|
|
||||||
client: &Client,
|
|
||||||
namespace: &str,
|
|
||||||
name: &str,
|
|
||||||
) -> Result<(), BringUpError> {
|
|
||||||
// Temporary readiness gate. ADR-023 principle 4 promises that a
|
|
||||||
// future smoke-test contract on the Score itself will replace this
|
|
||||||
// — `deploy` will only return after the agent is up. Today the
|
|
||||||
// Score returns when the manifest is applied; we poll for Ready
|
|
||||||
// here so each test can rely on the pods being reachable.
|
|
||||||
let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
|
|
||||||
let deadline = std::time::Instant::now() + Duration::from_secs(120);
|
|
||||||
while std::time::Instant::now() < deadline {
|
|
||||||
match api.get(name).await {
|
|
||||||
Ok(d) => {
|
|
||||||
if let Some(status) = d.status
|
|
||||||
&& status.ready_replicas.unwrap_or(0) >= 1
|
|
||||||
{
|
|
||||||
tracing::info!(namespace = %namespace, deployment = %name, "deployment ready");
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(kube::Error::Api(e)) if e.code == 404 => {}
|
|
||||||
Err(e) => {
|
|
||||||
return Err(BringUpError::WaitTimeout {
|
|
||||||
namespace: namespace.to_string(),
|
|
||||||
name: name.to_string(),
|
|
||||||
detail: format!("get deployment failed: {e}"),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
}
|
|
||||||
Err(BringUpError::WaitTimeout {
|
|
||||||
namespace: namespace.to_string(),
|
|
||||||
name: name.to_string(),
|
|
||||||
detail: "timed out after 120s".to_string(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn connect_nats_admin(url: &str) -> Result<async_nats::Client, BringUpError> {
|
async fn connect_nats_admin(url: &str) -> Result<async_nats::Client, BringUpError> {
|
||||||
// NATS helm chart cold-start is ~30-60s on a freshly-pulled
|
// NATS helm chart cold-start is ~30-60s on a freshly-pulled
|
||||||
// image. Keep retrying for 2 minutes before giving up so the
|
// image. Keep retrying for 2 minutes before giving up so the
|
||||||
|
|||||||
@@ -23,15 +23,17 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::{
|
use harmony::modules::fleet::{
|
||||||
DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
|
HARMONY_FLEET_POOL_NAME, ProvisionVmScore, ensure_fleet_ssh_keypair, ensure_harmony_fleet_pool,
|
||||||
HARMONY_FLEET_POOL_NAME, HostsEntry, ProvisionVmScore, ensure_fleet_ssh_keypair,
|
read_public_key,
|
||||||
ensure_harmony_fleet_pool, read_public_key,
|
|
||||||
};
|
};
|
||||||
use harmony::modules::kvm::KvmVirtualMachineHost;
|
use harmony::modules::kvm::KvmVirtualMachineHost;
|
||||||
use harmony::modules::kvm::config::init_executor;
|
use harmony::modules::kvm::config::init_executor;
|
||||||
use harmony::modules::linux::{LinuxHostTopology, SshCommandOutput, SshCredentials, ssh_exec};
|
use harmony::modules::linux::{LinuxHostTopology, SshCommandOutput, SshCredentials, ssh_exec};
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
|
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
|
||||||
|
use harmony_fleet_deploy::{
|
||||||
|
DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry,
|
||||||
|
};
|
||||||
use harmony_types::id::Id;
|
use harmony_types::id::Id;
|
||||||
use harmony_types::net::IpAddress;
|
use harmony_types::net::IpAddress;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
//! NATS the Pod harness deploys.
|
//! NATS the Pod harness deploys.
|
||||||
//!
|
//!
|
||||||
//! Per ADR-023 P2 the harness composes the same production Scores
|
//! Per ADR-023 P2 the harness composes the same production Scores
|
||||||
//! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `FleetNatsScore`)
|
//! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `NatsScore`)
|
||||||
//! — the only thing the test layer owns is the test-fixture wiring
|
//! — the only thing the test layer owns is the test-fixture wiring
|
||||||
//! (per-binary `OnceCell`, drop-time VM cleanup, KV admin helpers).
|
//! (per-binary `OnceCell`, drop-time VM cleanup, KV admin helpers).
|
||||||
//!
|
//!
|
||||||
|
|||||||
@@ -26,14 +26,14 @@ use tokio::process::Command;
|
|||||||
use tokio::sync::OnceCell;
|
use tokio::sync::OnceCell;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use harmony::modules::fleet::{DeviceOpenbao, FleetDeviceAuth, HostsEntry};
|
use harmony_fleet_deploy::{DeviceOpenbao, FleetDeviceAuth, HostsEntry};
|
||||||
|
|
||||||
use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack};
|
use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack};
|
||||||
use crate::vm::agent_build::{AgentBuildError, build_agent_for};
|
use crate::vm::agent_build::{AgentBuildError, build_agent_for};
|
||||||
use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions};
|
use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions};
|
||||||
use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip};
|
use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip};
|
||||||
|
|
||||||
/// NATS device user the [`crate::stack`] `FleetNatsScore` provisions
|
/// NATS device user the [`crate::stack`] `NatsScore` provisions
|
||||||
/// in user-pass mode. Mirrors `stack::NATS_DEVICE_USER` (which is
|
/// in user-pass mode. Mirrors `stack::NATS_DEVICE_USER` (which is
|
||||||
/// private to that module) so VM agents authenticate against the
|
/// private to that module) so VM agents authenticate against the
|
||||||
/// same user-pass account.
|
/// same user-pass account.
|
||||||
@@ -300,14 +300,8 @@ impl VmStack {
|
|||||||
|
|
||||||
/// Wait until every agent in the stack answers `Verb::Ping`.
|
/// Wait until every agent in the stack answers `Verb::Ping`.
|
||||||
///
|
///
|
||||||
/// `FleetDeviceSetupScore` returns the moment `systemctl
|
/// Deployment readiness verifies the systemd unit. This assertion also
|
||||||
/// start fleet-agent` exits — *seconds* before the agent's NATS
|
/// covers the first OIDC exchange, NATS connection, and command round-trip.
|
||||||
/// subscription has actually propagated. Pinging in that window
|
|
||||||
/// hits `no_responders` immediately (NATS knows there's no
|
|
||||||
/// subscriber and short-circuits). Tests that want the agent
|
|
||||||
/// reachable must wait through that window; this helper does it
|
|
||||||
/// by retrying ping with a 3 s per-attempt timeout up to
|
|
||||||
/// `per_device_budget`.
|
|
||||||
pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> {
|
pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> {
|
||||||
// Fresh connection on the caller's runtime — see
|
// Fresh connection on the caller's runtime — see
|
||||||
// `Stack::connect_admin` for why the shared client can't be
|
// `Stack::connect_admin` for why the shared client can't be
|
||||||
|
|||||||
@@ -5,15 +5,14 @@
|
|||||||
//! live Zitadel, and tests mint their own device tokens.
|
//! live Zitadel, and tests mint their own device tokens.
|
||||||
|
|
||||||
use harmony::inventory::Inventory;
|
use harmony::inventory::Inventory;
|
||||||
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
|
|
||||||
use harmony::modules::openbao::{
|
use harmony::modules::openbao::{
|
||||||
OpenBaoDeploymentSecretGrants, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore,
|
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
|
||||||
OpenbaoSetupScore, cached_root_token,
|
|
||||||
};
|
};
|
||||||
use harmony::score::Score;
|
use harmony::score::Score;
|
||||||
use harmony::topology::{K8sAnywhereTopology, K8sclient};
|
use harmony::topology::{K8sAnywhereTopology, K8sclient};
|
||||||
use harmony_fleet_e2e::{StackOptions, shared_stack};
|
use harmony_fleet_e2e::{StackOptions, shared_stack};
|
||||||
use harmony_reconciler_contracts::DeploymentName;
|
use harmony_reconciler_contracts::{DeploymentName, DeploymentSecretGrants};
|
||||||
|
use harmony_secret::OpenBaoDeploymentSecretGrants;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
const E2E_ENV: &str = "HARMONY_FLEET_E2E";
|
const E2E_ENV: &str = "HARMONY_FLEET_E2E";
|
||||||
@@ -169,6 +168,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
|
|||||||
host: "openbao-groups-test.local".to_string(),
|
host: "openbao-groups-test.local".to_string(),
|
||||||
openshift: false,
|
openshift: false,
|
||||||
tls_issuer: None,
|
tls_issuer: None,
|
||||||
|
node_port: None,
|
||||||
}),
|
}),
|
||||||
Box::new(OpenbaoSetupScore {
|
Box::new(OpenbaoSetupScore {
|
||||||
instance: instance.clone(),
|
instance: instance.clone(),
|
||||||
@@ -187,6 +187,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
|
|||||||
ttl: "1h".to_string(),
|
ttl: "1h".to_string(),
|
||||||
max_ttl: "1h".to_string(),
|
max_ttl: "1h".to_string(),
|
||||||
}),
|
}),
|
||||||
|
oidc_application: None,
|
||||||
}),
|
}),
|
||||||
];
|
];
|
||||||
for score in scores {
|
for score in scores {
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
|
|||||||
host: "openbao-policy-test.local".to_string(),
|
host: "openbao-policy-test.local".to_string(),
|
||||||
openshift: false,
|
openshift: false,
|
||||||
tls_issuer: None,
|
tls_issuer: None,
|
||||||
|
node_port: None,
|
||||||
}),
|
}),
|
||||||
Box::new(OpenbaoSetupScore {
|
Box::new(OpenbaoSetupScore {
|
||||||
instance: instance.clone(),
|
instance: instance.clone(),
|
||||||
@@ -103,6 +104,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
|
|||||||
policies: vec![],
|
policies: vec![],
|
||||||
}],
|
}],
|
||||||
jwt_auth: None,
|
jwt_auth: None,
|
||||||
|
oidc_application: None,
|
||||||
}),
|
}),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
use harmony::modules::fleet::operator::crd::{
|
use harmony_fleet_e2e::{StackOptions, shared_stack};
|
||||||
|
use harmony_fleet_operator::crd::{
|
||||||
Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore,
|
Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore,
|
||||||
Rollout, RolloutStrategy,
|
Rollout, RolloutStrategy,
|
||||||
};
|
};
|
||||||
use harmony_fleet_e2e::{StackOptions, shared_stack};
|
|
||||||
use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key};
|
use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key};
|
||||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
|
|||||||
@@ -13,10 +13,9 @@
|
|||||||
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use harmony::modules::podman::{PodmanService, PodmanV0Score};
|
|
||||||
use harmony::topology::RestartPolicy;
|
|
||||||
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
|
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
|
||||||
use harmony_reconciler_contracts::{DeploymentName, Phase};
|
use harmony_reconciler_contracts::{DeploymentName, Phase};
|
||||||
|
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy};
|
||||||
|
|
||||||
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
||||||
|
|
||||||
|
|||||||
@@ -19,12 +19,11 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use harmony::modules::podman::{PodmanService, PodmanV0Score};
|
use harmony_fleet_auth::AgentObservation;
|
||||||
use harmony::topology::RestartPolicy;
|
|
||||||
use harmony_fleet_auth::{AgentConfig, AgentSection, CredentialsSection, NatsSection};
|
use harmony_fleet_auth::{AgentConfig, AgentSection, CredentialsSection, NatsSection};
|
||||||
use harmony_fleet_deploy::companion::AgentObservation;
|
|
||||||
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
|
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
|
||||||
use harmony_reconciler_contracts::{DeploymentName, Id, desired_state_key};
|
use harmony_reconciler_contracts::{DeploymentName, Id, desired_state_key};
|
||||||
|
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy};
|
||||||
|
|
||||||
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
||||||
|
|
||||||
|
|||||||
@@ -17,15 +17,16 @@
|
|||||||
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
|
use harmony::topology::{K8sAnywhereTopology, K8sclient};
|
||||||
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
|
|
||||||
use harmony::modules::podman::{EnvVar, PodmanService, PodmanV0Score, SecretEnvVar};
|
|
||||||
use harmony::topology::{K8sAnywhereTopology, K8sclient, RestartPolicy};
|
|
||||||
use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX};
|
use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX};
|
||||||
use harmony_fleet_e2e::{
|
use harmony_fleet_e2e::{
|
||||||
AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack,
|
AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack,
|
||||||
};
|
};
|
||||||
use harmony_reconciler_contracts::{DeploymentName, Phase};
|
use harmony_reconciler_contracts::{DeploymentName, Phase};
|
||||||
|
use harmony_reconciler_contracts::{
|
||||||
|
DeploymentSecretGrants, EnvVar, PodmanService, PodmanV0Score, RestartPolicy, SecretEnvVar,
|
||||||
|
};
|
||||||
|
use harmony_secret::OpenBaoDeploymentSecretGrants;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ default = []
|
|||||||
web-frontend = ["dep:axum", "dep:axum-extra", "dep:maud", "dep:tokio-stream", "harmony_zitadel_auth/axum"]
|
web-frontend = ["dep:axum", "dep:axum-extra", "dep:maud", "dep:tokio-stream", "harmony_zitadel_auth/axum"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
harmony = { path = "../../harmony", features = ["podman"] }
|
|
||||||
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
|
harmony_secret = { path = "../../harmony_secret" }
|
||||||
harmony_config = { path = "../../harmony_config" }
|
harmony_config = { path = "../../harmony_config" }
|
||||||
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
|
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
|
||||||
toml = { workspace = true }
|
toml = { workspace = true }
|
||||||
|
|||||||
110
fleet/harmony-fleet-operator/src/access.rs
Normal file
110
fleet/harmony-fleet-operator/src/access.rs
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use harmony_reconciler_contracts::{
|
||||||
|
DeploymentName, DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct StaticDeviceGroups {
|
||||||
|
map: HashMap<String, HashSet<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StaticDeviceGroups {
|
||||||
|
pub fn parse(spec: &str) -> Self {
|
||||||
|
let map = spec
|
||||||
|
.split(';')
|
||||||
|
.filter_map(|entry| {
|
||||||
|
let (device, groups) = entry.split_once('=')?;
|
||||||
|
let device = device.trim();
|
||||||
|
(!device.is_empty()).then(|| {
|
||||||
|
(
|
||||||
|
device.to_string(),
|
||||||
|
groups
|
||||||
|
.split('|')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|group| !group.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Self { map }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl DeviceGroupSource for StaticDeviceGroups {
|
||||||
|
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
|
||||||
|
Ok(self.map.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct InMemoryDeploymentSecretGrants {
|
||||||
|
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InMemoryDeploymentSecretGrants {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn groups_for(&self, deployment: &DeploymentName) -> Vec<String> {
|
||||||
|
self.grants
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(deployment)
|
||||||
|
.map(|groups| groups.iter().cloned().collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
|
||||||
|
async fn set_deployment_groups(
|
||||||
|
&self,
|
||||||
|
grants: &[(DeploymentName, Vec<String>)],
|
||||||
|
) -> Result<(), SecretAccessError> {
|
||||||
|
let mut current = self.grants.lock().unwrap();
|
||||||
|
for (deployment, groups) in grants {
|
||||||
|
if groups.is_empty() {
|
||||||
|
current.remove(deployment);
|
||||||
|
} else {
|
||||||
|
current.insert(deployment.clone(), groups.iter().cloned().collect());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn static_groups_parse_membership() {
|
||||||
|
let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
|
||||||
|
let groups = source.device_groups().await.unwrap();
|
||||||
|
assert_eq!(groups.len(), 2);
|
||||||
|
assert_eq!(groups["pi-01"].len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn grants_replace_and_clear_membership() {
|
||||||
|
let grants = InMemoryDeploymentSecretGrants::new();
|
||||||
|
let deployment = DeploymentName::try_new("web").unwrap();
|
||||||
|
grants
|
||||||
|
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()])])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(grants.groups_for(&deployment), vec!["edge-a"]);
|
||||||
|
grants
|
||||||
|
.set_deployment_groups(&[(deployment.clone(), vec![])])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(grants.groups_for(&deployment).is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,7 +33,7 @@ use kube::runtime::finalizer::{Event as FinalizerEvent, finalizer};
|
|||||||
use kube::runtime::watcher::Config as WatcherConfig;
|
use kube::runtime::watcher::Config as WatcherConfig;
|
||||||
use kube::{Api, Client, ResourceExt};
|
use kube::{Api, Client, ResourceExt};
|
||||||
|
|
||||||
use harmony::modules::fleet::operator::Deployment;
|
use harmony_fleet_operator::Deployment;
|
||||||
|
|
||||||
const FINALIZER: &str = "fleet.nationtech.io/finalizer";
|
const FINALIZER: &str = "fleet.nationtech.io/finalizer";
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use kube::CustomResource;
|
|||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
pub use crate::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
|
pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
|
||||||
|
|
||||||
/// Deployment intent. Targets devices by label selector — identical
|
/// Deployment intent. Targets devices by label selector — identical
|
||||||
/// to the pattern K8s itself uses for DaemonSet nodeSelector, Service
|
/// to the pattern K8s itself uses for DaemonSet nodeSelector, Service
|
||||||
@@ -18,7 +18,7 @@ use kube::Client;
|
|||||||
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
use kube::api::{Api, DeleteParams, Patch, PatchParams};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use harmony::modules::fleet::operator::{Device, DeviceSpec};
|
use crate::crd::{Device, DeviceSpec};
|
||||||
|
|
||||||
const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler";
|
const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler";
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ use kube::{Client, ResourceExt};
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use harmony::modules::fleet::operator::{Device, Reachability};
|
use crate::crd::{Device, Reachability};
|
||||||
|
|
||||||
/// A device with no heartbeat within this window is `Stale`. Agents
|
/// A device with no heartbeat within this window is `Stale`. Agents
|
||||||
/// ping every 30 s, so this tolerates ~2 missed pings.
|
/// ping every 30 s, so this tolerates ~2 missed pings.
|
||||||
|
|||||||
@@ -33,11 +33,9 @@ use kube::{Client, ResourceExt};
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use harmony::modules::fleet::groups::DeviceGroupSource;
|
use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
|
||||||
use harmony::modules::fleet::operator::{
|
|
||||||
AggregateLastError, Deployment, DeploymentAggregate, Device,
|
use crate::crd::{AggregateLastError, Deployment, DeploymentAggregate, Device};
|
||||||
};
|
|
||||||
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
|
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
const PATCH_TICK: Duration = Duration::from_secs(1);
|
const PATCH_TICK: Duration = Duration::from_secs(1);
|
||||||
@@ -355,38 +353,20 @@ async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow:
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push the allowed-groups grant for `deployments` to the secrets
|
async fn sync_grant_batch(
|
||||||
/// backend in one batch. A deployment absent from the cache (deleted, or
|
grants: &Arc<dyn DeploymentSecretGrants>,
|
||||||
/// no `allowedGroups` declared) resolves to the empty set = revocation.
|
batch: &[(DeploymentName, Vec<String>)],
|
||||||
/// Non-fatal: a backend hiccup logs and is retried on the next
|
|
||||||
/// deployment change rather than tearing down reconciliation. Skipped
|
|
||||||
/// entirely when no backend is configured (dev without OpenBao).
|
|
||||||
async fn sync_deployment_grants(
|
|
||||||
state: &SharedFleetState,
|
|
||||||
grants: Option<&Arc<dyn DeploymentSecretGrants>>,
|
|
||||||
deployments: impl IntoIterator<Item = DeploymentName>,
|
|
||||||
) {
|
) {
|
||||||
let Some(grants) = grants else { return };
|
let mut delay = Duration::from_secs(1);
|
||||||
let batch: Vec<(DeploymentName, Vec<String>)> = {
|
loop {
|
||||||
let guard = state.lock().await;
|
match grants.set_deployment_groups(batch).await {
|
||||||
deployments
|
Ok(()) => return,
|
||||||
.into_iter()
|
Err(error) => {
|
||||||
.map(|dn| {
|
warn!(%error, ?delay, "aggregator: secret grant sync failed; retrying");
|
||||||
let groups = guard
|
tokio::time::sleep(delay).await;
|
||||||
.deployments
|
delay = (delay * 2).min(Duration::from_secs(30));
|
||||||
.values()
|
}
|
||||||
.find(|c| c.deployment_name == dn)
|
}
|
||||||
.and_then(|c| c.allowed_groups.clone())
|
|
||||||
.unwrap_or_default();
|
|
||||||
(dn, groups)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
};
|
|
||||||
if batch.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Err(e) = grants.set_deployment_groups(&batch).await {
|
|
||||||
warn!(error = %e, "aggregator: secret grant sync failed");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -513,6 +493,19 @@ async fn on_deployment_upsert(
|
|||||||
score_json: score_json.clone(),
|
score_json: score_json.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Keep the deployment out of schedulable state until its secret grant
|
||||||
|
// exists; device and group watchers share the cache below.
|
||||||
|
if let Some(grants) = grants {
|
||||||
|
sync_grant_batch(
|
||||||
|
grants,
|
||||||
|
&[(
|
||||||
|
deployment_name.clone(),
|
||||||
|
cached.allowed_groups.clone().unwrap_or_default(),
|
||||||
|
)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
let (new_targets, previous_targets) = {
|
let (new_targets, previous_targets) = {
|
||||||
let mut guard = state.lock().await;
|
let mut guard = state.lock().await;
|
||||||
let new_targets = matched_devices(&cached, &guard);
|
let new_targets = matched_devices(&cached, &guard);
|
||||||
@@ -528,10 +521,6 @@ async fn on_deployment_upsert(
|
|||||||
(new_targets, previous)
|
(new_targets, previous)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Grant before workload signal: never tell a device to run a
|
|
||||||
// deployment whose secrets it cannot yet read.
|
|
||||||
sync_deployment_grants(state, grants, std::iter::once(deployment_name.clone())).await;
|
|
||||||
|
|
||||||
reconcile_kv(
|
reconcile_kv(
|
||||||
desired,
|
desired,
|
||||||
&deployment_name,
|
&deployment_name,
|
||||||
@@ -555,6 +544,10 @@ async fn on_deployment_delete(
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if let Some(grants) = grants {
|
||||||
|
sync_grant_batch(grants, &[(deployment_name.clone(), Vec::new())]).await;
|
||||||
|
}
|
||||||
|
|
||||||
let previous = {
|
let previous = {
|
||||||
let mut guard = state.lock().await;
|
let mut guard = state.lock().await;
|
||||||
guard.deployments.remove(&key);
|
guard.deployments.remove(&key);
|
||||||
@@ -575,9 +568,6 @@ async fn on_deployment_delete(
|
|||||||
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed");
|
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gone from the cache, so the grant resolves to empty = revocation.
|
|
||||||
sync_deployment_grants(state, grants, std::iter::once(deployment_name)).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -1041,19 +1031,19 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sync_pushes_declared_groups_per_deployment() {
|
async fn sync_pushes_declared_groups_per_deployment() {
|
||||||
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants;
|
use crate::access::InMemoryDeploymentSecretGrants;
|
||||||
|
|
||||||
let mut s = FleetState::default();
|
|
||||||
let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a", "edge-b"]);
|
|
||||||
let db = cached("ns", "db", "zone", "lab"); // no allowedGroups → no grant
|
|
||||||
s.deployments.insert(web.key.clone(), web);
|
|
||||||
s.deployments.insert(db.key.clone(), db);
|
|
||||||
let state: SharedFleetState = Arc::new(Mutex::new(s));
|
|
||||||
|
|
||||||
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
|
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
|
||||||
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
|
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
|
||||||
|
|
||||||
sync_deployment_grants(&state, Some(&grants), [dn("web"), dn("db")]).await;
|
sync_grant_batch(
|
||||||
|
&grants,
|
||||||
|
&[
|
||||||
|
(dn("web"), vec!["edge-a".into(), "edge-b".into()]),
|
||||||
|
(dn("db"), vec![]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]);
|
assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]);
|
||||||
assert!(inmem.groups_for(&dn("db")).is_empty());
|
assert!(inmem.groups_for(&dn("db")).is_empty());
|
||||||
@@ -1061,7 +1051,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sync_revokes_deleted_deployment() {
|
async fn sync_revokes_deleted_deployment() {
|
||||||
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants;
|
use crate::access::InMemoryDeploymentSecretGrants;
|
||||||
|
|
||||||
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
|
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
|
||||||
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
|
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
|
||||||
@@ -1070,13 +1060,38 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Deployment no longer cached (CR deleted) → grant resolves empty.
|
sync_grant_batch(&grants, &[(dn("web"), vec![])]).await;
|
||||||
let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default()));
|
|
||||||
sync_deployment_grants(&state, Some(&grants), std::iter::once(dn("web"))).await;
|
|
||||||
|
|
||||||
assert!(inmem.groups_for(&dn("web")).is_empty());
|
assert!(inmem.groups_for(&dn("web")).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn grant_sync_retries_before_returning() {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
struct FlakyGrants(AtomicUsize);
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl DeploymentSecretGrants for FlakyGrants {
|
||||||
|
async fn set_deployment_groups(
|
||||||
|
&self,
|
||||||
|
_grants: &[(DeploymentName, Vec<String>)],
|
||||||
|
) -> Result<(), harmony_reconciler_contracts::SecretAccessError> {
|
||||||
|
if self.0.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||||
|
return Err(harmony_reconciler_contracts::SecretAccessError::Backend(
|
||||||
|
"temporary failure".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let flaky = Arc::new(FlakyGrants(AtomicUsize::new(0)));
|
||||||
|
let grants: Arc<dyn DeploymentSecretGrants> = flaky.clone();
|
||||||
|
sync_grant_batch(&grants, &[(dn("web"), vec!["edge-a".into()])]).await;
|
||||||
|
assert_eq!(flaky.0.load(Ordering::SeqCst), 2);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn membership_change_reschedules_device() {
|
fn membership_change_reschedules_device() {
|
||||||
let mut s = FleetState::default();
|
let mut s = FleetState::default();
|
||||||
|
|||||||
@@ -6,13 +6,17 @@
|
|||||||
//! documentation generators, alternative entry points — can drive
|
//! documentation generators, alternative entry points — can drive
|
||||||
//! the reconcilers without `main.rs`'s bootstrap.
|
//! the reconcilers without `main.rs`'s bootstrap.
|
||||||
//!
|
//!
|
||||||
//! CRD type definitions (`Deployment`, `Device`, `DeploymentSpec`,
|
//! CRD type definitions live in this runtime crate and are reused by
|
||||||
//! `ScorePayload`, …) live in
|
//! `harmony-fleet-deploy` when installing the operator.
|
||||||
//! [`harmony::modules::fleet::operator`] so that
|
|
||||||
//! `harmony::modules::fleet::FleetOperatorScore` can reach them
|
|
||||||
//! without a workspace dependency cycle. Import them from there.
|
|
||||||
|
|
||||||
|
pub mod access;
|
||||||
pub mod commands;
|
pub mod commands;
|
||||||
|
pub mod crd;
|
||||||
pub mod device_reconciler;
|
pub mod device_reconciler;
|
||||||
pub mod device_status;
|
pub mod device_status;
|
||||||
pub mod fleet_aggregator;
|
pub mod fleet_aggregator;
|
||||||
|
|
||||||
|
pub use crd::{
|
||||||
|
AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
|
||||||
|
DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
|
||||||
|
};
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ mod frontend;
|
|||||||
#[cfg(feature = "web-frontend")]
|
#[cfg(feature = "web-frontend")]
|
||||||
mod service;
|
mod service;
|
||||||
|
|
||||||
use harmony::modules::fleet::groups::{DeviceGroupSource, StaticDeviceGroups};
|
use harmony_fleet_operator::access::StaticDeviceGroups;
|
||||||
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
|
|
||||||
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
|
|
||||||
use harmony::modules::zitadel::ZitadelDeviceGroups;
|
|
||||||
use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
|
use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
|
||||||
|
use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
|
||||||
|
use harmony_secret::OpenBaoDeploymentSecretGrants;
|
||||||
|
use harmony_zitadel_auth::ZitadelDeviceGroups;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use async_nats::jetstream;
|
use async_nats::jetstream;
|
||||||
@@ -20,7 +20,6 @@ use harmony_fleet_auth::{
|
|||||||
};
|
};
|
||||||
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
|
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
#[cfg(feature = "web-frontend")]
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -62,6 +61,15 @@ struct Cli {
|
|||||||
)]
|
)]
|
||||||
credentials_toml: String,
|
credentials_toml: String,
|
||||||
|
|
||||||
|
#[arg(long, env = "FLEET_OPERATOR_OIDC_ISSUER", global = true)]
|
||||||
|
oidc_issuer: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long, env = "FLEET_OPERATOR_OIDC_AUDIENCE", global = true)]
|
||||||
|
oidc_audience: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long, env = "FLEET_OPERATOR_OIDC_KEY_PATH", global = true)]
|
||||||
|
oidc_key_path: Option<PathBuf>,
|
||||||
|
|
||||||
#[arg(long, env = "OPENBAO_URL", global = true)]
|
#[arg(long, env = "OPENBAO_URL", global = true)]
|
||||||
openbao_url: Option<String>,
|
openbao_url: Option<String>,
|
||||||
|
|
||||||
@@ -108,12 +116,28 @@ async fn main() -> Result<()> {
|
|||||||
.init();
|
.init();
|
||||||
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
let credentials_toml = if cli.credentials_toml.is_empty() {
|
||||||
|
match (&cli.oidc_issuer, &cli.oidc_audience, &cli.oidc_key_path) {
|
||||||
|
(Some(issuer), Some(audience), Some(key_path)) => {
|
||||||
|
toml::to_string(&CredentialsSection::ZitadelJwt {
|
||||||
|
key_path: key_path.clone(),
|
||||||
|
key_json: None,
|
||||||
|
oidc_issuer_url: issuer.clone(),
|
||||||
|
audience: audience.clone(),
|
||||||
|
danger_accept_invalid_certs: false,
|
||||||
|
})?
|
||||||
|
}
|
||||||
|
_ => String::new(),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cli.credentials_toml.clone()
|
||||||
|
};
|
||||||
match cli.command.unwrap_or(Command::Run) {
|
match cli.command.unwrap_or(Command::Run) {
|
||||||
Command::Run => {
|
Command::Run => {
|
||||||
run(
|
run(
|
||||||
&cli.nats_url,
|
&cli.nats_url,
|
||||||
&cli.kv_bucket,
|
&cli.kv_bucket,
|
||||||
&cli.credentials_toml,
|
&credentials_toml,
|
||||||
&cli.openbao_url,
|
&cli.openbao_url,
|
||||||
&cli.openbao_token,
|
&cli.openbao_token,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,11 +17,11 @@ use kube::api::{Api, ListParams, Patch, PatchParams};
|
|||||||
use kube::{Client, ResourceExt};
|
use kube::{Client, ResourceExt};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use harmony::modules::fleet::operator::{
|
use harmony_fleet_operator::crd::{
|
||||||
Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability,
|
Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability,
|
||||||
};
|
};
|
||||||
use harmony::modules::podman::ReconcileScore;
|
|
||||||
use harmony_fleet_operator::fleet_aggregator::selector_matches;
|
use harmony_fleet_operator::fleet_aggregator::selector_matches;
|
||||||
|
use harmony_reconciler_contracts::ReconcileScore;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
|
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
|
||||||
@@ -390,7 +390,7 @@ impl FleetService for RealFleetService {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use harmony::modules::podman::{PodmanService, PodmanV0Score};
|
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score};
|
||||||
|
|
||||||
fn liveness(r: Reachability) -> DeviceLiveness {
|
fn liveness(r: Reachability) -> DeviceLiveness {
|
||||||
DeviceLiveness {
|
DeviceLiveness {
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use k8s_openapi::api::{
|
use k8s_openapi::api::{
|
||||||
apps::v1::Deployment,
|
apps::v1::{Deployment, StatefulSet},
|
||||||
core::v1::{Namespace, Node, ServiceAccount},
|
core::v1::{Namespace, Node, Pod, ServiceAccount},
|
||||||
};
|
};
|
||||||
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
||||||
use kube::api::ApiResource;
|
use kube::api::ApiResource;
|
||||||
@@ -268,10 +268,84 @@ impl K8sClient {
|
|||||||
};
|
};
|
||||||
let timeout = timeout.unwrap_or(Duration::from_secs(120));
|
let timeout = timeout.unwrap_or(Duration::from_secs(120));
|
||||||
let establish = await_condition(api, name, conditions::is_deployment_completed());
|
let establish = await_condition(api, name, conditions::is_deployment_completed());
|
||||||
tokio::time::timeout(timeout, establish)
|
match tokio::time::timeout(timeout, establish).await {
|
||||||
.await
|
Ok(Ok(_)) => Ok(()),
|
||||||
.map(|_| ())
|
Ok(Err(error)) => Err(format!("Failed waiting for deployment {name}: {error}")),
|
||||||
.map_err(|_| "Timed out waiting for deployment".to_string())
|
Err(_) => Err(format!(
|
||||||
|
"Timed out after {}s waiting for deployment {name}",
|
||||||
|
timeout.as_secs()
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn wait_until_statefulset_ready(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
namespace: Option<&str>,
|
||||||
|
timeout: Option<Duration>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let timeout = timeout.unwrap_or(Duration::from_secs(120));
|
||||||
|
match tokio::time::timeout(timeout, async {
|
||||||
|
loop {
|
||||||
|
match self.get_resource::<StatefulSet>(name, namespace).await {
|
||||||
|
Ok(Some(statefulset))
|
||||||
|
if statefulset.status.as_ref().is_some_and(|status| {
|
||||||
|
status.ready_replicas.unwrap_or_default() > 0
|
||||||
|
}) =>
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(error) => {
|
||||||
|
return Err(format!("Failed waiting for statefulset {name}: {error}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(format!(
|
||||||
|
"Timed out after {}s waiting for statefulset {name}",
|
||||||
|
timeout.as_secs()
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn wait_until_pod_running(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
namespace: Option<&str>,
|
||||||
|
timeout: Option<Duration>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let timeout = timeout.unwrap_or(Duration::from_secs(120));
|
||||||
|
match tokio::time::timeout(timeout, async {
|
||||||
|
loop {
|
||||||
|
match self.get_resource::<Pod>(name, namespace).await {
|
||||||
|
Ok(Some(pod))
|
||||||
|
if pod
|
||||||
|
.status
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|status| status.phase.as_deref())
|
||||||
|
== Some("Running") =>
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(error) => return Err(format!("Failed waiting for pod {name}: {error}")),
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(format!(
|
||||||
|
"Timed out after {}s waiting for pod {name}",
|
||||||
|
timeout.as_secs()
|
||||||
|
)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Gets a single named resource, using the correct API scope for `K`.
|
/// Gets a single named resource, using the correct API scope for `K`.
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ license.workspace = true
|
|||||||
# module. None of those consumers should pay for the others' deps.
|
# module. None of those consumers should pay for the others' deps.
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
async-trait.workspace = true
|
||||||
chrono = { workspace = true, features = ["serde"] }
|
chrono = { workspace = true, features = ["serde"] }
|
||||||
harmony_types = { path = "../harmony_types" }
|
harmony_types = { path = "../harmony_types" }
|
||||||
schemars = "0.8.22"
|
schemars = "0.8.22"
|
||||||
|
|||||||
30
harmony-reconciler-contracts/src/access.rs
Normal file
30
harmony-reconciler-contracts/src/access.rs
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
use crate::DeploymentName;
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum GroupSourceError {
|
||||||
|
#[error("group source error: {0}")]
|
||||||
|
Source(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait DeviceGroupSource: Send + Sync {
|
||||||
|
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum SecretAccessError {
|
||||||
|
#[error("secret access backend error: {0}")]
|
||||||
|
Backend(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait DeploymentSecretGrants: Send + Sync {
|
||||||
|
async fn set_deployment_groups(
|
||||||
|
&self,
|
||||||
|
grants: &[(DeploymentName, Vec<String>)],
|
||||||
|
) -> Result<(), SecretAccessError>;
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ use crate::fleet::DeploymentName;
|
|||||||
|
|
||||||
/// Operator-written bucket. One entry per `(device, deployment)` pair.
|
/// Operator-written bucket. One entry per `(device, deployment)` pair.
|
||||||
/// Values are the JSON-serialized Score envelope — today
|
/// Values are the JSON-serialized Score envelope — today
|
||||||
/// `harmony::modules::podman::ReconcileScore`, tomorrow any variant of
|
/// `crate::ReconcileScore`, tomorrow any variant of
|
||||||
/// a polymorphic `Score` enum the framework ships.
|
/// a polymorphic `Score` enum the framework ships.
|
||||||
pub const BUCKET_DESIRED_STATE: &str = "desired-state";
|
pub const BUCKET_DESIRED_STATE: &str = "desired-state";
|
||||||
|
|
||||||
|
|||||||
@@ -9,18 +9,22 @@
|
|||||||
//!
|
//!
|
||||||
//! This crate holds the wire-format bits both sides must agree on:
|
//! This crate holds the wire-format bits both sides must agree on:
|
||||||
//! NATS bucket names, KV key formats, and the typed payloads
|
//! NATS bucket names, KV key formats, and the typed payloads
|
||||||
//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`). The Score
|
//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`, and Podman
|
||||||
//! types themselves live in their respective harmony modules.
|
//! desired state).
|
||||||
//!
|
//!
|
||||||
//! **Deliberately lean** — no tokio, no async-nats, no harmony.
|
//! **Deliberately lean** — no tokio, no async-nats, no harmony.
|
||||||
//! The on-device agent build pulls it in alongside a minimal
|
//! The on-device agent build pulls it in alongside a minimal
|
||||||
//! async-nats client; the operator pulls it alongside kube-rs.
|
//! async-nats client; the operator pulls it alongside kube-rs.
|
||||||
|
|
||||||
|
pub mod access;
|
||||||
pub mod commands;
|
pub mod commands;
|
||||||
pub mod fleet;
|
pub mod fleet;
|
||||||
pub mod kv;
|
pub mod kv;
|
||||||
|
pub mod podman;
|
||||||
pub mod status;
|
pub mod status;
|
||||||
|
|
||||||
|
pub use access::{DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError};
|
||||||
|
|
||||||
pub use commands::{
|
pub use commands::{
|
||||||
CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB,
|
CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB,
|
||||||
HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject,
|
HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject,
|
||||||
@@ -34,6 +38,9 @@ pub use kv::{
|
|||||||
desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key,
|
desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key,
|
||||||
device_state_key,
|
device_state_key,
|
||||||
};
|
};
|
||||||
|
pub use podman::{
|
||||||
|
EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount,
|
||||||
|
};
|
||||||
pub use status::{InventorySnapshot, Phase};
|
pub use status::{InventorySnapshot, Phase};
|
||||||
|
|
||||||
// Re-exports so consumers (agent, operator) don't need a direct
|
// Re-exports so consumers (agent, operator) don't need a direct
|
||||||
|
|||||||
208
harmony-reconciler-contracts/src/podman.rs
Normal file
208
harmony-reconciler-contracts/src/podman.rs
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct EnvVar {
|
||||||
|
pub name: String,
|
||||||
|
pub value: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EnvVar {
|
||||||
|
pub fn new(name: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
name: name.into(),
|
||||||
|
value: value.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<(String, String)> for EnvVar {
|
||||||
|
fn from((name, value): (String, String)) -> Self {
|
||||||
|
Self { name, value }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<(&str, &str)> for EnvVar {
|
||||||
|
fn from((name, value): (&str, &str)) -> Self {
|
||||||
|
Self::new(name, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct VolumeMount {
|
||||||
|
pub host_path: String,
|
||||||
|
pub container_path: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub read_only: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum RestartPolicy {
|
||||||
|
No,
|
||||||
|
#[default]
|
||||||
|
UnlessStopped,
|
||||||
|
OnFailure,
|
||||||
|
Always,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestartPolicy {
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::No => "no",
|
||||||
|
Self::UnlessStopped => "unless-stopped",
|
||||||
|
Self::OnFailure => "on-failure",
|
||||||
|
Self::Always => "always",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
|
||||||
|
pub struct SecretEnvVar {
|
||||||
|
pub name: String,
|
||||||
|
pub secret: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
|
||||||
|
pub struct PodmanService {
|
||||||
|
pub name: String,
|
||||||
|
pub image: String,
|
||||||
|
pub ports: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub env: Vec<EnvVar>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub secret_env: Vec<SecretEnvVar>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub volumes: Vec<VolumeMount>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub restart_policy: RestartPolicy,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
|
||||||
|
pub struct PodmanV0Score {
|
||||||
|
pub services: Vec<PodmanService>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PodmanV0Score {
|
||||||
|
pub fn deployment_label(&self) -> String {
|
||||||
|
self.services
|
||||||
|
.iter()
|
||||||
|
.map(|service| service.name.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
|
||||||
|
#[serde(tag = "type", content = "data")]
|
||||||
|
pub enum ReconcileScore {
|
||||||
|
PodmanV0(PodmanV0Score),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn service(name: &str, image: &str) -> PodmanService {
|
||||||
|
PodmanService {
|
||||||
|
name: name.into(),
|
||||||
|
image: image.into(),
|
||||||
|
ports: vec![],
|
||||||
|
env: vec![],
|
||||||
|
secret_env: vec![],
|
||||||
|
volumes: vec![],
|
||||||
|
restart_policy: RestartPolicy::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_wire_format_and_roundtrip_are_stable() {
|
||||||
|
let score = ReconcileScore::PodmanV0(PodmanV0Score {
|
||||||
|
services: vec![PodmanService {
|
||||||
|
ports: vec!["8080:80".into()],
|
||||||
|
env: vec![EnvVar::new("LOG_LEVEL", "info")],
|
||||||
|
volumes: vec![VolumeMount {
|
||||||
|
host_path: "/var/lib/web/data".into(),
|
||||||
|
container_path: "/data".into(),
|
||||||
|
read_only: false,
|
||||||
|
}],
|
||||||
|
restart_policy: RestartPolicy::Always,
|
||||||
|
..service("web", "nginx:latest")
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
let json = serde_json::to_string(&score).unwrap();
|
||||||
|
assert!(json.contains("\"type\":\"PodmanV0\""));
|
||||||
|
assert!(json.contains("\"data\""));
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<ReconcileScore>(&json).unwrap(),
|
||||||
|
score
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_service_defaults_are_stable() {
|
||||||
|
let legacy = r#"{"type":"PodmanV0","data":{"services":[{"name":"web","image":"nginx","ports":["8080:80"]}]}}"#;
|
||||||
|
let ReconcileScore::PodmanV0(score) = serde_json::from_str(legacy).unwrap();
|
||||||
|
let service = &score.services[0];
|
||||||
|
assert!(service.env.is_empty());
|
||||||
|
assert!(service.secret_env.is_empty());
|
||||||
|
assert!(service.volumes.is_empty());
|
||||||
|
assert_eq!(service.restart_policy, RestartPolicy::UnlessStopped);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_secret_env_remains_omitted() {
|
||||||
|
let json = serde_json::to_string(&service("web", "nginx")).unwrap();
|
||||||
|
assert!(!json.contains("secret_env"));
|
||||||
|
|
||||||
|
let with_secret = PodmanService {
|
||||||
|
secret_env: vec![SecretEnvVar {
|
||||||
|
name: "DB_PASSWORD".into(),
|
||||||
|
secret: "db_password".into(),
|
||||||
|
}],
|
||||||
|
..service("api", "myapp")
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<PodmanService>(&serde_json::to_string(&with_secret).unwrap())
|
||||||
|
.unwrap(),
|
||||||
|
with_secret
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restart_policy_wire_names_are_stable() {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&RestartPolicy::UnlessStopped).unwrap(),
|
||||||
|
"\"unless-stopped\""
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_string(&RestartPolicy::OnFailure).unwrap(),
|
||||||
|
"\"on-failure\""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn env_order_is_preserved() {
|
||||||
|
let service = PodmanService {
|
||||||
|
env: vec![EnvVar::new("B", "2"), EnvVar::new("A", "1")],
|
||||||
|
..service("api", "myapp")
|
||||||
|
};
|
||||||
|
let roundtrip: PodmanService =
|
||||||
|
serde_json::from_str(&serde_json::to_string(&service).unwrap()).unwrap();
|
||||||
|
assert_eq!(roundtrip.env, service.env);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn env_schema_is_not_a_tuple() {
|
||||||
|
let schema = serde_json::to_value(schemars::schema_for!(PodmanService)).unwrap();
|
||||||
|
assert!(schema.pointer("/properties/env/items").unwrap().is_object());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deployment_label_joins_service_names() {
|
||||||
|
let score = PodmanV0Score {
|
||||||
|
services: vec![service("web", "nginx"), service("api", "myapp")],
|
||||||
|
};
|
||||||
|
assert_eq!(score.deployment_label(), "web,api");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,13 +6,9 @@ readme.workspace = true
|
|||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
# `podman` lights up the podman module AND the fleet operator's
|
|
||||||
# CRD types (which embed `ReconcileScore` from the podman module).
|
|
||||||
# Without it the operator CRD fails to compile, so it's part of
|
|
||||||
# the default build.
|
|
||||||
default = ["podman"]
|
default = ["podman"]
|
||||||
kvm = ["dep:virt"]
|
kvm = ["dep:virt"]
|
||||||
podman = ["dep:podman-api"]
|
podman = []
|
||||||
testing = []
|
testing = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -99,7 +95,7 @@ inquire.workspace = true
|
|||||||
brocade = { path = "../brocade" }
|
brocade = { path = "../brocade" }
|
||||||
option-ext = "0.2.0"
|
option-ext = "0.2.0"
|
||||||
rand.workspace = true
|
rand.workspace = true
|
||||||
podman-api = { version = "0.9", optional = true }
|
nkeys = "0.4"
|
||||||
sysinfo = "0.30"
|
sysinfo = "0.30"
|
||||||
|
|
||||||
# virt (libvirt bindings) uses std::os::unix and only compiles on Unix targets.
|
# virt (libvirt bindings) uses std::os::unix and only compiles on Unix targets.
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use schemars::JsonSchema;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::executors::ExecutorError;
|
use crate::executors::ExecutorError;
|
||||||
|
|
||||||
|
pub use harmony_reconciler_contracts::{EnvVar, RestartPolicy, VolumeMount};
|
||||||
|
|
||||||
/// Capability: run containers on a node-level runtime.
|
/// Capability: run containers on a node-level runtime.
|
||||||
///
|
///
|
||||||
/// Models the subset of container-runtime operations Harmony Scores need in
|
/// Models the subset of container-runtime operations Harmony Scores need in
|
||||||
@@ -38,41 +39,6 @@ pub trait ContainerRuntime: Send + Sync {
|
|||||||
async fn list_managed_services(&self) -> Result<Vec<ContainerState>, ExecutorError>;
|
async fn list_managed_services(&self) -> Result<Vec<ContainerState>, ExecutorError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A single environment variable, name/value pair.
|
|
||||||
///
|
|
||||||
/// Modeled as a struct rather than `(String, String)` because k8s CRD
|
|
||||||
/// validation rejects tuple schemas (`items: [<schema>, <schema>]`) —
|
|
||||||
/// schemars generates that for tuples and apiextensions returns
|
|
||||||
/// `Forbidden: items must be a schema object and not an array`. The
|
|
||||||
/// struct shape produces `items: { type: object, properties: { name,
|
|
||||||
/// value } }`, which validates cleanly.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
|
|
||||||
pub struct EnvVar {
|
|
||||||
pub name: String,
|
|
||||||
pub value: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EnvVar {
|
|
||||||
pub fn new(name: impl Into<String>, value: impl Into<String>) -> Self {
|
|
||||||
Self {
|
|
||||||
name: name.into(),
|
|
||||||
value: value.into(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<(String, String)> for EnvVar {
|
|
||||||
fn from((name, value): (String, String)) -> Self {
|
|
||||||
Self { name, value }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<(&str, &str)> for EnvVar {
|
|
||||||
fn from((name, value): (&str, &str)) -> Self {
|
|
||||||
Self::new(name, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Desired state for a single container.
|
/// Desired state for a single container.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ContainerSpec {
|
pub struct ContainerSpec {
|
||||||
@@ -109,51 +75,6 @@ impl ContainerSpec {
|
|||||||
pub const MANAGED_BY_VALUE: &'static str = "harmony";
|
pub const MANAGED_BY_VALUE: &'static str = "harmony";
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A single host-path → container-path bind mount. Bind mounts are the only
|
|
||||||
/// volume kind supported in v0 — they cover ~95% of compose use cases and
|
|
||||||
/// don't depend on a runtime-managed volume namespace.
|
|
||||||
#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct VolumeMount {
|
|
||||||
/// Absolute path on the host.
|
|
||||||
pub host_path: String,
|
|
||||||
/// Absolute path inside the container.
|
|
||||||
pub container_path: String,
|
|
||||||
/// Mount as read-only. Defaults to false (read-write) to match
|
|
||||||
/// docker-compose's default.
|
|
||||||
#[serde(default)]
|
|
||||||
pub read_only: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Restart policy for a managed container. Names follow podman/docker
|
|
||||||
/// conventions so docker-compose translation is mechanical.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)]
|
|
||||||
#[serde(rename_all = "kebab-case")]
|
|
||||||
pub enum RestartPolicy {
|
|
||||||
/// Don't restart on exit.
|
|
||||||
No,
|
|
||||||
/// Restart unless the user explicitly stopped the container.
|
|
||||||
/// Docker-compose's default for long-running services and what most
|
|
||||||
/// fleet workloads want.
|
|
||||||
#[default]
|
|
||||||
UnlessStopped,
|
|
||||||
/// Restart only if the container exits with a non-zero status.
|
|
||||||
OnFailure,
|
|
||||||
/// Always restart, even on clean exits and after host reboot.
|
|
||||||
Always,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RestartPolicy {
|
|
||||||
/// Canonical string podman + docker accept on the CLI / in their APIs.
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
RestartPolicy::No => "no",
|
|
||||||
RestartPolicy::UnlessStopped => "unless-stopped",
|
|
||||||
RestartPolicy::OnFailure => "on-failure",
|
|
||||||
RestartPolicy::Always => "always",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Observed state of a container on the runtime.
|
/// Observed state of a container on the runtime.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ContainerState {
|
pub struct ContainerState {
|
||||||
|
|||||||
@@ -97,6 +97,12 @@ pub trait SystemdManager: Send + Sync {
|
|||||||
scope: SystemdScope,
|
scope: SystemdScope,
|
||||||
) -> Result<ChangeReport, ExecutorError>;
|
) -> Result<ChangeReport, ExecutorError>;
|
||||||
|
|
||||||
|
async fn systemd_unit_active(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
scope: SystemdScope,
|
||||||
|
) -> Result<bool, ExecutorError>;
|
||||||
|
|
||||||
/// Enable+start a user-scoped unit (e.g. `podman.socket` under
|
/// Enable+start a user-scoped unit (e.g. `podman.socket` under
|
||||||
/// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has
|
/// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has
|
||||||
/// already been called for the user.
|
/// already been called for the user.
|
||||||
|
|||||||
@@ -1,81 +1 @@
|
|||||||
//! Device group membership: the admin-managed identity input that gates
|
pub use harmony_reconciler_contracts::{DeviceGroupSource, GroupSourceError};
|
||||||
//! scheduling and secret access (ADR-025).
|
|
||||||
//!
|
|
||||||
//! Groups are the security boundary — they must come from identity the
|
|
||||||
//! device cannot self-assert (Zitadel role grants), never from
|
|
||||||
//! device-reported labels. The operator polls a [`DeviceGroupSource`]
|
|
||||||
//! and feeds the snapshot into scheduling.
|
|
||||||
|
|
||||||
use std::collections::{HashMap, HashSet};
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
|
||||||
pub enum GroupSourceError {
|
|
||||||
#[error("group source error: {0}")]
|
|
||||||
Source(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Complete device → groups view. Snapshot-based: membership changes
|
|
||||||
/// rarely and per-device, so a periodic full fetch beats per-device
|
|
||||||
/// queries.
|
|
||||||
#[async_trait]
|
|
||||||
pub trait DeviceGroupSource: Send + Sync {
|
|
||||||
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Fixed membership parsed from a `dev=g1|g2;dev2=g3` string — dev
|
|
||||||
/// setups and e2e, where no Zitadel exists.
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
pub struct StaticDeviceGroups {
|
|
||||||
map: HashMap<String, HashSet<String>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl StaticDeviceGroups {
|
|
||||||
pub fn parse(spec: &str) -> Self {
|
|
||||||
let map = spec
|
|
||||||
.split(';')
|
|
||||||
.filter_map(|entry| {
|
|
||||||
let (device, groups) = entry.split_once('=')?;
|
|
||||||
let device = device.trim();
|
|
||||||
if device.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some((
|
|
||||||
device.to_string(),
|
|
||||||
groups
|
|
||||||
.split('|')
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|g| !g.is_empty())
|
|
||||||
.map(str::to_string)
|
|
||||||
.collect(),
|
|
||||||
))
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
Self { map }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl DeviceGroupSource for StaticDeviceGroups {
|
|
||||||
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
|
|
||||||
Ok(self.map.clone())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn parses_devices_and_groups() {
|
|
||||||
let src = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
|
|
||||||
let map = src.device_groups().await.unwrap();
|
|
||||||
assert_eq!(map.len(), 2);
|
|
||||||
assert_eq!(
|
|
||||||
map["pi-01"],
|
|
||||||
HashSet::from(["edge-a".to_string(), "edge-b".to_string()])
|
|
||||||
);
|
|
||||||
assert_eq!(map["pi-02"], HashSet::from(["edge-b".to_string()]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,12 +1,5 @@
|
|||||||
//! Harmony-side Scores for fleet device onboarding.
|
//! Harmony-side Scores for fleet device onboarding.
|
||||||
//!
|
//!
|
||||||
//! Today this module exposes [`FleetDeviceSetupScore`] — a customer
|
|
||||||
//! runs it against a freshly-booted device (Pi, VM, bare-metal node
|
|
||||||
//! later) to install podman, place the `fleet-agent` binary, drop
|
|
||||||
//! the TOML config, and bring up the agent under systemd. Re-running
|
|
||||||
//! with a changed config (different labels, new NATS URL, new
|
|
||||||
//! credentials) is how a device is moved between fleet partitions.
|
|
||||||
//!
|
|
||||||
//! The operator + agent crates live outside `harmony/` under
|
//! The operator + agent crates live outside `harmony/` under
|
||||||
//! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`.
|
//! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`.
|
||||||
//! What belongs here is the harmony-framework side: the Scores a
|
//! What belongs here is the harmony-framework side: the Scores a
|
||||||
@@ -22,10 +15,8 @@ pub mod assets;
|
|||||||
pub mod groups;
|
pub mod groups;
|
||||||
#[cfg(all(feature = "kvm", unix))]
|
#[cfg(all(feature = "kvm", unix))]
|
||||||
pub mod libvirt_pool;
|
pub mod libvirt_pool;
|
||||||
pub mod operator;
|
|
||||||
pub mod preflight;
|
pub mod preflight;
|
||||||
pub mod secret_access;
|
pub mod secret_access;
|
||||||
mod setup_score;
|
|
||||||
#[cfg(all(feature = "kvm", unix))]
|
#[cfg(all(feature = "kvm", unix))]
|
||||||
mod vm_score;
|
mod vm_score;
|
||||||
|
|
||||||
@@ -42,9 +33,5 @@ pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony
|
|||||||
// `harmony-fleet-deploy` per ADR-023 — see that crate for the
|
// `harmony-fleet-deploy` per ADR-023 — see that crate for the
|
||||||
// composite + per-component Scores.
|
// composite + per-component Scores.
|
||||||
pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch};
|
pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch};
|
||||||
pub use setup_score::{
|
|
||||||
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
|
|
||||||
HostsEntry, merge_hosts_file,
|
|
||||||
};
|
|
||||||
#[cfg(all(feature = "kvm", unix))]
|
#[cfg(all(feature = "kvm", unix))]
|
||||||
pub use vm_score::ProvisionVmScore;
|
pub use vm_score::ProvisionVmScore;
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
//! Operator CRD types (`Deployment`, `Device`, …).
|
|
||||||
//!
|
|
||||||
//! Per ADR-023, the deploy-side bits (`FleetOperatorScore`, the helm
|
|
||||||
//! chart generator, `install_crds`) moved to
|
|
||||||
//! `fleet/harmony-fleet-deploy/src/operator/`. Only the CRD type
|
|
||||||
//! definitions remain here — they're consumed at runtime by the
|
|
||||||
//! operator binary's reconcile loop. Lifting the CRDs out of
|
|
||||||
//! `harmony` core is in scope for a follow-up PR (they currently
|
|
||||||
//! couple to `crate::modules::podman` via the `ReconcileScore` payload
|
|
||||||
//! field, which needs to move alongside them).
|
|
||||||
|
|
||||||
pub mod crd;
|
|
||||||
|
|
||||||
pub use crd::{
|
|
||||||
AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
|
|
||||||
DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
|
|
||||||
};
|
|
||||||
@@ -1,132 +1 @@
|
|||||||
//! Deployment → secret access control: the domain boundary the fleet
|
pub use harmony_reconciler_contracts::{DeploymentSecretGrants, SecretAccessError};
|
||||||
//! operator drives when a `Deployment`'s allowed groups change.
|
|
||||||
//!
|
|
||||||
//! Per ADR-025, groups are the security boundary: a deployment declares
|
|
||||||
//! which device groups may read its secrets, and devices prove group
|
|
||||||
//! membership through their signed identity at login. How a grant maps
|
|
||||||
//! onto a backend's primitive (an OpenBao policy attached to an external
|
|
||||||
//! group, …) is the *implementation's* concern; callers speak only
|
|
||||||
//! deployments and group names. That separation is what lets the
|
|
||||||
//! operator be unit-tested against [`InMemoryDeploymentSecretGrants`]
|
|
||||||
//! with no live secret store — the production backend is
|
|
||||||
//! [`crate::modules::openbao::OpenBaoDeploymentSecretGrants`].
|
|
||||||
|
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
|
||||||
use std::sync::Mutex;
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use harmony_reconciler_contracts::DeploymentName;
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
|
||||||
pub enum SecretAccessError {
|
|
||||||
#[error("secret access backend error: {0}")]
|
|
||||||
Backend(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sets which device groups may read each deployment's secrets.
|
|
||||||
#[async_trait]
|
|
||||||
pub trait DeploymentSecretGrants: Send + Sync {
|
|
||||||
/// Set the *complete* set of groups allowed to read each listed
|
|
||||||
/// deployment's secrets. Absolute and idempotent: afterwards exactly
|
|
||||||
/// the given groups hold the grant — others are revoked, an empty
|
|
||||||
/// list revokes everywhere. Deployments not listed are untouched.
|
|
||||||
async fn set_deployment_groups(
|
|
||||||
&self,
|
|
||||||
grants: &[(DeploymentName, Vec<String>)],
|
|
||||||
) -> Result<(), SecretAccessError>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// In-memory [`DeploymentSecretGrants`] for tests: records the latest
|
|
||||||
/// group set per deployment so a test can assert the operator synced the
|
|
||||||
/// right grants on each flow, with nothing deployed.
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
pub struct InMemoryDeploymentSecretGrants {
|
|
||||||
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl InMemoryDeploymentSecretGrants {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self::default()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Current groups for `deployment`, sorted. Empty when no group has
|
|
||||||
/// access (never granted, or revoked).
|
|
||||||
pub fn groups_for(&self, deployment: &DeploymentName) -> Vec<String> {
|
|
||||||
self.grants
|
|
||||||
.lock()
|
|
||||||
.unwrap()
|
|
||||||
.get(deployment)
|
|
||||||
.map(|s| s.iter().cloned().collect())
|
|
||||||
.unwrap_or_default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
|
|
||||||
async fn set_deployment_groups(
|
|
||||||
&self,
|
|
||||||
grants: &[(DeploymentName, Vec<String>)],
|
|
||||||
) -> Result<(), SecretAccessError> {
|
|
||||||
let mut store = self.grants.lock().unwrap();
|
|
||||||
for (deployment, groups) in grants {
|
|
||||||
if groups.is_empty() {
|
|
||||||
store.remove(deployment);
|
|
||||||
} else {
|
|
||||||
store.insert(deployment.clone(), groups.iter().cloned().collect());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn dn(s: &str) -> DeploymentName {
|
|
||||||
DeploymentName::try_new(s).expect("valid test name")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn sets_overwrites_and_clears_grants() {
|
|
||||||
let grants = InMemoryDeploymentSecretGrants::new();
|
|
||||||
let web = dn("web");
|
|
||||||
|
|
||||||
grants
|
|
||||||
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into(), "edge-b".into()])])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(grants.groups_for(&web), vec!["edge-a", "edge-b"]);
|
|
||||||
|
|
||||||
// Absolute, not additive: a smaller set revokes the rest.
|
|
||||||
grants
|
|
||||||
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
|
|
||||||
|
|
||||||
// Empty revokes everywhere.
|
|
||||||
grants
|
|
||||||
.set_deployment_groups(&[(web.clone(), vec![])])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(grants.groups_for(&web).is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn batch_touches_only_listed_deployments() {
|
|
||||||
let grants = InMemoryDeploymentSecretGrants::new();
|
|
||||||
let web = dn("web");
|
|
||||||
let db = dn("db");
|
|
||||||
grants
|
|
||||||
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
grants
|
|
||||||
.set_deployment_groups(&[(db.clone(), vec!["edge-b".into()])])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
|
|
||||||
assert_eq!(grants.groups_for(&db), vec!["edge-b"]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user