docs: Score composition via refs, major architectural progress for simple yet type safe infrastructure component composition #342

Open
johnride wants to merge 5 commits from docs/score-composition-via-refs into master
140 changed files with 5935 additions and 6725 deletions

10
.harmony/contexts.toml Normal file
View File

@@ -0,0 +1,10 @@
[contexts.local]
profile = "local"
autoprovision = true
domain = "localhost"
# Customer contexts belong in their private repository and can be selected with
# `--config path/to/contexts.toml`. A production context also sets `registry`,
# `project`, `domain`, `image_pull_secret`, and brokered OpenBao cluster access.
# Its cluster contains the immutable `fleet-system/fleet-callout-credentials`
# Secret described in `fleet/README.md`.

104
Cargo.lock generated
View File

@@ -2872,33 +2872,9 @@ name = "example-fleet-e2e-demo"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-nats",
"clap", "clap",
"directories", "harmony-fleet-e2e",
"env_logger",
"example-fleet-auth-callout",
"futures-util",
"harmony",
"harmony-fleet-auth",
"harmony-fleet-deploy",
"harmony-fleet-operator",
"harmony-k8s",
"harmony-nats-callout",
"harmony-reconciler-contracts",
"harmony_types",
"k3d-rs",
"k8s-openapi",
"kube",
"log",
"nkeys",
"serde",
"serde_json",
"tempfile",
"tokio", "tokio",
"tokio-test",
"tracing",
"tracing-subscriber",
"url",
] ]
[[package]] [[package]]
@@ -2917,31 +2893,6 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "example-fleet-staging-deploy"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"clap",
"env_logger",
"harmony",
"harmony-k8s",
"harmony-nats-callout",
"harmony_types",
"k8s-openapi",
"kube",
"log",
"nkeys",
"reqwest 0.12.28",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"url",
]
[[package]] [[package]]
name = "example-grafana" name = "example-grafana"
version = "0.1.0" version = "0.1.0"
@@ -3447,6 +3398,7 @@ dependencies = [
"clap", "clap",
"env_logger", "env_logger",
"harmony", "harmony",
"harmony-fleet-deploy",
"harmony_types", "harmony_types",
"log", "log",
"tokio", "tokio",
@@ -3460,7 +3412,7 @@ dependencies = [
"async-nats", "async-nats",
"chrono", "chrono",
"clap", "clap",
"harmony", "harmony-fleet-operator",
"harmony-reconciler-contracts", "harmony-reconciler-contracts",
"k8s-openapi", "k8s-openapi",
"kube", "kube",
@@ -3489,6 +3441,7 @@ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"clap", "clap",
"harmony", "harmony",
"harmony-fleet-deploy",
"harmony_cli", "harmony_cli",
"harmony_secret", "harmony_secret",
"harmony_types", "harmony_types",
@@ -3511,28 +3464,6 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "example_fleet_staging_install"
version = "0.1.0"
dependencies = [
"anyhow",
"base64 0.22.1",
"clap",
"harmony",
"harmony-fleet-deploy",
"harmony-k8s",
"harmony-nats-callout",
"harmony_cli",
"harmony_config",
"harmony_secret",
"nkeys",
"rand 0.9.4",
"schemars 0.8.22",
"serde",
"tokio",
"tracing",
]
[[package]] [[package]]
name = "example_fleet_vm_setup" name = "example_fleet_vm_setup"
version = "0.1.0" version = "0.1.0"
@@ -3541,6 +3472,7 @@ dependencies = [
"clap", "clap",
"env_logger", "env_logger",
"harmony", "harmony",
"harmony-fleet-deploy",
"harmony_types", "harmony_types",
"log", "log",
"tokio", "tokio",
@@ -3552,7 +3484,8 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
"harmony", "harmony-fleet-operator",
"harmony-reconciler-contracts",
"k8s-openapi", "k8s-openapi",
"kube", "kube",
"serde_json", "serde_json",
@@ -4029,13 +3962,13 @@ dependencies = [
"kube-derive", "kube-derive",
"lazy_static", "lazy_static",
"log", "log",
"nkeys",
"non-blank-string-rs", "non-blank-string-rs",
"once_cell", "once_cell",
"opnsense-api", "opnsense-api",
"opnsense-config", "opnsense-config",
"opnsense-config-xml", "opnsense-config-xml",
"option-ext", "option-ext",
"podman-api",
"pretty_assertions", "pretty_assertions",
"rand 0.9.4", "rand 0.9.4",
"reqwest 0.11.27", "reqwest 0.11.27",
@@ -4079,12 +4012,13 @@ dependencies = [
"chrono", "chrono",
"clap", "clap",
"futures-util", "futures-util",
"harmony",
"harmony-fleet-auth", "harmony-fleet-auth",
"harmony-reconciler-contracts", "harmony-reconciler-contracts",
"harmony_secret", "harmony_secret",
"podman-api",
"serde", "serde",
"serde_json", "serde_json",
"sha2 0.10.9",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"toml", "toml",
@@ -4121,12 +4055,14 @@ dependencies = [
"fqdn", "fqdn",
"harmony", "harmony",
"harmony-fleet-auth", "harmony-fleet-auth",
"harmony-reconciler-contracts", "harmony-fleet-operator",
"harmony_app",
"harmony_cli", "harmony_cli",
"harmony_config", "harmony_config",
"harmony_macros", "harmony_macros",
"harmony_types", "harmony_types",
"harmony_zitadel_auth", "harmony_zitadel_auth",
"inquire 0.7.5",
"k8s-openapi", "k8s-openapi",
"kube", "kube",
"log", "log",
@@ -4135,11 +4071,13 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"similar",
"tempfile", "tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"toml", "toml",
"tracing", "tracing",
"url",
] ]
[[package]] [[package]]
@@ -4157,6 +4095,7 @@ dependencies = [
"harmony-fleet-operator", "harmony-fleet-operator",
"harmony-k8s", "harmony-k8s",
"harmony-reconciler-contracts", "harmony-reconciler-contracts",
"harmony_secret",
"harmony_types", "harmony_types",
"jsonwebtoken", "jsonwebtoken",
"k3d-rs", "k3d-rs",
@@ -4188,10 +4127,10 @@ dependencies = [
"clap", "clap",
"dotenvy", "dotenvy",
"futures-util", "futures-util",
"harmony",
"harmony-fleet-auth", "harmony-fleet-auth",
"harmony-reconciler-contracts", "harmony-reconciler-contracts",
"harmony_config", "harmony_config",
"harmony_secret",
"harmony_zitadel_auth", "harmony_zitadel_auth",
"k8s-openapi", "k8s-openapi",
"kube", "kube",
@@ -4269,6 +4208,7 @@ dependencies = [
name = "harmony-reconciler-contracts" name = "harmony-reconciler-contracts"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-trait",
"chrono", "chrono",
"harmony_types", "harmony_types",
"schemars 0.8.22", "schemars 0.8.22",
@@ -4317,7 +4257,6 @@ dependencies = [
name = "harmony_app" name = "harmony_app"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow",
"async-trait", "async-trait",
"docker-compose-types", "docker-compose-types",
"fqdn", "fqdn",
@@ -4327,8 +4266,10 @@ dependencies = [
"harmony_types", "harmony_types",
"k8s-openapi", "k8s-openapi",
"log", "log",
"reqwest 0.12.28",
"schemars 0.8.22", "schemars 0.8.22",
"serde", "serde",
"serde_json",
"serde_yaml", "serde_yaml",
"tempfile", "tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
@@ -4377,6 +4318,8 @@ dependencies = [
"indicatif", "indicatif",
"inquire 0.7.5", "inquire 0.7.5",
"lazy_static", "lazy_static",
"serde",
"serde_json",
"tokio", "tokio",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
@@ -4518,6 +4461,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"directories", "directories",
"harmony-reconciler-contracts",
"harmony_secret_derive", "harmony_secret_derive",
"harmony_zitadel_jwt", "harmony_zitadel_jwt",
"http 1.4.0", "http 1.4.0",
@@ -4584,10 +4528,12 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"arc-swap", "arc-swap",
"async-trait",
"axum", "axum",
"axum-extra", "axum-extra",
"base64 0.22.1", "base64 0.22.1",
"chrono", "chrono",
"harmony-reconciler-contracts",
"harmony_config", "harmony_config",
"harmony_zitadel_jwt", "harmony_zitadel_jwt",
"jsonwebtoken", "jsonwebtoken",

View File

@@ -37,7 +37,6 @@ members = [
"fleet/harmony-fleet-e2e", "fleet/harmony-fleet-e2e",
"harmony-reconciler-contracts", "harmony-reconciler-contracts",
"examples/fleet_server_install", "examples/fleet_server_install",
"examples/fleet_staging_install",
"nats/jwt", "nats/jwt",
"nats/callout", "nats/callout",
"nats/integration-test-callout", "nats/integration-test-callout",
@@ -97,6 +96,7 @@ futures-util = "0.3"
thiserror = "2.0.14" thiserror = "2.0.14"
serde = { version = "1.0.209", features = ["derive", "rc"] } serde = { version = "1.0.209", features = ["derive", "rc"] }
serde_json = "1.0.127" serde_json = "1.0.127"
sha2 = "0.10"
askama = "0.14" askama = "0.14"
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
reqwest = { version = "0.12", features = [ reqwest = { version = "0.12", features = [

View File

@@ -0,0 +1,221 @@
# Public Fleet E2E environment
## Goal
Run public-facing Fleet regression tests directly on the production-adjacent
OKD cluster. Keep the environment definition, identities, context, and CI
workflow in a standalone private repository checked out under
`private_repos/`. The public Harmony repository contains only reusable Scores,
Refs, lifecycle code, and generic test harnesses.
Build and publish run on the same CI runner. Deployment receives only
digest-pinned image references published to an OCI registry.
## Private repository
The private repository owns:
```text
Cargo.toml
src/main.rs # Fleet app + compiled context catalog
.github/workflows/e2e.yml # or the equivalent CI system
README.md # operator setup and recovery
```
It is checked out beside Harmony under `private_repos/`, which the public
repository ignores. No customer or private project identifier belongs in a
tracked Harmony file, fixture, log sample, or test name.
The private deploy binary defines its contexts as typed Rust values. Contexts
are compiled into the binary; loading additional context definitions at runtime
is deferred until a real requirement justifies that interface.
```rust
let contexts = ContextCatalog::new([
NamedContext {
name: context_name!("e2e"),
target: ContextSpec::Remote(RemoteContext {
profile: Profile::Prod,
registry: oci_registry!("registry.example.com"),
registry_project: oci_repository!("harmony/e2e"),
domain: domain!("e2e.example.com"),
image_pull_secret: k8s_name!("registry-pull"),
access: OpenBaoClusterAccess {
namespace: openbao_namespace!("example"),
url: http_url!("https://secrets.nationtech.io"),
role: OpenBaoRoleName::new("example-ci")?,
zitadel_url: http_url!("https://identity.example.com"),
zitadel_audience: OidcAudience::new("example")?,
},
}),
},
])?;
```
Concrete names and identity details stay private.
### Context type design
`ContextSpec` should encode the access-mode choice instead of representing it
with unrelated optional fields:
```rust
pub enum ContextSpec {
Local(LocalContext),
Remote(RemoteContext),
}
```
`RemoteContext` carries required registry, domain, image-pull, and OpenBao
values directly. `LocalContext` carries only its k3d settings. This makes
impossible combinations unrepresentable and removes checks such as "exactly one
of autoprovision, k3d, or OpenBao must be set."
Use existing domain types where they are already correct:
- `Profile` remains an enum. It does not need a string macro.
- Kubernetes namespaces, Secret names, and other DNS-label resource names use
`K8sName` and a compile-time `k8s_name!` literal.
- IP and MAC literals continue using the existing Harmony macros.
Add types only for distinct grammars or values that are easy to transpose:
- `ContextName`: CLI selection key. It is not implicitly a Kubernetes name.
- `DomainName`: normalized DNS name without scheme, path, port, or wildcard.
- `HttpUrl`: HTTP(S)-only remote URL. The existing `Url` also permits local
folders and is too broad for OpenBao and Zitadel endpoints.
- `OciRegistry`: registry authority with optional port, but no repository path.
- `OciRepository`: validated OCI repository path.
- `OpenBaoNamespace`: a safe OpenBao path namespace, distinct from `K8sName`.
- `OpenBaoRoleName`: auth-role identifier, distinct from the namespace.
- `OidcAudience`: non-empty OIDC audience, kept semantically distinct even
though the protocol permits several string shapes.
Do not add a macro for every newtype. Literal macros are justified for the
grammars where compile-time diagnostics materially help: DNS names, HTTP URLs,
OCI registries/repositories, Kubernetes names, and OpenBao path namespaces.
Opaque identifiers can use typed constructors.
The runtime parser and literal macro must share one validator in
`harmony_types`; the macro must not copy the grammar. `harmony_macros` already
depends on `harmony_types`, so it can validate the literal during expansion and
emit the typed value. Each type needs accepted/rejected unit cases plus
compile-fail macro tests.
### Application integration
Change the lifecycle entry point from discovering `.harmony/contexts.toml` to
receiving the catalog:
```rust
harmony_cli::app::app_main(FleetApp, contexts).await
```
`--context e2e` and `HARMONY_CONTEXT=e2e` select only from contexts compiled
into that deploy binary. The public Fleet binary can compile its local context;
the private binary compiles the remote E2E context. Both use the same exported
`FleetApp` and Scores.
## CI trust bootstrap
The CI runner receives one secret:
```text
HARMONY_ZITADEL_KEY_JSON
```
It is the JSON key for a dedicated Zitadel machine identity. Harmony uses that
identity to obtain an OpenBao token from `secrets.nationtech.io`. `ConfigClient`
then loads the namespace-scoped kubeconfig and deployment secrets. The runner
holds no standing kubeconfig, OpenBao token, registry password, or application
secret.
Required server-side setup:
- [ ] Create a dedicated Zitadel machine identity for E2E CI.
- [ ] Grant only the role accepted by the E2E OpenBao auth role.
- [ ] Create an OpenBao policy restricted to the E2E config and secret paths.
- [ ] Store `ClusterAccess` with the namespace-scoped kubeconfig.
- [ ] Store registry credentials and deploy inputs under the E2E namespace.
- [ ] Add the machine key JSON as the CI repository's only secret.
- [ ] Compile the typed `e2e` context into the private deploy binary.
- [ ] Select it with `HARMONY_CONTEXT=e2e`.
## Known implementation gap
Cluster access already follows this flow. Image publication currently reads
`REGISTRY_USER` and `REGISTRY_TOKEN` directly from the environment. Before the
single-secret workflow is complete, registry authentication must be loaded
through the resolved `AppContext` and `ConfigClient` like the other secrets.
The context should resolve once at the start of the combined build-and-publish
job. Build itself remains credential-free; publish consumes the registry
credential fetched from OpenBao.
The current `harmony_app::context` TOML parser and `Option<String>` context
definition must also be replaced by the typed catalog above. This is a public
application-layer change; concrete context values remain private.
## Phase 1: cluster and identity
- [ ] Add the context domain types and their shared validators to
`harmony_types`.
- [ ] Add only the justified literal macros to `harmony_macros`, including
compile-fail tests.
- [ ] Replace TOML discovery with a caller-supplied `ContextCatalog`.
- [ ] Export the canonical `FleetApp` composition so the private deploy binary
does not duplicate it.
- [ ] Create a dedicated E2E namespace or namespace prefix on the public OKD
cluster.
- [ ] Configure namespace-scoped RBAC for the OpenBao-brokered kubeconfig.
- [ ] Add public DNS and exact-host OKD Routes for E2E services.
- [ ] Verify the CI machine identity can read `ClusterAccess` and no unrelated
OpenBao path.
- [ ] Run a read-only context check from CI with only
`HARMONY_ZITADEL_KEY_JSON` present.
## Phase 2: OCI publication
- [ ] Move registry credentials behind `ConfigClient`.
- [ ] Create a disposable registry project or repository scope for E2E images.
- [ ] Run `build` and `publish` in one CI job.
- [ ] Verify the pushed manifest digest matches the digest returned by build.
- [ ] Pull and run that exact digest in a fresh namespace.
- [ ] Apply retention rules so E2E images expire automatically.
The production publication path remains the same operation against the
production registry; only the context and credential policy differ.
## Phase 3: NATS regression
- [ ] Deploy the existing certificate and mTLS-capable NATS supercluster Scores
unchanged before modifying their APIs.
- [ ] Exercise public HTTPS/WebSocket and native TLS/mTLS endpoints through OKD
Routes.
- [ ] Assert cluster formation, authenticated traffic, SNI routing, reconnect
after pod restart, and certificate validation from outside the cluster.
- [ ] Record failures as regressions against the existing module before changing
Fleet composition.
## Phase 4: Fleet lifecycle
- [ ] Build and publish the operator and callout images in one job.
- [ ] Pass their digest-pinned references to `deploy`.
- [ ] Deploy Fleet through the private `e2e` context using production Scores and
typed Refs.
- [ ] Enroll an external test agent through the public NATS endpoint.
- [ ] Apply, update, and delete a Podman deployment and verify reflected state.
- [ ] Exercise Zitadel group changes and OpenBao secret access.
- [ ] Verify failed reconciliation recovers after the dependency is restored.
- [ ] Remove per-run resources and verify no desired-state or grant residue
remains.
## Guardrails
- Use a dedicated machine identity, OpenBao role, registry scope, and Kubernetes
RBAC grant for E2E.
- Keep every concrete private identifier in the private repository or OpenBao.
- Put a TTL label on per-run namespaces and clean abandoned runs automatically.
- Never print the machine key, OpenBao token, kubeconfig, or registry credential.
- Split CI into identity, registry, NATS, and Fleet stages so failures identify
one boundary.

View File

@@ -3,6 +3,7 @@
[Harmony Documentation](./README.md) [Harmony Documentation](./README.md)
- [Core Concepts](./concepts.md) - [Core Concepts](./concepts.md)
- [Typed Score References](./concepts/score-references.md)
- [Getting Started Guide](./guides/getting-started.md) - [Getting Started Guide](./guides/getting-started.md)
## Use Cases ## Use Cases
@@ -33,6 +34,10 @@
- [Configuration](./concepts/configuration.md) - [Configuration](./concepts/configuration.md)
## Reference Designs
- [Fleet Score References](./reference/fleet-score-references.md)
## Architecture Decision Records ## Architecture Decision Records
- [ADR Overview](./adr/README.md) - [ADR Overview](./adr/README.md)
@@ -57,5 +62,11 @@
- [019 · Network Bond Setup](./adr/019-Network-bond-setup.md) - [019 · Network Bond Setup](./adr/019-Network-bond-setup.md)
- [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md) - [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md)
- [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md) - [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md)
- [022 · Fleet Agent Upgrade Procedure](./adr/022-fleet-agent-upgrade.md)
- [023 · Deploy Architecture](./adr/023-deploy-architecture.md)
- [024 · Fleet Platform Capability Decomposition](./adr/024-fleet-platform-capability-decomposition.md)
- [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md) - [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md)
- [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md) - [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md)
- [026 · Application Lifecycle CLI](./adr/026-application-lifecycle-cli.md)
- [027 · Multi-Tenant Cloud Identity](./adr/027-multi-tenant-identity.md)
- [028 · Typed Score References](./adr/028-typed-score-references.md)

View File

@@ -0,0 +1,176 @@
# Architecture Decision Record: Typed Score References
Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-07-15
Last Updated Date: 2026-07-15
## Status
Proposed. Fleet is the first application.
## Context
ADR-023 establishes one Score per deployable component and requires deploy
crates to compose Scores instead of rebuilding their operations. It does not
define how one Score refers to desired state owned by another component.
Today, deploy code often passes URLs, names, IDs, and credential locations as
strings. Consumers then parse those strings or reproduce the producer's naming
conventions. Rust cannot distinguish different string contracts, and a refactor
can leave valid code that fails only at runtime.
Earlier composition design attempts treated these relationships as a dependency
graph or pipeline. That caused too strong coupling between each component's
lifecycle. Harmony needs a smaller contract: a consumer should know what it
references without depending on the producer Score or controlling when it runs.
Infrastructure also does not converge once and remain stable. A referenced
component may not exist yet, may still be converging, or may become unavailable
later. The reference model must not imply otherwise.
## Decision
Scores will exchange concrete typed Refs for desired state owned by another
component:
[![A producer Score passes a typed Ref to a dependent Score.](../diagrams/score-ref-composition.svg)](../diagrams/score-ref-composition.svg)
The complete API is small:
```rust
pub struct DbRef {
// Desired-state properties needed to connect.
}
let db_score = SomeDbScore::new(...);
let db_ref = db_score.connection();
let app_score = AppScore::new(...).with_db(&db_ref);
```
The producer derives a Ref from its desired state. Creating or passing the Ref
performs no I/O and does not require the producer to have run.
The consumer depends on the Ref type, not the producer Score. The Ref contains
the typed identifiers needed to describe the relationship. It does not contain
a client, execute provider operations, or transfer ownership of the producer's
lifecycle.
A Ref is eventually consistent. It does not prove that its target exists, has
converged, or remains available. The consumer's `Interpret`/`Topology` path must
handle that uncertainty. Where Kubernetes already understands the relationship,
the consumer should emit the native Kubernetes reference and let Kubernetes
reconcile it.
Refs do not impose execution order. They do not define a dependency graph,
pipeline, scheduler, or continuous convergence mechanism.
### Cyclic relationships
A cyclic relationship uses the same operation in both directions:
[![BrokerScore and AuthScore exchange BrokerRef and AuthRef before either Score runs.](../diagrams/score-ref-cycle.svg)](../diagrams/score-ref-cycle.svg)
```rust
let broker_score = BrokerScore::new(...);
let broker_ref = broker_score.connection();
let auth_score = AuthScore::new(...).with_broker(&broker_ref);
let auth_ref = auth_score.endpoint();
let broker_score = broker_score.with_auth(&auth_ref);
```
Both Refs derive from desired state before either Score runs. Declaring the
cycle therefore needs no graph traversal or scheduler.
### Values produced later
A Ref may identify a value that will be generated later, such as an OIDC client
ID. Composition can pass the Ref before that value exists. This is the same
eventual-consistency rule, not a separate output mechanism.
### Guarantees
| Statement | Guarantee |
|---|---|
| Consumer accepts the intended Ref type | Rust compile time |
| Ref construction performs no I/O | Ref API contract |
| Target exists or has converged | Not guaranteed |
| Runtime systems are compatible or authorized | Not guaranteed(*) |
| Target remains available | Not guaranteed |
> (*) The runtime compatibility or authorization is not guaranteed by the **Ref** itself. But it is usually Harmony's job to provide ref types in a way that makes them compatible. For example, a PostgreSQL score exposing a PostgreSQLRootAccountRef will have the responsibility to provision it.
>
> In other words, __correct provisioning of a Ref is the score's developer's job, not the ref's job.__
Changing a Ref's type-level contract produces compiler errors in affected
consumers. Rust cannot catch a provider returning incorrect live state, so
runtime validation and readiness remain necessary.
## Rationale
- Typed Refs replace string parsing and copied naming conventions.
- Scores share only the desired-state contract needed by the consumer.
- Refs can be composed before any component runs.
- Consumers remain responsible for infrastructure that is not yet ready or is
temporarily unavailable.
- Kubernetes keeps reconciling relationships it already understands.
- Harmony does not need a graph or pipeline abstraction to express a typed
relationship.
## Consequences
### Positive
- Refactoring a relationship's typed contract reaches its consumers through the
compiler.
- Components remain independent of each other's Score implementations and
execution order.
- Production and E2E can compose the same Score APIs against different
Topologies.
- Values produced later need not become string dependencies in deploy code.
### Negative
- Each real relationship needs a concrete Ref type.
- Consumers must handle unresolved and unavailable targets.
- Types do not remove runtime readiness, compatibility, or authorization checks.
- Existing string-based wiring must be migrated as its owning Scores gain Refs.
## Alternatives considered
### Continue passing strings
Rejected. Strings erase domain meaning and force consumers to know producer
naming and parsing rules.
### Pass producer Scores to consumers
Rejected. This couples component lifecycle and implementation when the consumer
only needs a desired-state reference.
### Make execution return values the composition mechanism
Rejected for this decision. It requires the producer to run before composition
can continue and turns execution order into the dependency model.
### Introduce a generic graph or output registry
Rejected. Typed Refs solve the current problem without either abstraction.
## Out of scope
ADR-028 does not define continuous convergence, retries, scheduling, graph
execution, Ref persistence, or a generic `Ref<T>`. Those decisions require their
own evidence and designs.
## References
- [Typed Score references](../concepts/score-references.md): concept guide.
- [Fleet Score references](../reference/fleet-score-references.md): first application.
- [ADR-002](./002-hexagonal-architecture.md): domain and adapter boundaries.
- [ADR-015](./015-higher-order-topologies.md): compile-time Topology
composition.
- [ADR-023](./023-deploy-architecture.md): component Scores and deploy crates.

View File

@@ -59,6 +59,7 @@ Every ADR follows this structure:
| [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted | | [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted |
| [026](./026-application-lifecycle-cli.md) | Application Lifecycle CLI | Accepted | | [026](./026-application-lifecycle-cli.md) | Application Lifecycle CLI | Accepted |
| [027](./027-multi-tenant-identity.md) | Multi-Tenant Cloud Identity | Accepted | | [027](./027-multi-tenant-identity.md) | Multi-Tenant Cloud Identity | Accepted |
| [028](./028-typed-score-references.md) | Typed Score References | Proposed |
## Contributing ## Contributing

View File

@@ -422,9 +422,8 @@ what vocabulary should the new fleet shape stay consistent with?
apply via typed code. apply via typed code.
- **Implication for fleet:** chart-rendering helpers - **Implication for fleet:** chart-rendering helpers
(`build_operator_deployment` et al.) are template-hydration (`build_operator_deployment` et al.) are template-hydration
edges. They *should* be hidden from domain code. Today they're edges. They *should* be hidden from domain code. Exposing them to
`pub` — visible to consumers like `fleet_staging_install` who deploy callers instead of composing `FleetOperatorScore` is adapter
reach in and grab `operator_secret(opts)`. That's adapter
leakage. leakage.
### §2.4 — Synthesis: principles for the redesign ### §2.4 — Synthesis: principles for the redesign

View File

@@ -0,0 +1,125 @@
# Typed Score references
Harmony builds loosely dependent but strongly typed infrastructure components
using Refs.
This means that a `ZitadelScore` does not directly depend on a
`PostgreSQLScore`. It can instead accept a `PostgreSQLRootAccountRef` in its
builder to know how to connect to PostgreSQL. This is safer than passing strings
around: the Ref has a type Rust can check, and consumers do not need to parse or
reconstruct the producer's naming conventions.
The loose dependency matters. Previous design proposals treated composition as a
dependency graph or pipeline, which forced components to know too much about one
another. A Ref is only a typed reference to declarative desired state. It does
not give the consumer control over the producer's lifecycle.
Most importantly, a Ref does not guarantee that its target exists or has
converged when it is passed. Like a Kubernetes reference, it is eventually
consistent. Consumers must tolerate the target being absent, starting, or
temporarily unavailable.
Kubernetes gets this right: infrastructure is chaotic. A component can fail
after it becomes ready. Drives fail, power goes out, and networks split. The
long-term answer is continuous convergence, not an assumption that deployment
finishes infrastructure forever.
The convergence mechanism is outside ADR-028. Choosing eventually consistent
Refs now lets Scores declare relationships without committing Harmony to a
scheduler, dependency engine, or reconciliation design.
## The whole pattern
[![SomeDbScore produces DbRef for AppScore.](../diagrams/score-ref-composition.svg)](../diagrams/score-ref-composition.svg)
```rust
pub struct DbRef {
// Desired-state properties needed to connect.
}
let db_score = SomeDbScore::new(...);
let some_db_ref = db_score.connection();
let app_score = AppScore::new(...).with_db(&some_db_ref);
```
That is the mechanism. `connection()` derives `DbRef` from the database's desired
state. It performs no I/O and does not require `SomeDbScore` to have been
interpreted.
`AppScore` knows `DbRef`. It does not need to know `SomeDbScore`, its provider,
or its readiness procedure.
Compare that with copying a connection string:
```rust
app.database_url("postgres://some-db.apps.svc:5432/app");
```
That string duplicates database naming and connection knowledge in the consumer.
A rename or layout change can leave valid Rust that fails at runtime. A typed
Ref removes that string contract and makes type-level contract changes visible
to the compiler.
## Cyclic relationships
A cycle is the same pattern in both directions:
[![BrokerScore and AuthScore exchange BrokerRef and AuthRef before either Score runs.](../diagrams/score-ref-cycle.svg)](../diagrams/score-ref-cycle.svg)
```rust
let broker_score = BrokerScore::new(...);
let broker_ref = broker_score.connection();
let auth_score = AuthScore::new(...).with_broker(&broker_ref);
let auth_ref = auth_score.endpoint();
let broker_score = broker_score.with_auth(&auth_ref);
```
No Score has run. Each Ref comes from desired state, so the two relationships
can be declared without traversing a graph or resolving either resource.
## Eventual consistency
Composition code passes Refs; it does not wait for them to become ready. The
consumer's `Interpret`/`Topology` path handles an unresolved target in the way
appropriate for that component. It may:
- emit a native Kubernetes reference and let Kubernetes reconcile it;
- wait or retry when the value is needed immediately;
- return a non-success outcome for the current interpretation attempt.
ADR-028 does not choose among those behaviors or define when another attempt
runs. It only requires consumers not to treat possession of a Ref as proof of
availability.
Refs may also identify values generated later, such as an OIDC client ID. The
same rule applies: passing the Ref does not claim that the value exists yet.
## What types guarantee
Concrete types such as `PostgreSQLRootAccountRef`, `OidcApplicationRef`, and
`NatsAccountRef` prevent unrelated references from being exchanged by accident.
They also remove string parsing and copied naming rules from consumers.
Types do not prove that:
- the referenced resource exists;
- the producer has converged;
- two live systems can communicate;
- credentials are authorized;
- the infrastructure will remain available.
Those remain runtime concerns.
## Authoring rule
Use a Ref when one Score would otherwise copy a name, URL, ID, credential
location, or other desired-state identifier owned by another component. Use an
ordinary typed value when there is no such relationship.
Refs express relationships. They do not impose execution order, create a
pipeline, or make Harmony responsible for continuous convergence.
See [ADR-028](../adr/028-typed-score-references.md) for the decision and the
[Fleet reference](../reference/fleet-score-references.md) for the first application.

View File

@@ -0,0 +1,45 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="330" viewBox="0 0 1200 330" role="img" aria-labelledby="title desc">
<title id="title">PostgreSQL and Zitadel compose identity functionality through a typed Ref</title>
<desc id="desc">PostgreSQLScore exports PostgreSQLRootAccountRef to ZitadelScore. Together they provide identity functionality.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
</marker>
<style>
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
</style>
</defs>
<rect width="1200" height="330" fill="#ffffff"/>
<g class="sans">
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Compose identity</text>
<text x="40" y="80" fill="#64748b" font-size="17">A consumer receives a typed contract, not the producer Score.</text>
<rect x="40" y="120" width="270" height="100" rx="12" fill="#172554"/>
<text x="65" y="150" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
<text x="65" y="184" fill="#ffffff" font-size="21" font-weight="700" class="mono">PostgreSQLScore</text>
<text x="65" y="207" fill="#bfdbfe" font-size="14">provisions PostgreSQL</text>
<path d="M310 170H385" fill="none" stroke="#0f766e" stroke-width="2.5"/>
<rect x="385" y="143" width="290" height="54" rx="27" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
<text x="530" y="165" text-anchor="middle" fill="#0f766e" font-size="11" font-weight="750" letter-spacing="1">REF TYPE</text>
<text x="530" y="184" text-anchor="middle" fill="#115e59" font-size="14" font-weight="650" class="mono">PostgreSQLRootAccountRef</text>
<path d="M675 170H750" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="750" y="120" width="230" height="100" rx="12" fill="#172554"/>
<text x="775" y="150" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
<text x="775" y="184" fill="#ffffff" font-size="21" font-weight="700" class="mono">ZitadelScore</text>
<text x="775" y="207" fill="#bfdbfe" font-size="14">provisions Zitadel</text>
<text x="1005" y="181" fill="#94a3b8" font-size="28" font-weight="500">=</text>
<rect x="1040" y="120" width="130" height="100" rx="12" fill="#eef2ff" stroke="#6366f1" stroke-width="1.5"/>
<text x="1105" y="158" text-anchor="middle" fill="#3730a3" font-size="12" font-weight="750" letter-spacing="1.2">IDENTITY</text>
<text x="1105" y="180" text-anchor="middle" fill="#3730a3" font-size="12" font-weight="750" letter-spacing="1.2">FUNCTIONALITY</text>
<text x="1105" y="204" text-anchor="middle" fill="#6366f1" font-size="13">composed</text>
<rect x="40" y="260" width="18" height="18" rx="4" fill="#172554"/>
<text x="70" y="275" fill="#64748b" font-size="14">Score owns behavior</text>
<rect x="238" y="259" width="54" height="20" rx="10" fill="#f0fdfa" stroke="#0f766e"/>
<text x="304" y="275" fill="#64748b" font-size="14">Ref carries typed desired state</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 3.2 KiB

View File

@@ -0,0 +1,50 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="390" viewBox="0 0 1200 390" role="img" aria-labelledby="title desc">
<title id="title">NATS and its authentication callout compose authenticated messaging</title>
<desc id="desc">NatsScore exports NatsAccountRef to NatsAuthCalloutScore. NatsAuthCalloutScore exports NatsAuthCalloutRef back to NatsScore. The composed functionality requires OidcProviderRef and exports NatsClientRef.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
</marker>
<style>
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
</style>
</defs>
<rect width="1200" height="390" fill="#ffffff"/>
<g class="sans">
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Compose authenticated messaging</text>
<text x="40" y="80" fill="#64748b" font-size="17">Two Scores can exchange desired-state Refs without imposing execution order.</text>
<rect x="40" y="108" width="1120" height="220" rx="16" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
<text x="70" y="137" fill="#64748b" font-size="12" font-weight="750" letter-spacing="1.3">AUTHENTICATED MESSAGING FUNCTIONALITY</text>
<text x="70" y="177" fill="#64748b" font-size="12" font-weight="750">REQUIRES</text>
<rect x="145" y="155" width="190" height="34" rx="17" fill="#f0fdfa" stroke="#0f766e"/>
<text x="240" y="177" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">OidcProviderRef</text>
<rect x="70" y="210" width="235" height="90" rx="12" fill="#172554"/>
<text x="94" y="238" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
<text x="94" y="270" fill="#ffffff" font-size="21" font-weight="700" class="mono">NatsScore</text>
<text x="94" y="291" fill="#bfdbfe" font-size="14">provisions NATS</text>
<path d="M305 236H650" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="365" y="216" width="220" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
<text x="475" y="241" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">NatsAccountRef</text>
<path d="M650 276H305" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="350" y="256" width="250" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
<text x="475" y="281" text-anchor="middle" fill="#115e59" font-size="13" font-weight="650" class="mono">NatsAuthCalloutRef</text>
<rect x="650" y="210" width="265" height="90" rx="12" fill="#172554"/>
<text x="674" y="238" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.5">SCORE</text>
<text x="674" y="270" fill="#ffffff" font-size="16" font-weight="700" class="mono">NatsAuthCalloutScore</text>
<text x="674" y="291" fill="#bfdbfe" font-size="14">provisions the callout</text>
<text x="950" y="220" fill="#64748b" font-size="12" font-weight="750">EXPORTS</text>
<rect x="950" y="232" width="180" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
<text x="1040" y="258" text-anchor="middle" fill="#115e59" font-size="14" font-weight="650" class="mono">NatsClientRef</text>
<text x="950" y="299" fill="#475569" font-size="14">One capability for consumers</text>
<text x="40" y="365" fill="#64748b" font-size="14">The common region is the functionality; the blue boxes remain independent Scores.</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 3.6 KiB

View File

@@ -0,0 +1,100 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="790" viewBox="0 0 1400 790" role="img" aria-labelledby="title desc">
<title id="title">Fleet reference implementation contract map</title>
<desc id="desc">A three-column map lists reusable Fleet functionality, the concrete Ref types it produces, and the Score types that consume each Ref.</desc>
<style>
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
.header { fill: #64748b; font-size: 13px; font-weight: 750; letter-spacing: 1.4px; }
.function { fill: #0f172a; font-size: 20px; font-weight: 720; }
.detail { fill: #64748b; font-size: 14px; }
.ref { fill: #115e59; font-size: 14px; font-weight: 650; }
.score { fill: #ffffff; font-size: 13px; font-weight: 650; }
</style>
<rect width="1400" height="790" fill="#ffffff"/>
<g class="sans">
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Fleet reference implementation</text>
<text x="40" y="80" fill="#64748b" font-size="17">Read each contract left to right. The map shows type compatibility, not execution order.</text>
<text x="40" y="122" class="header">FUNCTIONALITY</text>
<text x="360" y="122" class="header">PRODUCES REF TYPE</text>
<text x="740" y="122" class="header">CONSUMED BY SCORE TYPE</text>
<line x1="40" y1="135" x2="1360" y2="135" stroke="#94a3b8" stroke-width="1.5"/>
<!-- Identity contracts -->
<rect x="40" y="150" width="280" height="190" rx="12" fill="#eef2ff"/>
<rect x="40" y="150" width="6" height="190" rx="3" fill="#6366f1"/>
<text x="65" y="190" class="function">Identity</text>
<text x="65" y="218" class="detail mono">PostgreSQLScore</text>
<text x="65" y="240" class="detail">+</text>
<text x="65" y="262" class="detail mono">ZitadelScore</text>
<rect x="360" y="158" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="184" text-anchor="middle" class="mono ref">OidcProviderRef</text>
<rect x="360" y="218" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="244" text-anchor="middle" class="mono ref">OidcApplicationRef</text>
<rect x="360" y="278" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="304" text-anchor="middle" class="mono ref">OidcMachineIdentityRef</text>
<rect x="740" y="158" width="170" height="42" rx="10" fill="#172554"/>
<text x="825" y="184" text-anchor="middle" class="mono score">OpenBaoScore</text>
<rect x="920" y="158" width="210" height="42" rx="10" fill="#172554"/>
<text x="1025" y="184" text-anchor="middle" class="mono score" style="font-size:12px">NatsAuthCalloutScore</text>
<rect x="1140" y="158" width="220" height="42" rx="10" fill="#172554"/>
<text x="1250" y="184" text-anchor="middle" class="mono score" style="font-size:11px">FleetDeviceEnrollmentScore</text>
<rect x="740" y="218" width="220" height="42" rx="10" fill="#172554"/>
<text x="850" y="244" text-anchor="middle" class="mono score">FleetOperatorScore</text>
<rect x="740" y="278" width="220" height="42" rx="10" fill="#172554"/>
<text x="850" y="304" text-anchor="middle" class="mono score">FleetOperatorScore</text>
<line x1="330" y1="210" x2="1360" y2="210" stroke="#e2e8f0"/>
<line x1="330" y1="270" x2="1360" y2="270" stroke="#e2e8f0"/>
<!-- Messaging contract -->
<rect x="40" y="360" width="280" height="75" rx="12" fill="#ecfeff"/>
<rect x="40" y="360" width="6" height="75" rx="3" fill="#06b6d4"/>
<text x="65" y="390" class="function">Messaging</text>
<text x="65" y="415" class="detail mono" style="font-size:12px">NatsScore + NatsAuthCalloutScore</text>
<rect x="360" y="376" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="402" text-anchor="middle" class="mono ref">NatsClientRef</text>
<rect x="740" y="376" width="220" height="42" rx="10" fill="#172554"/>
<text x="850" y="402" text-anchor="middle" class="mono score">FleetOperatorScore</text>
<rect x="975" y="376" width="205" height="42" rx="10" fill="#172554"/>
<text x="1077" y="402" text-anchor="middle" class="mono score">FleetDeviceScore</text>
<!-- Secret contracts -->
<rect x="40" y="455" width="280" height="130" rx="12" fill="#f0fdf4"/>
<rect x="40" y="455" width="6" height="130" rx="3" fill="#22c55e"/>
<text x="65" y="495" class="function">Secrets</text>
<text x="65" y="523" class="detail mono">OpenBaoScore</text>
<rect x="360" y="463" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="489" text-anchor="middle" class="mono ref">OpenBaoRef</text>
<rect x="360" y="523" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="549" text-anchor="middle" class="mono ref">OpenBaoJwtAuthRef</text>
<rect x="740" y="463" width="220" height="42" rx="10" fill="#172554"/>
<text x="850" y="489" text-anchor="middle" class="mono score">FleetOperatorScore</text>
<rect x="740" y="523" width="205" height="42" rx="10" fill="#172554"/>
<text x="842" y="549" text-anchor="middle" class="mono score">FleetDeviceScore</text>
<line x1="330" y1="515" x2="1360" y2="515" stroke="#e2e8f0"/>
<!-- Device identity contract -->
<rect x="40" y="605" width="280" height="75" rx="12" fill="#fff7ed"/>
<rect x="40" y="605" width="6" height="75" rx="3" fill="#f97316"/>
<text x="65" y="635" class="function">Device identity</text>
<text x="65" y="660" class="detail mono" style="font-size:12px">FleetDeviceEnrollmentScore</text>
<rect x="360" y="621" width="300" height="42" rx="21" fill="#f0fdfa" stroke="#0f766e"/>
<text x="510" y="647" text-anchor="middle" class="mono ref">DeviceIdentityRef</text>
<rect x="740" y="621" width="205" height="42" rx="10" fill="#172554"/>
<text x="842" y="647" text-anchor="middle" class="mono score">FleetDeviceScore</text>
<line x1="40" y1="705" x2="1360" y2="705" stroke="#cbd5e1"/>
<rect x="40" y="730" width="120" height="36" rx="9" fill="#172554"/>
<text x="100" y="753" text-anchor="middle" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.2">SCORE TYPE</text>
<text x="178" y="753" fill="#64748b" font-size="14">owns behavior</text>
<rect x="320" y="728" width="145" height="40" rx="20" fill="#f0fdfa" stroke="#0f766e"/>
<text x="392" y="753" text-anchor="middle" class="mono ref">SomeRef</text>
<text x="485" y="753" fill="#64748b" font-size="14">carries typed desired state, not readiness</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 6.5 KiB

View File

@@ -0,0 +1,38 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="300" viewBox="0 0 1100 300" role="img" aria-labelledby="title desc">
<title id="title">A Score passes a typed Ref to a dependent Score</title>
<desc id="desc">SomeDbScore produces DbRef from desired state. AppScore accepts DbRef without depending on SomeDbScore.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
</marker>
<style>
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
</style>
</defs>
<rect width="1100" height="300" fill="#ffffff"/>
<g class="sans">
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">Score → Ref → dependent Score</text>
<text x="40" y="80" fill="#64748b" font-size="17">The Ref is the complete relationship between otherwise independent components.</text>
<text x="55" y="128" fill="#64748b" font-size="12" font-weight="750">PRODUCER</text>
<rect x="40" y="140" width="255" height="90" rx="12" fill="#172554"/>
<text x="64" y="168" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
<text x="64" y="201" fill="#ffffff" font-size="22" font-weight="700" class="mono">SomeDbScore</text>
<text x="64" y="221" fill="#bfdbfe" font-size="14">owns database behavior</text>
<path d="M295 185H390" fill="none" stroke="#0f766e" stroke-width="2.5"/>
<rect x="390" y="158" width="250" height="54" rx="27" fill="#f0fdfa" stroke="#0f766e" stroke-width="1.5"/>
<text x="515" y="180" text-anchor="middle" fill="#0f766e" font-size="11" font-weight="750" letter-spacing="1.2">REF TYPE</text>
<text x="515" y="200" text-anchor="middle" fill="#115e59" font-size="18" font-weight="700" class="mono">DbRef</text>
<path d="M640 185H735" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<text x="750" y="128" fill="#64748b" font-size="12" font-weight="750">CONSUMER</text>
<rect x="735" y="140" width="255" height="90" rx="12" fill="#172554"/>
<text x="759" y="168" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
<text x="759" y="201" fill="#ffffff" font-size="22" font-weight="700" class="mono">AppScore</text>
<text x="759" y="221" fill="#bfdbfe" font-size="14">knows DbRef, not SomeDbScore</text>
<text x="40" y="270" fill="#64748b" font-size="14">Creating and passing DbRef performs no I/O and says nothing about readiness.</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.6 KiB

View File

@@ -0,0 +1,41 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="380" viewBox="0 0 1100 380" role="img" aria-labelledby="title desc">
<title id="title">Two Scores exchange typed Refs in a cyclic relationship</title>
<desc id="desc">BrokerScore passes BrokerRef to AuthScore. AuthScore passes AuthRef to BrokerScore. Both Refs come from desired state before either Score runs.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto">
<path d="M0 0 10 5 0 10z" fill="#0f766e"/>
</marker>
<style>
.sans { font-family: Inter, ui-sans-serif, system-ui, sans-serif; }
.mono { font-family: "JetBrains Mono", Consolas, monospace; }
</style>
</defs>
<rect width="1100" height="380" fill="#ffffff"/>
<g class="sans">
<text x="40" y="52" fill="#0f172a" font-size="30" font-weight="750">A cycle is the same pattern twice</text>
<text x="40" y="80" fill="#64748b" font-size="17">Both Refs come from desired state before either Score runs.</text>
<rect x="40" y="108" width="1020" height="220" rx="16" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
<text x="70" y="140" fill="#64748b" font-size="12" font-weight="750" letter-spacing="1.3">COMPOSED FUNCTIONALITY</text>
<rect x="85" y="178" width="260" height="100" rx="12" fill="#172554"/>
<text x="110" y="208" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
<text x="110" y="242" fill="#ffffff" font-size="22" font-weight="700" class="mono">BrokerScore</text>
<text x="110" y="265" fill="#bfdbfe" font-size="14">owns the broker</text>
<path d="M345 204H755" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="440" y="181" width="220" height="46" rx="23" fill="#f0fdfa" stroke="#0f766e"/>
<text x="550" y="210" text-anchor="middle" fill="#115e59" font-size="15" font-weight="650" class="mono">BrokerRef</text>
<path d="M755 252H345" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="440" y="229" width="220" height="46" rx="23" fill="#f0fdfa" stroke="#0f766e"/>
<text x="550" y="258" text-anchor="middle" fill="#115e59" font-size="15" font-weight="650" class="mono">AuthRef</text>
<rect x="755" y="178" width="260" height="100" rx="12" fill="#172554"/>
<text x="780" y="208" fill="#bfdbfe" font-size="12" font-weight="750" letter-spacing="1.4">SCORE</text>
<text x="780" y="242" fill="#ffffff" font-size="22" font-weight="700" class="mono">AuthScore</text>
<text x="780" y="265" fill="#bfdbfe" font-size="14">owns authentication</text>
<text x="40" y="360" fill="#64748b" font-size="14">The cycle describes desired relationships; it does not require a graph scheduler.</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.8 KiB

View File

@@ -1,460 +1,105 @@
# Fleet staging install on OKD # Deploy Fleet to a remote cluster
End-to-end runbook for deploying the fleet stack (Zitadel + NATS + Fleet uses the same lifecycle commands and Scores for local, staging, and
auth callout + operator) on an OKD cluster, with a remote agent production clusters. The selected context changes the target and publication
connecting through the public WSS endpoint. Targets the staging behavior; there is no separate staging installer.
shape — single-instance NATS, public Zitadel + NATS WS Routes with
edge-TLS via cert-manager, env-only Secret config (no volume
mounts) so default `restricted-v2` SCC is enough.
Time budget: ~30 min on a warm cluster, ~60 min cold. ## Prerequisites
## 0. Prereqs - Docker with Buildx, Helm, and kubectl are on `PATH`.
- The remote cluster has cert-manager and the CloudNativePG operator.
- The image registry contains credentials for `REGISTRY_USER` and
`REGISTRY_TOKEN`.
- The deploy identity can obtain the cluster kubeconfig through OpenBao.
- Public DNS points the context domain at the cluster ingress.
- `oc` CLI logged in with cluster-admin (or at least ## Define the context
cluster-scoped privileges on the namespaces below — namespace
create, CRD apply, ClusterRole create).
- `podman` on your laptop, authenticated to the destination registry
(default `hub.nationtech.io/harmony``podman login` if needed).
- `helm` on PATH (used by Harmony's helm chart Scores).
- The staging cluster has:
- cert-manager installed and a `ClusterIssuer` ready for the
cluster's base domain (default name: `letsencrypt-prod`
override with `--cluster-issuer` if yours differs).
- CNPG (cloudnative-pg) operator installed (Zitadel relies on it
for its Postgres cluster).
- DNS: the chosen `--base-domain` resolves to the OKD ingress
router. For `cb1.nationtech.io`, that means `*.cb1.nationtech.io`
or at least `sso-staging.cb1.nationtech.io` and
`nats-fleet-staging.cb1.nationtech.io` must point at the OKD
router VIP. If you're using the cluster's apps domain
(`apps.cb1.nationtech.io`), set `--base-domain` accordingly.
- Access to write a `[credentials]` TOML on whichever machine will
run the agent (your laptop is fine for the demo).
## 1. Build and push images Remote contexts belong in the customer's private repository, not this public
repository. Create a context file like this:
The staging install pulls operator + auth-callout images from your ```toml
registry. The helper script builds both, tags them, and pushes: [contexts.customer-prod]
profile = "prod"
registry = "hub.example.com"
project = "customer"
domain = "fleet.example.com"
image_pull_secret = "registry-pull"
openbao_namespace = "customer-prod"
openbao_url = "https://openbao.example.com"
zitadel_url = "https://sso.example.com"
openbao_role = "customer-prod-cd"
zitadel_audience = "openbao"
```
Set `HARMONY_ZITADEL_KEY_JSON` to the deploy service user's Zitadel key JSON.
The context uses that identity to obtain its OpenBao-brokered kubeconfig.
## Provision callout credentials
Before the first deploy, create the immutable
`fleet-system/fleet-callout-credentials` Secret. It must contain
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass`.
```bash ```bash
cd /path/to/harmony kubectl create namespace fleet-system --dry-run=client -o yaml | kubectl apply -f -
./fleet/scripts/build_and_push_images.sh kubectl -n fleet-system create secret generic fleet-callout-credentials \
--from-literal=issuer-nkey-seed="$ISSUER_NKEY_SEED" \
--from-literal=issuer-public-key="$ISSUER_PUBLIC_KEY" \
--from-literal=nats-auth-pass="$NATS_AUTH_PASS" \
--dry-run=client -o yaml | kubectl apply -f -
kubectl -n fleet-system patch secret fleet-callout-credentials \
--type=merge -p '{"immutable":true}'
``` ```
Defaults: `REGISTRY=hub.nationtech.io/harmony`, `IMAGE_TAG=dev`, The registry pull Secret named by `image_pull_secret` must also exist in each
`PUSH=1`. Override with environment variables. Skip the push (e.g. namespace that pulls private images.
to inspect the images locally first) with `PUSH=0`.
Output ends with the exact `--operator-image` / `--callout-image` ## Deploy
flags to paste into step 4.
**Verify:** `ship` builds the operator and callout images, pushes their digest-pinned
references, then converges the Fleet Scores serially:
```bash ```bash
podman images | grep harmony # both refs present locally export REGISTRY_USER=...
podman pull hub.nationtech.io/harmony/harmony-fleet-operator:dev # registry confirmed export REGISTRY_TOKEN=...
export HARMONY_ZITADEL_KEY_JSON=...
cargo run -p harmony-fleet-deploy -- \
ship --context customer-prod --config path/to/contexts.toml
``` ```
## 2. Create namespaces For CI, the stages can run independently. Pass the digest references printed by
`build` into `publish` and `deploy`:
```bash ```bash
oc new-project zitadel-staging cargo run -p harmony-fleet-deploy -- build --context customer-prod --config path/to/contexts.toml
oc new-project fleet-staging cargo run -p harmony-fleet-deploy -- publish --context customer-prod --config path/to/contexts.toml \
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
cargo run -p harmony-fleet-deploy -- deploy --context customer-prod --config path/to/contexts.toml \
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
``` ```
If `hub.nationtech.io` requires authentication, add the imagePullSecret `deploy` never builds or publishes images. It returns only after each Score's
to both namespaces (each pod that pulls from the registry needs it): readiness check passes, including the operator's authenticated NATS startup.
## Verify
```bash ```bash
# adjust to whatever you have for hub.nationtech.io cargo run -p harmony-fleet-deploy -- status --context customer-prod --config path/to/contexts.toml
oc -n fleet-staging secrets link default <hub-pull-secret> --for=pull cargo run -p harmony-fleet-deploy -- logs --context customer-prod --config path/to/contexts.toml
oc -n zitadel-staging secrets link default <hub-pull-secret> --for=pull kubectl -n fleet-system get deployments,pods
kubectl get deployments.fleet.nationtech.io,devices.fleet.nationtech.io
``` ```
(For Zitadel + Postgres the chart pulls from public registries, so For local validation before a remote deploy:
the secret is only strictly required in `fleet-staging` for the
operator + callout images. Linking both is safest.)
## 3. Set KUBECONFIG and verify cluster context
```bash ```bash
export KUBECONFIG=$ADMIN_KUBECONFIG cargo run -p harmony-fleet-deploy -- ship --context local
oc whoami HARMONY_FLEET_E2E=1 cargo test -p harmony-fleet-e2e --test ping -- --nocapture
oc config current-context
oc get clusterversion # confirm OKD reachable + healthy
``` ```
The install runs with this `KUBECONFIG`. **Double-check** before See [`fleet/README.md`](../../fleet/README.md) for component details and the E2E
running step 4 — Harmony's `K8sAnywhereTopology::from_env()` honors workflow.
this and there's no second confirmation prompt.
## 4. Run `fleet_staging_install`
```bash
cargo run --release -p example_fleet_staging_install -- \
--base-domain cb1.nationtech.io \
--operator-image hub.nationtech.io/harmony/harmony-fleet-operator:dev \
--callout-image hub.nationtech.io/harmony/harmony-nats-callout:dev
```
Optional flags (defaults shown):
```
--cluster-issuer letsencrypt-prod
--fleet-namespace fleet-staging
--zitadel-namespace zitadel-staging
--nats-account FLEET
--zitadel-version v4.12.1
--project-name fleet
--admin-role fleet-admin
--device-role device
--operator-username fleet-operator
--admin-username fleet-ops
```
Step-by-step the binary does:
1. **Zitadel helm install** — Postgres (CNPG) + Zitadel chart into
`--zitadel-namespace`. Edge-TLS Route at `sso-staging.<base>`
with cert-manager-driven certificate.
2. **Zitadel setup** — project, two roles (`fleet-admin`, `device`),
API app `nats`, and two machine users (`fleet-ops` for manual
admin work, `fleet-operator` for the operator pod). Both get
JSON keys cached at `~/.local/share/harmony/zitadel/client-config.json`.
3. **NATS install** — single-instance JetStream, `auth_callout`
block referencing the issuer NKey pubkey, WebSocket listener on
8080. Edge-TLS Route at `nats-fleet-staging.<base>`.
4. **Auth callout deployment** — env-only Secret config (no mounts),
wired to the same issuer + Zitadel project audience.
5. **Operator deployment** — single Secret holding the credentials
TOML (with the operator's JSON keyfile inlined). One env var,
`FLEET_OPERATOR_CREDENTIALS_TOML`, no volumes.
The binary prints the URLs + project_id at the end. Save that block
— you'll need the project_id for the agent config.
**Expected output tail:**
```
=== fleet-staging install complete ===
Zitadel: https://sso-staging.cb1.nationtech.io/
NATS WS public: wss://nats-fleet-staging.cb1.nationtech.io/
NATS in-cluster: nats://fleet-nats.fleet-staging.svc.cluster.local:4222
Operator: oc -n fleet-staging get deploy/harmony-fleet-operator
Auth callout: oc -n fleet-staging get deploy/fleet-callout
Project id: 371xxxxxxxxxxxxxxx
Admin user: fleet-ops (machine key in ~/.local/share/harmony/zitadel/client-config.json)
Operator user: fleet-operator (machine key embedded in operator's Secret)
```
## 5. Verify each layer
### 5.1 Zitadel reachable, certificate provisioned
```bash
# pod up
oc -n zitadel-staging get pods
# expect: zitadel-* Running, zitadel-pg-1/2 Running
# Route + certificate (cert-manager creates the secret)
oc -n zitadel-staging get route
oc -n zitadel-staging get certificate
# OIDC discovery from the public URL
curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer
# expect: "https://sso-staging.cb1.nationtech.io"
```
If `curl` fails with TLS errors, the cert-manager certificate isn't
ready yet. Watch its status:
```bash
oc -n zitadel-staging describe certificate
oc -n cert-manager logs deploy/cert-manager --tail=50
```
A `Ready` condition `True` + `secretName: zitadel-tls` populated
means the Route can serve HTTPS.
### 5.2 NATS pod up, callout connected
```bash
oc -n fleet-staging get pods
# expect:
# fleet-nats-0 2/2 Running (NATS + reloader sidecar)
# fleet-callout-... 1/1 Running
oc -n fleet-staging logs deploy/fleet-callout --tail=30 | grep -E "starting|JWKS|listening"
# expect:
# starting harmony NATS auth callout
# JWKS refreshed count=2
# auth callout service listening subject="$SYS.REQ.USER.AUTH"
```
If the callout pod CrashLoopBackOff:
```bash
oc -n fleet-staging logs deploy/fleet-callout --previous --tail=30
```
Most common: OIDC issuer URL mismatch. The callout's
`OIDC_ISSUER_URL` env must byte-equal what Zitadel emits as `iss` in
its discovery doc. Check both:
```bash
oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_ISSUER_URL
# vs.
curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer
```
### 5.3 Operator authenticated and running
```bash
oc -n fleet-staging get pods -l app.kubernetes.io/name=harmony-fleet-operator
oc -n fleet-staging logs deploy/harmony-fleet-operator --tail=30
```
Look for, in order:
```
minted fresh Zitadel access token audience=<project_id>
connected successfully server=4222
NATS connected
KV bucket ready bucket=desired-state
starting Deployment controller
device-reconciler: watching device-info KV
aggregator: startup complete
```
If you see `Permissions Violation` errors, the callout's
`OIDC_AUDIENCE` (project_id at deploy time) doesn't match the
project_id in Zitadel today. Re-run step 4 — the live-query fix
in the Zitadel setup will refresh.
### 5.4 NATS WSS reachable from outside the cluster
```bash
curl -sSI https://nats-fleet-staging.cb1.nationtech.io/ | head -5
```
Expect a 4xx (NATS doesn't speak HTTP, but the TLS handshake should
succeed and you'll get back a WebSocket-upgrade-related response).
A connection refused or TLS handshake error means the Route or
cert-manager is unhappy.
### 5.5 CRDs registered
```bash
oc get crd | grep fleet.nationtech.io
# expect:
# deployments.fleet.nationtech.io
# devices.fleet.nationtech.io
```
## 6. Connect a remote agent
The fleet agent runs on the device (laptop, Pi, anywhere with
outbound HTTPS). It needs:
- Its own Zitadel machine user with the `device` role grant.
- The JSON keyfile from that user.
- A `[credentials]` TOML pointing at the public Zitadel + the WSS
NATS URL.
### 6.1 Mint a per-device machine user
Use `oc port-forward` or a helper to call Zitadel's API. Easier
path: drop a quick Score that adds one machine user. For tonight,
do it from the Zitadel UI:
1. Browse to `https://sso-staging.cb1.nationtech.io/ui/console/`,
log in as the human admin (password from Zitadel ConfigMap on
first install — see `docs/guides/fleet-zitadel-faq.md`).
2. Pick the `Default` org → `fleet` project → Roles → confirm
`device` exists.
3. Org → Users → Service Users → New: name `device-laptop-01`,
userName `device-laptop-01`. Save.
4. The user's "Personal Information" tab → Authorizations or
"Authorization" → "+New" — grant the `fleet` project's `device`
role to this user.
5. The user's "Keys" tab → "+New", type `JSON`, expiration future
date. **Download the keyfile JSON** — Zitadel only shows the
private half once. Save as `~/.local/share/harmony/fleet/agents/device-laptop-01.json`.
### 6.2 Build the agent locally
```bash
cargo build --release -p harmony-fleet-agent
ls -la target/release/harmony-fleet-agent
```
### 6.3 Render the agent's config TOML
```bash
PROJECT_ID=$(oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_AUDIENCE)
cat > /tmp/fleet-agent-config.toml <<EOF
[agent]
device_id = "device-laptop-01"
[nats]
urls = ["wss://nats-fleet-staging.cb1.nationtech.io"]
[credentials]
type = "zitadel-jwt"
key_path = "/etc/fleet-agent/zitadel-key.json"
oidc_issuer_url = "https://sso-staging.cb1.nationtech.io"
audience = "$PROJECT_ID"
[labels]
env = "staging"
location = "laptop"
arch = "$(uname -m)"
EOF
```
The agent's username convention is `device-<device_id>`, matching
the callout's `DEVICE_ID_PREFIX_STRIP=device-`. The Zitadel machine
user must literally be `device-laptop-01` for the JWT-bearer flow
to extract the right device id.
### 6.4 Run the agent
```bash
sudo mkdir -p /etc/fleet-agent
sudo cp ~/.local/share/harmony/fleet/agents/device-laptop-01.json \
/etc/fleet-agent/zitadel-key.json
sudo chown $(id -u):$(id -g) /etc/fleet-agent/zitadel-key.json
sudo chmod 0400 /etc/fleet-agent/zitadel-key.json
FLEET_AGENT_CONFIG=/tmp/fleet-agent-config.toml \
RUST_LOG=info \
./target/release/harmony-fleet-agent
```
Watch the log:
```
fleet-agent-v0 starting device_id=device-laptop-01
podman socket ready
inventory loaded hostname=...
connecting to NATS ["wss://nats-fleet-staging.cb1.nationtech.io"]
minted fresh Zitadel access token audience=<project_id>
connected successfully server=...
NATS connected
fleet publisher ready
watching KV keys filter=device-laptop-01.>
```
If you hit `Permissions Violation` errors after `connected`:
- check `oc -n fleet-staging logs deploy/fleet-callout --tail=20`
it'll show why the JWT was rejected (audience, role claim,
device_id format).
### 6.5 Verify the operator created a Device CR
```bash
oc get devices
# expect:
# NAME AGE
# device-laptop-01 Xs
oc describe device device-laptop-01
# labels block reflects what the agent sent in [labels]
```
## 7. Drive a deployment end to end
```bash
cat > /tmp/hello-web.yaml <<'EOF'
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: hello-web
spec:
score:
type: PodmanV0
data:
services:
- name: testnginx
image: docker.io/nginx:latest
ports:
- "8080:80"
targetSelector:
matchLabels:
env: staging
rollout:
strategy: Immediate
EOF
oc apply -f /tmp/hello-web.yaml
# Status reflect-back from the agent (takes ~5-15s)
oc get deployment.fleet.nationtech.io hello-web -o yaml | yq '.status'
# expect:
# aggregate:
# matchedDeviceCount: 1
# succeeded: 1
# failed: 0
# pending: 0
# On the device:
podman ps
# expect: testnginx running, port 8080→80
curl -sS http://localhost:8080 | head -3
```
## 8. Common failure modes
| Symptom | Cause / fix |
| --- | --- |
| `cert-manager` Certificate stuck `False` for 5+ min | DNS for the host doesn't resolve to the OKD router yet. `dig sso-staging.<base> +short` should match the cluster's ingress IP. Or your `letsencrypt-prod` ClusterIssuer is using HTTP01 and the route isn't reachable from Let's Encrypt. |
| Operator pod `Error: constructing CredentialSource` | The credentials TOML in the Secret is malformed. `oc -n fleet-staging get secret harmony-fleet-operator-secrets -o jsonpath='{.data.credentials\.toml}' \| base64 -d` and inspect; the `key_json` field must be a valid JSON keyfile string (multi-line triple-quoted in TOML is fine). |
| Operator pod `Permissions Violation` after `NATS connected` | Issuer pubkey or project_id mismatch between callout and NATS chart values, or Zitadel was reset and the operator's machine key no longer authenticates. Re-run `cargo run -p example_fleet_staging_install`. |
| Agent: `Zitadel token endpoint returned 400: invalid_grant_type` | TOML scope assembly bug or wrong `audience`. Confirm `audience` matches `oc exec deploy/fleet-callout -- printenv OIDC_AUDIENCE`. |
| Agent: connects, then `Permissions Violation for Publish to "$KV.device-info..."` | The device's machine user has no `device` role grant. Add via Zitadel UI → user → Authorizations. |
| `Deployment.fleet.nationtech.io` CR applied but `matchedDeviceCount: 0` | `targetSelector.matchLabels` doesn't match any Device CR's `metadata.labels`. `oc get devices --show-labels`. |
| Container redeploys every 30s on the device | Known FIXME — the agent's `matches_spec` returns false for any spec with env or volumes. For the demo, use trivial specs (the hello-web above is fine). Tracked in `harmony/src/modules/podman/topology.rs`. |
## 9. Tear down
The Helm releases own the bulk of the resources, so the cleanest
recovery from a broken state is:
```bash
helm -n zitadel-staging uninstall zitadel
helm -n fleet-staging uninstall fleet-nats
oc -n fleet-staging delete deploy/harmony-fleet-operator deploy/fleet-callout
oc -n fleet-staging delete secret harmony-fleet-operator-secrets fleet-callout-secrets
oc -n zitadel-staging delete pgcluster zitadel-pg --ignore-not-found
oc delete project zitadel-staging fleet-staging
# CRDs persist (helm.sh/resource-policy: keep). Delete by hand if you
# really want a clean slate:
oc delete crd deployments.fleet.nationtech.io devices.fleet.nationtech.io
```
The host-side `~/.local/share/harmony/zitadel/client-config.json`
caches machine keys + project IDs from this install. Wipe it before
re-installing against a freshly reset Zitadel:
```bash
rm -f ~/.local/share/harmony/zitadel/client-config.json
```
(The cache-vs-live drift bug is fixed — `ZitadelSetupScore` now
re-queries Zitadel for IDs on every apply — but stale machine-key
material from a deleted Zitadel project will fail at JWT-bearer
mint until you delete + re-create.)
## 10. Cross-reference
- [`fleet-zitadel-faq.md`](./fleet-zitadel-faq.md) — concepts behind
Zitadel projects, roles, machine users, audit-trail decisions.
- [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) —
worked recipe for minting an admin token by hand and using it
with `nats kv` commands.
- `examples/fleet_staging_install/src/main.rs` — the install code
itself; the comments narrate every step.
- `harmony/src/modules/fleet/server.rs::FleetServerScore`
composable form of the same install for callers that don't need
the intermediate read of `ZitadelClientConfig`.

View File

@@ -0,0 +1,108 @@
# Fleet Score references
Fleet is the first application of [ADR-028](../adr/028-typed-score-references.md).
Its deployment should read like a declaration of what Fleet needs and how those
components connect.
## Target experience
Deploying a configured Fleet should require one command:
```bash
./harmony-fleet-deploy ship --context prod
```
The context selects the desired configuration. The command composes the Scores,
passes their Refs, and interprets the result. It does not expose the deployment
sequence to the operator.
The corresponding Rust should stay close to the same statement:
```rust
let postgres = PostgreSQLScore::new(config.postgres);
let zitadel = ZitadelScore::new(config.zitadel)
.database(postgres.root_account_ref());
let nats = NatsScore::new(config.nats);
let callout = NatsAuthCalloutScore::new(config.callout)
.nats(nats.account_ref());
let operator = FleetOperatorScore::new(config.operator)
.identity(zitadel.application_ref())
.messaging(nats.client_ref());
```
The API names are illustrative. The required shape is a short component
declaration followed by typed connections to other components.
## How composition works
### Identity functionality
[![PostgreSQLScore passes PostgreSQLRootAccountRef to ZitadelScore, composing identity functionality.](../diagrams/fleet-identity-composition.svg)](../diagrams/fleet-identity-composition.svg)
### Messaging functionality
[![NatsScore and NatsAuthCalloutScore exchange typed Refs, composing authenticated messaging.](../diagrams/fleet-messaging-composition.svg)](../diagrams/fleet-messaging-composition.svg)
The bidirectional messaging relationship is still declarative: constructing
either Ref does not require its target to exist.
## Complete reference
[![Fleet functionality, exported Ref types, and consuming Score types.](../diagrams/fleet-score-references.svg)](../diagrams/fleet-score-references.svg)
The final map lists each functionality, the Ref types it exports, and the Score
types that consume them. The names identify intended contracts. They become
public APIs only as implementation replaces the corresponding string wiring;
the map does not prescribe a generic Ref hierarchy or registry.
## Composition boundary
`harmony-fleet-deploy` is composition only. It may:
- load the selected context;
- declare component configuration;
- construct Scores;
- connect Scores through typed Refs;
- interpret the composed deployment.
It must not:
- call provider APIs or commands directly;
- build Kubernetes or Helm resources;
- generate credentials or reconstruct provider outputs;
- implement deployment ordering, retries, or readiness checks;
- copy provider URLs, IDs, credential locations, or naming conventions.
All deployment behavior belongs to the Score that owns the component. If a
required operation has no Score, the missing work is to add or extend that
Score, not to implement the operation in Fleet composition.
## Ref behavior
Refs are available from desired state before their resources exist. Passing a
Ref is therefore not a readiness check. Each consuming Score handles an absent
or unavailable target through its own `Interpret`/`Topology` path, often by
emitting a native Kubernetes reference.
ADR-028 does not require a particular non-success outcome, retry policy, or
continuous convergence mechanism.
## Acceptance criteria
Fleet validates this design when:
1. A configured deployment runs as
`./harmony-fleet-deploy ship --context prod`.
2. Declarations and Ref wiring average at most five non-blank lines per logical
component in the canonical Fleet composition. PostgreSQL, Zitadel, each
Zitadel application or account, NATS, the auth callout, OpenBao, the Fleet
operator, and device enrollment each count as logical components.
3. `harmony-fleet-deploy` contains only context loading, configuration,
component declarations, Ref wiring, and interpretation.
4. Each migrated Ref removes the string or cache contract it replaces.
5. Constructing and passing Refs performs no I/O and implies no execution order.
6. Production and E2E compose the same Score and Ref APIs.
7. No generic graph, scheduler, or convergence mechanism is added as part of
ADR-028.

View File

@@ -241,6 +241,7 @@ pub async fn bring_up_stack() -> Result<StackHandles> {
info!("[5/8] provisioning project + roles + machine users in Zitadel"); info!("[5/8] provisioning project + roles + machine users in Zitadel");
let setup = ZitadelSetupScore { let setup = ZitadelSetupScore {
outputs_namespace: None,
host: ZITADEL_HOST.to_string(), host: ZITADEL_HOST.to_string(),
scheme: Default::default(), scheme: Default::default(),
port: None, port: None,

View File

@@ -27,6 +27,7 @@ vm-rehearsal = ["harmony/kvm"]
# `podman` types via the reconciler-contracts shape). `kvm` is the # `podman` types via the reconciler-contracts shape). `kvm` is the
# only feature that pulls libvirt and stays opt-in via `vm-rehearsal`. # only feature that pulls libvirt and stays opt-in via `vm-rehearsal`.
harmony = { path = "../../harmony", default-features = false, features = ["podman"] } harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
harmony_types = { path = "../../harmony_types" } harmony_types = { path = "../../harmony_types" }
tokio.workspace = true tokio.workspace = true
log.workspace = true log.workspace = true

View File

@@ -25,11 +25,11 @@ use std::path::PathBuf;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::{ use harmony::modules::fleet::ensure_fleet_ssh_keypair;
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
ensure_fleet_ssh_keypair,
};
use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials}; use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials};
use harmony_fleet_deploy::{
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
};
use harmony_types::id::Id; use harmony_types::id::Id;
// VM-rehearsal-only imports. Hidden behind a feature so `cargo build // VM-rehearsal-only imports. Hidden behind a feature so `cargo build

View File

@@ -4,46 +4,10 @@ edition = "2024"
version.workspace = true version.workspace = true
readme.workspace = true readme.workspace = true
license.workspace = true license.workspace = true
description = "VM-based end-to-end rehearsal: k3d + Zitadel + NATS auth callout + libvirt VM agents + operator → CR → podman → status" description = "VM-based end-to-end rehearsal of the Fleet Scores"
[lib]
name = "example_fleet_e2e_demo"
path = "src/lib.rs"
[[bin]]
name = "fleet-e2e-demo"
path = "src/main.rs"
[[test]]
name = "e2e_walking_skeleton"
path = "tests/e2e_walking_skeleton.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", features = ["kvm"] } harmony-fleet-e2e = { path = "../../fleet/harmony-fleet-e2e" }
harmony-k8s = { path = "../../harmony-k8s" }
harmony_types = { path = "../../harmony_types" }
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
harmony-nats-callout = { path = "../../nats/callout" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" }
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
k3d-rs = { path = "../../k3d" }
async-nats.workspace = true
nkeys = "0.4"
tokio = { workspace = true, features = ["full"] }
tokio-test.workspace = true
serde.workspace = true
serde_json.workspace = true
anyhow.workspace = true anyhow.workspace = true
log.workspace = true tokio = { workspace = true, features = ["full"] }
env_logger.workspace = true clap = { workspace = true, features = ["derive"] }
tracing.workspace = true
tracing-subscriber.workspace = true
futures-util.workspace = true
k8s-openapi.workspace = true
kube.workspace = true
clap = { version = "4", features = ["derive", "env"] }
directories = "6.0.0"
tempfile = "3"
url.workspace = true

View File

@@ -1,312 +0,0 @@
# Local fleet rehearsal runbook
End-to-end walkthrough of the IoT fleet platform on your laptop:
k3d-hosted control plane (Zitadel + NATS + auth callout) plus two
libvirt VMs running the fleet-agent. Mirrors the production topology
closely enough that you can watch the auth callout flow, the
JetStream KV traffic, and the per-device permission boundary in a
real cluster.
This is not the integration-test harness (that runs unattended). It
is a step-by-step sequence with inspection points in between. Run
each section, look at what happened, then continue.
## 0. Prerequisites
- Linux host with KVM (the user running the commands in `libvirt` /
`kvm` group; check with `id`).
- `podman`, `qemu-system-x86_64` (and `qemu-system-aarch64` if you
pick `--arch aarch64`), `mdbook` (optional), `kubectl`, `nats` CLI
(optional, for the manual subscribe step). Most other tooling
(k3d, ansible venv, cloud images) is auto-provisioned under
`~/.local/share/harmony/`.
- `/etc/hosts`: `127.0.0.1 sso.fleet.local` so you can hit Zitadel
from your browser through the cluster's HTTP_PORT (see
`examples/fleet_auth_callout/src/lib.rs` for the constant).
- Free TCP ports `8080` and `30422` on the host.
Source map for the things you'll inspect:
| Component | File |
| --- | --- |
| Bring-up flow | `examples/fleet_e2e_demo/src/lib.rs` |
| Per-device Zitadel + agent install | same, `provision_device()` |
| NATS Score (auth-callout mode) | `fleet/harmony-fleet-deploy/src/nats.rs::FleetNatsScore::callout` |
| Shared agent config schema | `fleet/harmony-fleet-auth/src/agent_config.rs` |
| Auth callout deployment Score | `harmony/src/modules/nats_auth_callout/mod.rs` |
| Callout decision logic | `nats/callout/src/handler.rs::decide` |
| Per-device permissions template | `nats/callout/src/permissions.rs::device_default` |
| Agent NATS auth (JWT-bearer mint) | `fleet/harmony-fleet-auth/src/credentials.rs` |
| Agent KV publishers + direct pulse | `fleet/harmony-fleet-agent/src/fleet_publisher.rs` |
| Walking-skeleton tests | `examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs` |
The NATS server's helm values are rendered from typed Rust structs
via `serde_yaml::to_string` (see `FleetNatsScore::values_yaml`),
not by `format!()` string interpolation. Same with the agent's
`/etc/fleet-agent/config.toml` — typed `AgentConfig`
`toml::to_string` → ConfigMap. Per ADR-023 principle 2 the e2e
demo composes the same `*Score` types the production deploy uses.
## 1. Provision the VMs
Each VM is one libvirt domain on the default network
(`192.168.122.0/24`). Run `fleet_vm_setup` once per VM. Pass
`--only-vm` so it stops at the cloud-init step (the agent install
happens later from the e2e bring-up — keeps the two phases legible).
```bash
# VM 0
cargo run --release -p example-fleet-vm-setup -- \
--arch aarch64 \
--vm-name vm-device-00 \
--only-vm
# VM 1
cargo run --release -p example-fleet-vm-setup -- \
--arch aarch64 \
--vm-name vm-device-01 \
--only-vm
```
Use `--arch x86_64` for native KVM speed; `aarch64` runs under
qemu-system-aarch64 TCG emulation on x86_64 hosts and is slower but
matches Pi targets.
**Inspect:**
```bash
virsh list --all
virsh domifaddr vm-device-00
virsh domifaddr vm-device-01
```
Note the IPs — you'll pass them in step 2. Confirm SSH works:
```bash
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
fleet-admin@<vm0-ip> uptime
```
The keypair lives under `~/.local/share/harmony/fleet/ssh/`,
generated on first run.
## 2. Bring up the control-plane stack
This single command does everything: k3d cluster, Zitadel,
ZitadelSetupScore (project + roles + 2 device machine users +
`fleet-ops` admin), NATS with `auth_callout`, callout image build &
sideload, callout Deployment, and finally `FleetDeviceSetupScore`
over SSH for each VM (packages, agent binary, JWT keyfile,
systemd unit).
```bash
FLEET_E2E_VM_0_IP=<vm0-ip> FLEET_E2E_VM_1_IP=<vm1-ip> \
cargo run --release -p example-fleet-e2e-demo -- --num-devices 2
```
The bring-up logs each step as `[e2e-demo X/9]`. Read along with
`examples/fleet_e2e_demo/src/lib.rs::bring_up_full_stack` to see
what's happening at each line. Stops at `STACK READY` and waits on
Ctrl-C (the cluster stays up after Ctrl-C — this is just the
foreground holder).
**Inspect:**
```bash
export KUBECONFIG=$(k3d kubeconfig write fleet-auth-callout)
# All workloads up?
kubectl get pods -n fleet-system
kubectl get pods -n zitadel
# Callout config the deployment is using:
kubectl get deployment -n fleet-system fleet-callout \
-o jsonpath='{.spec.template.spec.containers[0].env}' | jq
```
Open Zitadel in the browser: <http://sso.fleet.local:8080/ui/console>
(login with `root@zitadel.local` / the bootstrap password printed
during step `[e2e-demo 3/9]`). Click into the `fleet` project →
`Users` to see the two `device-vm-device-0X` machine users with
`device` role grants and the `fleet-ops` admin.
## 3. Watch the auth callout in action
The callout is the security boundary: every NATS connect attempt
hits `$SYS.REQ.USER.AUTH`, the callout validates the Zitadel JWT
in `connect_opts.auth_token`, applies the decision tree in
`nats/callout/src/handler.rs::decide`, and signs back a user JWT
with role-scoped permissions.
Tail it while the agents reconnect:
```bash
kubectl logs -n fleet-system -l app=fleet-callout -f
```
You'll see one set of lines per (re)connect:
```
received auth callout request user_nkey=U…
Zitadel JWT validated, generating user JWT device_id=vm-device-00 role=device
sending auth response
```
The `device_id` field is the value AFTER `device_id_prefix_strip`
runs (Zitadel emits `client_id=device-vm-device-00`; the callout
strips `device-` so permissions are interpolated against the bare
device id the agent uses for KV keys). See
`nats/callout/src/zitadel.rs::extract_device_id` for the strip.
**Force a reconnect to make a callout fire on demand:**
```bash
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
fleet-admin@<vm0-ip> 'sudo systemctl restart fleet-agent'
```
Watch the callout pod log emit one fresh request/response.
## 4. Watch the agent
```bash
ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \
fleet-admin@<vm0-ip> 'sudo journalctl -u fleet-agent -f'
```
What good looks like, in order:
| Log line | Where it comes from |
| --- | --- |
| `minted fresh Zitadel access token audience=…` | `credentials.rs::zitadel_mint` — RFC 7523 JWT-bearer flow, signed with the per-device machine key under `/etc/fleet-agent/zitadel-key.json` |
| `connected successfully server=4222` | NATS accepted the JWT minted by the callout |
| `fleet publisher ready` | KV buckets opened; `device-info` write succeeded |
| `watching KV keys filter=vm-device-00.>` | desired-state subscriber is up |
Absence of `Permissions Violation` lines is the success signal —
those mean the JWT's perms don't match what the agent tried to
publish (you'd hit them if `device_id_prefix_strip` were
misconfigured, for example).
## 5. Observe fleet traffic as admin
The harness mints a `fleet-ops` admin machine user with the
`fleet-admin` role; the callout maps that role to
`pub/sub allow: [">"]`. The integration test
`admin_jwt_reads_any_device_subject` exercises this — easiest path
to see it live is to run it with output. The test is
`#[ignore]`d on `cargo test` so a developer box doesn't burn a
10-minute Zitadel bring-up by accident; `--ignored` opts in:
```bash
FLEET_E2E_VM_0_IP=<vm0-ip> FLEET_E2E_VM_1_IP=<vm1-ip> \
cargo test -p example-fleet-e2e-demo \
--test e2e_walking_skeleton \
admin_jwt_reads_any_device_subject \
-- --test-threads=1 --nocapture --ignored
```
It subscribes admin to `device-state.>` (the direct, non-JetStream
fan-out subject the agent emits a pulse on every 30s — see
`fleet_publisher.rs::publish_state_pulse`) and asserts a message
arrives within 30s.
**Inspect KV state directly** using a bare admin client. The
underlying mechanism is in
`examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs::admin_nats_client`:
mint a JWT-bearer token from `stack.admin_machine_key`, hand it to
`async_nats` as `auth_token`. The test
`both_devices_heartbeat_within_60s` then reads `device-info` keys
directly:
```rust
let js = async_nats::jetstream::new(admin);
let bucket = js.get_key_value(BUCKET_DEVICE_INFO).await?;
let entry = bucket.entry(&device_info_key("vm-device-00")).await?;
```
To do it from a shell, port-forward NATS and use the `nats` CLI
with admin creds — but creds for an auth-callout server take a
JWT-bearer token, which the `nats` CLI doesn't speak natively;
running the test is the path of least friction.
## 6. Verify cross-device isolation (currently `#[ignore]`)
`cross_device_isolation_enforced_in_vm` is an empty test marked
`#[ignore = "requires E2eHandles::device_machine_key plumbing"]`
in `e2e_walking_skeleton.rs` — the test is a placeholder. The
plumbing it's waiting on is straightforward: the existing
`DeviceHandle` struct (`examples/fleet_e2e_demo/src/lib.rs:106`)
exposes `device_id` + `vm_ip` + `labels` but not the per-device
Zitadel machine key the test would need to mint a `device`-role
JWT and try cross-device subjects. `provision_device` already
creates the key (line ~324, `machine_key_json`) — wiring it through
into `DeviceHandle.machine_key` and implementing the test body
(mint JWT-bearer for vm-device-00, sub to
`device-commands.vm-device-01`, expect `Permissions Violation`)
is a single follow-up commit. I haven't touched it because nothing
in this branch's scope required it.
**You can verify the boundary manually right now**, even without
the test wired up: tail the callout pod, then SSH onto vm-device-00
and run the agent with a tampered config that points it at
vm-device-01's keyfile. The callout will issue a JWT for
`vm-device-01` (because the JWT-bearer assertion is signed with
that user's key); the agent on vm-device-00 will then publish on
`$KV.device-info.info.vm-device-00`, which is NOT in the JWT's
allow list — NATS rejects with `Permissions Violation`. This is
the same gate the test would automate.
The permissions template is in
`nats/callout/src/permissions.rs::device_default` — every allowed
subject contains `{device_id}` and is interpolated per-request, so
device A's JWT physically cannot publish to device B's subjects.
## 7. Drive the desired-state loop
(Not yet covered by a walking-skeleton test, but the agent's
reconciler is wired and observable.) From an admin client, write a
desired state for vm-device-00:
```rust
// pseudocode — see harmony-reconciler-contracts for the exact types
let kv = jetstream.create_key_value(kv::Config {
bucket: BUCKET_DESIRED_STATE.into(),
history: 1,
..Default::default()
}).await?;
kv.put(
&desired_state_key("vm-device-00", &dn("hello-web")),
payload.into(),
).await?;
```
What happens, observable from the agent's journal:
1. Agent's KV watcher (filter `vm-device-00.>`) fires.
2. Reconciler computes the diff and runs the podman create.
3. `write_deployment_state(&state)` fires:
- puts `state.vm-device-00.hello-web` into the `device-state`
KV bucket (operator-side watch picks it up)
- publishes the same payload on direct subject
`device-state.vm-device-00` (admin observers see it live)
You can subscribe to the latter with admin and watch reconcile
events stream in real time.
## 8. Teardown
The cluster persists across runs (re-running `fleet_e2e_demo`
converges drift, doesn't recreate). When you want a clean slate:
```bash
k3d cluster delete fleet-auth-callout
virsh destroy vm-device-00; virsh undefine vm-device-00 --remove-all-storage
virsh destroy vm-device-01; virsh undefine vm-device-01 --remove-all-storage
```
Cached assets (cloud images, k3d binary, ansible venv, SSH key,
fleet secrets) live under `~/.local/share/harmony/` and survive
cluster/VM destruction by design — first run after a clean reuses
them.

View File

@@ -1,847 +0,0 @@
//! VM-based end-to-end rehearsal of the customer demo flow.
//!
//! Goal: prove the JWT-auth chain works on a real-system agent
//! before pointing the demo at OKD. See
//! `ROADMAP/fleet_platform/v0_demo_e2e.md` for the full plan.
//!
//! Bring-up sequence:
//! 1. k3d cluster with HTTP + NATS port mappings (re-uses
//! fleet_auth_callout's k3d helpers — same cluster name so
//! re-runs of either example reuse the same cluster).
//! 2. Zitadel + Postgres via ZitadelScore.
//! 3. Wait for Zitadel HTTP and the chart-provisioned `iam-admin-pat`
//! secret (the chart's setup job is async).
//! 4. ZitadelSetupScore for the project + API app + roles + admin
//! machine user (no per-device users yet).
//! 5. NATS with auth_callout block + the callout pod.
//! 6. For each device i:
//! - ZitadelSetupScore minting a per-device machine user with
//! the `device` role grant. The JSON keyfile is cached in
//! `ZitadelClientConfig` and read back here for the agent.
//! - libvirt VM via `ProvisionVmScore`.
//! - SSH-inject `/etc/hosts` so the VM resolves
//! `sso.fleet.local` to the libvirt host.
//! - `FleetDeviceSetupScore` with `FleetDeviceAuth::ZitadelJwt`
//! pointing at the dropped keyfile.
//!
//! Tests in `tests/e2e_walking_skeleton.rs` share a single bring-up
//! via `OnceCell` and exercise: heartbeats, label-selector targeting,
//! status reflect-back, env+volume propagation, admin cross-device
//! read, per-device isolation, NATS-pod-restart reconnect.
use std::path::PathBuf;
use std::time::Duration;
use anyhow::{Context, Result};
use example_fleet_auth_callout::{
ADMIN_ROLE_KEY, API_APP_NAME, CALLOUT_DEPLOYMENT_NAME, CALLOUT_IMAGE_TAG, DEVICE_ROLE_KEY,
FLEET_NAMESPACE, HTTP_PORT, NATS_ACCOUNT, NATS_AUTH_PASS, NATS_AUTH_USER, NATS_NAMESPACE,
NATS_NODE_PORT, NATS_RELEASE, PROJECT_NAME, ZITADEL_HOST, build_and_load_callout_image,
create_k3d, create_topology, deploy_zitadel, ensure_issuer_seed, wait_for_callout_ready,
wait_for_zitadel_ready,
};
use harmony::inventory::Inventory;
use harmony::modules::fleet::{
FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry,
ensure_fleet_ssh_keypair,
};
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv};
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::zitadel::{
MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole,
ZitadelSetupScore,
};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
use harmony_fleet_deploy::FleetNatsScore;
use harmony_types::id::Id;
use log::{info, warn};
use nkeys::KeyPair;
// ---- constants -------------------------------------------------------------
/// Libvirt's default NAT gateway. The host's IP from inside any VM
/// attached to the `default` libvirt network. We bake this in because
/// every smoke-a* harness assumes it; if a customer runs their own
/// libvirt with a different bridge they can override via env.
pub const DEFAULT_LIBVIRT_HOST_IP: &str = "192.168.122.1";
pub const ADMIN_USERNAME: &str = "fleet-ops";
/// Separate machine user for the in-cluster operator. Distinct from
/// `fleet-ops` (manual admin tooling) so the audit trail can tell
/// operator-driven actions apart from human operator actions. Same
/// `fleet-admin` role grant — only the identity differs.
pub const OPERATOR_USERNAME: &str = "fleet-operator";
pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:dev";
/// Per-device username convention: `device-${device_id}`. Matches what
/// `fleet_rpi_setup` produces, so callout's `device_id_claim =
/// "client_id"` extracts the device id verbatim from the `client_id`
/// claim Zitadel emits in machine-user access tokens.
pub fn device_username(device_id: &str) -> String {
format!("device-{device_id}")
}
// ---- options + handles -----------------------------------------------------
#[derive(Debug, Clone)]
pub struct E2eDemoOpts {
/// Number of VM-as-device agents to provision.
pub num_devices: usize,
/// Path to the cross-compiled `fleet-agent` binary uploaded to
/// each VM. Defaults to `target/release/fleet-agent` (the same
/// path that smoke-a4 produces).
pub agent_binary: PathBuf,
/// Override for the libvirt host IP (the address VMs see as the
/// gateway). Defaults to [`DEFAULT_LIBVIRT_HOST_IP`].
pub libvirt_host_ip: String,
}
impl Default for E2eDemoOpts {
fn default() -> Self {
Self {
num_devices: 2,
agent_binary: workspace_target_path("release/harmony-fleet-agent"),
libvirt_host_ip: DEFAULT_LIBVIRT_HOST_IP.to_string(),
}
}
}
#[derive(Debug, Clone)]
pub struct DeviceHandle {
pub index: usize,
pub device_id: String,
pub vm_ip: String,
pub labels: std::collections::BTreeMap<String, String>,
}
#[derive(Debug, Clone)]
pub struct E2eHandles {
pub cluster_name: String,
pub nats_url_external: String,
pub zitadel_url: String,
pub project_id: String,
pub issuer_pubkey: String,
pub admin_machine_key: String,
pub devices: Vec<DeviceHandle>,
}
// ---- bring up --------------------------------------------------------------
pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result<E2eHandles> {
let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.try_init();
info!("[e2e-demo 1/9] ensuring k3d cluster");
let k3d = create_k3d();
k3d.ensure_installed()
.await
.map_err(|e| anyhow::anyhow!("k3d ensure: {e}"))?;
let topology = create_topology(&k3d);
topology.ensure_ready().await.context("topology init")?;
info!("[e2e-demo 2/9] deploying Zitadel (cold start: ~5 min)");
deploy_zitadel(&topology).await?;
info!("[e2e-demo 3/9] CoreDNS rewrite + waiting for Zitadel HTTP + iam-admin-pat secret");
CoreDNSRewriteScore {
rewrites: vec![CoreDNSRewrite {
hostname: ZITADEL_HOST.to_string(),
target: "zitadel.zitadel.svc.cluster.local".to_string(),
}],
}
.interpret(&Inventory::autoload(), &topology)
.await
.context("CoreDNSRewriteScore")?;
wait_for_zitadel_ready().await?;
wait_for_iam_admin_pat_secret(&topology).await?;
info!("[e2e-demo 4/9] provisioning project, API app, roles, admin machine user");
let admin_setup = ZitadelSetupScore {
host: ZITADEL_HOST.to_string(),
scheme: Default::default(),
port: None,
skip_tls: true,
endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")),
admin_org_id: None,
namespace: "zitadel".to_string(),
applications: vec![],
api_apps: vec![ZitadelApiApp {
project_name: PROJECT_NAME.to_string(),
app_name: API_APP_NAME.to_string(),
}],
roles: vec![
ZitadelRole {
project_name: PROJECT_NAME.to_string(),
key: ADMIN_ROLE_KEY.to_string(),
display_name: "Fleet Admin".to_string(),
group: None,
},
ZitadelRole {
project_name: PROJECT_NAME.to_string(),
key: DEVICE_ROLE_KEY.to_string(),
display_name: "Device".to_string(),
group: None,
},
],
groups_claim_action: false,
human_users: vec![],
org_members: vec![],
instance_members: vec![],
machine_secrets: vec![],
port_forward_service: None,
machine_users: vec![
ZitadelMachineUser {
username: ADMIN_USERNAME.to_string(),
name: "Fleet Operations".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(PROJECT_NAME.to_string()),
grant_roles: vec![ADMIN_ROLE_KEY.to_string()],
},
// Separate machine user for the in-cluster operator pod.
// Same `fleet-admin` role grant as the manual admin
// identity, but distinct username so JWT `client_id` lets
// log analysis tell operator-driven actions apart from
// human operator actions.
ZitadelMachineUser {
username: OPERATOR_USERNAME.to_string(),
name: "Fleet Operator (in-cluster)".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(PROJECT_NAME.to_string()),
grant_roles: vec![ADMIN_ROLE_KEY.to_string()],
},
],
};
admin_setup
.interpret(&Inventory::autoload(), &topology)
.await
.context("admin ZitadelSetupScore")?;
let zcfg = ZitadelClientConfig::load()
.context("ZitadelSetupScore did not produce a client config cache")?;
let project_id = zcfg
.project_id_by_name(PROJECT_NAME)
.or(zcfg.project_id.as_ref())
.context("project_id missing from cache")?
.clone();
let admin_machine_key = zcfg
.machine_key(ADMIN_USERNAME)
.context("admin machine key missing from cache")?
.clone();
info!("[e2e-demo 5/9] generating issuer NKey, deploying NATS with auth_callout");
let issuer_seed = ensure_issuer_seed(&topology).await?;
let issuer_kp = KeyPair::from_seed(&issuer_seed)
.map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?;
let issuer_pubkey = issuer_kp.public_key();
// Per ADR-023 principle 2 — e2e uses the same Scores as production.
// `FleetNatsScore::callout` renders the auth-callout values block
// typed (serde_yaml) rather than the legacy `render_nats_values`
// string interpolation. Same upstream chart, same wire format,
// schema-checked at compile time.
FleetNatsScore::callout(
NATS_NAMESPACE,
NATS_NODE_PORT as u16,
&issuer_pubkey,
NATS_ACCOUNT,
NATS_AUTH_USER,
NATS_AUTH_PASS,
)
.release_name(NATS_RELEASE)
.interpret(&Inventory::autoload(), &topology)
.await
.context("NATS deploy")?;
info!("[e2e-demo 6/9] building + sideloading callout image into k3d");
build_and_load_callout_image(&k3d).await?;
info!("[e2e-demo 7/9] deploying NatsAuthCalloutScore");
let mut callout = NatsAuthCalloutScore::new(
CALLOUT_DEPLOYMENT_NAME,
FLEET_NAMESPACE,
format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"),
format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
project_id.clone(),
NATS_AUTH_USER,
NATS_AUTH_PASS,
issuer_seed.clone(),
)
.image(CALLOUT_IMAGE_TAG)
.target_account(NATS_ACCOUNT)
.admin_role(ADMIN_ROLE_KEY)
.device_role(DEVICE_ROLE_KEY)
.danger_accept_invalid_certs(true);
// Same convention as fleet_auth_callout: the username is in the
// access token's `client_id` claim. The role claim path is
// project-scoped because the JWT-bearer flow requests project
// audience scope.
callout.device_id_claim = "client_id".to_string();
// Zitadel's `client_id` for a machine user equals its userName, so a
// user created as `device-vm-device-00` (the convention shared with
// fleet_rpi_setup and fleet_auth_callout) lands in the JWT verbatim.
// Strip the `device-` prefix so the callout interpolates permissions
// against the bare device id (`vm-device-00`) the agent uses for KV
// keys + direct subjects.
callout.device_id_prefix_strip = "device-".to_string();
callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles");
callout
.interpret(&Inventory::autoload(), &topology)
.await
.context("callout deploy")?;
wait_for_callout_ready(&topology).await?;
info!("[e2e-demo 8/10] building + sideloading operator image into k3d");
build_and_load_operator_image(&k3d).await?;
info!("[e2e-demo 9/10] deploying fleet operator with Zitadel JWT auth");
let operator_machine_key = zcfg
.machine_key(OPERATOR_USERNAME)
.with_context(|| format!("machine key for {OPERATOR_USERNAME} missing from cache"))?
.clone();
deploy_operator(&topology, &project_id, &operator_machine_key).await?;
wait_for_operator_ready(&topology).await?;
info!(
"[e2e-demo 10/10] provisioning {} VM(s) and onboarding agent(s)",
opts.num_devices
);
let mut devices = Vec::with_capacity(opts.num_devices);
for i in 0..opts.num_devices {
let handle = provision_device(i, &opts, &topology, &project_id).await?;
devices.push(handle);
}
info!(
"full stack ready: {} device(s), operator + admin role configured",
devices.len()
);
Ok(E2eHandles {
cluster_name: example_fleet_auth_callout::CLUSTER_NAME.to_string(),
nats_url_external: format!("nats://127.0.0.1:{NATS_NODE_PORT}"),
zitadel_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
project_id,
issuer_pubkey,
admin_machine_key,
devices,
})
}
// ---- per-device provisioning ----------------------------------------------
async fn provision_device(
index: usize,
opts: &E2eDemoOpts,
topology: &K8sAnywhereTopology,
project_id: &str,
) -> Result<DeviceHandle> {
let device_id = format!("vm-device-{index:02}");
let username = device_username(&device_id);
info!("[device {index}] minting Zitadel machine user {username}");
// Per-device ZitadelSetupScore (search-then-create — running this
// for an existing user is a NOOP that just refreshes the cache
// entry pointing at the persisted machine key). The keyfile is
// re-minted because Zitadel doesn't expose the private half of
// an existing key — accept that any prior key drifts to "stale
// until expiry" on the previous device installation.
let device_setup = ZitadelSetupScore {
host: ZITADEL_HOST.to_string(),
scheme: Default::default(),
port: None,
skip_tls: true,
endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")),
admin_org_id: None,
namespace: "zitadel".to_string(),
applications: vec![],
api_apps: vec![],
roles: vec![],
groups_claim_action: false,
human_users: vec![],
org_members: vec![],
instance_members: vec![],
machine_secrets: vec![],
port_forward_service: None,
machine_users: vec![ZitadelMachineUser {
username: username.clone(),
name: format!("Fleet Device {device_id}"),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(PROJECT_NAME.to_string()),
grant_roles: vec![DEVICE_ROLE_KEY.to_string()],
}],
};
device_setup
.interpret(&Inventory::autoload(), topology)
.await
.with_context(|| format!("ZitadelSetupScore for {username}"))?;
let zcfg = ZitadelClientConfig::load()
.context("ZitadelClientConfig disappeared between admin and device setup")?;
let machine_key_json = zcfg
.machine_key(&username)
.with_context(|| format!("machine key for {username} missing from cache"))?
.clone();
// -- VM provisioning would go here. Deferred to keep the harness
// cold-start observable in pieces — the kvm bits (ProvisionVmScore)
// require root + libvirtd + the cloud image. Today the harness
// expects the operator to have provisioned VMs out-of-band (e.g.
// via fleet_vm_setup, or a pre-existing libvirt domain). We read
// the IP from a convention path (see `discover_vm_ip`) so the
// test driver can iterate on the agent path without re-paying VM
// boot every test cycle.
//
// Follow-up: fold ProvisionVmScore::ensure_vm here once the
// bring-up has been demonstrated end-to-end at least once.
let vm_ip = discover_vm_ip(index)
.with_context(|| format!("could not resolve IP for device {index}"))?;
info!("[device {index}] {device_id} at {vm_ip} — installing agent with Zitadel JWT auth");
let labels = build_device_labels(&device_id, index);
let agent_score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig {
device_id: Id::from(device_id.clone()),
labels: labels.clone(),
// Agent connects to NATS at the libvirt host's IP via the
// NodePort. The libvirt default network NATs the VM through
// the host so the host's port mapping is reachable.
nats_urls: vec![format!("nats://{}:{NATS_NODE_PORT}", opts.libvirt_host_ip)],
auth: FleetDeviceAuth::ZitadelJwt {
machine_key_json,
// Issuer URL the agent uses MUST match the issuer
// string Zitadel returns — Zitadel derives that from
// the request's Host header. We hit Zitadel via the
// host's port mapping, so the agent's URL is
// `http://sso.fleet.local:<host-port>`. The /etc/hosts
// entry below points sso.fleet.local at the libvirt
// host so the VM resolves it.
oidc_issuer_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"),
audience: project_id.to_string(),
// Local rehearsal hits Zitadel over plain HTTP through
// the cluster ingress; no TLS validation needed.
danger_accept_invalid_certs: true,
},
agent_binary_path: opts.agent_binary.clone(),
hosts_entries: vec![HostsEntry {
ip: opts.libvirt_host_ip.clone(),
hostname: ZITADEL_HOST.to_string(),
}],
openbao: None,
});
// Apply the score over SSH against the VM. Same pattern as
// fleet_rpi_setup, but synthesized inline so the harness can drive
// multiple VMs in sequence without copying the CLI plumbing.
apply_fleet_setup_to_vm(index, &vm_ip, agent_score).await?;
Ok(DeviceHandle {
index,
device_id,
vm_ip,
labels,
})
}
async fn apply_fleet_setup_to_vm(
index: usize,
vm_ip: &str,
score: FleetDeviceSetupScore,
) -> Result<()> {
ensure_ansible_venv()
.await
.map_err(|e| anyhow::anyhow!("ansible venv: {e}"))?;
let ssh = ensure_fleet_ssh_keypair()
.await
.map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?;
let ip = vm_ip
.parse()
.with_context(|| format!("VM IP '{vm_ip}' is not a valid IP address"))?;
let creds = SshCredentials {
// Matches the cloud-init admin user that fleet_vm_setup +
// smoke-a4 create. If the operator overrode that during
// out-of-band VM provisioning, follow-up: thread the
// username through E2eDemoOpts.
user: "fleet-admin".to_string(),
private_key_path: ssh.private_key.clone(),
remote_python: Some("/usr/bin/python3".to_string()),
sudo_password: None,
};
let topology = LinuxHostTopology::new(format!("vm-device-{index:02}"), ip, creds);
use harmony::score::Score;
score
.create_interpret()
.execute(&Inventory::empty(), &topology)
.await
.with_context(|| format!("FleetDeviceSetupScore against VM {index} ({vm_ip})"))?;
Ok(())
}
fn build_device_labels(
device_id: &str,
index: usize,
) -> std::collections::BTreeMap<String, String> {
// Two devices, two distinct group labels by default — lets
// selector tests target "exactly one device". Label scheme
// matches the demo runbook.
let mut labels = std::collections::BTreeMap::new();
labels.insert(
"group".to_string(),
if index == 0 {
"group-a".to_string()
} else {
"group-b".to_string()
},
);
labels.insert("arch".to_string(), std::env::consts::ARCH.to_string());
labels.insert("role".to_string(), "rehearsal".to_string());
labels.insert("device-id".to_string(), device_id.to_string());
labels
}
fn discover_vm_ip(index: usize) -> Result<String> {
// Convention: a `FLEET_E2E_VM_<i>_IP` env var points at the
// pre-provisioned VM's IP. This keeps the harness usable on a
// workstation where the operator runs `fleet_vm_setup` once per
// device out-of-band, then re-runs the e2e harness against the
// already-booted VMs.
let key = format!("FLEET_E2E_VM_{index}_IP");
std::env::var(&key)
.with_context(|| format!("set {key} to the libvirt VM's IP (default network)"))
}
// ---- iam-admin-pat readiness ----------------------------------------------
/// Wait for the Zitadel chart's setup job to write the `iam-admin-pat`
/// secret. The Helm release reports Ready before the job completes,
/// so calling ZitadelSetupScore immediately after Zitadel deploy
/// races. ZitadelSetupScore itself reads this secret to authenticate
/// to the management API.
async fn wait_for_iam_admin_pat_secret(topology: &K8sAnywhereTopology) -> Result<()> {
use k8s_openapi::api::core::v1::Secret;
let k8s = topology
.k8s_client()
.await
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
for attempt in 1..=120 {
if let Some(secret) = k8s
.get_resource::<Secret>("iam-admin-pat", Some("zitadel"))
.await?
&& let Some(data) = secret.data
&& data.contains_key("pat")
{
return Ok(());
}
if attempt % 10 == 0 {
warn!("iam-admin-pat secret not yet present in zitadel ns ({attempt}/120)");
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
anyhow::bail!(
"timed out waiting for iam-admin-pat secret in 'zitadel' namespace — \
is FirstInstance.Org.Machine.Pat configured in ZitadelScore Helm values?"
)
}
// ---- operator deploy -------------------------------------------------------
const OPERATOR_NAMESPACE: &str = FLEET_NAMESPACE;
/// k3d's data directory under `$XDG_DATA_HOME`. Mirrors
/// `example_fleet_auth_callout::data_dir` (the latter is private —
/// duplicated here rather than re-exported so the operator wiring is
/// self-contained).
fn k3d_data_dir() -> PathBuf {
directories::BaseDirs::new()
.map(|dirs| dirs.data_dir().join("harmony").join("k3d"))
.unwrap_or_else(|| PathBuf::from("/tmp/harmony"))
}
/// Build the operator's release binary, package it into an OCI image,
/// and sideload into the k3d cluster. Mirrors
/// `build_and_load_callout_image`. The Dockerfile lives in the
/// operator crate.
async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> {
use std::process::Stdio;
let workspace_root = std::env::var("CARGO_MANIFEST_DIR")
.map(|d| PathBuf::from(d).join("..").join(".."))
.unwrap_or_else(|_| PathBuf::from("."));
let workspace_root = workspace_root.canonicalize().unwrap_or(workspace_root);
info!("cargo build --release -p harmony-fleet-operator");
let status = tokio::process::Command::new("cargo")
.args(["build", "--release", "-p", "harmony-fleet-operator"])
.current_dir(&workspace_root)
.status()
.await?;
if !status.success() {
anyhow::bail!("cargo build for fleet operator failed");
}
// Stage the binary + Dockerfile into a clean temp dir so podman
// build doesn't drag the whole target/ tree across.
let ctx = tempfile::tempdir()?;
let bin_dst = ctx.path().join("target/release");
std::fs::create_dir_all(&bin_dst)?;
std::fs::copy(
workspace_root.join("target/release/harmony-fleet-operator"),
bin_dst.join("harmony-fleet-operator"),
)
.context("staging operator binary into build context")?;
let dockerfile_src = workspace_root.join("fleet/harmony-fleet-operator/Dockerfile");
if !dockerfile_src.exists() {
anyhow::bail!(
"missing fleet/harmony-fleet-operator/Dockerfile — operator image staging \
expects it next to Cargo.toml; either add it or update the bring-up."
);
}
std::fs::copy(&dockerfile_src, ctx.path().join("Dockerfile"))?;
info!("podman build → {OPERATOR_IMAGE_TAG}");
let status = tokio::process::Command::new("podman")
.args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."])
.current_dir(ctx.path())
.stderr(Stdio::inherit())
.status()
.await?;
if !status.success() {
anyhow::bail!("podman build for operator failed");
}
let tar_path =
std::env::temp_dir().join(format!("harmony-operator-image-{}.tar", std::process::id()));
let _ = std::fs::remove_file(&tar_path);
let status = tokio::process::Command::new("podman")
.args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG])
.status()
.await?;
if !status.success() {
anyhow::bail!("podman save for operator failed");
}
info!("k3d image import {OPERATOR_IMAGE_TAG}");
let cluster_name = k3d
.cluster_name()
.unwrap_or(example_fleet_auth_callout::CLUSTER_NAME)
.to_string();
let tar_path_str = tar_path.to_str().unwrap().to_string();
let cluster_for_blocking = cluster_name.clone();
let data_dir = k3d_data_dir();
tokio::task::spawn_blocking(move || {
k3d_rs::K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([
"image",
"import",
tar_path_str.as_str(),
"-c",
cluster_for_blocking.as_str(),
])
})
.await?
.map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?;
let _ = std::fs::remove_file(&tar_path);
Ok(())
}
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
/// ClusterRoleBinding + Secret + Deployment via Harmony's
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
async fn deploy_operator(
topology: &K8sAnywhereTopology,
project_id: &str,
operator_machine_key: &str,
) -> Result<()> {
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
use harmony_fleet_deploy::operator::chart::{
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
build_cluster_role_binding, build_operator_deployment, build_service_account,
operator_secret,
};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
// Render the [credentials] TOML the operator pod consumes via the
// OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret
// key). The Zitadel JSON keyfile is embedded inline under
// `key_json`; the operator never sees a file. Triple-quoted TOML
// string keeps the JSON's `"`s untouched.
let credentials_toml = format!(
r#"type = "zitadel-jwt"
oidc_issuer_url = "http://{host}:{port}"
audience = "{project_id}"
danger_accept_invalid_certs = true
key_json = """
{key_json}
"""
"#,
host = ZITADEL_HOST,
port = HTTP_PORT,
key_json = operator_machine_key,
);
let opts = ChartOptions {
output_dir: PathBuf::new(), // unused on this code path
image: OPERATOR_IMAGE_TAG.to_string(),
image_pull_policy: "IfNotPresent".to_string(),
nats_url: format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"),
log_level: "info,kube_runtime=warn".to_string(),
credentials: Some(OperatorCredentials { credentials_toml }),
chart_version: None,
web_auth_config_json: None,
web_cookie_key_json: None,
};
// CRDs first — the operator watches them on startup.
let crds: Vec<CustomResourceDefinition> = vec![FleetDeployment::crd(), Device::crd()];
K8sResourceScore::<CustomResourceDefinition> {
resource: crds,
namespace: None,
}
.interpret(&Inventory::autoload(), topology)
.await
.context("operator CRD apply")?;
// RBAC.
K8sResourceScore::single(
build_service_account(),
Some(OPERATOR_NAMESPACE.to_string()),
)
.interpret(&Inventory::autoload(), topology)
.await
.context("operator ServiceAccount apply")?;
K8sResourceScore::single(build_cluster_role(), None)
.interpret(&Inventory::autoload(), topology)
.await
.context("operator ClusterRole apply")?;
K8sResourceScore::single(build_cluster_role_binding(OPERATOR_NAMESPACE), None)
.interpret(&Inventory::autoload(), topology)
.await
.context("operator ClusterRoleBinding apply")?;
// Secret holding both the credentials TOML and the keyfile.
let secret = operator_secret(&opts).expect("credentials present in opts");
K8sResourceScore::single(secret, Some(OPERATOR_NAMESPACE.to_string()))
.interpret(&Inventory::autoload(), topology)
.await
.context("operator Secret apply")?;
// Deployment last so it pulls the up-to-date Secret.
K8sResourceScore::single(
build_operator_deployment(&opts),
Some(OPERATOR_NAMESPACE.to_string()),
)
.interpret(&Inventory::autoload(), topology)
.await
.context("operator Deployment apply")?;
info!("operator deployment {OPERATOR_NAMESPACE}/{RELEASE_NAME} applied");
Ok(())
}
async fn wait_for_operator_ready(topology: &K8sAnywhereTopology) -> Result<()> {
use harmony_fleet_deploy::operator::chart::RELEASE_NAME;
use k8s_openapi::api::apps::v1::Deployment as K8sDeployment;
let k8s = topology
.k8s_client()
.await
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
for attempt in 1..=120 {
if let Some(d) = k8s
.get_resource::<K8sDeployment>(RELEASE_NAME, Some(OPERATOR_NAMESPACE))
.await?
&& let Some(status) = d.status
&& status.ready_replicas.unwrap_or(0) >= 1
{
return Ok(());
}
if attempt % 10 == 0 {
warn!("operator Deployment not yet Ready ({attempt}/120)");
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
anyhow::bail!("timed out waiting for operator Deployment to become Ready")
}
// ---- helpers ---------------------------------------------------------------
fn workspace_target_path(rel: &str) -> PathBuf {
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("."));
manifest_dir.join("..").join("..").join("target").join(rel)
}
// ---- next-steps panel ------------------------------------------------------
impl E2eHandles {
pub fn print_next_steps(&self) {
println!();
println!("============================================================");
println!(" E2E DEMO REHEARSAL — STACK READY");
println!("============================================================");
println!(" k3d cluster: {}", self.cluster_name);
println!(" Zitadel: {}", self.zitadel_url);
println!(" NATS (host): {}", self.nats_url_external);
println!(" Project ID: {}", self.project_id);
println!(" Issuer pubkey: {}", self.issuer_pubkey);
println!();
println!(" Devices ({}):", self.devices.len());
for d in &self.devices {
let labels: Vec<String> = d.labels.iter().map(|(k, v)| format!("{k}={v}")).collect();
println!(
" [{}] {} @ {} ({})",
d.index,
d.device_id,
d.vm_ip,
labels.join(",")
);
}
println!();
println!(" Run the test suite:");
println!();
println!(" cargo test -p example-fleet-e2e-demo \\");
println!(" --test e2e_walking_skeleton -- --test-threads=1 --nocapture");
println!();
println!(" Ctrl-C exits without tearing the cluster down — re-run");
println!(" the bring-up to converge any drift.");
println!("============================================================");
}
}
#[cfg(test)]
mod unit_tests {
use super::*;
#[test]
fn device_username_matches_callout_convention() {
// Callout's device_id_claim is `client_id`, which Zitadel
// populates from the machine user's username. The test we
// run later asserts the agent's per-device subjects match
// its device_id, which therefore must equal the username
// minus the "device-" prefix the callout knows about.
assert_eq!(device_username("vm-device-00"), "device-vm-device-00");
}
#[test]
fn device_labels_split_into_distinct_groups() {
let l0 = build_device_labels("vm-device-00", 0);
let l1 = build_device_labels("vm-device-01", 1);
assert_eq!(l0.get("group").unwrap(), "group-a");
assert_eq!(l1.get("group").unwrap(), "group-b");
assert_ne!(l0.get("group"), l1.get("group"));
// Ubiquitous labels: device-id + arch + role on both.
for l in [&l0, &l1] {
assert!(l.contains_key("device-id"));
assert!(l.contains_key("arch"));
assert_eq!(l.get("role").unwrap(), "rehearsal");
}
}
}

View File

@@ -1,51 +1,22 @@
//! `cargo run -p example-fleet-e2e-demo -- --num-devices 2 ...` use anyhow::Result;
//!
//! Brings up the full E2E rehearsal stack: k3d + Zitadel + NATS auth
//! callout + per-device Zitadel machine users + (out-of-band)
//! libvirt VMs + agents authenticating via JWT-bearer.
//!
//! See `src/lib.rs` and `ROADMAP/fleet_platform/v0_demo_e2e.md`.
use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use example_fleet_e2e_demo::{DEFAULT_LIBVIRT_HOST_IP, E2eDemoOpts, bring_up_full_stack}; use harmony_fleet_e2e::{AuthMode, VmStack, VmStackOptions};
use std::path::PathBuf;
#[derive(Parser, Debug)] #[derive(Parser)]
#[command(
name = "fleet-e2e-demo",
about = "VM-based end-to-end rehearsal of the fleet platform demo flow"
)]
struct Cli { struct Cli {
/// Number of VM-as-device agents to bring up. Each one needs its #[arg(long, default_value_t = 1)]
/// own libvirt domain (provisioned out-of-band today via
/// `fleet_vm_setup` — see `FLEET_E2E_VM_<i>_IP` env vars below).
#[arg(long, default_value_t = 2)]
num_devices: usize, num_devices: usize,
/// Path to the cross-compiled `fleet-agent` binary uploaded to
/// each VM. Same binary that smoke-a4 produces.
#[arg(long, default_value = "target/release/harmony-fleet-agent")]
agent_binary: PathBuf,
/// Override for the libvirt host IP (the address VMs see as the
/// gateway). Defaults to the libvirt default network's gateway.
#[arg(long, default_value = DEFAULT_LIBVIRT_HOST_IP)]
libvirt_host_ip: String,
} }
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let cli = Cli::parse(); let cli = Cli::parse();
let handles = bring_up_full_stack(E2eDemoOpts { let _stack = VmStack::bring_up(VmStackOptions {
num_devices: cli.num_devices, num_devices: cli.num_devices,
agent_binary: cli.agent_binary, auth_mode: AuthMode::Callout,
libvirt_host_ip: cli.libvirt_host_ip, ..Default::default()
}) })
.await .await?;
.context("bring_up_full_stack")?;
handles.print_next_steps();
println!();
println!(" Press Ctrl-C to exit (cluster keeps running).");
tokio::signal::ctrl_c().await?; tokio::signal::ctrl_c().await?;
Ok(()) Ok(())
} }

View File

@@ -1,161 +0,0 @@
//! End-to-end walking-skeleton tests for the VM-based demo rehearsal.
//!
//! Shares one bring-up across the whole suite via `OnceCell`. Run
//! sequentially — they touch shared k3d + libvirt VM state.
//!
//! Pre-flight (manual, before `cargo test`):
//!
//! - libvirt + qemu installed; default network active.
//! - Two cloud-init Ubuntu VMs provisioned (e.g. via
//! `cargo run -p example_fleet_vm_setup`). Their IPs exported as
//! `FLEET_E2E_VM_0_IP` and `FLEET_E2E_VM_1_IP`.
//! - SSH keypair the VMs trust at `~/.ssh/id_ed25519` (or
//! override path; harness reads the standard pair).
//!
//! Run:
//!
//! ```bash
//! FLEET_E2E_VM_0_IP=192.168.122.42 \
//! FLEET_E2E_VM_1_IP=192.168.122.43 \
//! cargo test -p example-fleet-e2e-demo --test e2e_walking_skeleton \
//! -- --test-threads=1 --nocapture
//! ```
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use async_nats::ConnectOptions;
use example_fleet_auth_callout::{mint_access_token, scopes_for_project};
use example_fleet_e2e_demo::{E2eDemoOpts, E2eHandles, bring_up_full_stack};
use futures_util::StreamExt;
use tokio::sync::OnceCell;
static STACK: OnceCell<Arc<E2eHandles>> = OnceCell::const_new();
async fn shared_stack() -> Result<Arc<E2eHandles>> {
let cell = STACK
.get_or_try_init(|| async {
let h = bring_up_full_stack(E2eDemoOpts::default()).await?;
anyhow::Ok(Arc::new(h))
})
.await?;
Ok(cell.clone())
}
async fn admin_nats_client(stack: &E2eHandles) -> Result<async_nats::Client> {
let token = mint_access_token(
&stack.zitadel_url,
&stack.admin_machine_key,
&scopes_for_project(&stack.project_id),
)
.await
.context("mint admin Zitadel token")?;
ConnectOptions::with_token(token)
.connection_timeout(Duration::from_secs(5))
.connect(&stack.nats_url_external)
.await
.map_err(|e| anyhow::anyhow!("admin connect: {e}"))
}
// -- Test 1 -------------------------------------------------------------
/// Each provisioned VM publishes a DeviceInfo within the heartbeat
/// window. Reads from the `device-info` KV bucket via the admin
/// client (admin role can subscribe to anything).
#[tokio::test]
#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"]
async fn both_devices_heartbeat_within_60s() -> Result<()> {
let _ = tracing_subscriber::fmt().with_env_filter("info").try_init();
let stack = shared_stack().await?;
let admin = admin_nats_client(&stack).await?;
let js = async_nats::jetstream::new(admin);
let bucket = js
.get_key_value(harmony_reconciler_contracts::BUCKET_DEVICE_INFO)
.await
.context("device-info bucket")?;
let deadline = std::time::Instant::now() + Duration::from_secs(60);
let expected: std::collections::HashSet<String> =
stack.devices.iter().map(|d| d.device_id.clone()).collect();
let mut seen = std::collections::HashSet::new();
while std::time::Instant::now() < deadline && seen != expected {
for d in &stack.devices {
let key = harmony_reconciler_contracts::device_info_key(&d.device_id);
if let Some(_e) = bucket.entry(&key).await? {
seen.insert(d.device_id.clone());
}
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
assert_eq!(
seen, expected,
"each provisioned device must publish DeviceInfo within 60s; saw {seen:?}"
);
Ok(())
}
// -- Test 5 (admin cross-device read) -----------------------------------
/// The admin's Zitadel JWT carries `fleet-admin` role. Callout maps
/// that to `pub/sub allow: [">"]`, so subscribing to `device-state.>`
/// is admitted and observes every device's traffic.
#[tokio::test]
#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"]
async fn admin_jwt_reads_any_device_subject() -> Result<()> {
let _ = tracing_subscriber::fmt().with_env_filter("info").try_init();
let stack = shared_stack().await?;
let admin = admin_nats_client(&stack).await?;
let mut sub = admin.subscribe("device-state.>").await?;
admin.flush().await?;
// Hold the subscription open long enough that any device's
// periodic state publication should land. We don't pump traffic
// ourselves — the agents themselves publish per-deployment state
// on every reconcile tick. If no traffic arrives in 30s it means
// either the agents aren't connected or they're not publishing,
// both of which are fatal for the demo.
let result = tokio::time::timeout(Duration::from_secs(30), sub.next()).await;
assert!(
result.is_ok() && result.as_ref().unwrap().is_some(),
"admin must observe at least one device-state.* message in 30s"
);
Ok(())
}
// -- Test 6 (per-device isolation) ---------------------------------------
/// A per-device JWT has subject permissions scoped to its own
/// `device-state.{device_id}` and `device-commands.{device_id}`. The
/// callout enforces this; subscribing to a sibling device's commands
/// must fail at NATS connect-time or at SUB-time.
///
/// Skipped here because the per-device JWT minting helper (analogous
/// to `mint_access_token` but for a `device` role user) needs the
/// per-device machine key to be plumbed back from `bring_up_full_stack`
/// through `E2eHandles`. Follow-up commit adds
/// `E2eHandles::device_machine_key(idx)` so this test can be
/// implemented without re-running `ZitadelSetupScore` from the test
/// body.
#[tokio::test]
#[ignore = "requires E2eHandles::device_machine_key plumbing"]
async fn cross_device_isolation_enforced_in_vm() {}
// -- Test 7 (load-bearing reconnect) -------------------------------------
/// Kill the NATS pod, wait for the new one to come up, verify both
/// agents reconnect with fresh JWTs and resume publishing within
/// 30 seconds. This is the test that validates the "never lose
/// connectivity to a device" guarantee under realistic disturbance.
///
/// Skipped pending operator install in the harness — without the
/// operator the agents have no `desired-state` to publish status
/// against, so verifying "publishing resumed" needs a separate
/// signal. Follow-up commit observes the agents' periodic
/// heartbeat publication directly via the device-heartbeat KV.
#[tokio::test]
#[ignore = "requires NATS-pod-restart driver and heartbeat-presence assertion"]
async fn agent_recovers_from_nats_pod_restart() {}

View File

@@ -9,7 +9,7 @@ name = "fleet_load_test"
path = "src/main.rs" path = "src/main.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", default-features = false } harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
async-nats = { workspace = true } async-nats = { workspace = true }
chrono = { workspace = true } chrono = { workspace = true }

View File

@@ -28,13 +28,13 @@ use anyhow::{Context, Result};
use async_nats::jetstream::{self, kv}; use async_nats::jetstream::{self, kv};
use chrono::Utc; use chrono::Utc;
use clap::Parser; use clap::Parser;
use harmony::modules::fleet::operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy}; use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
use harmony_reconciler_contracts::{ use harmony_reconciler_contracts::{
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key, DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key,
device_info_key, device_state_key, device_info_key, device_state_key,
}; };
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::api::core::v1::Namespace;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client; use kube::Client;

View File

@@ -1,5 +1,5 @@
//! Install a single-node NATS server into the cluster `KUBECONFIG` //! Install a single-node NATS server into the cluster `KUBECONFIG`
//! points at, using harmony's `NatsBasicScore` + `K8sBareTopology`. //! points at, using harmony's `NatsScore` + `K8sBareTopology`.
//! //!
//! This binary is the glue between the smoke harness (`smoke-a4.sh`) //! This binary is the glue between the smoke harness (`smoke-a4.sh`)
//! and the framework Score. Typical usage from a demo script: //! and the framework Score. Typical usage from a demo script:
@@ -20,13 +20,13 @@ use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::k8s::K8sBareTopology; use harmony::modules::k8s::K8sBareTopology;
use harmony::modules::nats::NatsBasicScore; use harmony::modules::nats::NatsScore;
use harmony::score::Score; use harmony::score::Score;
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command( #[command(
name = "fleet_nats_install", name = "fleet_nats_install",
about = "Install single-node NATS (JetStream) via NatsBasicScore" about = "Install single-node NATS (JetStream) via NatsScore"
)] )]
struct Cli { struct Cli {
/// Target namespace. Created if missing. /// Target namespace. Created if missing.
@@ -45,7 +45,7 @@ struct Cli {
/// NodePort when `--expose=node-port`. Must be in the cluster's /// NodePort when `--expose=node-port`. Must be in the cluster's
/// nodeport range (default 30000-32767). Ignored otherwise. /// nodeport range (default 30000-32767). Ignored otherwise.
#[arg(long, default_value_t = 30422)] #[arg(long, default_value_t = 30422)]
node_port: i32, node_port: u16,
/// Override the NATS container image. /// Override the NATS container image.
#[arg(long)] #[arg(long)]
image: Option<String>, image: Option<String>,
@@ -67,7 +67,7 @@ async fn main() -> Result<()> {
.map_err(|e| anyhow::anyhow!(e)) .map_err(|e| anyhow::anyhow!(e))
.context("building K8sBareTopology from KUBECONFIG")?; .context("building K8sBareTopology from KUBECONFIG")?;
let mut score = NatsBasicScore::new(&cli.name, &cli.namespace); let mut score = NatsScore::new(&cli.name, &cli.namespace);
match cli.expose { match cli.expose {
ExposeMode::ClusterIp => {} ExposeMode::ClusterIp => {}
ExposeMode::NodePort => score = score.node_port(cli.node_port), ExposeMode::NodePort => score = score.node_port(cli.node_port),
@@ -81,7 +81,7 @@ async fn main() -> Result<()> {
let outcome = interpret let outcome = interpret
.execute(&Inventory::empty(), &topology) .execute(&Inventory::empty(), &topology)
.await .await
.map_err(|e| anyhow::anyhow!("execute NatsBasicScore: {e}"))?; .map_err(|e| anyhow::anyhow!("execute NatsScore: {e}"))?;
println!( println!(
"NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}", "NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}",

View File

@@ -10,6 +10,7 @@ path = "src/main.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony" } harmony = { path = "../../harmony" }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
harmony_cli = { path = "../../harmony_cli" } harmony_cli = { path = "../../harmony_cli" }
harmony_secret = { path = "../../harmony_secret" } harmony_secret = { path = "../../harmony_secret" }
harmony_types = { path = "../../harmony_types" } harmony_types = { path = "../../harmony_types" }

View File

@@ -37,8 +37,8 @@ use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use harmony::config::secret::SudoPassword; use harmony::config::secret::SudoPassword;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec}; use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec};
use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
use harmony_secret::SecretManager; use harmony_secret::SecretManager;
use harmony_types::id::Id; use harmony_types::id::Id;
use log::info; use log::info;

View File

@@ -21,7 +21,7 @@
//! //!
//! Behaviour: //! Behaviour:
//! - Installs single-node NATS (JetStream) into `--nats-namespace` //! - Installs single-node NATS (JetStream) into `--nats-namespace`
//! using `NatsBasicScore`, exposed per `--nats-expose`. //! using `NatsScore`, exposed per `--nats-expose`.
//! - Installs the operator chart into `--operator-namespace` via //! - Installs the operator chart into `--operator-namespace` via
//! `FleetOperatorScore` (which renders the chart in a tempdir //! `FleetOperatorScore` (which renders the chart in a tempdir
//! and helm-installs it). //! and helm-installs it).
@@ -47,7 +47,7 @@
use anyhow::Result; use anyhow::Result;
use clap::Parser; use clap::Parser;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::nats::NatsBasicScore; use harmony::modules::nats::NatsScore;
use harmony::modules::zitadel::ZitadelScore; use harmony::modules::zitadel::ZitadelScore;
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
@@ -72,7 +72,7 @@ struct Cli {
nats_expose: NatsExpose, nats_expose: NatsExpose,
/// NodePort when `--nats-expose=node-port`. Ignored otherwise. /// NodePort when `--nats-expose=node-port`. Ignored otherwise.
#[arg(long, default_value_t = 30422)] #[arg(long, default_value_t = 30422)]
nats_node_port: i32, nats_node_port: u16,
/// Optional NATS image override (`repository:tag`). /// Optional NATS image override (`repository:tag`).
#[arg(long)] #[arg(long)]
nats_image: Option<String>, nats_image: Option<String>,
@@ -124,7 +124,7 @@ async fn main() -> Result<()> {
let topology = K8sAnywhereTopology::from_env(); let topology = K8sAnywhereTopology::from_env();
let mut nats = NatsBasicScore::new(&cli.nats_name, &cli.nats_namespace); let mut nats = NatsScore::new(&cli.nats_name, &cli.nats_namespace);
match cli.nats_expose { match cli.nats_expose {
NatsExpose::ClusterIp => {} NatsExpose::ClusterIp => {}
NatsExpose::NodePort => nats = nats.node_port(cli.nats_node_port), NatsExpose::NodePort => nats = nats.node_port(cli.nats_node_port),
@@ -135,17 +135,14 @@ async fn main() -> Result<()> {
} }
// Point the operator at NATS via the in-cluster service DNS the // Point the operator at NATS via the in-cluster service DNS the
// NatsBasicScore install creates. ClusterIP and LoadBalancer both // NatsScore install creates. ClusterIP and LoadBalancer both
// expose the same `<release>.<namespace>:4222` for in-cluster // expose the same `<release>.<namespace>:4222` for in-cluster
// callers. // callers.
let nats_url = format!("nats://{}.{}:4222", cli.nats_name, cli.nats_namespace); let operator = FleetOperatorScore::new(&cli.operator_image)
let operator = FleetOperatorScore::new()
.namespace(&cli.operator_namespace) .namespace(&cli.operator_namespace)
.release_name(&cli.operator_release) .release_name(&cli.operator_release)
.image(&cli.operator_image)
.image_pull_policy(&cli.operator_image_pull_policy) .image_pull_policy(&cli.operator_image_pull_policy)
.nats_url(&nats_url) .messaging(&nats.client_ref())
.log_level(&cli.log_level); .log_level(&cli.log_level);
// FleetServerScore now takes NatsK8sScore (auth-callout-aware, // FleetServerScore now takes NatsK8sScore (auth-callout-aware,

View File

@@ -1,36 +0,0 @@
[package]
name = "example-fleet-staging-deploy"
edition = "2024"
version.workspace = true
readme.workspace = true
license.workspace = true
description = "Deploy the fleet platform stack (Zitadel + NATS + auth callout) onto an OKD/Kubernetes cluster. Operator-side, run-once-per-customer."
[lib]
name = "example_fleet_staging_deploy"
path = "src/lib.rs"
[[bin]]
name = "fleet-staging-deploy"
path = "src/main.rs"
[dependencies]
harmony = { path = "../../harmony" }
harmony-k8s = { path = "../../harmony-k8s" }
harmony_types = { path = "../../harmony_types" }
harmony-nats-callout = { path = "../../nats/callout" }
nkeys = "0.4"
async-nats.workspace = true
reqwest = { workspace = true }
tokio = { workspace = true, features = ["full"] }
serde.workspace = true
serde_json.workspace = true
anyhow.workspace = true
log.workspace = true
env_logger.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
clap = { version = "4", features = ["derive", "env"] }
k8s-openapi.workspace = true
kube.workspace = true
url.workspace = true

View File

@@ -1,579 +0,0 @@
//! Operator-side staging deploy harness.
//!
//! Runs once per customer instance against an OKD / Kubernetes cluster
//! to bring up the fleet platform's central services:
//!
//! 1. Zitadel + Postgres (HTTPS via OKD HAProxy ingress, edge TLS).
//! 2. The fleet project + roles (`fleet-admin`, `device`) + an API app
//! (so the project ID can be the JWT-bearer audience).
//! 3. NATS with `auth_callout` and a WSS ingress (so Pis on a customer
//! LAN connect through `wss://nats.<base>/`).
//! 4. The auth callout Deployment, configured to validate Zitadel JWTs
//! and emit per-device permissions on user JWTs to NATS.
//!
//! Everything keys off [`FleetDomainConfig::base_domain`] —
//! `zitadel.<base>`, `nats.<base>`, `api.<base>` are the only
//! customer-visible hostnames. Pi-side onboarding (see
//! `examples/fleet_rpi_setup/`) consumes the Zitadel admin PAT plus
//! the project ID this harness prints, so the operator's flow is:
//!
//! ```text
//! cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io
//! ↓ prints PROJECT_ID, NATS WSS URL, instructions to extract iam-admin-pat
//! HARMONY_ZITADEL_ADMIN_PAT=$(kubectl -n zitadel get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d) \
//! cargo run -p example-fleet-rpi-setup -- \
//! --pi-host 192.168.1.42 \
//! --bootstrap-token "$HARMONY_ZITADEL_ADMIN_PAT" \
//! --zitadel-issuer-url https://zitadel.customer1.nationtech.io \
//! --zitadel-project-id <PROJECT_ID printed above> \
//! --nats-url wss://nats.customer1.nationtech.io/ \
//! --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent
//! ```
//!
//! The harness is **idempotent** by design — re-running picks up
//! existing resources via the new helm-upgrade-by-default behavior +
//! ZitadelSetupScore's search-then-create flow + a persisted issuer
//! NKey in a K8s secret so user JWTs survive restarts.
use std::time::Duration;
use anyhow::{Context, Result};
use harmony::inventory::Inventory;
use harmony::modules::nats::NatsHelmChartScore;
use harmony::modules::nats_auth_callout::{NatsAuthCalloutScore, render_auth_callout_block};
use harmony::modules::zitadel::{
ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelRole,
ZitadelScore, ZitadelSetupScore,
};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
use log::info;
use nkeys::KeyPair;
// ---- domain config ---------------------------------------------------------
/// Single source of truth for all customer-visible hostnames. Every
/// `<app>.<customer>.<base>` URL the staging deploy emits derives from
/// the one base domain — no hostnames are hardcoded so the same code
/// runs across customers / staging / canary instances.
#[derive(Debug, Clone)]
pub struct FleetDomainConfig {
/// e.g. `customer1.nationtech.io`. The deploy emits
/// `zitadel.<base>`, `nats.<base>`, `api.<base>` against it.
pub base_domain: String,
}
impl FleetDomainConfig {
pub fn new(base_domain: impl Into<String>) -> Self {
Self {
base_domain: base_domain.into(),
}
}
pub fn zitadel_host(&self) -> String {
format!("zitadel.{}", self.base_domain)
}
pub fn nats_wss_host(&self) -> String {
format!("nats.{}", self.base_domain)
}
pub fn zitadel_issuer_url(&self) -> String {
format!("https://{}", self.zitadel_host())
}
pub fn nats_wss_url(&self) -> String {
format!("wss://{}/", self.nats_wss_host())
}
}
// ---- naming + constants ----------------------------------------------------
pub const FLEET_NAMESPACE: &str = "fleet-staging";
pub const ZITADEL_NAMESPACE: &str = "zitadel-staging";
pub const NATS_RELEASE: &str = "fleet-nats";
pub const CALLOUT_DEPLOYMENT_NAME: &str = "fleet-callout";
pub const PROJECT_NAME: &str = "fleet";
pub const API_APP_NAME: &str = "nats";
pub const CLI_APP_NAME: &str = "harmony-cli";
pub const ADMIN_ROLE_KEY: &str = "fleet-admin";
pub const DEVICE_ROLE_KEY: &str = "device";
pub const NATS_AUTH_USER: &str = "auth";
pub const NATS_ACCOUNT: &str = "DEVICES";
pub const NATS_SYSTEM_USER: &str = "sys-admin";
pub const ISSUER_SEED_SECRET: &str = "callout-issuer-seed";
// ---- handles ---------------------------------------------------------------
#[derive(Debug, Clone)]
pub struct StagingHandles {
pub domain: FleetDomainConfig,
pub project_id: String,
pub issuer_pubkey: String,
/// Tag of the callout image expected to exist in a registry the
/// cluster pulls from. The operator pushes it before running the
/// deploy; this field is just the name we put on the Deployment
/// for traceability.
pub callout_image: String,
/// OIDC client_id of the `harmony-cli` Device Code app — what the
/// `fleet_sso_login` CLI sends in its device-authorization request.
/// `None` if the app pre-existed without the cache picking it up
/// (re-running the staging deploy after `rm -rf
/// ~/.local/share/harmony/zitadel/`).
pub cli_client_id: Option<String>,
}
// ---- bring up --------------------------------------------------------------
pub struct StagingDeployOpts {
pub domain: FleetDomainConfig,
pub kubeconfig_context: Option<String>,
/// Image reference the cluster will pull. Operator must have
/// pushed this beforehand (e.g. `quay.io/customer/harmony-nats-callout:demo`).
pub callout_image: String,
/// Per-NATS-account password for the callout's own NATS connection.
/// Stored in a K8s secret + listed in the chart's
/// `accounts.<account>.users` so the callout bypasses callout to
/// connect (otherwise it'd deadlock authenticating itself).
pub nats_auth_pass: String,
/// SYS account password (for `kubectl exec nats-box` debugging).
pub nats_system_pass: String,
}
pub async fn bring_up_staging(opts: StagingDeployOpts) -> Result<StagingHandles> {
let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.try_init();
if let Some(ctx) = &opts.kubeconfig_context {
unsafe {
std::env::set_var("HARMONY_K8S_CONTEXT", ctx);
std::env::set_var("HARMONY_USE_LOCAL_K3D", "false");
std::env::set_var("HARMONY_AUTOINSTALL", "false");
}
}
let topology = K8sAnywhereTopology::from_env();
topology.ensure_ready().await.context("topology init")?;
info!(
"[1/5] deploying Zitadel at https://{}",
opts.domain.zitadel_host()
);
deploy_zitadel(&opts.domain, &topology).await?;
info!("[2/5] waiting for Zitadel HTTPS to respond");
wait_for_zitadel_ready(&opts.domain).await?;
info!("[3/5] provisioning project '{PROJECT_NAME}', api app, CLI device-code app, and roles");
provision_zitadel_project(&opts.domain, &topology).await?;
let project_id = read_project_id()?;
let cli_client_id = read_cli_client_id();
info!(" → project_id = {project_id}");
if let Some(cid) = &cli_client_id {
info!(" → cli_client_id = {cid}");
} else {
log::warn!(
" → cli_client_id missing from cache; CLI login won't work until you reset the local zitadel cache"
);
}
info!("[4/5] generating issuer NKey + deploying NATS with auth_callout + WSS ingress");
let issuer_seed = ensure_issuer_seed(&topology).await?;
let issuer_kp = KeyPair::from_seed(&issuer_seed)
.map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?;
let issuer_pubkey = issuer_kp.public_key();
NatsHelmChartScore::new(
NATS_RELEASE.to_string(),
FLEET_NAMESPACE.to_string(),
render_nats_values(
&opts.domain,
&issuer_pubkey,
&opts.nats_auth_pass,
&opts.nats_system_pass,
),
)
.interpret(&Inventory::autoload(), &topology)
.await
.context("NATS deploy")?;
info!(
"[5/5] deploying NatsAuthCalloutScore (image: {})",
opts.callout_image
);
NatsAuthCalloutScore::new(
CALLOUT_DEPLOYMENT_NAME,
FLEET_NAMESPACE,
format!("nats://{NATS_RELEASE}.{FLEET_NAMESPACE}.svc.cluster.local:4222"),
opts.domain.zitadel_issuer_url(),
// The aud the callout validates against is the project ID —
// Zitadel emits it in access tokens minted via the
// project-id-audience scope.
project_id.clone(),
NATS_AUTH_USER,
opts.nats_auth_pass.clone(),
issuer_seed,
)
.image(&opts.callout_image)
.target_account(NATS_ACCOUNT)
.admin_role(ADMIN_ROLE_KEY)
.device_role(DEVICE_ROLE_KEY)
.interpret(&Inventory::autoload(), &topology)
.await
.context("callout deploy")?;
Ok(StagingHandles {
domain: opts.domain,
project_id,
issuer_pubkey,
callout_image: opts.callout_image,
cli_client_id,
})
}
fn read_cli_client_id() -> Option<String> {
ZitadelClientConfig::load()?
.client_id(CLI_APP_NAME)
.cloned()
}
async fn deploy_zitadel(domain: &FleetDomainConfig, topology: &K8sAnywhereTopology) -> Result<()> {
let z = ZitadelScore {
host: domain.zitadel_host(),
zitadel_version: "v4.12.1".to_string(),
// OKD HAProxy edge-terminates TLS for us, so the issuer URL
// is `https://zitadel.<base>` (port 443 implied) — leave
// external_port at None so Zitadel's emitted issuer omits the
// port, matching what clients reach.
external_secure: true,
external_port: None,
namespace: ZITADEL_NAMESPACE.to_string(),
..Default::default()
};
z.interpret(&Inventory::autoload(), topology)
.await
.context("ZitadelScore")?;
Ok(())
}
async fn provision_zitadel_project(
domain: &FleetDomainConfig,
topology: &K8sAnywhereTopology,
) -> Result<()> {
let setup = ZitadelSetupScore {
host: domain.zitadel_host(),
// Direct HTTPS through OKD's HAProxy ingress — operator runs
// anywhere with kubeconfig + DNS access. Defaults give
// `https://<host>` (port 443).
scheme: Default::default(),
port: None,
skip_tls: false,
endpoint: None,
admin_org_id: None,
namespace: ZITADEL_NAMESPACE.to_string(),
applications: vec![ZitadelApplication {
project_name: PROJECT_NAME.to_string(),
app_name: CLI_APP_NAME.to_string(),
// Device Code grant — the only browser-driven OIDC flow
// that fits a CLI tool: prints a verification URL + user
// code, polls for a token, no embedded web server / open
// listener required.
app_type: ZitadelAppType::DeviceCode,
}],
api_apps: vec![ZitadelApiApp {
project_name: PROJECT_NAME.to_string(),
app_name: API_APP_NAME.to_string(),
}],
roles: vec![
ZitadelRole {
project_name: PROJECT_NAME.to_string(),
key: ADMIN_ROLE_KEY.to_string(),
display_name: "Fleet Admin".to_string(),
group: None,
},
ZitadelRole {
project_name: PROJECT_NAME.to_string(),
key: DEVICE_ROLE_KEY.to_string(),
display_name: "Device".to_string(),
group: None,
},
],
// No machine users provisioned here — `fleet_rpi_setup` mints
// them on demand per device, so the staging deploy stays
// device-count-agnostic.
groups_claim_action: false,
machine_users: vec![],
..Default::default()
};
setup
.interpret(&Inventory::autoload(), topology)
.await
.context("ZitadelSetupScore")?;
Ok(())
}
fn read_project_id() -> Result<String> {
let cfg = ZitadelClientConfig::load()
.context("ZitadelSetupScore did not produce a client config cache")?;
cfg.project_id_by_name(PROJECT_NAME)
.or(cfg.project_id.as_ref())
.context("project_id missing from ZitadelClientConfig cache")
.cloned()
}
/// Persist the callout's issuer NKey seed in a K8s secret so re-runs
/// of the staging deploy don't invalidate previously-issued user JWTs
/// already in flight on customer Pis.
async fn ensure_issuer_seed(topology: &K8sAnywhereTopology) -> Result<String> {
use k8s_openapi::ByteString;
use k8s_openapi::api::core::v1::{Namespace, Secret};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
use std::collections::BTreeMap;
let k8s = topology
.k8s_client()
.await
.map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?;
if k8s
.get_resource::<Namespace>(FLEET_NAMESPACE, None)
.await?
.is_none()
{
let ns = Namespace {
metadata: ObjectMeta {
name: Some(FLEET_NAMESPACE.to_string()),
..Default::default()
},
..Default::default()
};
k8s.create(&ns, None).await.ok();
}
if let Some(existing) = k8s
.get_resource::<Secret>(ISSUER_SEED_SECRET, Some(FLEET_NAMESPACE))
.await?
&& let Some(data) = existing.data
&& let Some(seed_bytes) = data.get("seed")
{
let seed = String::from_utf8(seed_bytes.0.clone())?;
return Ok(seed.trim().to_string());
}
let seed = KeyPair::new_account()
.seed()
.map_err(|e| anyhow::anyhow!("nkey seed: {e}"))?;
let mut data = BTreeMap::new();
data.insert("seed".to_string(), ByteString(seed.as_bytes().to_vec()));
let secret = Secret {
metadata: ObjectMeta {
name: Some(ISSUER_SEED_SECRET.to_string()),
namespace: Some(FLEET_NAMESPACE.to_string()),
..Default::default()
},
data: Some(data),
type_: Some("Opaque".to_string()),
..Default::default()
};
k8s.create(&secret, Some(FLEET_NAMESPACE)).await.ok();
Ok(seed)
}
// ---- NATS values -----------------------------------------------------------
/// Render NATS Helm values for an OKD-flavored deployment with WSS
/// ingress + auth callout + JetStream.
///
/// **Why WSS rather than plain NATS-on-TLS:** OKD's default ingress
/// controller (HAProxy) is HTTP-aware and edge-terminates TLS. NATS
/// over WebSocket goes through that ingress unchanged; native NATS
/// TCP would require a TCP loadbalancer service or a passthrough
/// Route, both of which are extra infra the customer's cluster may
/// not have. WSS is also the default async-nats client transport on
/// `wss://...` URLs — no special agent code needed.
pub fn render_nats_values(
domain: &FleetDomainConfig,
issuer_pubkey: &str,
nats_auth_pass: &str,
nats_system_pass: &str,
) -> String {
let auth_callout = render_auth_callout_block(issuer_pubkey, NATS_AUTH_USER, NATS_ACCOUNT);
let auth_callout_indented = auth_callout
.lines()
.enumerate()
.map(|(i, l)| {
if i == 0 {
l.to_string()
} else {
format!(" {l}")
}
})
.collect::<Vec<_>>()
.join("\n");
format!(
r#"fullnameOverride: {nats_release}
config:
cluster:
enabled: false
jetstream:
enabled: true
fileStorage:
enabled: true
size: 5Gi
websocket:
enabled: true
port: 8443
ingress:
enabled: true
className: openshift-default
pathType: Prefix
hosts:
- {nats_wss_host}
annotations:
# OKD HAProxy edge-terminates TLS — the chart's default Route
# generation needs `route.openshift.io/termination: edge` so
# the Route's TLS block is "edge", matching the cluster's wildcard
# cert behavior. Switch to `reencrypt` if you need TLS all the
# way to the NATS pod.
route.openshift.io/termination: edge
haproxy.router.openshift.io/timeout: "1h"
merge:
{auth_callout_indented}
accounts:
{nats_account}:
jetstream: enabled
users:
- user: "{auth_user}"
password: "{auth_pass}"
SYS:
users:
- user: "{sys_user}"
password: "{sys_pass}"
system_account: SYS
service:
ports:
nats:
enabled: true
"#,
nats_release = NATS_RELEASE,
nats_wss_host = domain.nats_wss_host(),
nats_account = NATS_ACCOUNT,
auth_user = NATS_AUTH_USER,
auth_pass = nats_auth_pass,
sys_user = NATS_SYSTEM_USER,
sys_pass = nats_system_pass,
)
}
// ---- readiness -------------------------------------------------------------
async fn wait_for_zitadel_ready(domain: &FleetDomainConfig) -> Result<()> {
let issuer = domain.zitadel_issuer_url();
let well_known = format!("{issuer}/.well-known/openid-configuration");
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
for attempt in 1..=180 {
match client.get(&well_known).send().await {
Ok(r) if r.status().is_success() => return Ok(()),
Ok(r) if attempt % 30 == 0 => {
info!("Zitadel HTTPS {} (attempt {attempt}/180)", r.status());
}
Err(e) if attempt % 30 == 0 => {
info!("Zitadel unreachable: {e} (attempt {attempt}/180)");
}
_ => {}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
anyhow::bail!("timed out waiting for Zitadel at {well_known}")
}
// ---- helpful printout ------------------------------------------------------
impl StagingHandles {
/// Print the operator's "what to do next" panel after a successful
/// staging deploy. Pasted at the end of the binary's run.
pub fn print_next_steps(&self) {
let zitadel = self.domain.zitadel_issuer_url();
let nats = self.domain.nats_wss_url();
println!();
println!("============================================================");
println!(" STAGING DEPLOY COMPLETE");
println!("============================================================");
println!(" Base domain: {}", self.domain.base_domain);
println!(" Zitadel: {zitadel}");
println!(" NATS (WSS): {nats}");
println!(" Project ID: {}", self.project_id);
println!(" Callout image: {}", self.callout_image);
println!(" Issuer pubkey: {}", self.issuer_pubkey);
if let Some(cid) = &self.cli_client_id {
println!(" CLI client_id: {cid}");
println!();
println!(" CLI SSO login (developer-side):");
println!();
println!(" cargo run -p example-fleet-sso-login -- \\");
println!(" --base-domain {} \\", self.domain.base_domain);
println!(" --client-id {cid}");
}
println!();
println!(" Onboard a Pi:");
println!();
println!(" PAT=$(kubectl -n zitadel get secret iam-admin-pat \\");
println!(" -o jsonpath='{{.data.pat}}' | base64 -d)");
println!();
println!(" cargo run -p example-fleet-rpi-setup -- \\");
println!(" --pi-host <PI_IP> \\");
println!(" --bootstrap-token \"$PAT\" \\");
println!(" --zitadel-issuer-url {zitadel} \\");
println!(" --zitadel-project-id {} \\", self.project_id);
println!(" --nats-url {nats} \\");
println!(" --agent-binary <path-to-aarch64-fleet-agent>");
println!();
println!("============================================================");
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn domain_config_derives_hostnames() {
let d = FleetDomainConfig::new("customer1.nationtech.io");
assert_eq!(d.zitadel_host(), "zitadel.customer1.nationtech.io");
assert_eq!(d.nats_wss_host(), "nats.customer1.nationtech.io");
assert_eq!(
d.zitadel_issuer_url(),
"https://zitadel.customer1.nationtech.io"
);
assert_eq!(d.nats_wss_url(), "wss://nats.customer1.nationtech.io/");
}
#[test]
fn nats_values_render_includes_wss_ingress_and_auth_callout() {
let d = FleetDomainConfig::new("acme.io");
let yaml = render_nats_values(&d, "ABCDEF", "auth-pass", "sys-pass");
// WSS plumbing.
assert!(yaml.contains("websocket:"));
assert!(yaml.contains("port: 8443"));
assert!(yaml.contains("nats.acme.io"));
// OKD edge-TLS annotations.
assert!(yaml.contains("openshift-default"));
assert!(yaml.contains("route.openshift.io/termination: edge"));
// Auth callout wired through with the issuer pubkey.
assert!(yaml.contains("auth_callout"));
assert!(yaml.contains("issuer: ABCDEF"));
assert!(yaml.contains("auth_users: [ auth ]"));
assert!(yaml.contains("system_account: SYS"));
// Account user.
assert!(yaml.contains("password: \"auth-pass\""));
}
#[test]
fn nats_values_inline_account_block_under_merge() {
// Prevent regressions where the auth_callout block leaks
// outside the `merge:` indentation level — chart expects it
// under config.merge.
let d = FleetDomainConfig::new("x.io");
let yaml = render_nats_values(&d, "K", "p", "s");
let idx_merge = yaml.find("\n merge:\n").expect("merge block present");
let idx_callout = yaml.find("auth_callout:").expect("auth_callout present");
assert!(idx_callout > idx_merge, "auth_callout must follow merge:");
}
}

View File

@@ -1,71 +0,0 @@
//! `cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io ...`
//!
//! Operator-side, run-once-per-customer-instance harness. Brings up
//! the central fleet platform services (Zitadel + NATS + auth callout)
//! against an OKD/K8s cluster pointed to by `KUBECONFIG`. Prints the
//! exact follow-up command the operator runs against a Pi to onboard
//! the first device.
//!
//! See `src/lib.rs` for the architectural notes.
use anyhow::{Context, Result};
use clap::Parser;
use example_fleet_staging_deploy::{FleetDomainConfig, StagingDeployOpts, bring_up_staging};
#[derive(Parser, Debug)]
#[command(
name = "fleet-staging-deploy",
about = "Deploy Zitadel + NATS + auth callout onto an OKD cluster"
)]
struct Cli {
/// Base DNS domain. All cluster-visible services derive from this:
/// `zitadel.<base>`, `nats.<base>`. The customer's wildcard cert /
/// CoreDNS / DNS provider must already point this at the cluster.
#[arg(long, env = "FLEET_BASE_DOMAIN")]
base_domain: String,
/// kubeconfig context to deploy against. Defaults to the
/// kubeconfig's current-context. Set this when your kubeconfig
/// has multiple contexts and you don't want to rely on the
/// global current.
#[arg(long, env = "FLEET_KUBE_CONTEXT")]
kube_context: Option<String>,
/// Container image reference for the harmony-nats-callout binary.
/// The cluster pulls this; operator must have pushed it before
/// running the deploy. Defaults to a quay.io path that the
/// customer should override per their registry.
#[arg(
long,
env = "FLEET_CALLOUT_IMAGE",
default_value = "quay.io/nationtech/harmony-nats-callout:demo"
)]
callout_image: String,
/// Password for the NATS service-account user the callout uses on
/// its own NATS connection. Stored in a K8s secret + listed in
/// the chart's `accounts.DEVICES.users` (which bypass callout —
/// otherwise the callout would deadlock authenticating itself).
#[arg(long, env = "FLEET_NATS_AUTH_PASS")]
nats_auth_pass: String,
/// Password for the NATS SYS account (used for nats-box debugging
/// inside the cluster).
#[arg(long, env = "FLEET_NATS_SYSTEM_PASS")]
nats_system_pass: String,
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
let domain = FleetDomainConfig::new(cli.base_domain);
let handles = bring_up_staging(StagingDeployOpts {
domain,
kubeconfig_context: cli.kube_context,
callout_image: cli.callout_image,
nats_auth_pass: cli.nats_auth_pass,
nats_system_pass: cli.nats_system_pass,
})
.await
.context("staging deploy")?;
handles.print_next_steps();
Ok(())
}

View File

@@ -1,29 +0,0 @@
[package]
name = "example_fleet_staging_install"
edition = "2024"
version.workspace = true
readme.workspace = true
license.workspace = true
description = "Production-shape fleet install: Zitadel + NATS + auth callout + operator on OKD"
[[bin]]
name = "fleet_staging_install"
path = "src/main.rs"
[dependencies]
harmony = { path = "../../harmony" }
harmony_cli = { path = "../../harmony_cli" }
harmony_config = { path = "../../harmony_config" }
harmony_secret = { path = "../../harmony_secret" }
harmony-k8s = { path = "../../harmony-k8s" }
harmony-nats-callout = { path = "../../nats/callout" }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
nkeys = "0.4"
rand = "0.9"
base64.workspace = true
anyhow.workspace = true
clap = { version = "4", features = ["derive", "env"] }
tokio.workspace = true
tracing = { workspace = true }
serde = { workspace = true }
schemars = "0.8"

View File

@@ -1,417 +0,0 @@
//! Production-shape fleet install for OKD (or any cluster with the same
//! capabilities): Zitadel SSO + NATS (auth-callout) + operator + OpenBao,
//! composed from Scores.
//!
//! Tunables come from [`ConfigClient`] (`HARMONY_CONFIG_FleetStagingConfig`
//! env JSON → OpenBao → interactive prompt), not a bespoke CLI. The only
//! flags are `harmony_cli`'s: `--filter`/`--list`/`-y` select which workload
//! Scores to (re)deploy — e.g. `--filter FleetOperatorScore` bumps the
//! operator without touching NATS or the callout.
//!
//! Zitadel + OpenBao are an idempotent bootstrap: ZitadelSetupScore mints the
//! `project_id` + `fleet-operator` machine key that the callout and operator
//! Scores consume, so it must converge (and cache to disk) before they're
//! built. That data flow is why those two can't sit in the filterable batch.
use std::sync::Arc;
use anyhow::{Context, Result};
use clap::Parser;
use harmony::inventory::Inventory;
use harmony::modules::nats::capability::NatsCluster;
use harmony::modules::nats::score_nats_k8s::{AuthCalloutCfg, NatsK8sScore, WebSocketRouteCfg};
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore,
cached_root_token,
};
use harmony::modules::zitadel::{
MachineKeyType, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig,
ZitadelMachineUser, ZitadelRole, ZitadelScore, ZitadelSetupScore,
};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
use harmony_config::{Config, ConfigClient, StoreSource};
use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, OperatorCredentials};
use harmony_k8s::KubernetesDistribution;
use harmony_secret::{OpenbaoSecretStore, OpenbaoStoreOptions};
use nkeys::KeyPair;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use tracing::info;
/// Non-secret install tunables. `base_domain` drives every public hostname;
/// the image refs and `*-stg.<base>` hosts have no safe default, so an empty
/// value is rejected at startup. Everything else defaults to the staging
/// conventions.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
#[serde(default)]
struct FleetStagingConfig {
base_domain: String,
operator_image: String,
callout_image: String,
cluster_issuer: String,
fleet_namespace: String,
zitadel_namespace: String,
nats_account: String,
zitadel_version: String,
project_name: String,
admin_role: String,
device_role: String,
operator_username: String,
admin_username: String,
}
impl Default for FleetStagingConfig {
fn default() -> Self {
Self {
base_domain: String::new(),
operator_image: String::new(),
callout_image: String::new(),
cluster_issuer: "letsencrypt-prod".to_string(),
fleet_namespace: "fleet-staging".to_string(),
zitadel_namespace: "zitadel-staging".to_string(),
nats_account: "FLEET".to_string(),
zitadel_version: "v4.12.1".to_string(),
project_name: "fleet".to_string(),
admin_role: "fleet-admin".to_string(),
device_role: "device".to_string(),
operator_username: "fleet-operator".to_string(),
admin_username: "fleet-ops".to_string(),
}
}
}
#[tokio::main]
async fn main() -> Result<()> {
harmony_cli::cli_logger::init();
let args = harmony_cli::Args::parse();
let cfg: FleetStagingConfig = ConfigClient::for_namespace("harmony")
.await
.get_or_prompt()
.await
.context("loading FleetStagingConfig")?;
anyhow::ensure!(
!cfg.base_domain.is_empty()
&& !cfg.operator_image.is_empty()
&& !cfg.callout_image.is_empty(),
"base_domain, operator_image and callout_image must be set"
);
let topology = K8sAnywhereTopology::from_env();
topology.ensure_ready().await?;
let zitadel_host = format!("sso-stg.{}", cfg.base_domain);
let nats_ws_host = format!("nats-fleet-stg.{}", cfg.base_domain);
let secrets_host = format!("secrets-stg.{}", cfg.base_domain);
let nats_release = "fleet-nats";
let cli_app_name = "harmony-cli";
// ---- Bootstrap (idempotent): Zitadel mints the project + operator key ---
ZitadelScore {
host: zitadel_host.clone(),
zitadel_version: cfg.zitadel_version.clone(),
external_secure: true,
external_port: None,
namespace: cfg.zitadel_namespace.clone(),
cluster_issuer: cfg.cluster_issuer.clone(),
..Default::default()
}
.interpret(&Inventory::empty(), &topology)
.await
.context("Zitadel helm install")?;
ZitadelSetupScore {
host: zitadel_host.clone(),
scheme: Default::default(),
port: None,
skip_tls: false,
endpoint: None,
admin_org_id: None,
namespace: cfg.zitadel_namespace.clone(),
// Devices read per-deployment secrets gated by group membership;
// the Action surfaces role keys as the `groups` claim OpenBao
// binds against (ADR-025).
groups_claim_action: true,
human_users: vec![],
org_members: vec![],
instance_members: vec![],
machine_secrets: vec![],
port_forward_service: None,
// Device-code OIDC app for human admin login from
// `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel
// generates is read back below and printed for `--admin-oidc-client-id`.
applications: vec![ZitadelApplication {
project_name: cfg.project_name.clone(),
app_name: cli_app_name.to_string(),
app_type: ZitadelAppType::DeviceCode,
}],
api_apps: vec![ZitadelApiApp {
project_name: cfg.project_name.clone(),
app_name: "nats".to_string(),
}],
roles: vec![
ZitadelRole {
project_name: cfg.project_name.clone(),
key: cfg.admin_role.clone(),
display_name: "Fleet Admin".to_string(),
group: None,
},
ZitadelRole {
project_name: cfg.project_name.clone(),
key: cfg.device_role.clone(),
display_name: "Device".to_string(),
group: None,
},
],
machine_users: vec![
ZitadelMachineUser {
username: cfg.admin_username.clone(),
name: "Fleet Operations".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(cfg.project_name.clone()),
grant_roles: vec![cfg.admin_role.clone()],
},
ZitadelMachineUser {
username: cfg.operator_username.clone(),
name: "Fleet Operator (in-cluster)".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(cfg.project_name.clone()),
grant_roles: vec![cfg.admin_role.clone()],
},
],
}
.interpret(&Inventory::empty(), &topology)
.await
.context("Zitadel setup (project + roles + machine users)")?;
// Read back the project_id + operator key + device-code client_id.
let zcfg = ZitadelClientConfig::load()
.context("ZitadelSetupScore did not produce a client config cache")?;
let project_id = zcfg
.project_id_by_name(&cfg.project_name)
.or(zcfg.project_id.as_ref())
.context("project_id missing from cache after setup")?
.clone();
let operator_machine_key = zcfg
.machine_key(&cfg.operator_username)
.with_context(|| {
format!(
"machine key for {} missing from cache",
cfg.operator_username
)
})?
.clone();
let cli_client_id = zcfg
.client_id(cli_app_name)
.with_context(|| format!("OIDC client_id for app '{cli_app_name}' missing from cache"))?
.clone();
// ---- OpenBao: deploy + policy, co-located in the fleet namespace --------
// The operator's credentials are seeded here so a later
// `harmony-fleet-deploy --from-tag <tag>` upgrades the operator alone.
let openbao = OpenbaoInstance {
namespace: cfg.fleet_namespace.clone(),
release: "openbao".to_string(),
};
OpenbaoScore {
instance: openbao.clone(),
host: secrets_host.clone(),
openshift: true,
tls_issuer: Some(cfg.cluster_issuer.clone()),
}
.interpret(&Inventory::empty(), &topology)
.await
.context("OpenBao deploy")?;
OpenbaoSetupScore {
instance: openbao.clone(),
policies: vec![OpenbaoPolicy {
name: "fleet-deployer".to_string(),
hcl: r#"path "secret/data/harmony/*" { capabilities = ["read"] }
path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
.to_string(),
}],
// Device login (ADR-025): batch tokens so re-login is free, 1h
// TTL as the group-membership convergence bound, groups bound
// from the Action-flattened `groups` claim.
jwt_auth: Some(OpenbaoJwtAuth {
oidc_discovery_url: format!("https://{zitadel_host}"),
jwt_validation_pubkeys: String::new(),
bound_issuer: format!("https://{zitadel_host}"),
role_name: "fleet-device".to_string(),
bound_audiences: project_id.clone(),
user_claim: "sub".to_string(),
groups_claim: "groups".to_string(),
token_type: "batch".to_string(),
ttl: "1h".to_string(),
max_ttl: "1h".to_string(),
}),
..Default::default()
}
.interpret(&Inventory::empty(), &topology)
.await
.context("OpenBao setup")?;
// ---- Workload Scores: filterable via `harmony_cli::Args` ----------------
// The callout signs user JWTs with this account NKey; NATS is configured
// with the matching pubkey via the auth_callout block in its helm values.
let issuer_kp = KeyPair::new_account();
let issuer_seed = issuer_kp
.seed()
.map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?;
let nats_auth_user = "auth";
let nats_auth_pass = generate_alphanum(24);
let nats_url = format!(
"nats://{nats_release}.{}.svc.cluster.local:4222",
cfg.fleet_namespace
);
let nats = NatsK8sScore {
distribution: KubernetesDistribution::OpenshiftFamily,
cluster: NatsCluster {
namespace: cfg.fleet_namespace.clone(),
// `domain` and the static-string fields below are only read in the
// supercluster path (gateway Some), which staging doesn't take.
domain: cfg.base_domain.clone(),
replicas: 1,
name: nats_release.to_string(),
gateway_advertise: String::new(),
dns_name: nats_ws_host.clone(),
supercluster_ca_secret_name: "fleet-nats-supercluster-ca",
tls_cert_name: "fleet-nats-tls",
jetstream_enabled: "true",
},
peers: None,
ca_bundle: None,
gateway: None,
auth_callout: Some(AuthCalloutCfg {
issuer_pubkey: issuer_kp.public_key(),
auth_user: nats_auth_user.to_string(),
auth_pass: nats_auth_pass.clone(),
account: cfg.nats_account.clone(),
}),
websocket: Some(WebSocketRouteCfg {
host: nats_ws_host.clone(),
cluster_issuer: cfg.cluster_issuer.clone(),
}),
};
let mut callout = NatsAuthCalloutScore::new(
"fleet-callout",
&cfg.fleet_namespace,
nats_url.clone(),
format!("https://{zitadel_host}"),
project_id.clone(),
nats_auth_user,
&nats_auth_pass,
&issuer_seed,
)
.image(&cfg.callout_image)
.target_account(&cfg.nats_account)
.admin_role(&cfg.admin_role)
.device_role(&cfg.device_role)
.danger_accept_invalid_certs(false);
callout.device_id_claim = "client_id".to_string();
callout.device_id_prefix_strip = "device-".to_string();
callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles");
let credentials = OperatorCredentials::zitadel_jwt(
&format!("https://{zitadel_host}"),
&project_id,
&operator_machine_key,
);
let mut operator = FleetOperatorScore::new()
.namespace(&cfg.fleet_namespace)
.release_name("harmony-fleet-operator")
.image(&cfg.operator_image)
.image_pull_policy("Always")
.nats_url(nats_url.clone())
.log_level("info,kube_runtime=warn");
operator.credentials = Some(credentials.clone());
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> =
vec![Box::new(nats), Box::new(callout), Box::new(operator)];
harmony_cli::run(Inventory::empty(), topology.clone(), scores, Some(args))
.await
.map_err(|e| anyhow::anyhow!("{e}"))?;
// ---- Seed operator credentials as FleetDeploySecrets --------------------
// Reached via port-forward with the cached root token, so it doesn't wait
// on the public route/cert. No kubeconfig — CD callers use their own context.
let k8s = topology
.k8s_client()
.await
.map_err(|e| anyhow::anyhow!(e))?;
let pf = k8s
.port_forward(&openbao.pod(), &openbao.namespace, 8200, 8200)
.await
.context("port-forward to OpenBao")?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
let store = OpenbaoSecretStore::new(OpenbaoStoreOptions {
base_url: format!("http://127.0.0.1:{}", pf.port()),
kv_mount: "secret".to_string(),
auth_mount: "token".to_string(),
skip_tls: true,
token: Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?),
username: None,
password: None,
zitadel_sso_url: None,
zitadel_client_id: None,
jwt_role: None,
jwt_auth_mount: None,
zitadel_jwt_bearer: None,
})
.await
.context("OpenBao client")?;
ConfigClient::new(vec![
Arc::new(StoreSource::new("harmony".to_string(), store))
as Arc<dyn harmony_config::ConfigSource>,
])
.set(&FleetDeploySecrets {
operator_credentials_toml: credentials.credentials_toml.clone(),
kubeconfig: None,
operator_cookie_key_b64: generate_cookie_key_b64(),
})
.await
.context("seed FleetDeploySecrets")?;
info!("=== fleet-staging install complete ===");
info!(
"Zitadel: https://{zitadel_host}/ (admin user {})",
cfg.admin_username
);
info!("NATS WS public: wss://{nats_ws_host}/");
info!("OpenBao: https://{secrets_host}/");
info!("Project id: {project_id}");
info!("SSO client_id: {cli_client_id} (app '{cli_app_name}', device-code grant)");
info!(
"Enroll a device: fleet_device_enroll --target ssh://<user>@<device> \
--issuer-url https://{zitadel_host} --audience {project_id} \
--nats-url wss://{nats_ws_host} --admin-oidc-client-id {cli_client_id} \
--agent-binary <path>"
);
Ok(())
}
fn generate_alphanum(len: usize) -> String {
use rand::Rng;
const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let mut rng = rand::rng();
(0..len)
.map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char)
.collect()
}
/// Standard-base64 of 64 random bytes — the operator's session-cookie
/// signing key. Generated per install and seeded as a secret.
fn generate_cookie_key_b64() -> String {
use base64::Engine;
use rand::Rng;
let mut bytes = [0u8; 64];
rand::rng().fill(&mut bytes);
base64::engine::general_purpose::STANDARD.encode(bytes)
}

View File

@@ -10,6 +10,7 @@ path = "src/main.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", features = ["kvm"] } harmony = { path = "../../harmony", features = ["kvm"] }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
harmony_types = { path = "../../harmony_types" } harmony_types = { path = "../../harmony_types" }
tokio.workspace = true tokio.workspace = true
log.workspace = true log.workspace = true

View File

@@ -12,13 +12,13 @@ use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::{ use harmony::modules::fleet::{
FleetDeviceSetupConfig, FleetDeviceSetupScore, ProvisionVmScore, ProvisionVmScore, check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair,
check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair,
}; };
use harmony::modules::kvm::KvmVirtualMachineHost; use harmony::modules::kvm::KvmVirtualMachineHost;
use harmony::modules::kvm::config::init_executor; use harmony::modules::kvm::config::init_executor;
use harmony::modules::linux::{LinuxHostTopology, SshCredentials}; use harmony::modules::linux::{LinuxHostTopology, SshCredentials};
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig}; use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore};
use harmony_types::id::Id; use harmony_types::id::Id;
use std::path::PathBuf; use std::path::PathBuf;
@@ -213,7 +213,7 @@ async fn main() -> Result<()> {
nats_urls: vec![cli.nats_url.clone()], nats_urls: vec![cli.nats_url.clone()],
// VM smoke harness keeps shared-creds for v0; the customer- // VM smoke harness keeps shared-creds for v0; the customer-
// facing Pi flow uses Zitadel JWT (see fleet_rpi_setup). // facing Pi flow uses Zitadel JWT (see fleet_rpi_setup).
auth: harmony::modules::fleet::FleetDeviceAuth::TomlShared { auth: FleetDeviceAuth::TomlShared {
nats_user: cli.nats_user.clone(), nats_user: cli.nats_user.clone(),
nats_pass: cli.nats_pass.clone(), nats_pass: cli.nats_pass.clone(),
}, },

View File

@@ -9,7 +9,8 @@ name = "harmony_apply_deployment"
path = "src/main.rs" path = "src/main.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", default-features = false, features = ["podman"] } harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
kube = { workspace = true, features = ["runtime", "derive"] } kube = { workspace = true, features = ["runtime", "derive"] }
k8s-openapi = { workspace = true } k8s-openapi = { workspace = true }
serde_json.workspace = true serde_json.workspace = true

View File

@@ -12,8 +12,8 @@
//! (not `iot_`-anything), in line with the review call to position //! (not `iot_`-anything), in line with the review call to position
//! the operator as a generic fleet/reconcile tool. //! the operator as a generic fleet/reconcile tool.
//! //!
//! The CRD types live in `harmony::modules::fleet::operator`; the score types //! The CRD types live in `harmony_fleet_operator`; the score types
//! live in `harmony::modules::podman` (PodmanV0 being the first //! live in `harmony_reconciler_contracts` (PodmanV0 being the first
//! reconciler variant — future variants drop in alongside). //! reconciler variant — future variants drop in alongside).
//! //!
//! Typical demo-driver usage: //! Typical demo-driver usage:
@@ -38,11 +38,10 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use harmony::modules::fleet::operator::crd::{ use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
Deployment, DeploymentSpec, Rollout, RolloutStrategy, use harmony_reconciler_contracts::{
EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, VolumeMount,
}; };
use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
use harmony::topology::{EnvVar, RestartPolicy, VolumeMount};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client; use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams}; use kube::api::{Api, DeleteParams, Patch, PatchParams};
@@ -143,7 +142,7 @@ async fn main() -> Result<()> {
Ok(()) Ok(())
} }
/// Mirrors `harmony::topology::RestartPolicy` so we can keep the CLI /// Mirrors the contract `RestartPolicy` so we can keep the CLI
/// schema stable even if the underlying enum gains variants. /// schema stable even if the underlying enum gains variants.
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
enum CliRestart { enum CliRestart {

View File

@@ -249,6 +249,7 @@ async fn main() -> anyhow::Result<()> {
host: OPENBAO_HOST.to_string(), host: OPENBAO_HOST.to_string(),
openshift: false, openshift: false,
tls_issuer: None, tls_issuer: None,
node_port: None,
} }
.interpret(&Inventory::autoload(), &topology) .interpret(&Inventory::autoload(), &topology)
.await .await

View File

@@ -94,6 +94,7 @@ async fn main() -> Result<()> {
host: cfg.host.clone(), host: cfg.host.clone(),
openshift: cfg.openshift, openshift: cfg.openshift,
tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()), tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()),
node_port: None,
}; };
// JWT auth composes in only when both issuer and audience are set; it // JWT auth composes in only when both issuer and audience are set; it
@@ -128,6 +129,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
policies, policies,
users: vec![], users: vec![],
jwt_auth, jwt_auth,
oidc_application: None,
}; };
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)]; let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)];

View File

@@ -7,7 +7,7 @@ IoT / decentralized-edge orchestration for harmony. A fleet stack is:
| **Operator** | [`harmony-fleet-operator`](harmony-fleet-operator/) | Watches `Deployment` CRs, writes desired state into NATS JetStream KV, aggregates device state back into CR status. Runtime binary; no `harmony` dep. | | **Operator** | [`harmony-fleet-operator`](harmony-fleet-operator/) | Watches `Deployment` CRs, writes desired state into NATS JetStream KV, aggregates device state back into CR status. Runtime binary; no `harmony` dep. |
| **Agent** | [`harmony-fleet-agent`](harmony-fleet-agent/) | One per device. Watches the desired-state KV, drives the local runtime (podman today), publishes heartbeats + per-deployment state, answers `device-commands.*` request/reply. | | **Agent** | [`harmony-fleet-agent`](harmony-fleet-agent/) | One per device. Watches the desired-state KV, drives the local runtime (podman today), publishes heartbeats + per-deployment state, answers `device-commands.*` request/reply. |
| **Auth** | [`harmony-fleet-auth`](harmony-fleet-auth/) | Shared NATS credential plumbing — `TomlShared` (dev) and `ZitadelJwt` (prod with auth-callout). | | **Auth** | [`harmony-fleet-auth`](harmony-fleet-auth/) | Shared NATS credential plumbing — `TomlShared` (dev) and `ZitadelJwt` (prod with auth-callout). |
| **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. Imports `harmony` and exposes one `*Score` per component (`FleetOperatorScore`, `FleetAgentScore`, `FleetNatsScore`, `FleetServerScore`). Both the production CLI and the e2e harness compose these — see [ADR-023](../docs/adr/023-deploy-architecture.md). | | **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. It composes independent component Scores through typed references; there is no Fleet-wide aggregate Score. See [ADR-023](../docs/adr/023-deploy-architecture.md) and [ADR-028](../docs/adr/028-typed-score-references.md). |
| **E2E harness** | [`harmony-fleet-e2e`](harmony-fleet-e2e/) | Brings the stack up in a fresh k3d namespace and runs integration tests against it. | | **E2E harness** | [`harmony-fleet-e2e`](harmony-fleet-e2e/) | Brings the stack up in a fresh k3d namespace and runs integration tests against it. |
The on-the-wire types both ends agree on (KV bucket names, key formats, command-protocol payloads) live in [`../harmony-reconciler-contracts`](../harmony-reconciler-contracts/). The on-the-wire types both ends agree on (KV bucket names, key formats, command-protocol payloads) live in [`../harmony-reconciler-contracts`](../harmony-reconciler-contracts/).
@@ -16,6 +16,20 @@ The on-the-wire types both ends agree on (KV bucket names, key formats, command-
`FleetOperatorScore`, `FleetAgentScore`, etc. are real Rust types with capability-bound `Topology` parameters. Production deploys, the e2e harness, and any future control-plane tool all compose the **same** Scores; the only thing that changes is the `Topology` instance. **No handrolled YAML or imperative manifest factories anywhere.** Read [ADR-023](../docs/adr/023-deploy-architecture.md) before adding deploy logic. `FleetOperatorScore`, `FleetAgentScore`, etc. are real Rust types with capability-bound `Topology` parameters. Production deploys, the e2e harness, and any future control-plane tool all compose the **same** Scores; the only thing that changes is the `Topology` instance. **No handrolled YAML or imperative manifest factories anywhere.** Read [ADR-023](../docs/adr/023-deploy-architecture.md) before adding deploy logic.
## Local Fleet
The local context creates or reuses the `harmony` k3d cluster, builds and imports
the Fleet images, and deploys PostgreSQL, Zitadel, NATS with auth callout,
OpenBao, and the Fleet operator:
```bash
cargo run -p harmony-fleet-deploy -- ship --context local
```
The command returns only after the operator has authenticated through Zitadel
and initialized its NATS JetStream state. The credentials Score creates an
immutable local Kubernetes Secret on the first run and reuses it thereafter.
--- ---
## Quickstart — run the e2e ping test ## Quickstart — run the e2e ping test
@@ -31,10 +45,10 @@ What it does, in order:
1. Ensures a `fleet-e2e` k3d cluster exists (creates one if not). NodePort `30423` on the host forwards to NATS inside the cluster. 1. Ensures a `fleet-e2e` k3d cluster exists (creates one if not). NodePort `30423` on the host forwards to NATS inside the cluster.
2. Builds `harmony-fleet-agent` in release mode, packages it into `localhost/harmony-fleet-agent:e2e`, and sideloads the image into the k3d cluster's containerd store. 2. Builds `harmony-fleet-agent` in release mode, packages it into `localhost/harmony-fleet-agent:e2e`, and sideloads the image into the k3d cluster's containerd store.
3. Mints a per-bring-up namespace `e2e-<uuid8>` and prunes any leftover `e2e-*` namespaces from prior runs (NodePort `30423` is cluster-scoped, so a stuck `Terminating` namespace would block the new bring-up — the prune waits up to 90 s for full cleanup before proceeding). 3. Mints a per-bring-up namespace `e2e-<uuid8>` and prunes any leftover `e2e-*` namespaces from prior runs (NodePort `30423` is cluster-scoped, so a stuck `Terminating` namespace would block the new bring-up — the prune waits up to 90 s for full cleanup before proceeding).
4. Deploys NATS via `FleetNatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service). 4. Deploys NATS via `NatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service).
5. Waits for NATS to be reachable from the host on `nats://localhost:30423` (admin/e2e-admin). 5. Waits for NATS to be reachable from the host on `nats://localhost:30423` (admin/e2e-admin).
6. Deploys one `FleetAgentScore { target: Pod }` — runs with `runtime_enabled = false` so it skips podman and only runs the command-server + heartbeat loop. 6. Deploys one `FleetAgentScore { target: Pod }` — runs with `runtime_enabled = false` so it skips podman and only runs the command-server + heartbeat loop.
7. Waits for the agent Deployment to be Ready. 7. `FleetAgentScore` waits for the agent Deployment to be Ready.
8. The test publishes `device-commands.<device_id>.ping` via `FleetCommandsClient::ping` and asserts the agent replies with `{ device_id, agent_version, uptime_s }`. 8. The test publishes `device-commands.<device_id>.ping` via `FleetCommandsClient::ping` and asserts the agent replies with `{ device_id, agent_version, uptime_s }`.
Cold first run: ~80 s (release build of the agent dominates). Warm: ~25 s. Cold first run: ~80 s (release build of the agent dominates). Warm: ~25 s.
@@ -102,14 +116,17 @@ k3d cluster delete fleet-e2e
## Production deploys ## Production deploys
`harmony-fleet-deploy` puts the published operator chart on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach) — the `harmony apply` / CD path. It loads `FleetDeploySecrets` from config (Env → OpenBao) and runs one `FleetOperatorScore`; auth is Zitadel-SSO-only. The full bring-up stack (`FleetNatsScore` + `FleetAgentScore` + …) is composed by the e2e harness directly over the same lib Scores, not by this binary. Production contexts live in the customer's private repository. They provide a
registry, project, domain, image-pull Secret, and OpenBao-brokered kubeconfig.
The cluster must also contain the immutable
`fleet-system/fleet-callout-credentials` Secret with
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass` keys. The deploy
validates this Secret before configuring NATS.
```bash ```bash
# Deploy a released tag (version parsed from it in Rust):
cargo run -p harmony-fleet-deploy -- \ cargo run -p harmony-fleet-deploy -- \
--filter FleetOperatorScore \ ship --context customer-prod \
--from-tag harmony-fleet-operator-v0.0.2 \ --config path/to/contexts.toml
--namespace fleet-system --yes
``` ```
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap. See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
@@ -132,22 +149,12 @@ cargo run -p harmony-fleet-operator --features web-frontend -- serve-web --mock
--- ---
## Existing manual rehearsal — `examples/fleet_e2e_demo` ## VM rehearsal
`examples/fleet_e2e_demo` brings up a *fuller* stack than the e2e harness — real Zitadel, the auth-callout, libvirt VM agents over SSH — at the cost of a 5-min cold start. It's the manual rehearsal flow; not what you want during the dev loop. See the example's [`RUNBOOK.md`](../examples/fleet_e2e_demo/RUNBOOK.md). `examples/fleet_e2e_demo` is a small executable over
`harmony-fleet-e2e::VmStack`. It runs the same callout, OpenBao, operator, and
device enrollment Scores as the maintained VM tests:
The harness and the rehearsal will converge: the [follow-up PR](#whats-next) lifts `FleetCalloutScore` + a mock-OIDC fixture into `harmony-fleet-deploy`, at which point the harness can run the full production auth path in ~30 s instead of 5 min, and `fleet_e2e_demo` thins down to a caller over the same Scores. ```bash
cargo run -p example-fleet-e2e-demo -- --num-devices 2
--- ```
## What's next
This branch lands the deploy-architecture cleanup (ADR-023), the per-component Scores, and the ping path. Slated immediately after:
1. **Zitadel + auth callout in `harmony-fleet-deploy`.** New `FleetCalloutScore` (preset over `NatsAuthCalloutScore`) plus an in-cluster mock-OIDC fixture so the e2e harness can exercise the real auth-callout code path without paying Zitadel's 5-min cold-start cost. The harness's `AuthMode::Callout` variant is already on the public API for this.
2. **Operator pod in the e2e harness.** `FleetOperatorScore` is already in the deploy crate; wiring it into the harness gives integration tests against the actual `Deployment` / `Device` reconcile loops.
3. **`Verb::Logs` and `Verb::Exec`** — the next two verbs on the `device-commands.*` protocol. Same harness, same TDD shape as `ping`.
4. **CRD types out of `harmony` core.** `harmony::modules::fleet::operator::crd` is the last fleet-deploy thing still living in `harmony`. The `ReconcileScore` payload coupling is the only blocker.
5. **Smoke-test contract.** ADR-023 principle 4 — every Score blocks on a smoke test before `deploy` returns success. Today the e2e suite plays that role; the trait/companion shape lands once it's been validated in practice.
See [`PLAN_requests_over_nats.md`](PLAN_requests_over_nats.md) for the full TDD-style plan this branch implements.

View File

@@ -8,13 +8,13 @@ rust-version = "1.85"
harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony_secret = { path = "../../harmony_secret" } harmony_secret = { path = "../../harmony_secret" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
async-nats = { workspace = true } async-nats = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
chrono = { workspace = true } chrono = { workspace = true }
futures-util = { workspace = true } futures-util = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
sha2.workspace = true
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
tracing-subscriber = { workspace = true } tracing-subscriber = { workspace = true }
@@ -22,3 +22,4 @@ anyhow = { workspace = true }
clap = { workspace = true } clap = { workspace = true }
toml = { workspace = true } toml = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
podman-api = "0.9"

View File

@@ -1,6 +1,7 @@
mod command_server; mod command_server;
mod config; mod config;
mod fleet_publisher; mod fleet_publisher;
mod podman;
mod reconciler; mod reconciler;
use std::sync::Arc; use std::sync::Arc;
@@ -20,12 +21,9 @@ use harmony_reconciler_contracts::{
BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter, BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter,
}; };
use harmony::inventory::Inventory;
use harmony::modules::podman::PodmanTopology;
use harmony::topology::Topology;
use crate::command_server::CommandServer; use crate::command_server::CommandServer;
use crate::fleet_publisher::FleetPublisher; use crate::fleet_publisher::FleetPublisher;
use crate::podman::PodmanRuntime;
use crate::reconciler::Reconciler; use crate::reconciler::Reconciler;
/// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch /// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
@@ -146,9 +144,11 @@ async fn publish_heartbeat_loop(fleet: Arc<FleetPublisher>) {
/// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise. /// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise.
/// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state /// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state
/// on heartbeat but not always send it over the wire /// on heartbeat but not always send it over the wire
fn local_inventory(inventory: &Inventory) -> InventorySnapshot { fn local_inventory() -> InventorySnapshot {
InventorySnapshot { InventorySnapshot {
hostname: inventory.location.name.clone(), hostname: std::fs::read_to_string("/proc/sys/kernel/hostname")
.map(|hostname| hostname.trim().to_string())
.unwrap_or_else(|_| "localhost".to_string()),
arch: std::env::consts::ARCH.to_string(), arch: std::env::consts::ARCH.to_string(),
os: std::env::consts::OS.to_string(), os: std::env::consts::OS.to_string(),
kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease") kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease")
@@ -206,7 +206,7 @@ async fn main() -> Result<()> {
// reconciler depends on the topology. // reconciler depends on the topology.
let topology = if cfg.agent.runtime_enabled { let topology = if cfg.agent.runtime_enabled {
let t = Arc::new( let t = Arc::new(
PodmanTopology::from_default_socket() PodmanRuntime::from_default_socket()
.map_err(|e| anyhow::anyhow!("failed to open podman socket: {e}"))?, .map_err(|e| anyhow::anyhow!("failed to open podman socket: {e}"))?,
); );
t.ensure_ready().await.context("podman socket not ready")?; t.ensure_ready().await.context("podman socket not ready")?;
@@ -220,9 +220,8 @@ async fn main() -> Result<()> {
None None
}; };
let inventory = Arc::new(Inventory::from_localhost()); let inventory_snapshot = local_inventory();
tracing::info!(hostname = %inventory.location.name, "inventory loaded"); tracing::info!(hostname = %inventory_snapshot.hostname, "inventory loaded");
let inventory_snapshot = local_inventory(&inventory);
let creds = credential_source_from_config(&cfg.credentials) let creds = credential_source_from_config(&cfg.credentials)
.context("building NATS credential source")?; .context("building NATS credential source")?;
@@ -291,7 +290,6 @@ async fn main() -> Result<()> {
Arc::new(Reconciler::new( Arc::new(Reconciler::new(
device_id.clone(), device_id.clone(),
t.clone(), t.clone(),
inventory.clone(),
Some(fleet.clone()), Some(fleet.clone()),
secrets, secrets,
)) ))

View File

@@ -0,0 +1,302 @@
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::time::Duration;
use anyhow::{Result, anyhow, bail};
use futures_util::StreamExt;
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy, VolumeMount};
use podman_api::Podman;
use podman_api::models::{ContainerMount, PortMapping};
use podman_api::opts::{
ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts,
ContainerRestartPolicy, ContainerStopOpts, PullOpts,
};
use sha2::{Digest, Sha256};
const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment";
const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256";
const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by";
const MANAGED_BY_VALUE: &str = "harmony";
const STOP_TIMEOUT: Duration = Duration::from_secs(300);
pub struct PodmanRuntime {
podman: Podman,
}
impl PodmanRuntime {
pub fn from_default_socket() -> Result<Self> {
Self::from_unix_socket(default_user_socket())
}
pub fn from_unix_socket(socket: PathBuf) -> Result<Self> {
Ok(Self {
podman: Podman::unix(&socket),
})
}
pub async fn ensure_ready(&self) -> Result<()> {
self.podman
.info()
.await
.map_err(|error| anyhow!("podman socket unreachable: {error}"))?;
Ok(())
}
pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> {
for service in &score.services {
self.ensure_service_running(service, deployment).await?;
}
let desired = score
.services
.iter()
.map(|service| service.name.as_str())
.collect::<HashSet<_>>();
for container in self.deployment_containers(deployment).await? {
let name = container_name(&container);
if !desired.contains(name.as_str()) {
self.remove_service(&name).await?;
}
}
Ok(())
}
pub async fn remove_deployment(&self, deployment: &str) -> Result<()> {
for container in self.deployment_containers(deployment).await? {
self.remove_service(&container_name(&container)).await?;
}
Ok(())
}
pub async fn remove_service(&self, name: &str) -> Result<()> {
let opts = ContainerStopOpts::builder()
.timeout(STOP_TIMEOUT.as_secs() as usize)
.build();
let container = self.podman.containers().get(name);
if container.exists().await.unwrap_or(false) {
let _ = container.stop(&opts).await;
}
self.remove_container(name).await
}
async fn ensure_service_running(
&self,
service: &PodmanService,
deployment: &str,
) -> Result<()> {
let existing = self.get_by_name(&service.name).await?;
if let Some(existing) = existing.as_ref() {
if matches_spec(existing, service) {
if existing.state.as_deref() == Some("running") {
return Ok(());
}
let id = existing.id.clone().unwrap_or_else(|| service.name.clone());
self.podman.containers().get(id).start(None).await?;
return Ok(());
}
self.remove_container(&service.name).await?;
}
self.ensure_image_present(&service.image).await?;
let labels = HashMap::from([
(MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
(DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
(SPEC_LABEL.to_string(), spec_hash(service)?),
]);
let ports = service
.ports
.iter()
.map(|port| parse_port_mapping(port))
.collect::<Result<Vec<_>>>()?;
let env: HashMap<String, String> = service
.env
.iter()
.map(|env| (env.name.clone(), env.value.clone()))
.collect();
let mounts = service
.volumes
.iter()
.map(volume_to_mount)
.collect::<Vec<_>>();
let mut builder = ContainerCreateOpts::builder()
.name(&service.name)
.image(&service.image)
.labels(labels)
.portmappings(ports)
.env(env)
.restart_policy(map_restart_policy(service.restart_policy));
if !mounts.is_empty() {
builder = builder.mounts(mounts);
}
let created = self.podman.containers().create(&builder.build()).await?;
self.podman.containers().get(created.id).start(None).await?;
Ok(())
}
async fn get_by_name(&self, name: &str) -> Result<Option<podman_api::models::ListContainer>> {
let opts = ContainerListOpts::builder()
.all(true)
.filter([ContainerListFilter::Name(name.to_string())])
.build();
Ok(self
.podman
.containers()
.list(&opts)
.await?
.into_iter()
.next())
}
async fn deployment_containers(
&self,
deployment: &str,
) -> Result<Vec<podman_api::models::ListContainer>> {
let opts = ContainerListOpts::builder()
.all(true)
.filter([ContainerListFilter::LabelKeyVal(
DEPLOYMENT_LABEL.to_string(),
deployment.to_string(),
)])
.build();
Ok(self.podman.containers().list(&opts).await?)
}
async fn ensure_image_present(&self, image: &str) -> Result<()> {
let images = self.podman.images();
if images.get(image).exists().await? {
return Ok(());
}
let mut pull = images.pull(&PullOpts::builder().reference(image).build());
while let Some(event) = pull.next().await {
if let Some(error) = event?.error {
bail!("podman pull {image} failed: {error}");
}
}
Ok(())
}
async fn remove_container(&self, name: &str) -> Result<()> {
let opts = ContainerDeleteOpts::builder().force(true).build();
match self.podman.containers().get(name).delete(&opts).await {
Ok(()) => Ok(()),
Err(error) if is_not_found(&error) => Ok(()),
Err(error) => Err(error.into()),
}
}
}
fn default_user_socket() -> PathBuf {
if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") {
return PathBuf::from(format!("{dir}/podman/podman.sock"));
}
let uid = std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|status| {
status
.lines()
.find_map(|line| line.strip_prefix("Uid:"))
.and_then(|value| value.split_whitespace().next())
.and_then(|value| value.parse::<u32>().ok())
})
.unwrap_or(0);
PathBuf::from(format!("/run/user/{uid}/podman/podman.sock"))
}
fn parse_port_mapping(raw: &str) -> Result<PortMapping> {
let (host, container) = raw
.split_once(':')
.ok_or_else(|| anyhow!("invalid port mapping '{raw}' - expected 'HOST:CONTAINER'"))?;
Ok(PortMapping {
host_port: Some(
host.parse()
.map_err(|_| anyhow!("invalid host port '{host}' in mapping '{raw}'"))?,
),
container_port: Some(
container
.parse()
.map_err(|_| anyhow!("invalid container port '{container}' in mapping '{raw}'"))?,
),
host_ip: None,
protocol: None,
range: None,
})
}
fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanService) -> bool {
observed
.labels
.as_ref()
.and_then(|labels| labels.get(SPEC_LABEL))
.is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected))
}
fn spec_hash(service: &PodmanService) -> Result<String> {
Ok(format!(
"{:x}",
Sha256::digest(serde_json::to_vec(service)?)
))
}
fn container_name(container: &podman_api::models::ListContainer) -> String {
container
.names
.as_ref()
.and_then(|names| names.first())
.cloned()
.or_else(|| container.id.clone())
.unwrap_or_default()
}
fn volume_to_mount(volume: &VolumeMount) -> ContainerMount {
ContainerMount {
_type: Some("bind".to_string()),
source: Some(volume.host_path.clone()),
destination: Some(volume.container_path.clone()),
options: Some(vec![if volume.read_only { "ro" } else { "rw" }.to_string()]),
uid_mappings: None,
gid_mappings: None,
}
}
fn map_restart_policy(policy: RestartPolicy) -> ContainerRestartPolicy {
match policy {
RestartPolicy::No => ContainerRestartPolicy::No,
RestartPolicy::UnlessStopped => ContainerRestartPolicy::UnlessStopped,
RestartPolicy::OnFailure => ContainerRestartPolicy::OnFailure,
RestartPolicy::Always => ContainerRestartPolicy::Always,
}
}
fn is_not_found(error: &podman_api::Error) -> bool {
error
.to_string()
.to_lowercase()
.contains("no such container")
}
#[cfg(test)]
mod tests {
use super::*;
use harmony_reconciler_contracts::EnvVar;
fn service() -> PodmanService {
PodmanService {
name: "web".into(),
image: "nginx:latest".into(),
ports: vec!["8080:80".into()],
env: vec![EnvVar::new("MODE", "prod")],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::Always,
}
}
#[test]
fn spec_hash_is_stable_and_covers_desired_state() {
let original = service();
assert_eq!(spec_hash(&original).unwrap(), spec_hash(&original).unwrap());
let mut changed = original;
changed.env[0].value = "dev".into();
assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap());
}
}

View File

@@ -4,15 +4,15 @@ use std::time::Duration;
use anyhow::Result; use anyhow::Result;
use chrono::Utc; use chrono::Utc;
use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase}; use harmony_reconciler_contracts::{
DeploymentName, DeploymentState, EnvVar, Id, Phase, PodmanV0Score, ReconcileScore,
};
use tokio::sync::Mutex; use tokio::sync::Mutex;
use harmony::inventory::Inventory;
use harmony::modules::podman::{EnvVar, PodmanTopology, PodmanV0Score, ReconcileScore};
use harmony::score::Score;
use harmony_secret::SecretStore; use harmony_secret::SecretStore;
use crate::fleet_publisher::FleetPublisher; use crate::fleet_publisher::FleetPublisher;
use crate::podman::PodmanRuntime;
/// Where deployment secrets come from (ADR-025). `prefix` scopes the /// Where deployment secrets come from (ADR-025). `prefix` scopes the
/// KV reads to `<prefix>/<deployment>/<secret>` — it must match the /// KV reads to `<prefix>/<deployment>/<secret>` — it must match the
@@ -25,19 +25,21 @@ pub struct SecretSource {
/// Cache key → last-seen state, populated by `apply` and consulted by the /// Cache key → last-seen state, populated by `apply` and consulted by the
/// 30-second periodic tick and the delete path. /// 30-second periodic tick and the delete path.
struct CachedEntry { #[derive(Clone)]
/// Serialized score JSON. Used for string-compare idempotency per enum CachedEntry {
/// ROADMAP §5.5 — cheaper and more deterministic than a hash. Apply {
serialized: String, serialized: String,
/// Parsed score. Cached so the periodic reconcile tick and delete
/// handlers don't have to re-parse the JSON.
score: PodmanV0Score, score: PodmanV0Score,
resolved: bool,
},
Remove {
score: Option<PodmanV0Score>,
},
} }
pub struct Reconciler { pub struct Reconciler {
device_id: Id, device_id: Id,
topology: Arc<PodmanTopology>, runtime: Arc<PodmanRuntime>,
inventory: Arc<Inventory>,
/// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per /// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per
/// KV key — in v0 there is no fan-out from one key to many scores. /// KV key — in v0 there is no fan-out from one key to many scores.
state: Mutex<HashMap<String, CachedEntry>>, state: Mutex<HashMap<String, CachedEntry>>,
@@ -58,15 +60,13 @@ pub struct Reconciler {
impl Reconciler { impl Reconciler {
pub fn new( pub fn new(
device_id: Id, device_id: Id,
topology: Arc<PodmanTopology>, runtime: Arc<PodmanRuntime>,
inventory: Arc<Inventory>,
fleet: Option<Arc<FleetPublisher>>, fleet: Option<Arc<FleetPublisher>>,
secrets: Option<SecretSource>, secrets: Option<SecretSource>,
) -> Self { ) -> Self {
Self { Self {
device_id, device_id,
topology, runtime,
inventory,
state: Mutex::new(HashMap::new()), state: Mutex::new(HashMap::new()),
deployments: Mutex::new(HashMap::new()), deployments: Mutex::new(HashMap::new()),
fleet, fleet,
@@ -75,10 +75,9 @@ impl Reconciler {
} }
/// Materialize every `secret_env` reference into a plain env var. /// Materialize every `secret_env` reference into a plain env var.
/// Resolution happens once per spec change, never on the periodic /// Resolution is retried until it succeeds, then the resolved score is
/// tick: secret values bind at apply time (rotation = spec bump or /// reused until the spec changes. This keeps the secret store off the
/// re-apply), which keeps the secret store off the 30s hot path — /// periodic hot path for converged deployments.
/// fleet-scale login load is the ADR-025 constraint.
async fn resolve_secrets( async fn resolve_secrets(
&self, &self,
deployment: Option<&DeploymentName>, deployment: Option<&DeploymentName>,
@@ -150,13 +149,7 @@ impl Reconciler {
/// entry. Idempotent: a delete for a never-applied deployment is /// entry. Idempotent: a delete for a never-applied deployment is
/// a no-op in memory and a harmless tombstone write on the wire. /// a no-op in memory and a harmless tombstone write on the wire.
async fn drop_phase(&self, deployment: &DeploymentName) { async fn drop_phase(&self, deployment: &DeploymentName) {
let was_known = { self.deployments.lock().await.remove(deployment);
let mut phases = self.deployments.lock().await;
phases.remove(deployment).is_some()
};
if !was_known {
return;
}
if let Some(publisher) = &self.fleet { if let Some(publisher) = &self.fleet {
publisher.delete_deployment_state(deployment).await; publisher.delete_deployment_state(deployment).await;
} }
@@ -182,30 +175,32 @@ impl Reconciler {
{ {
let state = self.state.lock().await; let state = self.state.lock().await;
if let Some(existing) = state.get(key) { if let Some(CachedEntry::Apply {
if existing.serialized == serialized { serialized: existing,
..
}) = state.get(key)
{
if existing == &serialized {
tracing::debug!(key, "score unchanged — noop"); tracing::debug!(key, "score unchanged — noop");
return Ok(()); return Ok(());
} }
} }
} }
self.state.lock().await.insert(
key.to_string(),
CachedEntry::Apply {
serialized,
score: incoming,
resolved: false,
},
);
if let Some(name) = &deployment { if let Some(name) = &deployment {
self.apply_phase(name, Phase::Pending, None).await; self.apply_phase(name, Phase::Pending, None).await;
} }
let resolved = match self.resolve_secrets(deployment.as_ref(), incoming).await { match self.reconcile_cached(key).await {
Ok(score) => score,
Err(e) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
.await;
}
return Err(e);
}
};
match self.run_score(key, &resolved).await {
Ok(()) => { Ok(()) => {
if let Some(name) = &deployment { if let Some(name) = &deployment {
self.apply_phase(name, Phase::Running, None).await; self.apply_phase(name, Phase::Running, None).await;
@@ -219,79 +214,52 @@ impl Reconciler {
return Err(e); return Err(e);
} }
} }
let mut state = self.state.lock().await;
// Cache the *resolved* score: the periodic tick re-runs it
// as-is, so converged deployments never re-hit the secret
// store (see `resolve_secrets`).
state.insert(
key.to_string(),
CachedEntry {
serialized,
score: resolved,
},
);
Ok(()) Ok(())
} }
/// Handle a Delete/Purge event. Stops and removes every container /// Handle a Delete/Purge event. The tombstone remains cached until all
/// referenced by the last cached score for this key. Idempotent: if we /// deployment containers are removed, including when no score survived a
/// never saw a Put for this key (agent restart after delete), logs and /// restart.
/// returns ok.
pub async fn remove(&self, key: &str) -> Result<()> { pub async fn remove(&self, key: &str) -> Result<()> {
let deployment = deployment_from_key(key);
let mut state = self.state.lock().await; let mut state = self.state.lock().await;
let Some(entry) = state.remove(key) else { let score = match state.get(key) {
tracing::info!(key, "delete for unknown key — nothing to remove"); Some(CachedEntry::Apply { score, .. }) => Some(score.clone()),
if let Some(name) = &deployment { Some(CachedEntry::Remove { score }) => score.clone(),
self.drop_phase(name).await; None => None,
}
return Ok(());
}; };
state.insert(key.to_string(), CachedEntry::Remove { score });
drop(state); drop(state);
self.remove_cached(key).await
use harmony::topology::ContainerRuntime;
for service in &entry.score.services {
if let Err(e) = self.topology.remove_service(&service.name).await {
tracing::warn!(
key,
service = %service.name,
error = %e,
"failed to remove container"
);
} else {
tracing::info!(key, service = %service.name, "removed container");
}
}
if let Some(name) = &deployment {
self.drop_phase(name).await;
}
Ok(())
} }
/// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of /// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of
/// event-driven PLEG. Agent polls podman every 30s as ground truth; /// event-driven PLEG. Agent polls podman every 30s as ground truth;
/// KV watch events are accelerators." Re-runs each cached score against /// KV watch events are accelerators." Retries cached applies and removals
/// podman-api; the underlying `ensure_service_running` is idempotent /// against podman-api.
/// so a converged state produces no log noise.
pub async fn tick(&self) -> Result<()> { pub async fn tick(&self) -> Result<()> {
let snapshot: Vec<(String, PodmanV0Score)> = { let keys: Vec<String> = self.state.lock().await.keys().cloned().collect();
let state = self.state.lock().await; for key in keys {
state
.iter()
.map(|(k, v)| (k.clone(), v.score.clone()))
.collect()
};
for (key, score) in snapshot {
let deployment = deployment_from_key(&key); let deployment = deployment_from_key(&key);
match self.run_score(&key, &score).await { let removing = matches!(
self.state.lock().await.get(&key),
Some(CachedEntry::Remove { .. })
);
let result = if removing {
self.remove_cached(&key).await
} else {
self.reconcile_cached(&key).await
};
match result {
Ok(()) => { Ok(()) => {
if !removing {
if let Some(name) = &deployment { if let Some(name) = &deployment {
self.apply_phase(name, Phase::Running, None).await; self.apply_phase(name, Phase::Running, None).await;
} }
} }
}
Err(e) => { Err(e) => {
tracing::warn!(key, error = %e, "periodic reconcile failed"); tracing::warn!(key, error = %e, "periodic reconcile failed");
if !removing {
if let Some(name) = &deployment { if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
.await; .await;
@@ -299,6 +267,7 @@ impl Reconciler {
} }
} }
} }
}
Ok(()) Ok(())
} }
@@ -313,13 +282,78 @@ impl Reconciler {
} }
} }
async fn reconcile_cached(&self, key: &str) -> Result<()> {
let Some(CachedEntry::Apply {
serialized,
mut score,
resolved,
}) = self.state.lock().await.get(key).cloned()
else {
return Ok(());
};
if !resolved {
score = self
.resolve_secrets(deployment_from_key(key).as_ref(), score)
.await?;
let mut state = self.state.lock().await;
let Some(CachedEntry::Apply {
serialized: current,
score: cached_score,
resolved,
}) = state.get_mut(key)
else {
return Ok(());
};
if current != &serialized {
return Ok(());
}
*cached_score = score.clone();
*resolved = true;
}
self.run_score(key, &score).await
}
async fn remove_cached(&self, key: &str) -> Result<()> {
let Some(CachedEntry::Remove { score }) = self.state.lock().await.get(key).cloned() else {
return Ok(());
};
let deployment = deployment_from_key(key);
if let Some(name) = &deployment {
self.runtime.remove_deployment(name.as_str()).await?;
} else if let Some(score) = score {
for service in &score.services {
self.runtime.remove_service(&service.name).await?;
}
}
let removed = {
let mut state = self.state.lock().await;
if matches!(state.get(key), Some(CachedEntry::Remove { .. })) {
state.remove(key);
true
} else {
false
}
};
if removed {
if let Some(name) = &deployment {
self.drop_phase(name).await;
}
}
Ok(())
}
async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> { async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> {
let interpret = Score::<PodmanTopology>::create_interpret(score); let deployment = deployment_from_key(key)
let outcome = interpret .map(|name| name.to_string())
.execute(&self.inventory, &self.topology) .unwrap_or_else(|| key.to_string());
self.runtime
.reconcile(&deployment, score)
.await .await
.map_err(|e| anyhow::anyhow!("PodmanV0Score interpret failed for {key}: {e}"))?; .map_err(|e| anyhow::anyhow!("PodmanV0Score reconcile failed for {key}: {e}"))?;
tracing::info!(key, outcome = ?outcome, "reconciled"); tracing::info!(key, services = score.services.len(), "reconciled");
Ok(()) Ok(())
} }
} }
@@ -350,26 +384,19 @@ mod tests {
//! `drop_phase` directly with an inert topology (no real podman //! `drop_phase` directly with an inert topology (no real podman
//! socket) and a `None` FleetPublisher. //! socket) and a `None` FleetPublisher.
use super::*; use super::*;
use harmony::inventory::Inventory;
use harmony::modules::podman::PodmanTopology;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Mutex as StdMutex;
use std::sync::atomic::{AtomicUsize, Ordering};
fn reconciler() -> Reconciler { fn reconciler() -> Reconciler {
reconciler_with_secrets(None) reconciler_with_secrets(None)
} }
fn reconciler_with_secrets(secrets: Option<SecretSource>) -> Reconciler { fn reconciler_with_secrets(secrets: Option<SecretSource>) -> Reconciler {
let topology = Arc::new( let runtime = Arc::new(
PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(), PodmanRuntime::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(),
); );
let inventory = Arc::new(Inventory::empty()); Reconciler::new(Id::from("test-device".to_string()), runtime, None, secrets)
Reconciler::new(
Id::from("test-device".to_string()),
topology,
inventory,
None,
secrets,
)
} }
fn dn(s: &str) -> DeploymentName { fn dn(s: &str) -> DeploymentName {
@@ -403,8 +430,40 @@ mod tests {
} }
} }
#[derive(Debug, Default)]
struct FlakyStore {
calls: AtomicUsize,
value: StdMutex<Option<String>>,
}
#[async_trait::async_trait]
impl SecretStore for FlakyStore {
async fn get_raw(
&self,
_namespace: &str,
_key: &str,
) -> Result<Vec<u8>, harmony_secret::SecretStoreError> {
self.calls.fetch_add(1, Ordering::Relaxed);
self.value
.lock()
.unwrap()
.as_ref()
.map(|value| value.as_bytes().to_vec())
.ok_or_else(|| harmony_secret::SecretStoreError::Store("temporarily denied".into()))
}
async fn set_raw(
&self,
_namespace: &str,
_key: &str,
_value: &[u8],
) -> Result<(), harmony_secret::SecretStoreError> {
unimplemented!("tests never write")
}
}
fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score { fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score {
use harmony::modules::podman::{PodmanService, SecretEnvVar}; use harmony_reconciler_contracts::{PodmanService, SecretEnvVar};
PodmanV0Score { PodmanV0Score {
services: vec![PodmanService { services: vec![PodmanService {
name: "web-svc".to_string(), name: "web-svc".to_string(),
@@ -479,6 +538,73 @@ mod tests {
assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed)); assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed));
} }
#[tokio::test]
async fn tick_retries_secret_resolution_then_reuses_resolved_score() {
let store = Arc::new(FlakyStore::default());
let r = reconciler_with_secrets(Some(SecretSource {
store: store.clone(),
prefix: "fleet-test".to_string(),
}));
let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score(
"DB_PASSWORD",
"db_password",
)))
.unwrap();
r.apply("test-device.web", &payload)
.await
.expect_err("first secret read fails");
*store.value.lock().unwrap() = Some("hunter2".to_string());
r.tick().await.unwrap();
r.tick().await.unwrap();
assert_eq!(store.calls.load(Ordering::Relaxed), 2);
let state = r.state.lock().await;
let Some(CachedEntry::Apply {
score, resolved, ..
}) = state.get("test-device.web")
else {
panic!("apply retry state missing");
};
assert!(*resolved);
assert_eq!(
score.services[0].env[0],
EnvVar::new("DB_PASSWORD", "hunter2")
);
}
#[tokio::test]
async fn failed_removal_is_retried_and_later_put_cancels_it() {
let r = reconciler();
r.remove("test-device.web")
.await
.expect_err("unreachable Podman keeps the tombstone pending");
assert!(matches!(
r.state.lock().await.get("test-device.web"),
Some(CachedEntry::Remove { score: None })
));
r.tick().await.unwrap();
assert!(matches!(
r.state.lock().await.get("test-device.web"),
Some(CachedEntry::Remove { .. })
));
let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(PodmanV0Score {
services: vec![],
}))
.unwrap();
r.apply("test-device.web", &payload)
.await
.expect_err("unreachable Podman keeps the new apply pending");
assert!(matches!(
r.state.lock().await.get("test-device.web"),
Some(CachedEntry::Apply { .. })
));
}
#[tokio::test] #[tokio::test]
async fn plain_scores_resolve_without_store() { async fn plain_scores_resolve_without_store() {
// No secret_env → no store needed; resolution is a no-op. // No secret_env → no store needed; resolution is a no-op.

View File

@@ -24,9 +24,11 @@
mod agent_config; mod agent_config;
mod config; mod config;
mod credentials; mod credentials;
mod observation;
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config}; pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR}; pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
pub use observation::AgentObservation;
use std::sync::Arc; use std::sync::Arc;

View File

@@ -12,7 +12,7 @@
//! Per ADR-023 P7 this is a companion struct, not a new method on //! Per ADR-023 P7 this is a companion struct, not a new method on
//! `Score` / `Interpret`. The base trait surface stays unchanged. //! `Score` / `Interpret`. The base trait surface stays unchanged.
use harmony_fleet_auth::AgentConfig; use crate::AgentConfig;
use harmony_reconciler_contracts::{ use harmony_reconciler_contracts::{
BUCKET_DESIRED_STATE, DeploymentName, Id, desired_state_key, desired_state_watch_filter, BUCKET_DESIRED_STATE, DeploymentName, Id, desired_state_key, desired_state_watch_filter,
}; };
@@ -104,7 +104,7 @@ impl AgentObservation {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use harmony_fleet_auth::{AgentSection, CredentialsSection, NatsSection}; use crate::{AgentSection, CredentialsSection, NatsSection};
use std::collections::BTreeMap; use std::collections::BTreeMap;
fn cfg(device_id: &str, runtime_enabled: bool) -> AgentConfig { fn cfg(device_id: &str, runtime_enabled: bool) -> AgentConfig {

View File

@@ -5,6 +5,7 @@ version.workspace = true
readme.workspace = true readme.workspace = true
license.workspace = true license.workspace = true
description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, callout. Imports harmony for the Score-Topology-Interpret machinery; consumed by the production CLI, the e2e harness, and future control-plane tooling." description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, callout. Imports harmony for the Score-Topology-Interpret machinery; consumed by the production CLI, the e2e harness, and future control-plane tooling."
default-run = "harmony-fleet-deploy"
[lib] [lib]
path = "src/lib.rs" path = "src/lib.rs"
@@ -15,20 +16,15 @@ path = "src/lib.rs"
name = "harmony-fleet-deploy" name = "harmony-fleet-deploy"
path = "src/main.rs" path = "src/main.rs"
# `harmony-fleet-publish --from-tag <tag>` builds + publishes the
# operator's image + chart for a release.
[[bin]]
name = "harmony-fleet-publish"
path = "src/bin/harmony-fleet-publish.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", features = ["podman"] } harmony = { path = "../../harmony", features = ["podman"] }
harmony_cli = { path = "../../harmony_cli" } harmony_cli = { path = "../../harmony_cli" }
harmony_app = { path = "../../harmony_app" }
harmony_config = { path = "../../harmony_config" } harmony_config = { path = "../../harmony_config" }
harmony_types = { path = "../../harmony_types" } harmony_types = { path = "../../harmony_types" }
harmony_macros = { path = "../../harmony_macros" } harmony_macros = { path = "../../harmony_macros" }
harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony-fleet-operator = { path = "../harmony-fleet-operator" }
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
anyhow = { workspace = true } anyhow = { workspace = true }
@@ -40,12 +36,15 @@ kube = { workspace = true, features = ["runtime", "derive"] }
log = { workspace = true } log = { workspace = true }
env_logger = { workspace = true } env_logger = { workspace = true }
non-blank-string-rs = "1" non-blank-string-rs = "1"
inquire.workspace = true
schemars = "0.8" schemars = "0.8"
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_yaml = { workspace = true } serde_yaml = { workspace = true }
similar.workspace = true
tempfile = "3" tempfile = "3"
thiserror = { workspace = true } thiserror = { workspace = true }
tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] }
toml = { workspace = true } toml = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
url.workspace = true

View File

@@ -2,9 +2,8 @@
//! instance. //! instance.
//! //!
//! v1 supports the in-cluster Pod form factor (the e2e harness and //! v1 supports the in-cluster Pod form factor (the e2e harness and
//! same-cluster production deploys). The VM/SSH/RPi path stays in //! same-cluster production deploys). [`crate::FleetDeviceSetupScore`]
//! `harmony::modules::fleet::FleetDeviceSetupScore` until a follow-up //! owns the VM/SSH/RPi form factor.
//! migrates it into this crate.
//! //!
//! Composition: the Pod target Score interprets to two //! Composition: the Pod target Score interprets to two
//! [`K8sResourceScore`] applies in sequence — a `ConfigMap` carrying //! [`K8sResourceScore`] applies in sequence — a `ConfigMap` carrying
@@ -16,6 +15,7 @@
//! compile error — that's the point. //! compile error — that's the point.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::Duration;
use async_trait::async_trait; use async_trait::async_trait;
use harmony::data::Version; use harmony::data::Version;
@@ -217,6 +217,24 @@ impl<T: Topology + K8sclient> Interpret<T> for FleetAgentInterpret {
source: e, source: e,
})?; })?;
topology
.k8s_client()
.await
.map_err(InterpretError::new)?
.wait_until_deployment_ready(
&FleetAgentScore::resource_name(&device_id),
Some(&self.score.namespace),
Some(Duration::from_secs(120)),
)
.await
.map_err(|error| {
InterpretError::new(format!(
"agent deployment {}/{} not ready: {error}",
self.score.namespace,
FleetAgentScore::resource_name(&device_id)
))
})?;
Ok(Outcome::success_with_details( Ok(Outcome::success_with_details(
outcome.message, outcome.message,
vec![ vec![

View File

@@ -1,54 +0,0 @@
//! `harmony-fleet-publish` — build + publish the operator image + chart
//! at a version. `docker` / `helm` must be on PATH and logged in to the
//! registry (CI's login actions; dev's manual login).
use anyhow::{Result, bail};
use clap::Parser;
use harmony_fleet_deploy::release::{release_operator, version_from_tag};
#[derive(Parser, Debug)]
#[command(
name = "harmony-fleet-publish",
about = "Build + publish the operator image + chart at a version"
)]
struct Cli {
/// The image + chart version, e.g. `0.1.0` or `0.0.0-dev.1730000000`
/// (must be valid semver — helm rejects anything else). Wins over
/// `--from-tag`; the laptop/dev path.
#[arg(long)]
version: Option<String>,
/// Parse the version out of a release tag like
/// `harmony-fleet-operator-v0.1.0`. Defaults to `$GITHUB_REF_NAME`
/// so the release workflow passes nothing. Used when `--version`
/// is unset.
#[arg(long, env = "GITHUB_REF_NAME")]
from_tag: Option<String>,
#[arg(long, default_value = "hub.nationtech.io")]
registry: String,
#[arg(long, default_value = "harmony")]
project: String,
/// Build + package only; skip both pushes (local k3d smoke-test).
#[arg(long)]
no_push: bool,
}
impl Cli {
fn version(&self) -> Result<String> {
match (&self.version, &self.from_tag) {
(Some(v), _) => Ok(v.clone()),
(None, Some(tag)) => version_from_tag(tag),
(None, None) => bail!("set --version (e.g. 0.1.0) or --from-tag"),
}
}
}
fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let cli = Cli::parse();
let version = cli.version()?;
release_operator(&version, &cli.registry, &cli.project, !cli.no_push)
}

View File

@@ -1,19 +0,0 @@
//! Score companions — observe deployed-Score behavior without
//! widening the `Score` / `Interpret` trait surface (ADR-023 P7).
//!
//! A companion takes a Score (or, when the runtime contract lives
//! elsewhere, the typed config the Score emits) and exposes a
//! derived view a caller cares about — what the agent will watch on
//! NATS, which CRDs the operator owns, what the readiness probe
//! checks. Companions never call into the framework or topology;
//! they're pure functions over data the Score already exposes.
//!
//! Today: one companion, [`AgentObservation`], used by the e2e
//! harness to derive the device-scoped KV watch filter from a typed
//! [`harmony_fleet_auth::AgentConfig`]. Future companions for the
//! operator's CRD ownership + the readiness/smoke-test contract
//! land alongside this module.
pub mod agent;
pub use agent::AgentObservation;

View File

@@ -11,19 +11,17 @@ use harmony_types::id::Id;
use log::{debug, info, warn}; use log::{debug, info, warn};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::data::Version; use harmony::data::Version;
use crate::domain::interpret::{ use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
Interpret, InterpretError, InterpretName, InterpretStatus, Outcome, use harmony::inventory::Inventory;
}; use harmony::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig};
use crate::domain::inventory::Inventory; use harmony::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials};
use crate::domain::topology::{ use harmony::score::Score;
use harmony::topology::{
FileDelivery, FileFetcher, FileSource, FileSpec, HostReachable, LinuxHostConfiguration, FileDelivery, FileFetcher, FileSource, FileSpec, HostReachable, LinuxHostConfiguration,
PackageInstaller, SystemdManager, SystemdScope, SystemdUnitSpec, Topology, UnixUserManager, PackageInstaller, SystemdManager, SystemdScope, SystemdUnitSpec, Topology, UnixUserManager,
UserSpec, UserSpec,
}; };
use crate::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig};
use crate::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials};
use crate::score::Score;
/// User-visible configuration for the setup Score. Everything a customer /// User-visible configuration for the setup Score. Everything a customer
/// needs to tell us to bring a device into the fleet. /// needs to tell us to bring a device into the fleet.
@@ -760,6 +758,23 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
"✅ running unchanged" "✅ running unchanged"
}; };
let mut active = false;
for _ in 0..30 {
if SystemdManager::systemd_unit_active(topology, "fleet-agent", SystemdScope::System)
.await
.map_err(wrap)?
{
active = true;
break;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
if !active {
return Err(InterpretError::new(
"fleet-agent.service did not become active within 30s".to_string(),
));
}
// Build human-readable recap. Style matches OKDAddNodeScore: // Build human-readable recap. Style matches OKDAddNodeScore:
// one fact per line, "Key: value" prefix, all the operationally // one fact per line, "Key: value" prefix, all the operationally
// useful state in a single Outcome details block so a CLI run // useful state in a single Outcome details block so a CLI run
@@ -811,7 +826,7 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
} }
} }
fn wrap(e: crate::executors::ExecutorError) -> InterpretError { fn wrap(e: harmony::executors::ExecutorError) -> InterpretError {
InterpretError::new(e.to_string()) InterpretError::new(e.to_string())
} }

View File

@@ -7,38 +7,16 @@
//! depend on this crate — they're the targets being deployed, not //! depend on this crate — they're the targets being deployed, not
//! the deployer. //! the deployer.
//! //!
//! Coverage today: //! The binary composes independent component Scores through typed references;
//! //! it does not own provider operations or a Fleet-wide aggregate Score.
//! - [`operator`] — `FleetOperatorScore` (moved out of
//! `harmony::modules::fleet::operator`), helm chart generator,
//! CRD apply via [`operator::install_crds`].
//! - [`nats`] — `FleetNatsScore` (preset over `NatsHelmChartScore`
//! with the values the fleet stack needs).
//! - [`agent`] — `FleetAgentScore` (Pod-target only in v1; the
//! VM/SSH target is `FleetDeviceSetupScore` in `harmony` core,
//! slated to move here in a follow-up).
//!
//! Out of scope for now:
//!
//! - **Callout** (mock-OIDC fixture + `NatsAuthCalloutScore` wiring) —
//! PR 1.5.
//! - **Smoke-test contract**. ADR-023 principle 4 says `deploy`
//! returns only after a smoke test passes; the trait/struct shape
//! for that contract is open. The e2e harness's `ping` test plays
//! the role of the smoke test for now.
pub mod agent; pub mod agent;
pub mod companion; mod device_setup;
pub mod nats;
pub mod operator; pub mod operator;
pub mod release;
pub mod secrets;
pub mod server;
pub use agent::{FleetAgentScore, PodTarget}; pub use agent::{FleetAgentScore, PodTarget};
pub use companion::AgentObservation; pub use device_setup::{
pub use nats::{FleetNatsScore, UserPassCredentials}; AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart}; HostsEntry, merge_hosts_file,
pub use release::{release_operator, version_from_tag}; };
pub use secrets::{FleetDeployConfig, FleetDeploySecrets}; pub use operator::{FleetOperatorScore, OperatorCredentials};
pub use server::FleetServerScore;

View File

@@ -1,149 +1,153 @@
//! `harmony-fleet-deploy` — deploy the published fleet operator chart use anyhow::Result;
//! (`harmony apply`). Loads its secrets from config and runs one use async_trait::async_trait;
//! [`FleetOperatorScore`] against [`K8sAnywhereTopology`]. use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService};
//! use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
//! The full dev/e2e stack (NATS + agent, user/pass) is composed by the use harmony::modules::openbao::{OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore};
//! e2e harness directly over the `harmony_fleet_deploy` Scores — not use harmony::modules::postgresql::K8sPostgreSQLScore;
//! here. This binary is the prod/CD path: Zitadel-SSO-only, no use harmony::modules::zitadel::{ZitadelAppType, ZitadelScore, ZitadelSetupScore};
//! handrolled manifests. use harmony::score::Score;
use std::io::Write;
use anyhow::{Context, Result, bail};
use clap::Parser;
use harmony::inventory::Inventory;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_cli::Args as HarmonyCliArgs; use harmony_app::{AppContext, AppError, AppIdentity, HarmonyApp, ImageRefs, ImageSpec, Profile};
use harmony_config::ConfigClient; use harmony_fleet_deploy::FleetOperatorScore;
use harmony_fleet_deploy::{
FleetDeployConfig, FleetDeploySecrets, FleetOperatorScore, version_from_tag,
};
use harmony_zitadel_auth::{OperatorCookieKey, ZitadelAuthConfig};
use tracing::info;
#[derive(Parser, Debug)] const PROJECT: &str = "fleet";
#[command( const ADMIN_ROLE: &str = "fleet-admin";
name = "harmony-fleet-deploy", const DEVICE_ROLE: &str = "device";
about = "Deploy the published harmony fleet operator chart" const OPERATOR_APP: &str = "harmony-fleet-operator";
)] const OPERATOR_USER: &str = "fleet-operator";
struct CliConfig { const NATS_ACCOUNT: &str = "FLEET";
/// Override the k8s namespace from config (e.g. `fleet-staging`).
#[arg(long, env = "HARMONY_FLEET_NAMESPACE")]
namespace: Option<String>,
/// Release tag to deploy (e.g. `harmony-fleet-operator-v0.0.2`); the struct FleetApp;
/// version is parsed from it in Rust so the workflow passes a tag and
/// never parses one in YAML. Wins over `--operator-chart-version`.
#[arg(long, env = "HARMONY_FLEET_OPERATOR_TAG")]
from_tag: Option<String>,
/// Bare chart version, for the laptop path. #[async_trait]
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")] impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
operator_chart_version: Option<String>, fn identity(&self) -> AppIdentity {
AppIdentity {
/// Override the OCI chart registry from config. name: "fleet".to_string(),
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_REGISTRY")] namespace: "fleet-system".to_string(),
operator_chart_registry: Option<String>,
/// Override the OCI chart project from config.
#[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_PROJECT")]
operator_chart_project: Option<String>,
/// Config namespace `FleetDeploySecrets` and `FleetDeployConfig` resolve under (Env → OpenBao).
#[arg(
long,
env = "HARMONY_CONFIG_NAMESPACE",
default_value = "fleet-staging"
)]
config_namespace: String,
#[command(flatten)]
harmony_cli: HarmonyCliArgs,
}
impl CliConfig {
fn chart_version(&self) -> Result<String> {
match (&self.from_tag, &self.operator_chart_version) {
(Some(tag), _) => version_from_tag(tag),
(None, Some(v)) => Ok(v.clone()),
(None, None) => bail!("set --from-tag or --operator-chart-version"),
} }
} }
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
Ok(vec![
ImageSpec {
name: "operator".to_string(),
image: ctx.image("harmony-fleet-operator")?,
context: ".".into(),
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
build_args: Vec::new(),
},
ImageSpec {
name: "callout".to_string(),
image: ctx.image("harmony-nats-callout")?,
context: ".".into(),
dockerfile: "nats/callout/Dockerfile".into(),
build_args: Vec::new(),
},
])
}
async fn scores(
&self,
ctx: &AppContext,
images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let namespace = "fleet-system";
let zitadel_namespace = "zitadel";
let openbao_namespace = "openbao";
let image_pull_secret = ctx.image_pull_secret()?;
let postgres = K8sPostgreSQLScore::new(zitadel_namespace).cluster_name("zitadel-pg");
let database = postgres.root_account_ref();
let mut zitadel = ZitadelScore::new(
ctx.service_host("zitadel", zitadel_namespace)?,
zitadel_namespace,
)
.database(database);
if ctx.profile() == Profile::Local {
// Q : should that be handled by harmony internally?
zitadel = zitadel.http(Some(8080));
}
let provider = zitadel.provider_ref();
let identity = ZitadelSetupScore::for_provider(&provider, zitadel_namespace, namespace)
.application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
.api_application(PROJECT, "nats")
.role(PROJECT, ADMIN_ROLE, "Fleet Admin")
.role(PROJECT, DEVICE_ROLE, "Device")
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
.port_forward("zitadel")
.groups_claim();
let application = identity.application_ref(OPERATOR_APP);
let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
let credentials = match ctx.profile() {
// Q: this looks weird. Why do we need to generate creds locally but not in prod? I
// would think that in both cases creds should be "ensured". Created if not existing.
// Maybe in prod it should use harmony config with a remote openbao and store them there
// (as we plan on using external secrets asap)
Profile::Local => NatsAuthCalloutCredentialsScore::generated,
Profile::Prod => NatsAuthCalloutCredentialsScore::existing,
}(namespace, "fleet-callout-credentials");
let service = match ctx.profile() {
// FIXME this should definitely not surface up to the end user deployment crate. This is
// topology dependent and is the topology's job (local k3d vs regular cluster) to decide
// how to expose a port and then the score ref should be appropriate
Profile::Local => NatsService::NodePort(30422),
Profile::Prod => NatsService::LoadBalancer,
};
// I feel like this should not be a standalone nats score but rather be configuration passed
// to the main nats score that is installing the nats cluster
let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
.with_jetstream_size("2Gi");
let account = nats.account_ref();
let callout =
NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
.credentials(&credentials.credentials_ref())
.with_oidc(&provider, &application)
.image(images.require("callout")?)
.image_pull_secret(image_pull_secret)
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE)
.device_id_claim("client_id");
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
let mut openbao = OpenbaoScore::new(
openbao_namespace,
"openbao",
ctx.service_host("openbao", openbao_namespace)?,
);
if ctx.profile() == Profile::Prod {
openbao = openbao.tls("letsencrypt-prod");
}
let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
&provider,
&application,
OpenbaoJwtAuth::oidc("fleet-device"),
);
let operator = FleetOperatorScore::new(images.require("operator")?)
.namespace(namespace)
.image_pull_secret(image_pull_secret)
.messaging(&nats.client_ref())
.identity(&provider, &application, &operator_identity);
Ok(vec![
Box::new(postgres),
Box::new(zitadel),
Box::new(credentials),
Box::new(nats),
Box::new(identity),
Box::new(callout),
Box::new(openbao),
Box::new(openbao_setup),
Box::new(operator),
])
}
} }
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
harmony_cli::cli_logger::init(); harmony_cli::app::app_main(FleetApp).await
let cli = CliConfig::parse();
let version = cli.chart_version()?;
let config_client = ConfigClient::for_namespace(&cli.config_namespace).await;
let secrets: FleetDeploySecrets = config_client
.get()
.await
.context("loading FleetDeploySecrets (set HARMONY_CONFIG_FleetDeploySecrets or OpenBao)")?;
let config: FleetDeployConfig = config_client
.get_or_prompt()
.await
.context("loading FleetDeployConfig")?;
// Point KUBECONFIG at the scoped deployer credential before the
// topology reads it, so the runner pod needs no standing permissions.
// Held to end of scope so the tempfile outlives the deploy.
let _kubeconfig = match &secrets.kubeconfig {
Some(kubeconfig) => {
let mut f = tempfile::NamedTempFile::new().context("kubeconfig tempfile")?;
f.write_all(kubeconfig.as_bytes())
.context("writing kubeconfig")?;
// SAFETY: single-threaded startup, before any topology reads it.
unsafe { std::env::set_var("KUBECONFIG", f.path()) };
Some(f)
}
None => None,
};
let namespace = cli.namespace.unwrap_or(config.namespace);
let registry = cli
.operator_chart_registry
.unwrap_or(config.operator_chart_registry);
let project = cli
.operator_chart_project
.unwrap_or(config.operator_chart_project);
// Coherent with the other staging hosts (sso-stg., secrets-stg.).
let ui_host = format!("fleet-stg.{}", config.base_domain);
// Dashboard SSO config: hosts derive from base_domain; client_id +
// audiences come from config, cookie key from secrets. Baked into the
// operator Secret for the pod's ConfigClient to read.
let web_auth = ZitadelAuthConfig {
zitadel_base: format!("https://sso-stg.{}", config.base_domain),
base_url: format!("https://{ui_host}"),
client_id: config.operator_oidc_client_id,
scope: "openid profile email".to_string(),
trusted_audiences: config.operator_trusted_audiences,
logout_redirect_uri: format!("https://{ui_host}/"),
};
let cookie = OperatorCookieKey {
cookie_key_b64: secrets.operator_cookie_key_b64,
};
let operator = FleetOperatorScore::new()
.namespace(namespace)
.nats_url(config.nats_url)
.credentials(secrets.operator_credentials_toml)
.published_chart(registry, project, version)
.ingress(ui_host, Some(config.cluster_issuer))
.web_auth(web_auth, cookie);
harmony_cli::run(
Inventory::autoload(),
K8sAnywhereTopology::from_env(),
vec![Box::new(operator)],
Some(cli.harmony_cli),
)
.await
.map_err(|e| anyhow::anyhow!("{e}"))
} }

View File

@@ -16,6 +16,7 @@
//! operator Deployment into a helm chart the cluster runs itself. //! operator Deployment into a helm chart the cluster runs itself.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::hash::{DefaultHasher, Hash, Hasher};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
@@ -25,7 +26,8 @@ use k8s_openapi::api::apps::v1::{
}; };
use k8s_openapi::api::core::v1::{ use k8s_openapi::api::core::v1::{
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile, Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile,
Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, Secret, SecretKeySelector, SecretVolumeSource, SecurityContext, Service, ServiceAccount,
ServicePort, ServiceSpec, Volume, VolumeMount,
}; };
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject}; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
@@ -36,8 +38,9 @@ use kube::api::ObjectMeta;
use serde::Serialize; use serde::Serialize;
use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
use harmony::modules::fleet::operator::crd::{Deployment, Device}; use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
use harmony_fleet_operator::{Deployment, Device};
/// Inputs for chart generation. Default values are aimed at a /// Inputs for chart generation. Default values are aimed at a
/// local-dev k3d install; override via the `chart` subcommand flags. /// local-dev k3d install; override via the `chart` subcommand flags.
@@ -77,56 +80,24 @@ pub struct ChartOptions {
/// JSON of the dashboard's `OperatorCookieKey`, stored under /// JSON of the dashboard's `OperatorCookieKey`, stored under
/// [`ENV_WEB_COOKIE_KEY`]. /// [`ENV_WEB_COOKIE_KEY`].
pub web_cookie_key_json: Option<String>, pub web_cookie_key_json: Option<String>,
pub identity: Option<OperatorIdentityRefs>,
pub identity_version: Option<String>,
pub image_pull_secret: Option<String>,
} }
/// What the operator pod needs to authenticate to NATS via the auth #[derive(Debug, Clone, Serialize)]
/// callout: a TOML snippet matching the agent's `[credentials]` pub struct OperatorIdentityRefs {
/// table. pub provider: OidcProviderRef,
/// pub application: OidcApplicationRef,
/// **Single Secret key**: the credentials TOML is the only payload. pub machine: OidcMachineIdentityRef,
/// The Zitadel JSON keyfile content is embedded inline inside the }
/// TOML as the `key_json` field (multi-line triple-quoted string).
/// One env-var-from-Secret on the operator Pod /// Static NATS credentials for deployments without typed OIDC identity refs.
/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume
/// mounts, OKD restricted-v2 SCC compatible.
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
pub struct OperatorCredentials { pub struct OperatorCredentials {
/// TOML payload, e.g.
/// ```text
/// type = "zitadel-jwt"
/// oidc_issuer_url = "http://sso.fleet.local:8080"
/// audience = "<project_id>"
/// key_json = '''{"type":"serviceaccount","keyId":"...","key":"-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----\n","userId":"..."}'''
/// ```
///
/// `key_json` **must** use TOML *literal* multi-line strings
/// (`'''...'''`), NOT *basic* multi-line strings (`"""..."""`).
/// Basic strings interpret backslash escapes — every `\n` in the
/// embedded RSA private key would expand to a literal 0x0A before
/// JSON parsing, and JSON rejects raw control chars inside strings
/// ("control character … found while parsing a string"). Literal
/// strings preserve `\n` as the two chars `\\` + `n`, the JSON
/// parser then interprets that as a string escape and decodes the
/// multi-line PEM correctly.
pub credentials_toml: String, pub credentials_toml: String,
} }
impl OperatorCredentials {
/// Build the auth-callout credentials from a Zitadel machine key.
/// `key_json` goes in a TOML *literal* string — see the field docs
/// for why basic strings corrupt the embedded PEM.
pub fn zitadel_jwt(oidc_issuer_url: &str, audience: &str, key_json: &str) -> Self {
Self {
credentials_toml: format!(
"type = \"zitadel-jwt\"\n\
oidc_issuer_url = \"{oidc_issuer_url}\"\n\
audience = \"{audience}\"\n\
key_json = '''{key_json}'''\n"
),
}
}
}
impl Default for ChartOptions { impl Default for ChartOptions {
fn default() -> Self { fn default() -> Self {
Self { Self {
@@ -143,6 +114,9 @@ impl Default for ChartOptions {
chart_version: None, chart_version: None,
web_auth_config_json: None, web_auth_config_json: None,
web_cookie_key_json: None, web_cookie_key_json: None,
identity: None,
identity_version: None,
image_pull_secret: None,
} }
} }
} }
@@ -401,12 +375,32 @@ fn operator_service() -> Service {
} }
} }
pub(crate) fn config_hash(opts: &ChartOptions) -> String {
let mut secret_hash = DefaultHasher::new();
opts.credentials
.as_ref()
.map(|credentials| &credentials.credentials_toml)
.hash(&mut secret_hash);
opts.web_auth_config_json.hash(&mut secret_hash);
opts.web_cookie_key_json.hash(&mut secret_hash);
opts.identity
.as_ref()
.map(|identity| identity.machine.secret_name())
.hash(&mut secret_hash);
opts.identity_version.hash(&mut secret_hash);
format!("{:x}", secret_hash.finish())
}
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
let mut match_labels = BTreeMap::new(); let mut match_labels = BTreeMap::new();
match_labels.insert( match_labels.insert(
"app.kubernetes.io/name".to_string(), "app.kubernetes.io/name".to_string(),
RELEASE_NAME.to_string(), RELEASE_NAME.to_string(),
); );
let pod_annotations = BTreeMap::from([(
"harmony.nationtech.io/config-hash".to_string(),
config_hash(opts),
)]);
let mut env = vec![ let mut env = vec![
EnvVar { EnvVar {
@@ -420,6 +414,32 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
..Default::default() ..Default::default()
}, },
]; ];
if let Some(identity) = &opts.identity {
env.extend([
EnvVar {
name: "FLEET_OPERATOR_OIDC_ISSUER".to_string(),
value: Some(identity.provider.issuer().to_string()),
..Default::default()
},
EnvVar {
name: "FLEET_OPERATOR_OIDC_AUDIENCE".to_string(),
value_from: Some(EnvVarSource {
config_map_key_ref: Some(k8s_openapi::api::core::v1::ConfigMapKeySelector {
name: identity.application.config_map_name().to_string(),
key: identity.application.project_id_key().to_string(),
optional: Some(false),
}),
..Default::default()
}),
..Default::default()
},
EnvVar {
name: "FLEET_OPERATOR_OIDC_KEY_PATH".to_string(),
value: Some("/var/run/secrets/fleet/key.json".to_string()),
..Default::default()
},
]);
}
// The whole `[credentials]` TOML payload — including the JSON // The whole `[credentials]` TOML payload — including the JSON
// keyfile inlined under `key_json` — travels as a single env // keyfile inlined under `key_json` — travels as a single env
@@ -490,18 +510,47 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
template: PodTemplateSpec { template: PodTemplateSpec {
metadata: Some(ObjectMeta { metadata: Some(ObjectMeta {
labels: Some(match_labels), labels: Some(match_labels),
annotations: Some(pod_annotations),
..Default::default() ..Default::default()
}), }),
spec: Some(PodSpec { spec: Some(PodSpec {
service_account_name: Some(SERVICE_ACCOUNT.to_string()), service_account_name: Some(SERVICE_ACCOUNT.to_string()),
image_pull_secrets: opts.image_pull_secret.as_ref().map(|name| {
vec![k8s_openapi::api::core::v1::LocalObjectReference {
name: name.clone(),
}]
}),
containers: vec![Container { containers: vec![Container {
name: "operator".to_string(), name: "operator".to_string(),
image: Some(opts.image.clone()), image: Some(opts.image.clone()),
image_pull_policy: Some(opts.image_pull_policy.clone()), image_pull_policy: Some(opts.image_pull_policy.clone()),
env: Some(env), env: Some(env),
volume_mounts: opts.identity.as_ref().map(|_| {
vec![VolumeMount {
name: "operator-identity".to_string(),
mount_path: "/var/run/secrets/fleet".to_string(),
read_only: Some(true),
..Default::default()
}]
}),
security_context: Some(container_security_context()), security_context: Some(container_security_context()),
..Default::default() ..Default::default()
}], }],
volumes: opts.identity.as_ref().map(|identity| {
vec![Volume {
name: "operator-identity".to_string(),
secret: Some(SecretVolumeSource {
secret_name: Some(identity.machine.secret_name().to_string()),
items: Some(vec![k8s_openapi::api::core::v1::KeyToPath {
key: identity.machine.key_json_key().to_string(),
path: "key.json".to_string(),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
}]
}),
..Default::default() ..Default::default()
}), }),
}, },
@@ -511,27 +560,6 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
} }
} }
// Re-export the manifest builders so the e2e bring-up can apply the
// operator inline (Score-style) without re-implementing the manifests.
//
// The SA + Deployment helpers return namespace-neutral manifests;
// callers inject the target namespace through `K8sResourceScore::single`.
// The CRB takes the SA's namespace as an explicit argument because the
// CRB subject must reference a concrete namespace — there's no
// kube-side "current namespace" for cluster-scoped resources.
pub fn build_service_account() -> ServiceAccount {
service_account()
}
pub fn build_cluster_role() -> ClusterRole {
cluster_role()
}
pub fn build_cluster_role_binding(subject_namespace: &str) -> ClusterRoleBinding {
cluster_role_binding(subject_namespace)
}
pub fn build_operator_deployment(opts: &ChartOptions) -> K8sDeployment {
operator_deployment(opts)
}
/// Minimum-privilege container security context. /// Minimum-privilege container security context.
/// ///
/// - `runAsNonRoot: true` — a compromised operator pod with /// - `runAsNonRoot: true` — a compromised operator pod with

View File

@@ -1,53 +0,0 @@
//! Install the operator's CRDs into a target Kubernetes cluster via
//! a harmony Score.
//!
//! Moved from `fleet/harmony-fleet-operator/src/install.rs` per
//! ADR-023: this is deploy-side logic, so it lives in the deploy
//! crate, not in the operator runtime binary. The runtime binary's
//! `Install` subcommand is gone; users invoke `harmony-fleet-deploy
//! operator-install-crds` (or compose [`install_crds`] from another
//! deploy flow) instead.
use harmony::inventory::Inventory;
use harmony::modules::fleet::operator::crd::{Deployment, Device};
use harmony::modules::k8s::K8sBareTopology;
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::score::Score;
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum InstallCrdsError {
#[error("building K8sBareTopology from KUBECONFIG: {0}")]
Topology(String),
#[error("executing K8sResourceScore for Deployment + Device CRDs: {0}")]
Interpret(String),
}
/// Apply the operator's CRDs to whatever cluster `KUBECONFIG` points
/// at. Returns once the apply call completes — does **not** wait for
/// the apiserver to mark each CRD `Established`; the caller does that
/// (e.g. with `kubectl wait --for=condition=Established`) if it
/// cares.
pub async fn install_crds() -> Result<(), InstallCrdsError> {
let topology = K8sBareTopology::from_kubeconfig("harmony-fleet-operator-install")
.await
.map_err(InstallCrdsError::Topology)?;
let inventory = Inventory::empty();
let crds: Vec<CustomResourceDefinition> = vec![Deployment::crd(), Device::crd()];
let score = K8sResourceScore::<CustomResourceDefinition> {
resource: crds,
namespace: None,
};
let interpret = Score::<K8sBareTopology>::create_interpret(&score);
let outcome = interpret
.execute(&inventory, &topology)
.await
.map_err(|e| InstallCrdsError::Interpret(e.to_string()))?;
tracing::info!(?outcome, "CRDs installed");
Ok(())
}

View File

@@ -3,17 +3,12 @@
//! //!
//! Lives in `harmony-fleet-deploy` per ADR-023: deploy logic for any //! Lives in `harmony-fleet-deploy` per ADR-023: deploy logic for any
//! fleet component sits in this crate, not in `harmony` core. The //! fleet component sits in this crate, not in `harmony` core. The
//! operator's CRD types ([`harmony::modules::fleet::operator::crd`]) //! operator's CRD types ([`harmony_fleet_operator::crd`])
//! stay in `harmony` for the moment — a follow-up PR finishes lifting //! stay in `harmony` for the moment — a follow-up PR finishes lifting
//! them out — but the *Score* + chart generator are home here. //! them out — but the *Score* + chart generator are home here.
pub mod chart; pub(crate) mod chart;
pub mod install;
pub mod score; pub mod score;
pub use chart::{ pub use chart::OperatorCredentials;
ChartOptions, OperatorCredentials, RELEASE_NAME, SECRET_KEY_CREDENTIALS_TOML, SECRET_NAME, pub use score::FleetOperatorScore;
build_chart, operator_secret,
};
pub use install::install_crds;
pub use score::{FleetOperatorScore, PublishedChart};

View File

@@ -18,12 +18,12 @@
//! use harmony::inventory::Inventory; //! use harmony::inventory::Inventory;
//! //!
//! let topology = K8sBareTopology::from_kubeconfig("fleet-operator").await?; //! let topology = K8sBareTopology::from_kubeconfig("fleet-operator").await?;
//! let score = FleetOperatorScore::new() //! let score = FleetOperatorScore::new("hub.nationtech.io/harmony/harmony-fleet-operator:dev")
//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev"); //! .messaging(&nats.client_ref());
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?; //! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
//! ``` //! ```
use std::str::FromStr; use std::{str::FromStr, time::Duration};
use async_trait::async_trait; use async_trait::async_trait;
use harmony_types::id::Id; use harmony_types::id::Id;
@@ -36,22 +36,17 @@ use harmony::inventory::Inventory;
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString}; use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
use harmony::modules::k8s::ingress::K8sIngressScore; use harmony::modules::k8s::ingress::K8sIngressScore;
use harmony::modules::k8s::resource::K8sResourceScore; use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::modules::nats::NatsClientRef;
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, Topology}; use harmony::topology::{HelmCommand, K8sclient, Topology};
use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret};
use kube::api::ListParams;
use crate::operator::chart; use crate::operator::chart;
use crate::operator::chart::{ChartOptions, OperatorCredentials, build_chart, operator_secret}; use crate::operator::chart::{
ChartOptions, OperatorCredentials, OperatorIdentityRefs, build_chart, operator_secret,
/// The already-published OCI chart to install (the CD `harmony apply` };
/// path). When set, the operator installs
/// `oci://{registry}/{project}/harmony-fleet-operator-chart:{version}` and the
/// score's `image` is ignored (the image is baked into the chart).
#[derive(Debug, Clone, Serialize)]
pub struct PublishedChart {
pub registry: String,
pub project: String,
pub version: String,
}
/// Declarative install of the harmony fleet operator. Construct via /// Declarative install of the harmony fleet operator. Construct via
/// [`new`](Self::new), tune with the builder-style methods, hand to /// [`new`](Self::new), tune with the builder-style methods, hand to
@@ -66,12 +61,9 @@ pub struct FleetOperatorScore {
pub release_name: String, pub release_name: String,
pub image: String, pub image: String,
pub image_pull_policy: String, pub image_pull_policy: String,
pub nats_url: String, nats: Option<NatsClientRef>,
pub log_level: String, pub log_level: String,
pub credentials: Option<OperatorCredentials>, pub credentials: Option<OperatorCredentials>,
/// `None` renders + installs the chart from local source (dev/e2e);
/// `Some` installs the published OCI chart at a pinned version (CD).
pub published_chart: Option<PublishedChart>,
/// `Some(host)` exposes the UI via an Ingress after install (CD); /// `Some(host)` exposes the UI via an Ingress after install (CD);
/// `None` leaves it cluster-internal (dev/e2e harnesses). /// `None` leaves it cluster-internal (dev/e2e harnesses).
pub operator_ui_host: Option<String>, pub operator_ui_host: Option<String>,
@@ -82,6 +74,8 @@ pub struct FleetOperatorScore {
/// for the pod's `ConfigClient` to read. `None` leaves the dashboard /// for the pod's `ConfigClient` to read. `None` leaves the dashboard
/// unauthenticated (dev/e2e). /// unauthenticated (dev/e2e).
pub web_auth: Option<WebAuth>, pub web_auth: Option<WebAuth>,
pub identity: Option<OperatorIdentityRefs>,
pub image_pull_secret: Option<String>,
} }
/// The dashboard's auth inputs the operator reads via `ConfigClient`. /// The dashboard's auth inputs the operator reads via `ConfigClient`.
@@ -94,22 +88,23 @@ pub struct WebAuth {
impl FleetOperatorScore { impl FleetOperatorScore {
/// Build a score targeting the `fleet-system` namespace with the /// Build a score targeting the `fleet-system` namespace with the
/// dev-default image and NATS URL. Use the builders to override. /// dev-default image and NATS URL. Use the builders to override.
pub fn new() -> Self { pub fn new(image: impl Into<String>) -> Self {
let defaults = ChartOptions::default(); let defaults = ChartOptions::default();
Self { Self {
// FleetOperatorScore's own default; the chart itself is // FleetOperatorScore's own default; the chart itself is
// namespace-neutral. Callers override via `.namespace(..)`. // namespace-neutral. Callers override via `.namespace(..)`.
namespace: "fleet-system".to_string(), namespace: "fleet-system".to_string(),
release_name: "harmony-fleet-operator".to_string(), release_name: "harmony-fleet-operator".to_string(),
image: defaults.image, image: image.into(),
image_pull_policy: defaults.image_pull_policy, image_pull_policy: defaults.image_pull_policy,
nats_url: defaults.nats_url, nats: None,
log_level: defaults.log_level, log_level: defaults.log_level,
credentials: defaults.credentials, credentials: defaults.credentials,
published_chart: None,
operator_ui_host: None, operator_ui_host: None,
cluster_issuer: None, cluster_issuer: None,
web_auth: None, web_auth: None,
identity: None,
image_pull_secret: None,
} }
} }
@@ -133,22 +128,6 @@ impl FleetOperatorScore {
self self
} }
/// Install the published OCI chart at `version` instead of rendering
/// one from local source (the CD `harmony apply` path).
pub fn published_chart(
mut self,
registry: impl Into<String>,
project: impl Into<String>,
version: impl Into<String>,
) -> Self {
self.published_chart = Some(PublishedChart {
registry: registry.into(),
project: project.into(),
version: version.into(),
});
self
}
pub fn namespace(mut self, ns: impl Into<String>) -> Self { pub fn namespace(mut self, ns: impl Into<String>) -> Self {
self.namespace = ns.into(); self.namespace = ns.into();
self self
@@ -159,18 +138,32 @@ impl FleetOperatorScore {
self self
} }
pub fn image(mut self, image: impl Into<String>) -> Self {
self.image = image.into();
self
}
pub fn image_pull_policy(mut self, policy: impl Into<String>) -> Self { pub fn image_pull_policy(mut self, policy: impl Into<String>) -> Self {
self.image_pull_policy = policy.into(); self.image_pull_policy = policy.into();
self self
} }
pub fn nats_url(mut self, url: impl Into<String>) -> Self { pub fn messaging(mut self, nats: &NatsClientRef) -> Self {
self.nats_url = url.into(); self.nats = Some(nats.clone());
self
}
pub fn identity(
mut self,
provider: &OidcProviderRef,
application: &OidcApplicationRef,
machine: &OidcMachineIdentityRef,
) -> Self {
self.identity = Some(OperatorIdentityRefs {
provider: provider.clone(),
application: application.clone(),
machine: machine.clone(),
});
self
}
pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self {
self.image_pull_secret = secret.map(str::to_string);
self self
} }
@@ -190,12 +183,6 @@ impl FleetOperatorScore {
} }
} }
impl Default for FleetOperatorScore {
fn default() -> Self {
Self::new()
}
}
impl<T: Topology + HelmCommand + K8sclient> Score<T> for FleetOperatorScore { impl<T: Topology + HelmCommand + K8sclient> Score<T> for FleetOperatorScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> { fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(FleetOperatorInterpret { Box::new(FleetOperatorInterpret {
@@ -213,6 +200,51 @@ pub struct FleetOperatorInterpret {
score: FleetOperatorScore, score: FleetOperatorScore,
} }
async fn smoke_test_operator<T: K8sclient>(
namespace: &str,
expected_config_hash: &str,
topology: &T,
) -> Result<(), InterpretError> {
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
tokio::time::timeout(Duration::from_secs(180), async {
loop {
let pods = k8s
.list_resources::<Pod>(
Some(namespace),
Some(
ListParams::default()
.labels(&format!("app.kubernetes.io/name={}", chart::RELEASE_NAME)),
),
)
.await;
if let Ok(pods) = pods {
for pod in pods.items {
let has_expected_config = pod
.metadata
.annotations
.as_ref()
.and_then(|annotations| {
annotations.get("harmony.nationtech.io/config-hash")
})
.is_some_and(|hash| hash == expected_config_hash);
if let Some(name) = pod.metadata.name
&& has_expected_config
&& k8s
.pod_logs(namespace, &name, Some(100))
.await
.is_ok_and(|logs| logs.contains("KV bucket ready"))
{
return;
}
}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await
.map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string()))
}
#[async_trait] #[async_trait]
impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterpret { impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterpret {
async fn execute( async fn execute(
@@ -220,6 +252,74 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
inventory: &Inventory, inventory: &Inventory,
topology: &T, topology: &T,
) -> Result<Outcome, InterpretError> { ) -> Result<Outcome, InterpretError> {
let nats_url = self
.score
.nats
.as_ref()
.ok_or_else(|| InterpretError::new("NatsClientRef is required".to_string()))?
.url();
if let Some(identity) = &self.score.identity
&& (identity.application.namespace() != self.score.namespace
|| identity.machine.namespace() != self.score.namespace)
{
return Err(InterpretError::new(format!(
"operator identity outputs must be in namespace '{}'",
self.score.namespace
)));
}
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
k8s.ensure_namespace(&self.score.namespace)
.await
.map_err(|e| {
InterpretError::new(format!(
"failed to ensure operator namespace '{}': {e}",
self.score.namespace
))
})?;
let identity_version = if let Some(identity) = &self.score.identity {
Some(
tokio::time::timeout(Duration::from_secs(180), async {
loop {
let application = k8s
.get_resource::<ConfigMap>(
identity.application.config_map_name(),
Some(identity.application.namespace()),
)
.await
.ok()
.flatten();
let machine = k8s
.get_resource::<Secret>(
identity.machine.secret_name(),
Some(identity.machine.namespace()),
)
.await
.ok()
.flatten();
if let (Some(application), Some(machine)) = (application, machine)
&& let Some(project_id) = application
.data
.as_ref()
.and_then(|data| data.get(identity.application.project_id_key()))
{
return format!(
"{}:{}",
project_id,
machine.metadata.resource_version.unwrap_or_default()
);
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await
.map_err(|_| {
InterpretError::new("timed out waiting for operator identity refs".to_string())
})?,
)
} else {
None
};
let credentials = self.score.credentials.clone();
// Apply the credentials Secret BEFORE the helm install (the // Apply the credentials Secret BEFORE the helm install (the
// chart's Deployment references it via secretKeyRef). Applied // chart's Deployment references it via secretKeyRef). Applied
// directly, not via the chart — it's environment-specific. The // directly, not via the chart — it's environment-specific. The
@@ -236,11 +336,23 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
), ),
None => (None, None), None => (None, None),
}; };
if let Some(creds) = &self.score.credentials let expected_config_hash = chart::config_hash(&ChartOptions {
credentials: credentials.clone(),
web_auth_config_json: web_auth_config_json.clone(),
web_cookie_key_json: web_cookie_key_json.clone(),
identity: self.score.identity.clone(),
identity_version: identity_version.clone(),
image_pull_secret: self.score.image_pull_secret.clone(),
..ChartOptions::default()
});
if let Some(creds) = &credentials
&& let Some(secret) = operator_secret(&ChartOptions { && let Some(secret) = operator_secret(&ChartOptions {
credentials: Some(creds.clone()), credentials: Some(creds.clone()),
web_auth_config_json, web_auth_config_json: web_auth_config_json.clone(),
web_cookie_key_json, web_cookie_key_json: web_cookie_key_json.clone(),
identity: self.score.identity.clone(),
identity_version: identity_version.clone(),
image_pull_secret: self.score.image_pull_secret.clone(),
..ChartOptions::default() ..ChartOptions::default()
}) })
{ {
@@ -254,69 +366,57 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
.await?; .await?;
} }
// Install through HelmChartScore from either the published OCI
// chart (CD) or a freshly rendered local chart (dev/e2e). Each
// branch runs its own install so the local tempdir stays alive
// across it.
let helm_outcome = if let Some(p) = &self.score.published_chart {
let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, chart::CHART_NAME);
info!(
"Installing helm release '{}' from published chart {chart_ref} version {}",
self.score.release_name, p.version
);
HelmChartScore {
namespace: Some(non_blank(&self.score.namespace, "namespace")?),
release_name: non_blank(&self.score.release_name, "release_name")?,
chart_name: non_blank(&chart_ref, "chart_name")?,
chart_version: Some(non_blank(&p.version, "chart_version")?),
values_overrides: None,
values_yaml: None,
create_namespace: true,
install_only: false,
force_conflicts: false,
repository: None,
}
.interpret(inventory, topology)
.await?
} else {
let tmp = tempfile::tempdir() let tmp = tempfile::tempdir()
.map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?; .map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?;
let chart_path = build_chart(&ChartOptions { let chart_path = build_chart(&ChartOptions {
output_dir: tmp.path().to_path_buf(), output_dir: tmp.path().to_path_buf(),
image: self.score.image.clone(), image: self.score.image.clone(),
image_pull_policy: self.score.image_pull_policy.clone(), image_pull_policy: self.score.image_pull_policy.clone(),
nats_url: self.score.nats_url.clone(), nats_url: nats_url.to_string(),
log_level: self.score.log_level.clone(), log_level: self.score.log_level.clone(),
credentials: self.score.credentials.clone(), credentials: credentials.clone(),
chart_version: None, chart_version: None,
// The auth Secret is applied separately above; the web_auth_config_json: web_auth_config_json.clone(),
// rendered chart only needs the Deployment env wiring. web_cookie_key_json: web_cookie_key_json.clone(),
web_auth_config_json: None, identity: self.score.identity.clone(),
web_cookie_key_json: None, identity_version,
image_pull_secret: self.score.image_pull_secret.clone(),
}) })
.map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?; .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
let chart_path_str = chart_path.to_str().ok_or_else(|| { let chart_path_str = chart_path
InterpretError::new("operator chart path is not utf-8".to_string()) .to_str()
})?; .ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?;
info!( let helm_outcome = HelmChartScore {
"Installing helm release '{}' from rendered chart {}",
self.score.release_name, chart_path_str
);
HelmChartScore {
namespace: Some(non_blank(&self.score.namespace, "namespace")?), namespace: Some(non_blank(&self.score.namespace, "namespace")?),
release_name: non_blank(&self.score.release_name, "release_name")?, release_name: non_blank(&self.score.release_name, "release_name")?,
chart_name: non_blank(chart_path_str, "chart_name")?, chart_name: non_blank(chart_path_str, "chart_name")?,
chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?), chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?),
values_overrides: None, values_overrides: None,
values_yaml: None, values_yaml: None,
create_namespace: true, create_namespace: false,
install_only: false, install_only: false,
force_conflicts: false, force_conflicts: false,
repository: None, repository: None,
} }
.interpret(inventory, topology) .interpret(inventory, topology)
.await? .await?;
};
k8s.wait_until_deployment_ready(
&self.score.release_name,
Some(&self.score.namespace),
Some(Duration::from_secs(180)),
)
.await
.map_err(|e| {
InterpretError::new(format!(
"operator deployment {}/{} not ready: {e}",
self.score.namespace, self.score.release_name
))
})?;
if credentials.is_some() || self.score.identity.is_some() {
smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?;
}
// Expose the UI. Applied after the chart so the backing Service // Expose the UI. Applied after the chart so the backing Service
// (shipped in the chart) exists. Skipped when no host is set — // (shipped in the chart) exists. Skipped when no host is set —
@@ -324,7 +424,7 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
let mut details = vec![ let mut details = vec![
format!("operator namespace: {}", self.score.namespace), format!("operator namespace: {}", self.score.namespace),
format!("operator release: {}", self.score.release_name), format!("operator release: {}", self.score.release_name),
format!("operator NATS URL: {}", self.score.nats_url), format!("operator NATS URL: {nats_url}"),
]; ];
if let Some(host) = &self.score.operator_ui_host { if let Some(host) = &self.score.operator_ui_host {
let to_fqdn = |s: &str| { let to_fqdn = |s: &str| {
@@ -383,7 +483,7 @@ mod tests {
#[test] #[test]
fn defaults_target_fleet_system() { fn defaults_target_fleet_system() {
let s = FleetOperatorScore::new(); let s = FleetOperatorScore::new("operator:dev");
assert_eq!(s.namespace, "fleet-system"); assert_eq!(s.namespace, "fleet-system");
assert_eq!(s.release_name, "harmony-fleet-operator"); assert_eq!(s.release_name, "harmony-fleet-operator");
assert_eq!(s.image_pull_policy, "IfNotPresent"); assert_eq!(s.image_pull_policy, "IfNotPresent");
@@ -391,27 +491,19 @@ mod tests {
#[test] #[test]
fn builders_compose() { fn builders_compose() {
let s = FleetOperatorScore::new() let nats = harmony::modules::nats::NatsScore::new("nats", "default").client_ref();
let s = FleetOperatorScore::new("hub.example/op:v1")
.namespace("custom-ns") .namespace("custom-ns")
.image("hub.example/op:v1")
.image_pull_policy("Always") .image_pull_policy("Always")
.nats_url("nats://nats:4222") .messaging(&nats)
.log_level("debug"); .log_level("debug");
assert_eq!(s.namespace, "custom-ns"); assert_eq!(s.namespace, "custom-ns");
assert_eq!(s.image, "hub.example/op:v1"); assert_eq!(s.image, "hub.example/op:v1");
assert_eq!(s.image_pull_policy, "Always"); assert_eq!(s.image_pull_policy, "Always");
assert_eq!(s.nats_url, "nats://nats:4222"); assert_eq!(
s.nats.unwrap().url(),
"nats://nats.default.svc.cluster.local:4222"
);
assert_eq!(s.log_level, "debug"); assert_eq!(s.log_level, "debug");
} }
#[test]
fn credentials_set_on_published_chart_path() {
let s = FleetOperatorScore::new()
.credentials("type = \"zitadel-jwt\"")
.published_chart("hub.example", "proj", "1.2.3");
let creds = s.credentials.expect("credentials wired");
assert_eq!(creds.credentials_toml, "type = \"zitadel-jwt\"");
// The published-chart interpret applies this Secret before install.
assert!(s.published_chart.is_some());
}
} }

View File

@@ -1,119 +0,0 @@
//! Build + publish the operator's image + chart for a tagged release.
//!
//! A release is five commands — docker build/push, hydrate the chart,
//! helm package/push — plus parsing the version out of the git tag.
//! Kept as a plain function with `anyhow` (this is binary glue, not
//! library API). When a second component needs releasing, lift the
//! per-component constants below into a small spec then — not before.
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
use crate::operator::chart::{CHART_NAME, ChartOptions, build_chart};
const TAG_PREFIX: &str = "harmony-fleet-operator-";
const IMAGE_NAME: &str = "harmony-fleet-operator";
const DOCKERFILE: &str = "fleet/harmony-fleet-operator/Dockerfile";
/// Version from a release tag: `harmony-fleet-operator-v0.1.0` → `0.1.0`.
/// One version drives both the image tag and the chart version.
pub fn version_from_tag(tag: &str) -> Result<String> {
let rest = tag
.strip_prefix(TAG_PREFIX)
.with_context(|| format!("tag '{tag}' is not a '{TAG_PREFIX}*' release tag"))?;
let version = rest.strip_prefix('v').unwrap_or(rest);
if version.is_empty() {
bail!("tag '{tag}' has no version after the prefix");
}
Ok(version.to_string())
}
/// Build + push the operator image and helm chart at `version`. With
/// `push = false`, builds + packages only (a local k3d smoke-test).
/// `docker` and `helm` must be on PATH and logged in to the registry.
pub fn release_operator(version: &str, registry: &str, project: &str, push: bool) -> Result<()> {
let image = format!("{registry}/{project}/{IMAGE_NAME}:{version}");
let oci_repo = format!("oci://{registry}/{project}");
log::info!("docker build {image}");
run("docker", &["build", "-f", DOCKERFILE, "-t", &image, "."])?;
if push {
log::info!("docker push {image}");
run("docker", &["push", &image])?;
}
let tmp = tempfile::tempdir().context("chart tempdir")?;
let chart_dir = build_chart(&ChartOptions {
output_dir: tmp.path().to_path_buf(),
image: image.clone(),
chart_version: Some(version.to_string()),
..ChartOptions::default()
})
.map_err(|e| anyhow::anyhow!("hydrate operator chart: {e}"))?;
let tgz = helm_package(&chart_dir, tmp.path())?;
if push {
log::info!("helm push {} {oci_repo}", tgz.display());
run("helm", &["push", path_str(&tgz)?, &oci_repo])?;
}
log::info!("released image={image} chart={oci_repo}/{CHART_NAME}:{version} pushed={push}");
Ok(())
}
fn run(bin: &str, args: &[&str]) -> Result<()> {
let status = Command::new(bin)
.args(args)
.status()
.with_context(|| format!("spawn {bin} (is it installed and in PATH?)"))?;
if !status.success() {
bail!("`{bin} {}` failed ({status})", args.join(" "));
}
Ok(())
}
fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result<PathBuf> {
let output = Command::new("helm")
.args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?])
.output()
.context("spawn helm package")?;
if !output.status.success() {
bail!("helm package failed ({})", output.status);
}
// helm prints "…saved it to: <path>"; the last token is the path.
String::from_utf8_lossy(&output.stdout)
.split_whitespace()
.last()
.map(PathBuf::from)
.context("helm package printed no path")
}
fn path_str(p: &Path) -> Result<&str> {
p.to_str()
.with_context(|| format!("path not utf-8: {}", p.display()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn version_strips_prefix_and_leading_v() {
assert_eq!(
version_from_tag("harmony-fleet-operator-v0.1.0").unwrap(),
"0.1.0"
);
assert_eq!(
version_from_tag("harmony-fleet-operator-2.3.4").unwrap(),
"2.3.4"
);
}
#[test]
fn rejects_foreign_or_empty_tags() {
assert!(version_from_tag("harmony-fleet-agent-v0.1.0").is_err());
assert!(version_from_tag("harmony-fleet-operator-").is_err());
}
}

View File

@@ -1,116 +0,0 @@
//! Secrets and config for the published-chart (CD) operator deploy, via
//! [`harmony_config::ConfigClient`]. SSO-only by construction: no
//! user/pass field exists, so dev-only user/pass auth can't reach a prod
//! deploy. Resolved EnvSource → OpenBao, so the in-cluster runner pulls
//! them at job time with no plaintext on the pod.
use harmony_config::Config;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// `#[config(secret)]` keeps every field out of cleartext SQLite and
/// masks it in logs.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
#[config(secret)]
pub struct FleetDeploySecrets {
/// Operator's zitadel-jwt `[credentials]` TOML — its NATS auth. The
/// CD path's only credential; there is no user/pass fallback.
pub operator_credentials_toml: String,
/// Scoped fleet-deployer credential, fetched at job time so the
/// runner pod holds no standing permissions. `None` uses the ambient
/// `KUBECONFIG`.
#[serde(default)]
pub kubeconfig: Option<String>,
/// Operator dashboard session-cookie signing key (standard-base64 of
/// ≥64 bytes). `#[serde(default)]` so older seeds without it still
/// load — but the dashboard won't authenticate until it's set.
#[serde(default)]
pub operator_cookie_key_b64: String,
}
/// Non-secret deploy config: k8s namespaces + chart coords. Loaded via
/// `ConfigClient::for_namespace("fleet-staging")` alongside `FleetDeploySecrets`.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
pub struct FleetDeployConfig {
/// K8s namespace where the operator, NATS, and callout live.
pub namespace: String,
/// Full NATS URL the operator connects to (e.g. `nats://fleet-nats.fleet-staging:4222`).
pub nats_url: String,
/// K8s namespace where Zitadel lives (for operator UI SSO).
pub zitadel_namespace: String,
/// K8s namespace where OpenBao lives (for operator secret fetching).
pub openbao_namespace: String,
/// OCI chart registry (e.g. `hub.nationtech.io`).
pub operator_chart_registry: String,
/// OCI chart project (e.g. `harmony`).
pub operator_chart_project: String,
/// Public base domain. The operator UI host is derived as
/// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`.
pub base_domain: String,
/// cert-manager `ClusterIssuer` for the operator UI's TLS cert.
pub cluster_issuer: String,
/// Zitadel OIDC app the operator dashboard authenticates against.
#[serde(default)]
pub operator_oidc_client_id: String,
/// Token audiences the dashboard accepts (Zitadel project/app ids).
#[serde(default)]
pub operator_trusted_audiences: Vec<String>,
}
impl Default for FleetDeployConfig {
fn default() -> Self {
Self {
namespace: "fleet-staging".to_string(),
nats_url: "nats://fleet-nats.fleet-staging:4222".to_string(),
zitadel_namespace: "zitadel-staging".to_string(),
openbao_namespace: "openbao-staging".to_string(),
operator_chart_registry: "hub.nationtech.io".to_string(),
operator_chart_project: "harmony".to_string(),
base_domain: "cb1.nationtech.io".to_string(),
cluster_issuer: "letsencrypt-prod".to_string(),
operator_oidc_client_id: String::new(),
operator_trusted_audiences: Vec::new(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use harmony_config::ConfigClass;
#[test]
fn secret_class_and_optional_kubeconfig() {
let s: FleetDeploySecrets =
serde_json::from_str(r#"{ "operator_credentials_toml": "x" }"#).unwrap();
assert!(s.kubeconfig.is_none());
// Secrets must never land in cleartext SQLite.
assert_eq!(FleetDeploySecrets::CLASS, ConfigClass::Secret);
}
#[test]
fn config_class_is_not_secret() {
assert_eq!(FleetDeployConfig::CLASS, ConfigClass::Standard);
}
#[test]
fn config_defaults() {
let c = FleetDeployConfig::default();
assert_eq!(c.namespace, "fleet-staging");
assert_eq!(c.zitadel_namespace, "zitadel-staging");
assert_eq!(c.openbao_namespace, "openbao-staging");
assert_eq!(c.base_domain, "cb1.nationtech.io");
assert_eq!(c.cluster_issuer, "letsencrypt-prod");
}
}

View File

@@ -1,234 +0,0 @@
//! [`FleetServerScore`] — one-shot install of harmony's fleet
//! server-side stack into a Kubernetes cluster.
//!
//! Composes:
//! 1. [`NatsBasicScore`](crate::modules::nats::NatsBasicScore) — single-node
//! NATS with JetStream, in-cluster.
//! 2. [`FleetOperatorScore`](super::FleetOperatorScore) — operator
//! Deployment + CRDs (Deployment / Device) + RBAC.
//! 3. (Optional) [`ZitadelScore`] — central OIDC identity provider
//! (Postgres + Zitadel helm chart). Set `identity = Some(...)` to
//! install; `None` skips it. Adds the `K8sclient + PostgreSQL`
//! capability requirements to the topology either way (they're
//! static trait bounds; pick a topology that supplies them, e.g.
//! [`K8sAnywhereTopology`](crate::topology::K8sAnywhereTopology)).
//!
//! Argo is the next follow-up — same pattern, optional field, hangs
//! off the same sequential chain in [`Interpret::execute`].
//!
//! NATS goes first deliberately: the operator pod's `NATS_URL`
//! environment points at the in-cluster NATS service, so installing
//! the operator first leads to a few minutes of `connect_with_retry`
//! noise in the operator log until NATS comes up.
//!
//! Idempotency: each inner Score is itself idempotent (NatsBasicScore
//! delegates to the upstream `nats/nats` chart via HelmChartScore,
//! FleetOperatorScore pins `chart_version`, ZitadelScore relies on
//! HelmChartScore + CNPG idempotency). Running this Score twice is
//! safe and produces no churn.
//!
//! Typical usage:
//!
//! ```ignore
//! use harmony::modules::fleet::{FleetOperatorScore, FleetServerScore};
//! use harmony::modules::nats::NatsBasicScore;
//! use harmony::modules::zitadel::ZitadelScore;
//! use harmony::score::Score;
//! use harmony::inventory::Inventory;
//! use harmony::topology::K8sAnywhereTopology;
//!
//! let topology = K8sAnywhereTopology::from_env();
//! let nats = NatsBasicScore::new("fleet-nats", "fleet-system").load_balancer();
//! let operator = FleetOperatorScore::new()
//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev");
//! let mut score = FleetServerScore::new(nats, operator);
//! score.identity = Some(ZitadelScore {
//! host: "zitadel.localhost".into(),
//! external_secure: false,
//! ..Default::default()
//! });
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
//! ```
use async_trait::async_trait;
use harmony::data::Version;
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use harmony::inventory::Inventory;
use harmony::modules::nats::capability::Nats;
use harmony::modules::nats::score_nats_k8s::NatsK8sScore;
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::postgresql::capability::PostgreSQL;
use harmony::modules::zitadel::{ZitadelScore, ZitadelSetupScore};
use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, TlsRouter, Topology};
use harmony_types::id::Id;
use serde::Serialize;
use crate::operator::FleetOperatorScore;
/// Install the fleet server-side stack on Kubernetes: NATS (via
/// `NatsK8sScore`, OKD-aware Routes), operator, optionally Zitadel
/// (helm + post-install identity setup), optionally the auth callout.
///
/// Execution order is deliberate:
///
/// 1. `identity` (Zitadel helm chart + cert-manager Route) FIRST
/// so the Zitadel API is reachable when `identity_setup` runs.
/// 2. `identity_setup` provisions project + roles + machine users
/// via the Zitadel management API. Produces the keys the
/// operator needs to authenticate.
/// 3. `nats` brings up the NATS pod. When the score's
/// `auth_callout` field is `Some`, the rendered helm values
/// include the matching `authorization.auth_callout` block
/// pointing at the (still-missing) callout pod's issuer.
/// 4. `auth_callout` deploys the callout pod itself. Once it's
/// Ready, NATS auth handshakes start succeeding.
/// 5. `operator` last. By this point Zitadel, NATS, and the
/// callout are all up; the operator's pod boots, mints its
/// Zitadel token, connects through the callout, watches its
/// CRDs.
#[derive(Debug, Clone, Serialize)]
pub struct FleetServerScore {
pub nats: NatsK8sScore,
pub operator: FleetOperatorScore,
/// Helm install of Zitadel + Postgres. Required when the rest
/// of the chain wants Zitadel-backed auth.
pub identity: Option<ZitadelScore>,
/// Post-install Zitadel API calls — project, roles, machine
/// users, JWT keys. Without this the operator has no machine
/// user to authenticate as.
pub identity_setup: Option<ZitadelSetupScore>,
/// Auth callout deployment. Pair with
/// `nats.auth_callout = Some(...)` so the rendered NATS helm
/// values delegate to this callout's issuer pubkey.
pub auth_callout: Option<NatsAuthCalloutScore>,
// TODO(argo): pub gitops: Option<ArgoScore>,
}
impl FleetServerScore {
pub fn new(nats: NatsK8sScore, operator: FleetOperatorScore) -> Self {
Self {
nats,
operator,
identity: None,
identity_setup: None,
auth_callout: None,
}
}
}
impl<T: Topology + HelmCommand + Nats + K8sclient + PostgreSQL + TlsRouter + 'static> Score<T>
for FleetServerScore
{
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(FleetServerInterpret {
score: self.clone(),
})
}
fn name(&self) -> String {
"FleetServerScore".to_string()
}
}
#[derive(Debug)]
pub struct FleetServerInterpret {
score: FleetServerScore,
}
#[async_trait]
impl<T: Topology + HelmCommand + Nats + K8sclient + PostgreSQL + TlsRouter + 'static> Interpret<T>
for FleetServerInterpret
{
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
// Use Score::interpret (not create_interpret().execute()) so each
// inner score fires its own InterpretExecutionStarted/Finished
// events — that's what gives the cli_logger the per-score
// progress lines and what surfaces the inner outcomes' details
// in the cli_reporter end-of-run summary.
// 1. Identity: Zitadel helm chart needs to be up before the
// setup score can talk to its API.
if let Some(identity) = &self.score.identity {
identity.interpret(inventory, topology).await?;
}
// 2. Identity setup: project + roles + machine users via the
// Zitadel management API. Output (machine keys) feeds the
// operator's credentials at apply time. The operator
// Score has those bytes already (callers wire them in via
// `OperatorCredentials.credentials_toml` before passing
// `FleetServerScore` to the framework).
if let Some(setup) = &self.score.identity_setup {
setup.interpret(inventory, topology).await?;
}
// 3. NATS: helm-install NATS with whatever auth_callout
// config was baked into `nats` already.
self.score.nats.interpret(inventory, topology).await?;
// 4. Auth callout deployment.
if let Some(callout) = &self.score.auth_callout {
callout.interpret(inventory, topology).await?;
}
// 5. Operator last so it doesn't waste reconnect attempts
// while NATS / callout are still coming up. The pod's own
// `connect_with_retry` handles the small remaining race.
self.score.operator.interpret(inventory, topology).await?;
// TODO(argo): chain GitOps install here.
// Hand the cli_reporter a few next-step pointers so the
// post-install summary at the bottom of the run is genuinely
// useful.
let mut details = vec![
format!(
"NATS in-cluster URL: nats://{}.{}:4222",
self.score.nats.cluster.name, self.score.nats.cluster.namespace
),
format!(
"Operator deployment: kubectl -n {} get deploy",
self.score.operator.namespace
),
"CRDs: kubectl get crd | grep fleet.nationtech.io".to_string(),
];
if let Some(ws) = &self.score.nats.websocket {
details.push(format!("NATS WS public URL: wss://{}/", ws.host));
}
if let Some(identity) = &self.score.identity {
let scheme = if identity.external_secure {
"https"
} else {
"http"
};
details.push(format!(
"Zitadel: {scheme}://{}/ (kubectl -n zitadel get all)",
identity.host
));
}
if self.score.auth_callout.is_some() {
details
.push("Auth callout: kubectl -n fleet-system get deploy/fleet-callout".to_string());
}
Ok(Outcome::success_with_details(
"Fleet server stack installed".to_string(),
details,
))
}
fn get_name(&self) -> InterpretName {
InterpretName::Custom("FleetServerInterpret")
}
fn get_version(&self) -> Version {
Version::from("0.1.0").expect("static version literal")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::QUEUED
}
fn get_children(&self) -> Vec<Id> {
vec![]
}
}

View File

@@ -51,6 +51,7 @@ harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony-fleet-deploy = { path = "../harmony-fleet-deploy" } harmony-fleet-deploy = { path = "../harmony-fleet-deploy" }
harmony-fleet-operator = { path = "../harmony-fleet-operator" } harmony-fleet-operator = { path = "../harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony_secret = { path = "../../harmony_secret" }
harmony-k8s = { path = "../../harmony-k8s" } harmony-k8s = { path = "../../harmony-k8s" }
harmony_types = { path = "../../harmony_types" } harmony_types = { path = "../../harmony_types" }
k3d-rs = { path = "../../k3d" } k3d-rs = { path = "../../k3d" }

View File

@@ -24,22 +24,20 @@ use std::time::Duration;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
use harmony::modules::k8s::resource::K8sResourceScore; use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService};
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{ use harmony::modules::openbao::{
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
}; };
use harmony::modules::postgresql::K8sPostgreSQLScore;
use harmony::modules::zitadel::{ use harmony::modules::zitadel::{
MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef, ZitadelAppType,
ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials, ZitadelClientConfig, ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials,
}; };
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_fleet_deploy::FleetNatsScore;
use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode}; use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode};
use k8s_openapi::api::core::v1::{Secret, Service, ServicePort, ServiceSpec}; use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta;
use nkeys::KeyPair;
use serde::Deserialize; use serde::Deserialize;
use crate::images::CALLOUT_IMAGE_TAG; use crate::images::CALLOUT_IMAGE_TAG;
@@ -70,9 +68,9 @@ pub const ADMIN_USERNAME: &str = "e2e-admin";
const NATS_ACCOUNT: &str = "FLEET"; const NATS_ACCOUNT: &str = "FLEET";
const NATS_AUTH_USER: &str = "auth"; const NATS_AUTH_USER: &str = "auth";
const OPERATOR_APP: &str = "harmony-fleet-operator";
// Dev-only static credential for the callout's own NATS user, same // Dev-only static credential for the callout's own NATS user, same
// posture as the user-pass mode creds in `stack.rs`. // posture as the user-pass mode creds in `stack.rs`.
const NATS_AUTH_PASS: &str = "e2e-callout-auth";
const CALLOUT_DEPLOYMENT: &str = "fleet-callout"; const CALLOUT_DEPLOYMENT: &str = "fleet-callout";
/// Handles the callout bring-up leaves behind for tests. /// Handles the callout bring-up leaves behind for tests.
@@ -87,6 +85,9 @@ pub struct CalloutHandles {
/// Zitadel IAM admin PAT (from the chart's `iam-admin-pat` /// Zitadel IAM admin PAT (from the chart's `iam-admin-pat`
/// Secret) — used to enroll device machine users. /// Secret) — used to enroll device machine users.
pub admin_pat: String, pub admin_pat: String,
pub provider: OidcProviderRef,
pub application: OidcApplicationRef,
pub operator_identity: OidcMachineIdentityRef,
pub openbao: OpenbaoInstance, pub openbao: OpenbaoInstance,
pub openbao_root_token: String, pub openbao_root_token: String,
} }
@@ -95,72 +96,34 @@ pub fn issuer_url() -> String {
format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}") format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}")
} }
/// NodePort Service in front of Zitadel with in-cluster port ==
/// nodePort == [`ZITADEL_PORT`] (targetPort = the chart's 8080), so
/// the issuer URL's port number is valid both inside and outside the
/// cluster.
fn zitadel_nodeport_service() -> Service {
let selector = std::collections::BTreeMap::from([
("app.kubernetes.io/name".to_string(), "zitadel".to_string()),
(
"app.kubernetes.io/instance".to_string(),
"zitadel".to_string(),
),
]);
Service {
metadata: ObjectMeta {
name: Some("zitadel-e2e".to_string()),
namespace: Some(ZITADEL_NAMESPACE.to_string()),
..Default::default()
},
spec: Some(ServiceSpec {
type_: Some("NodePort".to_string()),
selector: Some(selector),
ports: Some(vec![ServicePort {
name: Some("http".to_string()),
port: ZITADEL_PORT as i32,
target_port: Some(
k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(8080),
),
node_port: Some(ZITADEL_PORT as i32),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
}
}
/// Deploy Zitadel + project/roles/admin-user, returning everything /// Deploy Zitadel + project/roles/admin-user, returning everything
/// except the OpenBao handles (filled by [`deploy_openbao`]). /// except the OpenBao handles (filled by [`deploy_openbao`]).
pub async fn bring_up_zitadel( pub async fn bring_up_zitadel(
topology: &K8sAnywhereTopology, topology: &K8sAnywhereTopology,
kube_client: &kube::Client, kube_client: &kube::Client,
outputs_namespace: &str,
) -> Result<CalloutHandles> { ) -> Result<CalloutHandles> {
ZitadelScore { let postgres = K8sPostgreSQLScore::new(ZITADEL_NAMESPACE).cluster_name("zitadel-pg");
host: ZITADEL_HOST.to_string(), let database = postgres.root_account_ref();
zitadel_version: "v4.12.1".to_string(), postgres
external_secure: false, .interpret(&Inventory::autoload(), topology)
external_port: Some(ZITADEL_PORT as u32), .await
namespace: ZITADEL_NAMESPACE.to_string(), .map_err(|e| anyhow::anyhow!("Zitadel PostgreSQL deploy: {e}"))?;
..Default::default()
} let zitadel = ZitadelScore::new(ZITADEL_HOST, ZITADEL_NAMESPACE)
.http(Some(ZITADEL_PORT.into()))
.database(database)
.node_port(ZITADEL_PORT);
let provider = zitadel.provider_ref();
zitadel
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
.await .await
.map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?; .map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?;
K8sResourceScore::single(
zitadel_nodeport_service(),
Some(ZITADEL_NAMESPACE.to_string()),
)
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("Zitadel NodePort service: {e}"))?;
CoreDNSRewriteScore { CoreDNSRewriteScore {
rewrites: vec![CoreDNSRewrite { rewrites: vec![CoreDNSRewrite {
hostname: ZITADEL_HOST.to_string(), hostname: ZITADEL_HOST.to_string(),
target: format!("zitadel-e2e.{ZITADEL_NAMESPACE}.svc.cluster.local"), target: format!("zitadel-nodeport.{ZITADEL_NAMESPACE}.svc.cluster.local"),
}], }],
} }
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
@@ -169,43 +132,23 @@ pub async fn bring_up_zitadel(
wait_for_zitadel_ready().await?; wait_for_zitadel_ready().await?;
let roles = [ADMIN_ROLE, DEVICE_ROLE] let mut identity =
ZitadelSetupScore::for_provider(&provider, ZITADEL_NAMESPACE, outputs_namespace)
.application(PROJECT_NAME, OPERATOR_APP, ZitadelAppType::DeviceCode)
.api_application(PROJECT_NAME, "nats")
.machine_identity(PROJECT_NAME, ADMIN_USERNAME, "E2E Admin", [ADMIN_ROLE])
.groups_claim();
identity.endpoint = Some(format!("http://127.0.0.1:{ZITADEL_PORT}"));
identity.skip_tls = true;
for role in [ADMIN_ROLE, DEVICE_ROLE]
.into_iter() .into_iter()
.chain(GROUP_ROLES.iter().copied()) .chain(GROUP_ROLES.iter().copied())
.map(|key| ZitadelRole { {
project_name: PROJECT_NAME.to_string(), identity = identity.role(PROJECT_NAME, role, role);
key: key.to_string(),
display_name: key.to_string(),
group: None,
})
.collect();
ZitadelSetupScore {
machine_users: vec![ZitadelMachineUser {
username: ADMIN_USERNAME.to_string(),
name: "E2E Admin".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(PROJECT_NAME.to_string()),
grant_roles: vec![ADMIN_ROLE.to_string()],
}],
roles,
// The flattening Action turns project role keys into the
// `groups` array claim OpenBao binds against (ADR-025) — this
// bring-up is its live verification.
groups_claim_action: true,
human_users: vec![],
org_members: vec![],
instance_members: vec![],
machine_secrets: vec![],
port_forward_service: None,
applications: vec![],
api_apps: vec![ZitadelApiApp {
project_name: PROJECT_NAME.to_string(),
app_name: "nats".to_string(),
}],
..zitadel_connection()
} }
let application = identity.application_ref(OPERATOR_APP);
let operator_identity = identity.machine_identity_ref(ADMIN_USERNAME);
identity
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
.await .await
.map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?; .map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?;
@@ -228,6 +171,9 @@ pub async fn bring_up_zitadel(
project_id, project_id,
admin_machine_key, admin_machine_key,
admin_pat, admin_pat,
provider,
application,
operator_identity,
// Placeholder until deploy_openbao fills them in. // Placeholder until deploy_openbao fills them in.
openbao: OpenbaoInstance { openbao: OpenbaoInstance {
namespace: String::new(), namespace: String::new(),
@@ -281,7 +227,7 @@ pub async fn enroll_device(
} }
/// Deploy NATS in callout mode + the callout service itself. /// Deploy NATS in callout mode + the callout service itself.
/// Returns the `FleetNatsScore` so the caller can derive the /// Returns the `NatsScore` so the caller can derive the
/// in-cluster URL the same way the user-pass path does. /// in-cluster URL the same way the user-pass path does.
pub async fn deploy_nats_callout( pub async fn deploy_nats_callout(
topology: &K8sAnywhereTopology, topology: &K8sAnywhereTopology,
@@ -289,44 +235,38 @@ pub async fn deploy_nats_callout(
release: &str, release: &str,
node_port: u16, node_port: u16,
handles: &CalloutHandles, handles: &CalloutHandles,
) -> Result<FleetNatsScore> { ) -> Result<NatsScore> {
let issuer_kp = KeyPair::new_account(); let credentials = NatsAuthCalloutCredentialsScore::generated(
let issuer_seed = issuer_kp namespace,
.seed() format!("{CALLOUT_DEPLOYMENT}-credentials"),
.map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?; );
let credentials_ref = credentials.credentials_ref();
let nats = FleetNatsScore::callout( credentials
namespace.to_string(), .interpret(&Inventory::autoload(), topology)
node_port, .await
issuer_kp.public_key(), .map_err(|e| anyhow::anyhow!("callout credentials deploy: {e}"))?;
let nats = NatsScore::callout_account(
release,
namespace,
NatsService::NodePort(node_port),
NATS_ACCOUNT, NATS_ACCOUNT,
);
let callout = NatsAuthCalloutScore::for_account(
CALLOUT_DEPLOYMENT,
namespace,
&nats.account_ref(),
NATS_AUTH_USER, NATS_AUTH_USER,
NATS_AUTH_PASS,
) )
.release_name(release); .credentials(&credentials_ref)
.with_oidc(&handles.provider, &handles.application)
.image(CALLOUT_IMAGE_TAG)
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE)
.device_id_claim("client_id");
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
nats.interpret(&Inventory::autoload(), topology) nats.interpret(&Inventory::autoload(), topology)
.await .await
.map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?; .map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?;
let mut callout = NatsAuthCalloutScore::new(
CALLOUT_DEPLOYMENT,
namespace,
nats.in_cluster_url(),
handles.issuer_url.clone(),
handles.project_id.clone(),
NATS_AUTH_USER,
NATS_AUTH_PASS,
issuer_seed,
)
.image(CALLOUT_IMAGE_TAG)
.target_account(NATS_ACCOUNT)
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE);
// Machine-user access tokens carry the userName in `client_id`;
// usernames ARE device ids here (see `enroll_device`).
callout.device_id_claim = "client_id".to_string();
callout.device_id_prefix_strip = String::new();
callout.roles_claim = format!("urn:zitadel:iam:org:project:{}:roles", handles.project_id);
callout callout
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
.await .await
@@ -335,10 +275,6 @@ pub async fn deploy_nats_callout(
Ok(nats) Ok(nats)
} }
pub fn callout_deployment_name() -> &'static str {
CALLOUT_DEPLOYMENT
}
/// Deploy OpenBao into the bring-up namespace: JWT auth bound to the /// Deploy OpenBao into the bring-up namespace: JWT auth bound to the
/// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging /// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging
/// shape) plus a NodePort Service so VMs reach it through the libvirt /// shape) plus a NodePort Service so VMs reach it through the libvirt
@@ -352,71 +288,22 @@ pub async fn deploy_openbao(
namespace: namespace.to_string(), namespace: namespace.to_string(),
release: OPENBAO_RELEASE.to_string(), release: OPENBAO_RELEASE.to_string(),
}; };
OpenbaoScore { OpenbaoScore::new(namespace, OPENBAO_RELEASE, "openbao-e2e.local")
instance: instance.clone(), .node_port(OPENBAO_NODE_PORT)
host: "openbao-e2e.local".to_string(),
openshift: false,
tls_issuer: None,
}
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
.await .await
.map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?; .map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?;
OpenbaoSetupScore { OpenbaoSetupScore::new(instance.clone())
instance: instance.clone(), .with_oidc_application(
kv_mount: KV_MOUNT.to_string(), &handles.provider,
jwt_auth: Some(OpenbaoJwtAuth { &handles.application,
oidc_discovery_url: handles.issuer_url.clone(), OpenbaoJwtAuth::oidc("fleet-device"),
jwt_validation_pubkeys: String::new(), )
bound_issuer: handles.issuer_url.clone(),
role_name: "fleet-device".to_string(),
bound_audiences: handles.project_id.clone(),
user_claim: "sub".to_string(),
groups_claim: "groups".to_string(),
token_type: "batch".to_string(),
ttl: "1h".to_string(),
max_ttl: "1h".to_string(),
}),
..Default::default()
}
.interpret(&Inventory::autoload(), topology) .interpret(&Inventory::autoload(), topology)
.await .await
.map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?; .map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?;
// NodePort for VM access. Same selector shape as the chart's own
// ClusterIP Service.
let selector = std::collections::BTreeMap::from([
("app.kubernetes.io/name".to_string(), "openbao".to_string()),
(
"app.kubernetes.io/instance".to_string(),
OPENBAO_RELEASE.to_string(),
),
("component".to_string(), "server".to_string()),
]);
let svc = Service {
metadata: ObjectMeta {
name: Some("openbao-nodeport".to_string()),
namespace: Some(namespace.to_string()),
..Default::default()
},
spec: Some(ServiceSpec {
type_: Some("NodePort".to_string()),
selector: Some(selector),
ports: Some(vec![ServicePort {
name: Some("http".to_string()),
port: 8200,
node_port: Some(OPENBAO_NODE_PORT as i32),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
};
K8sResourceScore::single(svc, Some(namespace.to_string()))
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("OpenBao NodePort service: {e}"))?;
handles.openbao = instance.clone(); handles.openbao = instance.clone();
handles.openbao_root_token = handles.openbao_root_token =
cached_root_token(&instance).map_err(|e| anyhow::anyhow!("OpenBao root token: {e}"))?; cached_root_token(&instance).map_err(|e| anyhow::anyhow!("OpenBao root token: {e}"))?;

View File

@@ -17,11 +17,11 @@ use std::time::{Duration, Instant};
use async_nats::Client; use async_nats::Client;
use async_nats::jetstream::{self, kv}; use async_nats::jetstream::{self, kv};
use harmony::modules::podman::{PodmanV0Score, ReconcileScore};
use harmony_reconciler_contracts::{ use harmony_reconciler_contracts::{
BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase, BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase,
desired_state_key, device_state_key, desired_state_key, device_state_key,
}; };
use harmony_reconciler_contracts::{PodmanV0Score, ReconcileScore};
use thiserror::Error; use thiserror::Error;
/// One handle per test, opens the buckets the test reads/writes. /// One handle per test, opens the buckets the test reads/writes.

View File

@@ -1,6 +1,6 @@
//! In-cluster end-to-end harness for the harmony fleet stack. //! In-cluster end-to-end harness for the harmony fleet stack.
//! //!
//! Brings up the fleet stack ([`FleetNatsScore`] + //! Brings up the fleet stack ([`NatsScore`] +
//! [`FleetAgentScore`]) in a fresh per-bring-up k3d namespace and //! [`FleetAgentScore`]) in a fresh per-bring-up k3d namespace and
//! hands the test a connected NATS admin client + the device ids the //! hands the test a connected NATS admin client + the device ids the
//! agents are running under. Bring-up is shared across tests in the //! agents are running under. Bring-up is shared across tests in the
@@ -10,7 +10,7 @@
//! Per ADR-023, the harness composes the same `*Score` types the //! Per ADR-023, the harness composes the same `*Score` types the
//! production CLI uses. It does **not** build manifests itself. //! production CLI uses. It does **not** build manifests itself.
//! //!
//! [`FleetNatsScore`]: harmony_fleet_deploy::FleetNatsScore //! [`NatsScore`]: harmony::modules::nats::NatsScore
//! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore //! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore
pub mod callout; pub mod callout;

View File

@@ -14,9 +14,13 @@
use std::sync::Arc; use std::sync::Arc;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use harmony::inventory::Inventory;
use harmony::modules::k8s::namespace::K8sNamespaceScore;
use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology;
use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::api::core::v1::Namespace;
use kube::Client; use kube::Client;
use kube::api::{Api, ObjectMeta, PostParams}; use kube::api::Api;
/// Env var that, when set to `"1"`, prevents the guard from deleting /// Env var that, when set to `"1"`, prevents the guard from deleting
/// the namespace on drop. Convenience for debugging hung tests. /// the namespace on drop. Convenience for debugging hung tests.
@@ -43,24 +47,15 @@ impl std::fmt::Debug for NamespaceGuard {
} }
impl NamespaceGuard { impl NamespaceGuard {
pub(crate) async fn create(client: Arc<Client>, namespace: String) -> Result<Self> { pub(crate) async fn create(
let api: Api<Namespace> = Api::all((*client).clone()); client: Arc<Client>,
let ns = Namespace { topology: &K8sAnywhereTopology,
metadata: ObjectMeta { namespace: String,
name: Some(namespace.clone()), ) -> Result<Self> {
labels: Some( K8sNamespaceScore::new(&namespace)
[ .label("harmony.io/managed-by", "fleet-e2e")
("harmony.io/managed-by".to_string(), "fleet-e2e".to_string()), .label("harmony.io/scope", "ephemeral")
("harmony.io/scope".to_string(), "ephemeral".to_string()), .interpret(&Inventory::autoload(), topology)
]
.into_iter()
.collect(),
),
..Default::default()
},
..Default::default()
};
api.create(&PostParams::default(), &ns)
.await .await
.with_context(|| format!("creating namespace {namespace}"))?; .with_context(|| format!("creating namespace {namespace}"))?;
let keep = matches!( let keep = matches!(

View File

@@ -4,7 +4,7 @@
//! same `*Score` types production uses. The only thing this module //! same `*Score` types production uses. The only thing this module
//! owns is the *plumbing* around them: ensure a k3d cluster exists, //! owns is the *plumbing* around them: ensure a k3d cluster exists,
//! sideload the agent image into it, mint a per-bring-up namespace, //! sideload the agent image into it, mint a per-bring-up namespace,
//! compose [`FleetNatsScore`] + [`FleetAgentScore`] against the test //! compose [`NatsScore`] + [`FleetAgentScore`] against the test
//! Topology, wait for pods to be Ready, and hand the test an admin //! Topology, wait for pods to be Ready, and hand the test an admin
//! NATS client. //! NATS client.
//! //!
@@ -24,15 +24,14 @@ use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::operator::crd::Deployment as FleetDeployment; use harmony::modules::nats::{NatsScore, NatsService, UserPassCredentials};
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_fleet_deploy::agent::PodTarget; use harmony_fleet_deploy::agent::PodTarget;
use harmony_fleet_deploy::nats::UserPassCredentials;
use harmony_fleet_deploy::operator::OperatorCredentials; use harmony_fleet_deploy::operator::OperatorCredentials;
use harmony_fleet_deploy::{FleetAgentScore, FleetNatsScore, FleetOperatorScore}; use harmony_fleet_deploy::{FleetAgentScore, FleetOperatorScore};
use harmony_fleet_operator::Deployment as FleetDeployment;
use k3d_rs::{K3d, PortMapping}; use k3d_rs::{K3d, PortMapping};
use k8s_openapi::api::apps::v1::Deployment;
use kube::Client; use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams}; use kube::api::{Api, DeleteParams, Patch, PatchParams};
use thiserror::Error; use thiserror::Error;
@@ -182,14 +181,10 @@ impl Stack {
/// use [`shared_stack`]. /// use [`shared_stack`].
pub async fn bring_up(opts: StackOptions) -> Result<Self, BringUpError> { pub async fn bring_up(opts: StackOptions) -> Result<Self, BringUpError> {
let callout_mode = opts.auth_mode == AuthMode::Callout; let callout_mode = opts.auth_mode == AuthMode::Callout;
// Static user-pass material is what the Pod agent + operator // Pod agents still use static credentials; callout-mode devices are VMs.
// paths inject today; behind the callout they'd be rejected if callout_mode && opts.num_devices > 0 {
// at connect. The VM harness brings callout-authenticated
// agents instead.
if callout_mode && (opts.num_devices > 0 || opts.deploy_operator) {
return Err(BringUpError::Callout(anyhow::anyhow!( return Err(BringUpError::Callout(anyhow::anyhow!(
"callout mode supports infra-only bring-up (VM agents); \ "callout mode uses VM agents; Pod agents still authenticate with static creds"
Pod agents / operator still authenticate with static creds"
))); )));
} }
@@ -236,12 +231,18 @@ impl Stack {
prune_stale_operator_cluster_resources(&kube_client).await; prune_stale_operator_cluster_resources(&kube_client).await;
} }
let namespace = unique_namespace();
let guard = NamespaceGuard::create(kube_client.clone(), &topology, namespace.clone())
.await
.map_err(BringUpError::Namespace)?;
tracing::info!(namespace = %namespace, "namespace created");
// Zitadel + project/roles/admin identity. Sits outside the // Zitadel + project/roles/admin identity. Sits outside the
// per-bring-up namespace (stable `zitadel` ns) so the // per-bring-up namespace (stable `zitadel` ns) so the
// multi-minute first install amortizes across runs. // multi-minute first install amortizes across runs.
let mut callout_handles = if callout_mode { let mut callout_handles = if callout_mode {
Some( Some(
crate::callout::bring_up_zitadel(&topology, &kube_client) crate::callout::bring_up_zitadel(&topology, &kube_client, &namespace)
.await .await
.map_err(BringUpError::Callout)?, .map_err(BringUpError::Callout)?,
) )
@@ -249,15 +250,9 @@ impl Stack {
None None
}; };
let namespace = unique_namespace();
let guard = NamespaceGuard::create(kube_client.clone(), namespace.clone())
.await
.map_err(BringUpError::Namespace)?;
tracing::info!(namespace = %namespace, "namespace created");
// Deploy NATS via the production Score. The harness owns no // Deploy NATS via the production Score. The harness owns no
// bespoke nats-server.conf, no NodePort Service — that's all // bespoke nats-server.conf, no NodePort Service — that's all
// inside `FleetNatsScore`. NATS readiness is gated by the // inside `NatsScore`. NATS readiness is gated by the
// host-side `connect_nats_admin` retry loop further down (the // host-side `connect_nats_admin` retry loop further down (the
// upstream NATS chart deploys a StatefulSet, not a Deployment, // upstream NATS chart deploys a StatefulSet, not a Deployment,
// so a generic Deployment-poll wouldn't match; the agent's // so a generic Deployment-poll wouldn't match; the agent's
@@ -267,19 +262,20 @@ impl Stack {
// deploy crate — keeping them out of `harmony-fleet-deploy` // deploy crate — keeping them out of `harmony-fleet-deploy`
// means production builds can't accidentally ship them. // means production builds can't accidentally ship them.
let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}"); let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}");
let (nats_client, nats_in_cluster_url) = match callout_handles.as_mut() { let (nats_client, nats_ref) = match callout_handles.as_mut() {
None => { None => {
let nats = FleetNatsScore::user_pass( let nats = NatsScore::user_pass(
NATS_RELEASE,
namespace.clone(), namespace.clone(),
NATS_NODE_PORT, NatsService::NodePort(NATS_NODE_PORT),
"APP",
UserPassCredentials { UserPassCredentials {
admin_user: NATS_ADMIN_USER.to_string(), admin_user: NATS_ADMIN_USER.to_string(),
admin_pass: NATS_ADMIN_PASS.to_string(), admin_pass: NATS_ADMIN_PASS.to_string(),
device_user: NATS_DEVICE_USER.to_string(), device_user: NATS_DEVICE_USER.to_string(),
device_pass: NATS_DEVICE_PASS.to_string(), device_pass: NATS_DEVICE_PASS.to_string(),
}, },
) );
.release_name(NATS_RELEASE);
nats.interpret(&Inventory::autoload(), &topology) nats.interpret(&Inventory::autoload(), &topology)
.await .await
.map_err(BringUpError::NatsDeploy)?; .map_err(BringUpError::NatsDeploy)?;
@@ -288,7 +284,7 @@ impl Stack {
// up, can't connect, hits CrashLoopBackOff for a // up, can't connect, hits CrashLoopBackOff for a
// while, and the agent Deployment-Ready wait // while, and the agent Deployment-Ready wait
// spuriously times out. // spuriously times out.
(connect_nats_admin(&nats_url).await?, nats.in_cluster_url()) (connect_nats_admin(&nats_url).await?, nats.client_ref())
} }
Some(handles) => { Some(handles) => {
let nats = crate::callout::deploy_nats_callout( let nats = crate::callout::deploy_nats_callout(
@@ -300,28 +296,28 @@ impl Stack {
) )
.await .await
.map_err(BringUpError::Callout)?; .map_err(BringUpError::Callout)?;
wait_for_deployment(
&kube_client,
&namespace,
crate::callout::callout_deployment_name(),
)
.await?;
let client = crate::callout::connect_nats_admin_sso(&nats_url, handles) let client = crate::callout::connect_nats_admin_sso(&nats_url, handles)
.await .await
.map_err(BringUpError::Callout)?; .map_err(BringUpError::Callout)?;
crate::callout::deploy_openbao(&topology, &namespace, handles) crate::callout::deploy_openbao(&topology, &namespace, handles)
.await .await
.map_err(BringUpError::Callout)?; .map_err(BringUpError::Callout)?;
(client, nats.in_cluster_url()) (client, nats.client_ref())
} }
}; };
if opts.deploy_operator { if opts.deploy_operator {
let mut operator = FleetOperatorScore::new() let mut operator = FleetOperatorScore::new(opts.operator_image.clone())
.namespace(namespace.clone()) .namespace(namespace.clone())
.image(opts.operator_image.clone()) .messaging(&nats_ref)
.nats_url(nats_in_cluster_url.clone())
.log_level(opts.log_level.clone()); .log_level(opts.log_level.clone());
if let Some(handles) = callout_handles.as_ref() {
operator = operator.identity(
&handles.provider,
&handles.application,
&handles.operator_identity,
);
} else {
operator.credentials = Some(OperatorCredentials { operator.credentials = Some(OperatorCredentials {
credentials_toml: format!( credentials_toml: format!(
r#"type = "toml-shared" r#"type = "toml-shared"
@@ -330,17 +326,16 @@ impl Stack {
"# "#
), ),
}); });
}
operator operator
.interpret(&Inventory::autoload(), &topology) .interpret(&Inventory::autoload(), &topology)
.await .await
.map_err(BringUpError::OperatorDeploy)?; .map_err(BringUpError::OperatorDeploy)?;
wait_for_deployment(&kube_client, &namespace, "harmony-fleet-operator").await?;
} }
// Deploy one agent Pod per requested device, all sharing the // Deploy one agent Pod per requested device, all sharing the
// in-cluster NATS DNS the FleetNatsScore exposes. // in-cluster NATS DNS the NatsScore exposes.
let device_ids: Vec<String> = if opts.deploy_agent { let device_ids: Vec<String> = if opts.deploy_agent {
(0..opts.num_devices) (0..opts.num_devices)
.map(|i| format!("vm-device-{i:02}-{}", &namespace["e2e-".len()..])) .map(|i| format!("vm-device-{i:02}-{}", &namespace["e2e-".len()..]))
@@ -356,7 +351,7 @@ impl Stack {
PodTarget::user_pass( PodTarget::user_pass(
device_id.clone(), device_id.clone(),
opts.agent_image.clone(), opts.agent_image.clone(),
nats_in_cluster_url.clone(), nats_ref.url().to_string(),
NATS_DEVICE_USER, NATS_DEVICE_USER,
NATS_DEVICE_PASS, NATS_DEVICE_PASS,
) )
@@ -367,14 +362,6 @@ impl Stack {
.await .await
.map_err(BringUpError::AgentDeploy)?; .map_err(BringUpError::AgentDeploy)?;
} }
for device_id in &device_ids {
wait_for_deployment(
&kube_client,
&namespace,
&FleetAgentScore::resource_name(device_id),
)
.await?;
}
} }
Ok(Self { Ok(Self {
@@ -591,46 +578,6 @@ async fn prune_stale_namespaces(client: &Client) {
} }
} }
async fn wait_for_deployment(
client: &Client,
namespace: &str,
name: &str,
) -> Result<(), BringUpError> {
// Temporary readiness gate. ADR-023 principle 4 promises that a
// future smoke-test contract on the Score itself will replace this
// — `deploy` will only return after the agent is up. Today the
// Score returns when the manifest is applied; we poll for Ready
// here so each test can rely on the pods being reachable.
let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
let deadline = std::time::Instant::now() + Duration::from_secs(120);
while std::time::Instant::now() < deadline {
match api.get(name).await {
Ok(d) => {
if let Some(status) = d.status
&& status.ready_replicas.unwrap_or(0) >= 1
{
tracing::info!(namespace = %namespace, deployment = %name, "deployment ready");
return Ok(());
}
}
Err(kube::Error::Api(e)) if e.code == 404 => {}
Err(e) => {
return Err(BringUpError::WaitTimeout {
namespace: namespace.to_string(),
name: name.to_string(),
detail: format!("get deployment failed: {e}"),
});
}
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
Err(BringUpError::WaitTimeout {
namespace: namespace.to_string(),
name: name.to_string(),
detail: "timed out after 120s".to_string(),
})
}
async fn connect_nats_admin(url: &str) -> Result<async_nats::Client, BringUpError> { async fn connect_nats_admin(url: &str) -> Result<async_nats::Client, BringUpError> {
// NATS helm chart cold-start is ~30-60s on a freshly-pulled // NATS helm chart cold-start is ~30-60s on a freshly-pulled
// image. Keep retrying for 2 minutes before giving up so the // image. Keep retrying for 2 minutes before giving up so the

View File

@@ -23,15 +23,17 @@ use std::sync::Arc;
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::{ use harmony::modules::fleet::{
DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HARMONY_FLEET_POOL_NAME, ProvisionVmScore, ensure_fleet_ssh_keypair, ensure_harmony_fleet_pool,
HARMONY_FLEET_POOL_NAME, HostsEntry, ProvisionVmScore, ensure_fleet_ssh_keypair, read_public_key,
ensure_harmony_fleet_pool, read_public_key,
}; };
use harmony::modules::kvm::KvmVirtualMachineHost; use harmony::modules::kvm::KvmVirtualMachineHost;
use harmony::modules::kvm::config::init_executor; use harmony::modules::kvm::config::init_executor;
use harmony::modules::linux::{LinuxHostTopology, SshCommandOutput, SshCredentials, ssh_exec}; use harmony::modules::linux::{LinuxHostTopology, SshCommandOutput, SshCredentials, ssh_exec};
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig}; use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
use harmony_fleet_deploy::{
DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry,
};
use harmony_types::id::Id; use harmony_types::id::Id;
use harmony_types::net::IpAddress; use harmony_types::net::IpAddress;
use thiserror::Error; use thiserror::Error;

View File

@@ -3,7 +3,7 @@
//! NATS the Pod harness deploys. //! NATS the Pod harness deploys.
//! //!
//! Per ADR-023 P2 the harness composes the same production Scores //! Per ADR-023 P2 the harness composes the same production Scores
//! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `FleetNatsScore`) //! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `NatsScore`)
//! — the only thing the test layer owns is the test-fixture wiring //! — the only thing the test layer owns is the test-fixture wiring
//! (per-binary `OnceCell`, drop-time VM cleanup, KV admin helpers). //! (per-binary `OnceCell`, drop-time VM cleanup, KV admin helpers).
//! //!

View File

@@ -26,14 +26,14 @@ use tokio::process::Command;
use tokio::sync::OnceCell; use tokio::sync::OnceCell;
use uuid::Uuid; use uuid::Uuid;
use harmony::modules::fleet::{DeviceOpenbao, FleetDeviceAuth, HostsEntry}; use harmony_fleet_deploy::{DeviceOpenbao, FleetDeviceAuth, HostsEntry};
use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack}; use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack};
use crate::vm::agent_build::{AgentBuildError, build_agent_for}; use crate::vm::agent_build::{AgentBuildError, build_agent_for};
use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions}; use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions};
use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip}; use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip};
/// NATS device user the [`crate::stack`] `FleetNatsScore` provisions /// NATS device user the [`crate::stack`] `NatsScore` provisions
/// in user-pass mode. Mirrors `stack::NATS_DEVICE_USER` (which is /// in user-pass mode. Mirrors `stack::NATS_DEVICE_USER` (which is
/// private to that module) so VM agents authenticate against the /// private to that module) so VM agents authenticate against the
/// same user-pass account. /// same user-pass account.
@@ -300,14 +300,8 @@ impl VmStack {
/// Wait until every agent in the stack answers `Verb::Ping`. /// Wait until every agent in the stack answers `Verb::Ping`.
/// ///
/// `FleetDeviceSetupScore` returns the moment `systemctl /// Deployment readiness verifies the systemd unit. This assertion also
/// start fleet-agent` exits — *seconds* before the agent's NATS /// covers the first OIDC exchange, NATS connection, and command round-trip.
/// subscription has actually propagated. Pinging in that window
/// hits `no_responders` immediately (NATS knows there's no
/// subscriber and short-circuits). Tests that want the agent
/// reachable must wait through that window; this helper does it
/// by retrying ping with a 3 s per-attempt timeout up to
/// `per_device_budget`.
pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> { pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> {
// Fresh connection on the caller's runtime — see // Fresh connection on the caller's runtime — see
// `Stack::connect_admin` for why the shared client can't be // `Stack::connect_admin` for why the shared client can't be

View File

@@ -5,15 +5,14 @@
//! live Zitadel, and tests mint their own device tokens. //! live Zitadel, and tests mint their own device tokens.
use harmony::inventory::Inventory; use harmony::inventory::Inventory;
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use harmony::modules::openbao::{ use harmony::modules::openbao::{
OpenBaoDeploymentSecretGrants, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
OpenbaoSetupScore, cached_root_token,
}; };
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient}; use harmony::topology::{K8sAnywhereTopology, K8sclient};
use harmony_fleet_e2e::{StackOptions, shared_stack}; use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_reconciler_contracts::DeploymentName; use harmony_reconciler_contracts::{DeploymentName, DeploymentSecretGrants};
use harmony_secret::OpenBaoDeploymentSecretGrants;
use serde_json::json; use serde_json::json;
const E2E_ENV: &str = "HARMONY_FLEET_E2E"; const E2E_ENV: &str = "HARMONY_FLEET_E2E";
@@ -169,6 +168,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
host: "openbao-groups-test.local".to_string(), host: "openbao-groups-test.local".to_string(),
openshift: false, openshift: false,
tls_issuer: None, tls_issuer: None,
node_port: None,
}), }),
Box::new(OpenbaoSetupScore { Box::new(OpenbaoSetupScore {
instance: instance.clone(), instance: instance.clone(),
@@ -187,6 +187,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
ttl: "1h".to_string(), ttl: "1h".to_string(),
max_ttl: "1h".to_string(), max_ttl: "1h".to_string(),
}), }),
oidc_application: None,
}), }),
]; ];
for score in scores { for score in scores {

View File

@@ -89,6 +89,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
host: "openbao-policy-test.local".to_string(), host: "openbao-policy-test.local".to_string(),
openshift: false, openshift: false,
tls_issuer: None, tls_issuer: None,
node_port: None,
}), }),
Box::new(OpenbaoSetupScore { Box::new(OpenbaoSetupScore {
instance: instance.clone(), instance: instance.clone(),
@@ -103,6 +104,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
policies: vec![], policies: vec![],
}], }],
jwt_auth: None, jwt_auth: None,
oidc_application: None,
}), }),
]; ];

View File

@@ -1,8 +1,8 @@
use harmony::modules::fleet::operator::crd::{ use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_fleet_operator::crd::{
Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore, Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore,
Rollout, RolloutStrategy, Rollout, RolloutStrategy,
}; };
use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key}; use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client; use kube::Client;

View File

@@ -13,10 +13,9 @@
use std::time::Duration; use std::time::Duration;
use harmony::modules::podman::{PodmanService, PodmanV0Score};
use harmony::topology::RestartPolicy;
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack}; use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
use harmony_reconciler_contracts::{DeploymentName, Phase}; use harmony_reconciler_contracts::{DeploymentName, Phase};
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy};
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";

View File

@@ -19,12 +19,11 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::Duration; use std::time::Duration;
use harmony::modules::podman::{PodmanService, PodmanV0Score}; use harmony_fleet_auth::AgentObservation;
use harmony::topology::RestartPolicy;
use harmony_fleet_auth::{AgentConfig, AgentSection, CredentialsSection, NatsSection}; use harmony_fleet_auth::{AgentConfig, AgentSection, CredentialsSection, NatsSection};
use harmony_fleet_deploy::companion::AgentObservation;
use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack}; use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack};
use harmony_reconciler_contracts::{DeploymentName, Id, desired_state_key}; use harmony_reconciler_contracts::{DeploymentName, Id, desired_state_key};
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy};
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";

View File

@@ -17,15 +17,16 @@
use std::time::Duration; use std::time::Duration;
use harmony::modules::fleet::secret_access::DeploymentSecretGrants; use harmony::topology::{K8sAnywhereTopology, K8sclient};
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
use harmony::modules::podman::{EnvVar, PodmanService, PodmanV0Score, SecretEnvVar};
use harmony::topology::{K8sAnywhereTopology, K8sclient, RestartPolicy};
use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX}; use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX};
use harmony_fleet_e2e::{ use harmony_fleet_e2e::{
AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack, AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack,
}; };
use harmony_reconciler_contracts::{DeploymentName, Phase}; use harmony_reconciler_contracts::{DeploymentName, Phase};
use harmony_reconciler_contracts::{
DeploymentSecretGrants, EnvVar, PodmanService, PodmanV0Score, RestartPolicy, SecretEnvVar,
};
use harmony_secret::OpenBaoDeploymentSecretGrants;
use serde_json::json; use serde_json::json;
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";

View File

@@ -14,9 +14,9 @@ default = []
web-frontend = ["dep:axum", "dep:axum-extra", "dep:maud", "dep:tokio-stream", "harmony_zitadel_auth/axum"] web-frontend = ["dep:axum", "dep:axum-extra", "dep:maud", "dep:tokio-stream", "harmony_zitadel_auth/axum"]
[dependencies] [dependencies]
harmony = { path = "../../harmony", features = ["podman"] }
harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony_secret = { path = "../../harmony_secret" }
harmony_config = { path = "../../harmony_config" } harmony_config = { path = "../../harmony_config" }
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
toml = { workspace = true } toml = { workspace = true }

View File

@@ -0,0 +1,110 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::sync::Mutex;
use async_trait::async_trait;
use harmony_reconciler_contracts::{
DeploymentName, DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError,
};
#[derive(Debug, Default)]
pub struct StaticDeviceGroups {
map: HashMap<String, HashSet<String>>,
}
impl StaticDeviceGroups {
pub fn parse(spec: &str) -> Self {
let map = spec
.split(';')
.filter_map(|entry| {
let (device, groups) = entry.split_once('=')?;
let device = device.trim();
(!device.is_empty()).then(|| {
(
device.to_string(),
groups
.split('|')
.map(str::trim)
.filter(|group| !group.is_empty())
.map(str::to_string)
.collect(),
)
})
})
.collect();
Self { map }
}
}
#[async_trait]
impl DeviceGroupSource for StaticDeviceGroups {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
Ok(self.map.clone())
}
}
#[derive(Debug, Default)]
pub struct InMemoryDeploymentSecretGrants {
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
}
impl InMemoryDeploymentSecretGrants {
pub fn new() -> Self {
Self::default()
}
pub fn groups_for(&self, deployment: &DeploymentName) -> Vec<String> {
self.grants
.lock()
.unwrap()
.get(deployment)
.map(|groups| groups.iter().cloned().collect())
.unwrap_or_default()
}
}
#[async_trait]
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError> {
let mut current = self.grants.lock().unwrap();
for (deployment, groups) in grants {
if groups.is_empty() {
current.remove(deployment);
} else {
current.insert(deployment.clone(), groups.iter().cloned().collect());
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn static_groups_parse_membership() {
let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
let groups = source.device_groups().await.unwrap();
assert_eq!(groups.len(), 2);
assert_eq!(groups["pi-01"].len(), 2);
}
#[tokio::test]
async fn grants_replace_and_clear_membership() {
let grants = InMemoryDeploymentSecretGrants::new();
let deployment = DeploymentName::try_new("web").unwrap();
grants
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&deployment), vec!["edge-a"]);
grants
.set_deployment_groups(&[(deployment.clone(), vec![])])
.await
.unwrap();
assert!(grants.groups_for(&deployment).is_empty());
}
}

View File

@@ -33,7 +33,7 @@ use kube::runtime::finalizer::{Event as FinalizerEvent, finalizer};
use kube::runtime::watcher::Config as WatcherConfig; use kube::runtime::watcher::Config as WatcherConfig;
use kube::{Api, Client, ResourceExt}; use kube::{Api, Client, ResourceExt};
use harmony::modules::fleet::operator::Deployment; use harmony_fleet_operator::Deployment;
const FINALIZER: &str = "fleet.nationtech.io/finalizer"; const FINALIZER: &str = "fleet.nationtech.io/finalizer";

View File

@@ -4,7 +4,7 @@ use kube::CustomResource;
use schemars::JsonSchema; use schemars::JsonSchema;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
pub use crate::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore}; pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
/// Deployment intent. Targets devices by label selector — identical /// Deployment intent. Targets devices by label selector — identical
/// to the pattern K8s itself uses for DaemonSet nodeSelector, Service /// to the pattern K8s itself uses for DaemonSet nodeSelector, Service

View File

@@ -18,7 +18,7 @@ use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams}; use kube::api::{Api, DeleteParams, Patch, PatchParams};
use std::collections::BTreeMap; use std::collections::BTreeMap;
use harmony::modules::fleet::operator::{Device, DeviceSpec}; use crate::crd::{Device, DeviceSpec};
const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler"; const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler";

View File

@@ -24,7 +24,7 @@ use kube::{Client, ResourceExt};
use serde_json::json; use serde_json::json;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use harmony::modules::fleet::operator::{Device, Reachability}; use crate::crd::{Device, Reachability};
/// A device with no heartbeat within this window is `Stale`. Agents /// A device with no heartbeat within this window is `Stale`. Agents
/// ping every 30 s, so this tolerates ~2 missed pings. /// ping every 30 s, so this tolerates ~2 missed pings.

View File

@@ -33,11 +33,9 @@ use kube::{Client, ResourceExt};
use serde_json::json; use serde_json::json;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use harmony::modules::fleet::groups::DeviceGroupSource; use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
use harmony::modules::fleet::operator::{
AggregateLastError, Deployment, DeploymentAggregate, Device, use crate::crd::{AggregateLastError, Deployment, DeploymentAggregate, Device};
};
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use tracing::warn; use tracing::warn;
const PATCH_TICK: Duration = Duration::from_secs(1); const PATCH_TICK: Duration = Duration::from_secs(1);
@@ -355,38 +353,20 @@ async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow:
Ok(()) Ok(())
} }
/// Push the allowed-groups grant for `deployments` to the secrets async fn sync_grant_batch(
/// backend in one batch. A deployment absent from the cache (deleted, or grants: &Arc<dyn DeploymentSecretGrants>,
/// no `allowedGroups` declared) resolves to the empty set = revocation. batch: &[(DeploymentName, Vec<String>)],
/// Non-fatal: a backend hiccup logs and is retried on the next
/// deployment change rather than tearing down reconciliation. Skipped
/// entirely when no backend is configured (dev without OpenBao).
async fn sync_deployment_grants(
state: &SharedFleetState,
grants: Option<&Arc<dyn DeploymentSecretGrants>>,
deployments: impl IntoIterator<Item = DeploymentName>,
) { ) {
let Some(grants) = grants else { return }; let mut delay = Duration::from_secs(1);
let batch: Vec<(DeploymentName, Vec<String>)> = { loop {
let guard = state.lock().await; match grants.set_deployment_groups(batch).await {
deployments Ok(()) => return,
.into_iter() Err(error) => {
.map(|dn| { warn!(%error, ?delay, "aggregator: secret grant sync failed; retrying");
let groups = guard tokio::time::sleep(delay).await;
.deployments delay = (delay * 2).min(Duration::from_secs(30));
.values() }
.find(|c| c.deployment_name == dn)
.and_then(|c| c.allowed_groups.clone())
.unwrap_or_default();
(dn, groups)
})
.collect()
};
if batch.is_empty() {
return;
} }
if let Err(e) = grants.set_deployment_groups(&batch).await {
warn!(error = %e, "aggregator: secret grant sync failed");
} }
} }
@@ -513,6 +493,19 @@ async fn on_deployment_upsert(
score_json: score_json.clone(), score_json: score_json.clone(),
}; };
// Keep the deployment out of schedulable state until its secret grant
// exists; device and group watchers share the cache below.
if let Some(grants) = grants {
sync_grant_batch(
grants,
&[(
deployment_name.clone(),
cached.allowed_groups.clone().unwrap_or_default(),
)],
)
.await;
}
let (new_targets, previous_targets) = { let (new_targets, previous_targets) = {
let mut guard = state.lock().await; let mut guard = state.lock().await;
let new_targets = matched_devices(&cached, &guard); let new_targets = matched_devices(&cached, &guard);
@@ -528,10 +521,6 @@ async fn on_deployment_upsert(
(new_targets, previous) (new_targets, previous)
}; };
// Grant before workload signal: never tell a device to run a
// deployment whose secrets it cannot yet read.
sync_deployment_grants(state, grants, std::iter::once(deployment_name.clone())).await;
reconcile_kv( reconcile_kv(
desired, desired,
&deployment_name, &deployment_name,
@@ -555,6 +544,10 @@ async fn on_deployment_delete(
return; return;
}; };
if let Some(grants) = grants {
sync_grant_batch(grants, &[(deployment_name.clone(), Vec::new())]).await;
}
let previous = { let previous = {
let mut guard = state.lock().await; let mut guard = state.lock().await;
guard.deployments.remove(&key); guard.deployments.remove(&key);
@@ -575,9 +568,6 @@ async fn on_deployment_delete(
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed"); tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed");
} }
} }
// Gone from the cache, so the grant resolves to empty = revocation.
sync_deployment_grants(state, grants, std::iter::once(deployment_name)).await;
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -1041,19 +1031,19 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn sync_pushes_declared_groups_per_deployment() { async fn sync_pushes_declared_groups_per_deployment() {
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; use crate::access::InMemoryDeploymentSecretGrants;
let mut s = FleetState::default();
let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a", "edge-b"]);
let db = cached("ns", "db", "zone", "lab"); // no allowedGroups → no grant
s.deployments.insert(web.key.clone(), web);
s.deployments.insert(db.key.clone(), db);
let state: SharedFleetState = Arc::new(Mutex::new(s));
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone(); let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
sync_deployment_grants(&state, Some(&grants), [dn("web"), dn("db")]).await; sync_grant_batch(
&grants,
&[
(dn("web"), vec!["edge-a".into(), "edge-b".into()]),
(dn("db"), vec![]),
],
)
.await;
assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]); assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]);
assert!(inmem.groups_for(&dn("db")).is_empty()); assert!(inmem.groups_for(&dn("db")).is_empty());
@@ -1061,7 +1051,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn sync_revokes_deleted_deployment() { async fn sync_revokes_deleted_deployment() {
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; use crate::access::InMemoryDeploymentSecretGrants;
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone(); let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
@@ -1070,13 +1060,38 @@ mod tests {
.await .await
.unwrap(); .unwrap();
// Deployment no longer cached (CR deleted) → grant resolves empty. sync_grant_batch(&grants, &[(dn("web"), vec![])]).await;
let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default()));
sync_deployment_grants(&state, Some(&grants), std::iter::once(dn("web"))).await;
assert!(inmem.groups_for(&dn("web")).is_empty()); assert!(inmem.groups_for(&dn("web")).is_empty());
} }
#[tokio::test]
async fn grant_sync_retries_before_returning() {
use std::sync::atomic::{AtomicUsize, Ordering};
struct FlakyGrants(AtomicUsize);
#[async_trait::async_trait]
impl DeploymentSecretGrants for FlakyGrants {
async fn set_deployment_groups(
&self,
_grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), harmony_reconciler_contracts::SecretAccessError> {
if self.0.fetch_add(1, Ordering::SeqCst) == 0 {
return Err(harmony_reconciler_contracts::SecretAccessError::Backend(
"temporary failure".into(),
));
}
Ok(())
}
}
let flaky = Arc::new(FlakyGrants(AtomicUsize::new(0)));
let grants: Arc<dyn DeploymentSecretGrants> = flaky.clone();
sync_grant_batch(&grants, &[(dn("web"), vec!["edge-a".into()])]).await;
assert_eq!(flaky.0.load(Ordering::SeqCst), 2);
}
#[test] #[test]
fn membership_change_reschedules_device() { fn membership_change_reschedules_device() {
let mut s = FleetState::default(); let mut s = FleetState::default();

View File

@@ -6,13 +6,17 @@
//! documentation generators, alternative entry points — can drive //! documentation generators, alternative entry points — can drive
//! the reconcilers without `main.rs`'s bootstrap. //! the reconcilers without `main.rs`'s bootstrap.
//! //!
//! CRD type definitions (`Deployment`, `Device`, `DeploymentSpec`, //! CRD type definitions live in this runtime crate and are reused by
//! `ScorePayload`, …) live in //! `harmony-fleet-deploy` when installing the operator.
//! [`harmony::modules::fleet::operator`] so that
//! `harmony::modules::fleet::FleetOperatorScore` can reach them
//! without a workspace dependency cycle. Import them from there.
pub mod access;
pub mod commands; pub mod commands;
pub mod crd;
pub mod device_reconciler; pub mod device_reconciler;
pub mod device_status; pub mod device_status;
pub mod fleet_aggregator; pub mod fleet_aggregator;
pub use crd::{
AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
};

View File

@@ -5,11 +5,11 @@ mod frontend;
#[cfg(feature = "web-frontend")] #[cfg(feature = "web-frontend")]
mod service; mod service;
use harmony::modules::fleet::groups::{DeviceGroupSource, StaticDeviceGroups}; use harmony_fleet_operator::access::StaticDeviceGroups;
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
use harmony::modules::zitadel::ZitadelDeviceGroups;
use harmony_fleet_operator::{device_reconciler, fleet_aggregator}; use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
use harmony_secret::OpenBaoDeploymentSecretGrants;
use harmony_zitadel_auth::ZitadelDeviceGroups;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use async_nats::jetstream; use async_nats::jetstream;
@@ -20,7 +20,6 @@ use harmony_fleet_auth::{
}; };
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE; use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
use kube::Client; use kube::Client;
#[cfg(feature = "web-frontend")]
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
@@ -62,6 +61,15 @@ struct Cli {
)] )]
credentials_toml: String, credentials_toml: String,
#[arg(long, env = "FLEET_OPERATOR_OIDC_ISSUER", global = true)]
oidc_issuer: Option<String>,
#[arg(long, env = "FLEET_OPERATOR_OIDC_AUDIENCE", global = true)]
oidc_audience: Option<String>,
#[arg(long, env = "FLEET_OPERATOR_OIDC_KEY_PATH", global = true)]
oidc_key_path: Option<PathBuf>,
#[arg(long, env = "OPENBAO_URL", global = true)] #[arg(long, env = "OPENBAO_URL", global = true)]
openbao_url: Option<String>, openbao_url: Option<String>,
@@ -108,12 +116,28 @@ async fn main() -> Result<()> {
.init(); .init();
let cli = Cli::parse(); let cli = Cli::parse();
let credentials_toml = if cli.credentials_toml.is_empty() {
match (&cli.oidc_issuer, &cli.oidc_audience, &cli.oidc_key_path) {
(Some(issuer), Some(audience), Some(key_path)) => {
toml::to_string(&CredentialsSection::ZitadelJwt {
key_path: key_path.clone(),
key_json: None,
oidc_issuer_url: issuer.clone(),
audience: audience.clone(),
danger_accept_invalid_certs: false,
})?
}
_ => String::new(),
}
} else {
cli.credentials_toml.clone()
};
match cli.command.unwrap_or(Command::Run) { match cli.command.unwrap_or(Command::Run) {
Command::Run => { Command::Run => {
run( run(
&cli.nats_url, &cli.nats_url,
&cli.kv_bucket, &cli.kv_bucket,
&cli.credentials_toml, &credentials_toml,
&cli.openbao_url, &cli.openbao_url,
&cli.openbao_token, &cli.openbao_token,
) )

View File

@@ -17,11 +17,11 @@ use kube::api::{Api, ListParams, Patch, PatchParams};
use kube::{Client, ResourceExt}; use kube::{Client, ResourceExt};
use serde_json::json; use serde_json::json;
use harmony::modules::fleet::operator::{ use harmony_fleet_operator::crd::{
Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability, Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability,
}; };
use harmony::modules::podman::ReconcileScore;
use harmony_fleet_operator::fleet_aggregator::selector_matches; use harmony_fleet_operator::fleet_aggregator::selector_matches;
use harmony_reconciler_contracts::ReconcileScore;
use super::{ use super::{
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail, Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
@@ -390,7 +390,7 @@ impl FleetService for RealFleetService {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use harmony::modules::podman::{PodmanService, PodmanV0Score}; use harmony_reconciler_contracts::{PodmanService, PodmanV0Score};
fn liveness(r: Reachability) -> DeviceLiveness { fn liveness(r: Reachability) -> DeviceLiveness {
DeviceLiveness { DeviceLiveness {

View File

@@ -1,8 +1,8 @@
use std::collections::HashMap; use std::collections::HashMap;
use k8s_openapi::api::{ use k8s_openapi::api::{
apps::v1::Deployment, apps::v1::{Deployment, StatefulSet},
core::v1::{Namespace, Node, ServiceAccount}, core::v1::{Namespace, Node, Pod, ServiceAccount},
}; };
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::api::ApiResource; use kube::api::ApiResource;
@@ -268,10 +268,84 @@ impl K8sClient {
}; };
let timeout = timeout.unwrap_or(Duration::from_secs(120)); let timeout = timeout.unwrap_or(Duration::from_secs(120));
let establish = await_condition(api, name, conditions::is_deployment_completed()); let establish = await_condition(api, name, conditions::is_deployment_completed());
tokio::time::timeout(timeout, establish) match tokio::time::timeout(timeout, establish).await {
Ok(Ok(_)) => Ok(()),
Ok(Err(error)) => Err(format!("Failed waiting for deployment {name}: {error}")),
Err(_) => Err(format!(
"Timed out after {}s waiting for deployment {name}",
timeout.as_secs()
)),
}
}
pub async fn wait_until_statefulset_ready(
&self,
name: &str,
namespace: Option<&str>,
timeout: Option<Duration>,
) -> Result<(), String> {
let timeout = timeout.unwrap_or(Duration::from_secs(120));
match tokio::time::timeout(timeout, async {
loop {
match self.get_resource::<StatefulSet>(name, namespace).await {
Ok(Some(statefulset))
if statefulset.status.as_ref().is_some_and(|status| {
status.ready_replicas.unwrap_or_default() > 0
}) =>
{
return Ok(());
}
Ok(_) => {}
Err(error) => {
return Err(format!("Failed waiting for statefulset {name}: {error}"));
}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await .await
.map(|_| ()) {
.map_err(|_| "Timed out waiting for deployment".to_string()) Ok(result) => result,
Err(_) => Err(format!(
"Timed out after {}s waiting for statefulset {name}",
timeout.as_secs()
)),
}
}
pub async fn wait_until_pod_running(
&self,
name: &str,
namespace: Option<&str>,
timeout: Option<Duration>,
) -> Result<(), String> {
let timeout = timeout.unwrap_or(Duration::from_secs(120));
match tokio::time::timeout(timeout, async {
loop {
match self.get_resource::<Pod>(name, namespace).await {
Ok(Some(pod))
if pod
.status
.as_ref()
.and_then(|status| status.phase.as_deref())
== Some("Running") =>
{
return Ok(());
}
Ok(_) => {}
Err(error) => return Err(format!("Failed waiting for pod {name}: {error}")),
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await
{
Ok(result) => result,
Err(_) => Err(format!(
"Timed out after {}s waiting for pod {name}",
timeout.as_secs()
)),
}
} }
/// Gets a single named resource, using the correct API scope for `K`. /// Gets a single named resource, using the correct API scope for `K`.

View File

@@ -14,6 +14,7 @@ license.workspace = true
# module. None of those consumers should pay for the others' deps. # module. None of those consumers should pay for the others' deps.
[dependencies] [dependencies]
async-trait.workspace = true
chrono = { workspace = true, features = ["serde"] } chrono = { workspace = true, features = ["serde"] }
harmony_types = { path = "../harmony_types" } harmony_types = { path = "../harmony_types" }
schemars = "0.8.22" schemars = "0.8.22"

View File

@@ -0,0 +1,30 @@
use std::collections::{HashMap, HashSet};
use async_trait::async_trait;
use crate::DeploymentName;
#[derive(Debug, thiserror::Error)]
pub enum GroupSourceError {
#[error("group source error: {0}")]
Source(String),
}
#[async_trait]
pub trait DeviceGroupSource: Send + Sync {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError>;
}
#[derive(Debug, thiserror::Error)]
pub enum SecretAccessError {
#[error("secret access backend error: {0}")]
Backend(String),
}
#[async_trait]
pub trait DeploymentSecretGrants: Send + Sync {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError>;
}

View File

@@ -11,7 +11,7 @@ use crate::fleet::DeploymentName;
/// Operator-written bucket. One entry per `(device, deployment)` pair. /// Operator-written bucket. One entry per `(device, deployment)` pair.
/// Values are the JSON-serialized Score envelope — today /// Values are the JSON-serialized Score envelope — today
/// `harmony::modules::podman::ReconcileScore`, tomorrow any variant of /// `crate::ReconcileScore`, tomorrow any variant of
/// a polymorphic `Score` enum the framework ships. /// a polymorphic `Score` enum the framework ships.
pub const BUCKET_DESIRED_STATE: &str = "desired-state"; pub const BUCKET_DESIRED_STATE: &str = "desired-state";

View File

@@ -9,18 +9,22 @@
//! //!
//! This crate holds the wire-format bits both sides must agree on: //! This crate holds the wire-format bits both sides must agree on:
//! NATS bucket names, KV key formats, and the typed payloads //! NATS bucket names, KV key formats, and the typed payloads
//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`). The Score //! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`, and Podman
//! types themselves live in their respective harmony modules. //! desired state).
//! //!
//! **Deliberately lean** — no tokio, no async-nats, no harmony. //! **Deliberately lean** — no tokio, no async-nats, no harmony.
//! The on-device agent build pulls it in alongside a minimal //! The on-device agent build pulls it in alongside a minimal
//! async-nats client; the operator pulls it alongside kube-rs. //! async-nats client; the operator pulls it alongside kube-rs.
pub mod access;
pub mod commands; pub mod commands;
pub mod fleet; pub mod fleet;
pub mod kv; pub mod kv;
pub mod podman;
pub mod status; pub mod status;
pub use access::{DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError};
pub use commands::{ pub use commands::{
CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB, CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB,
HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject, HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject,
@@ -34,6 +38,9 @@ pub use kv::{
desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key, desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key,
device_state_key, device_state_key,
}; };
pub use podman::{
EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount,
};
pub use status::{InventorySnapshot, Phase}; pub use status::{InventorySnapshot, Phase};
// Re-exports so consumers (agent, operator) don't need a direct // Re-exports so consumers (agent, operator) don't need a direct

View File

@@ -0,0 +1,208 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct EnvVar {
pub name: String,
pub value: String,
}
impl EnvVar {
pub fn new(name: impl Into<String>, value: impl Into<String>) -> Self {
Self {
name: name.into(),
value: value.into(),
}
}
}
impl From<(String, String)> for EnvVar {
fn from((name, value): (String, String)) -> Self {
Self { name, value }
}
}
impl From<(&str, &str)> for EnvVar {
fn from((name, value): (&str, &str)) -> Self {
Self::new(name, value)
}
}
#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)]
pub struct VolumeMount {
pub host_path: String,
pub container_path: String,
#[serde(default)]
pub read_only: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "kebab-case")]
pub enum RestartPolicy {
No,
#[default]
UnlessStopped,
OnFailure,
Always,
}
impl RestartPolicy {
pub fn as_str(&self) -> &'static str {
match self {
Self::No => "no",
Self::UnlessStopped => "unless-stopped",
Self::OnFailure => "on-failure",
Self::Always => "always",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
pub struct SecretEnvVar {
pub name: String,
pub secret: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
pub struct PodmanService {
pub name: String,
pub image: String,
pub ports: Vec<String>,
#[serde(default)]
pub env: Vec<EnvVar>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub secret_env: Vec<SecretEnvVar>,
#[serde(default)]
pub volumes: Vec<VolumeMount>,
#[serde(default)]
pub restart_policy: RestartPolicy,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
pub struct PodmanV0Score {
pub services: Vec<PodmanService>,
}
impl PodmanV0Score {
pub fn deployment_label(&self) -> String {
self.services
.iter()
.map(|service| service.name.as_str())
.collect::<Vec<_>>()
.join(",")
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(tag = "type", content = "data")]
pub enum ReconcileScore {
PodmanV0(PodmanV0Score),
}
#[cfg(test)]
mod tests {
use super::*;
fn service(name: &str, image: &str) -> PodmanService {
PodmanService {
name: name.into(),
image: image.into(),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::default(),
}
}
#[test]
fn score_wire_format_and_roundtrip_are_stable() {
let score = ReconcileScore::PodmanV0(PodmanV0Score {
services: vec![PodmanService {
ports: vec!["8080:80".into()],
env: vec![EnvVar::new("LOG_LEVEL", "info")],
volumes: vec![VolumeMount {
host_path: "/var/lib/web/data".into(),
container_path: "/data".into(),
read_only: false,
}],
restart_policy: RestartPolicy::Always,
..service("web", "nginx:latest")
}],
});
let json = serde_json::to_string(&score).unwrap();
assert!(json.contains("\"type\":\"PodmanV0\""));
assert!(json.contains("\"data\""));
assert_eq!(
serde_json::from_str::<ReconcileScore>(&json).unwrap(),
score
);
}
#[test]
fn legacy_service_defaults_are_stable() {
let legacy = r#"{"type":"PodmanV0","data":{"services":[{"name":"web","image":"nginx","ports":["8080:80"]}]}}"#;
let ReconcileScore::PodmanV0(score) = serde_json::from_str(legacy).unwrap();
let service = &score.services[0];
assert!(service.env.is_empty());
assert!(service.secret_env.is_empty());
assert!(service.volumes.is_empty());
assert_eq!(service.restart_policy, RestartPolicy::UnlessStopped);
}
#[test]
fn empty_secret_env_remains_omitted() {
let json = serde_json::to_string(&service("web", "nginx")).unwrap();
assert!(!json.contains("secret_env"));
let with_secret = PodmanService {
secret_env: vec![SecretEnvVar {
name: "DB_PASSWORD".into(),
secret: "db_password".into(),
}],
..service("api", "myapp")
};
assert_eq!(
serde_json::from_str::<PodmanService>(&serde_json::to_string(&with_secret).unwrap())
.unwrap(),
with_secret
);
}
#[test]
fn restart_policy_wire_names_are_stable() {
assert_eq!(
serde_json::to_string(&RestartPolicy::UnlessStopped).unwrap(),
"\"unless-stopped\""
);
assert_eq!(
serde_json::to_string(&RestartPolicy::OnFailure).unwrap(),
"\"on-failure\""
);
}
#[test]
fn env_order_is_preserved() {
let service = PodmanService {
env: vec![EnvVar::new("B", "2"), EnvVar::new("A", "1")],
..service("api", "myapp")
};
let roundtrip: PodmanService =
serde_json::from_str(&serde_json::to_string(&service).unwrap()).unwrap();
assert_eq!(roundtrip.env, service.env);
}
#[test]
fn env_schema_is_not_a_tuple() {
let schema = serde_json::to_value(schemars::schema_for!(PodmanService)).unwrap();
assert!(schema.pointer("/properties/env/items").unwrap().is_object());
}
#[test]
fn deployment_label_joins_service_names() {
let score = PodmanV0Score {
services: vec![service("web", "nginx"), service("api", "myapp")],
};
assert_eq!(score.deployment_label(), "web,api");
}
}

View File

@@ -6,13 +6,9 @@ readme.workspace = true
license.workspace = true license.workspace = true
[features] [features]
# `podman` lights up the podman module AND the fleet operator's
# CRD types (which embed `ReconcileScore` from the podman module).
# Without it the operator CRD fails to compile, so it's part of
# the default build.
default = ["podman"] default = ["podman"]
kvm = ["dep:virt"] kvm = ["dep:virt"]
podman = ["dep:podman-api"] podman = []
testing = [] testing = []
[dependencies] [dependencies]
@@ -99,7 +95,7 @@ inquire.workspace = true
brocade = { path = "../brocade" } brocade = { path = "../brocade" }
option-ext = "0.2.0" option-ext = "0.2.0"
rand.workspace = true rand.workspace = true
podman-api = { version = "0.9", optional = true } nkeys = "0.4"
sysinfo = "0.30" sysinfo = "0.30"
# virt (libvirt bindings) uses std::os::unix and only compiles on Unix targets. # virt (libvirt bindings) uses std::os::unix and only compiles on Unix targets.

View File

@@ -1,9 +1,10 @@
use async_trait::async_trait; use async_trait::async_trait;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::executors::ExecutorError; use crate::executors::ExecutorError;
pub use harmony_reconciler_contracts::{EnvVar, RestartPolicy, VolumeMount};
/// Capability: run containers on a node-level runtime. /// Capability: run containers on a node-level runtime.
/// ///
/// Models the subset of container-runtime operations Harmony Scores need in /// Models the subset of container-runtime operations Harmony Scores need in
@@ -38,41 +39,6 @@ pub trait ContainerRuntime: Send + Sync {
async fn list_managed_services(&self) -> Result<Vec<ContainerState>, ExecutorError>; async fn list_managed_services(&self) -> Result<Vec<ContainerState>, ExecutorError>;
} }
/// A single environment variable, name/value pair.
///
/// Modeled as a struct rather than `(String, String)` because k8s CRD
/// validation rejects tuple schemas (`items: [<schema>, <schema>]`) —
/// schemars generates that for tuples and apiextensions returns
/// `Forbidden: items must be a schema object and not an array`. The
/// struct shape produces `items: { type: object, properties: { name,
/// value } }`, which validates cleanly.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct EnvVar {
pub name: String,
pub value: String,
}
impl EnvVar {
pub fn new(name: impl Into<String>, value: impl Into<String>) -> Self {
Self {
name: name.into(),
value: value.into(),
}
}
}
impl From<(String, String)> for EnvVar {
fn from((name, value): (String, String)) -> Self {
Self { name, value }
}
}
impl From<(&str, &str)> for EnvVar {
fn from((name, value): (&str, &str)) -> Self {
Self::new(name, value)
}
}
/// Desired state for a single container. /// Desired state for a single container.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ContainerSpec { pub struct ContainerSpec {
@@ -109,51 +75,6 @@ impl ContainerSpec {
pub const MANAGED_BY_VALUE: &'static str = "harmony"; pub const MANAGED_BY_VALUE: &'static str = "harmony";
} }
/// A single host-path → container-path bind mount. Bind mounts are the only
/// volume kind supported in v0 — they cover ~95% of compose use cases and
/// don't depend on a runtime-managed volume namespace.
#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)]
pub struct VolumeMount {
/// Absolute path on the host.
pub host_path: String,
/// Absolute path inside the container.
pub container_path: String,
/// Mount as read-only. Defaults to false (read-write) to match
/// docker-compose's default.
#[serde(default)]
pub read_only: bool,
}
/// Restart policy for a managed container. Names follow podman/docker
/// conventions so docker-compose translation is mechanical.
#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "kebab-case")]
pub enum RestartPolicy {
/// Don't restart on exit.
No,
/// Restart unless the user explicitly stopped the container.
/// Docker-compose's default for long-running services and what most
/// fleet workloads want.
#[default]
UnlessStopped,
/// Restart only if the container exits with a non-zero status.
OnFailure,
/// Always restart, even on clean exits and after host reboot.
Always,
}
impl RestartPolicy {
/// Canonical string podman + docker accept on the CLI / in their APIs.
pub fn as_str(&self) -> &'static str {
match self {
RestartPolicy::No => "no",
RestartPolicy::UnlessStopped => "unless-stopped",
RestartPolicy::OnFailure => "on-failure",
RestartPolicy::Always => "always",
}
}
}
/// Observed state of a container on the runtime. /// Observed state of a container on the runtime.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ContainerState { pub struct ContainerState {

View File

@@ -97,6 +97,12 @@ pub trait SystemdManager: Send + Sync {
scope: SystemdScope, scope: SystemdScope,
) -> Result<ChangeReport, ExecutorError>; ) -> Result<ChangeReport, ExecutorError>;
async fn systemd_unit_active(
&self,
name: &str,
scope: SystemdScope,
) -> Result<bool, ExecutorError>;
/// Enable+start a user-scoped unit (e.g. `podman.socket` under /// Enable+start a user-scoped unit (e.g. `podman.socket` under
/// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has /// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has
/// already been called for the user. /// already been called for the user.

View File

@@ -1,81 +1 @@
//! Device group membership: the admin-managed identity input that gates pub use harmony_reconciler_contracts::{DeviceGroupSource, GroupSourceError};
//! scheduling and secret access (ADR-025).
//!
//! Groups are the security boundary — they must come from identity the
//! device cannot self-assert (Zitadel role grants), never from
//! device-reported labels. The operator polls a [`DeviceGroupSource`]
//! and feeds the snapshot into scheduling.
use std::collections::{HashMap, HashSet};
use async_trait::async_trait;
#[derive(Debug, thiserror::Error)]
pub enum GroupSourceError {
#[error("group source error: {0}")]
Source(String),
}
/// Complete device → groups view. Snapshot-based: membership changes
/// rarely and per-device, so a periodic full fetch beats per-device
/// queries.
#[async_trait]
pub trait DeviceGroupSource: Send + Sync {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError>;
}
/// Fixed membership parsed from a `dev=g1|g2;dev2=g3` string — dev
/// setups and e2e, where no Zitadel exists.
#[derive(Debug, Default)]
pub struct StaticDeviceGroups {
map: HashMap<String, HashSet<String>>,
}
impl StaticDeviceGroups {
pub fn parse(spec: &str) -> Self {
let map = spec
.split(';')
.filter_map(|entry| {
let (device, groups) = entry.split_once('=')?;
let device = device.trim();
if device.is_empty() {
return None;
}
Some((
device.to_string(),
groups
.split('|')
.map(str::trim)
.filter(|g| !g.is_empty())
.map(str::to_string)
.collect(),
))
})
.collect();
Self { map }
}
}
#[async_trait]
impl DeviceGroupSource for StaticDeviceGroups {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
Ok(self.map.clone())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn parses_devices_and_groups() {
let src = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
let map = src.device_groups().await.unwrap();
assert_eq!(map.len(), 2);
assert_eq!(
map["pi-01"],
HashSet::from(["edge-a".to_string(), "edge-b".to_string()])
);
assert_eq!(map["pi-02"], HashSet::from(["edge-b".to_string()]));
}
}

View File

@@ -1,12 +1,5 @@
//! Harmony-side Scores for fleet device onboarding. //! Harmony-side Scores for fleet device onboarding.
//! //!
//! Today this module exposes [`FleetDeviceSetupScore`] — a customer
//! runs it against a freshly-booted device (Pi, VM, bare-metal node
//! later) to install podman, place the `fleet-agent` binary, drop
//! the TOML config, and bring up the agent under systemd. Re-running
//! with a changed config (different labels, new NATS URL, new
//! credentials) is how a device is moved between fleet partitions.
//!
//! The operator + agent crates live outside `harmony/` under //! The operator + agent crates live outside `harmony/` under
//! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`. //! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`.
//! What belongs here is the harmony-framework side: the Scores a //! What belongs here is the harmony-framework side: the Scores a
@@ -22,10 +15,8 @@ pub mod assets;
pub mod groups; pub mod groups;
#[cfg(all(feature = "kvm", unix))] #[cfg(all(feature = "kvm", unix))]
pub mod libvirt_pool; pub mod libvirt_pool;
pub mod operator;
pub mod preflight; pub mod preflight;
pub mod secret_access; pub mod secret_access;
mod setup_score;
#[cfg(all(feature = "kvm", unix))] #[cfg(all(feature = "kvm", unix))]
mod vm_score; mod vm_score;
@@ -42,9 +33,5 @@ pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony
// `harmony-fleet-deploy` per ADR-023 — see that crate for the // `harmony-fleet-deploy` per ADR-023 — see that crate for the
// composite + per-component Scores. // composite + per-component Scores.
pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch}; pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch};
pub use setup_score::{
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
HostsEntry, merge_hosts_file,
};
#[cfg(all(feature = "kvm", unix))] #[cfg(all(feature = "kvm", unix))]
pub use vm_score::ProvisionVmScore; pub use vm_score::ProvisionVmScore;

View File

@@ -1,17 +0,0 @@
//! Operator CRD types (`Deployment`, `Device`, …).
//!
//! Per ADR-023, the deploy-side bits (`FleetOperatorScore`, the helm
//! chart generator, `install_crds`) moved to
//! `fleet/harmony-fleet-deploy/src/operator/`. Only the CRD type
//! definitions remain here — they're consumed at runtime by the
//! operator binary's reconcile loop. Lifting the CRDs out of
//! `harmony` core is in scope for a follow-up PR (they currently
//! couple to `crate::modules::podman` via the `ReconcileScore` payload
//! field, which needs to move alongside them).
pub mod crd;
pub use crd::{
AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
};

View File

@@ -1,132 +1 @@
//! Deployment → secret access control: the domain boundary the fleet pub use harmony_reconciler_contracts::{DeploymentSecretGrants, SecretAccessError};
//! operator drives when a `Deployment`'s allowed groups change.
//!
//! Per ADR-025, groups are the security boundary: a deployment declares
//! which device groups may read its secrets, and devices prove group
//! membership through their signed identity at login. How a grant maps
//! onto a backend's primitive (an OpenBao policy attached to an external
//! group, …) is the *implementation's* concern; callers speak only
//! deployments and group names. That separation is what lets the
//! operator be unit-tested against [`InMemoryDeploymentSecretGrants`]
//! with no live secret store — the production backend is
//! [`crate::modules::openbao::OpenBaoDeploymentSecretGrants`].
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Mutex;
use async_trait::async_trait;
use harmony_reconciler_contracts::DeploymentName;
#[derive(Debug, thiserror::Error)]
pub enum SecretAccessError {
#[error("secret access backend error: {0}")]
Backend(String),
}
/// Sets which device groups may read each deployment's secrets.
#[async_trait]
pub trait DeploymentSecretGrants: Send + Sync {
/// Set the *complete* set of groups allowed to read each listed
/// deployment's secrets. Absolute and idempotent: afterwards exactly
/// the given groups hold the grant — others are revoked, an empty
/// list revokes everywhere. Deployments not listed are untouched.
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError>;
}
/// In-memory [`DeploymentSecretGrants`] for tests: records the latest
/// group set per deployment so a test can assert the operator synced the
/// right grants on each flow, with nothing deployed.
#[derive(Debug, Default)]
pub struct InMemoryDeploymentSecretGrants {
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
}
impl InMemoryDeploymentSecretGrants {
pub fn new() -> Self {
Self::default()
}
/// Current groups for `deployment`, sorted. Empty when no group has
/// access (never granted, or revoked).
pub fn groups_for(&self, deployment: &DeploymentName) -> Vec<String> {
self.grants
.lock()
.unwrap()
.get(deployment)
.map(|s| s.iter().cloned().collect())
.unwrap_or_default()
}
}
#[async_trait]
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError> {
let mut store = self.grants.lock().unwrap();
for (deployment, groups) in grants {
if groups.is_empty() {
store.remove(deployment);
} else {
store.insert(deployment.clone(), groups.iter().cloned().collect());
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid test name")
}
#[tokio::test]
async fn sets_overwrites_and_clears_grants() {
let grants = InMemoryDeploymentSecretGrants::new();
let web = dn("web");
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into(), "edge-b".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a", "edge-b"]);
// Absolute, not additive: a smaller set revokes the rest.
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
// Empty revokes everywhere.
grants
.set_deployment_groups(&[(web.clone(), vec![])])
.await
.unwrap();
assert!(grants.groups_for(&web).is_empty());
}
#[tokio::test]
async fn batch_touches_only_listed_deployments() {
let grants = InMemoryDeploymentSecretGrants::new();
let web = dn("web");
let db = dn("db");
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
.await
.unwrap();
grants
.set_deployment_groups(&[(db.clone(), vec!["edge-b".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
assert_eq!(grants.groups_for(&db), vec!["edge-b"]);
}
}

Some files were not shown because too many files have changed in this diff Show More