diff --git a/.harmony/contexts.toml b/.harmony/contexts.toml new file mode 100644 index 00000000..1265f3bc --- /dev/null +++ b/.harmony/contexts.toml @@ -0,0 +1,10 @@ +[contexts.local] +profile = "local" +autoprovision = true +domain = "localhost" + +# Customer contexts belong in their private repository and can be selected with +# `--config path/to/contexts.toml`. A production context also sets `registry`, +# `project`, `domain`, `image_pull_secret`, and brokered OpenBao cluster access. +# Its cluster contains the immutable `fleet-system/fleet-callout-credentials` +# Secret described in `fleet/README.md`. diff --git a/Cargo.lock b/Cargo.lock index 4d57dd76..8d7c4f5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2872,33 +2872,9 @@ name = "example-fleet-e2e-demo" version = "0.1.0" dependencies = [ "anyhow", - "async-nats", "clap", - "directories", - "env_logger", - "example-fleet-auth-callout", - "futures-util", - "harmony", - "harmony-fleet-auth", - "harmony-fleet-deploy", - "harmony-fleet-operator", - "harmony-k8s", - "harmony-nats-callout", - "harmony-reconciler-contracts", - "harmony_types", - "k3d-rs", - "k8s-openapi", - "kube", - "log", - "nkeys", - "serde", - "serde_json", - "tempfile", + "harmony-fleet-e2e", "tokio", - "tokio-test", - "tracing", - "tracing-subscriber", - "url", ] [[package]] @@ -2917,31 +2893,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "example-fleet-staging-deploy" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-nats", - "clap", - "env_logger", - "harmony", - "harmony-k8s", - "harmony-nats-callout", - "harmony_types", - "k8s-openapi", - "kube", - "log", - "nkeys", - "reqwest 0.12.28", - "serde", - "serde_json", - "tokio", - "tracing", - "tracing-subscriber", - "url", -] - [[package]] name = "example-grafana" version = "0.1.0" @@ -3447,6 +3398,7 @@ dependencies = [ "clap", "env_logger", "harmony", + "harmony-fleet-deploy", "harmony_types", "log", "tokio", @@ -3460,7 +3412,7 @@ dependencies = [ "async-nats", "chrono", "clap", - "harmony", + "harmony-fleet-operator", "harmony-reconciler-contracts", "k8s-openapi", "kube", @@ -3489,6 +3441,7 @@ dependencies = [ "base64 0.22.1", "clap", "harmony", + "harmony-fleet-deploy", "harmony_cli", "harmony_secret", "harmony_types", @@ -3511,28 +3464,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "example_fleet_staging_install" -version = "0.1.0" -dependencies = [ - "anyhow", - "base64 0.22.1", - "clap", - "harmony", - "harmony-fleet-deploy", - "harmony-k8s", - "harmony-nats-callout", - "harmony_cli", - "harmony_config", - "harmony_secret", - "nkeys", - "rand 0.9.4", - "schemars 0.8.22", - "serde", - "tokio", - "tracing", -] - [[package]] name = "example_fleet_vm_setup" version = "0.1.0" @@ -3541,6 +3472,7 @@ dependencies = [ "clap", "env_logger", "harmony", + "harmony-fleet-deploy", "harmony_types", "log", "tokio", @@ -3552,7 +3484,8 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "harmony", + "harmony-fleet-operator", + "harmony-reconciler-contracts", "k8s-openapi", "kube", "serde_json", @@ -4029,13 +3962,13 @@ dependencies = [ "kube-derive", "lazy_static", "log", + "nkeys", "non-blank-string-rs", "once_cell", "opnsense-api", "opnsense-config", "opnsense-config-xml", "option-ext", - "podman-api", "pretty_assertions", "rand 0.9.4", "reqwest 0.11.27", @@ -4079,12 +4012,13 @@ dependencies = [ "chrono", "clap", "futures-util", - "harmony", "harmony-fleet-auth", "harmony-reconciler-contracts", "harmony_secret", + "podman-api", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", "toml", @@ -4121,12 +4055,14 @@ dependencies = [ "fqdn", "harmony", "harmony-fleet-auth", - "harmony-reconciler-contracts", + "harmony-fleet-operator", + "harmony_app", "harmony_cli", "harmony_config", "harmony_macros", "harmony_types", "harmony_zitadel_auth", + "inquire 0.7.5", "k8s-openapi", "kube", "log", @@ -4135,11 +4071,13 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "similar", "tempfile", "thiserror 2.0.18", "tokio", "toml", "tracing", + "url", ] [[package]] @@ -4157,6 +4095,7 @@ dependencies = [ "harmony-fleet-operator", "harmony-k8s", "harmony-reconciler-contracts", + "harmony_secret", "harmony_types", "jsonwebtoken", "k3d-rs", @@ -4188,10 +4127,10 @@ dependencies = [ "clap", "dotenvy", "futures-util", - "harmony", "harmony-fleet-auth", "harmony-reconciler-contracts", "harmony_config", + "harmony_secret", "harmony_zitadel_auth", "k8s-openapi", "kube", @@ -4269,6 +4208,7 @@ dependencies = [ name = "harmony-reconciler-contracts" version = "0.1.0" dependencies = [ + "async-trait", "chrono", "harmony_types", "schemars 0.8.22", @@ -4317,7 +4257,6 @@ dependencies = [ name = "harmony_app" version = "0.1.0" dependencies = [ - "anyhow", "async-trait", "docker-compose-types", "fqdn", @@ -4327,8 +4266,10 @@ dependencies = [ "harmony_types", "k8s-openapi", "log", + "reqwest 0.12.28", "schemars 0.8.22", "serde", + "serde_json", "serde_yaml", "tempfile", "thiserror 2.0.18", @@ -4377,6 +4318,8 @@ dependencies = [ "indicatif", "inquire 0.7.5", "lazy_static", + "serde", + "serde_json", "tokio", "tracing", "tracing-subscriber", @@ -4518,6 +4461,7 @@ version = "0.1.0" dependencies = [ "async-trait", "directories", + "harmony-reconciler-contracts", "harmony_secret_derive", "harmony_zitadel_jwt", "http 1.4.0", @@ -4584,10 +4528,12 @@ version = "0.1.0" dependencies = [ "anyhow", "arc-swap", + "async-trait", "axum", "axum-extra", "base64 0.22.1", "chrono", + "harmony-reconciler-contracts", "harmony_config", "harmony_zitadel_jwt", "jsonwebtoken", diff --git a/Cargo.toml b/Cargo.toml index 19b6987c..65af54da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,6 @@ members = [ "fleet/harmony-fleet-e2e", "harmony-reconciler-contracts", "examples/fleet_server_install", - "examples/fleet_staging_install", "nats/jwt", "nats/callout", "nats/integration-test-callout", @@ -97,6 +96,7 @@ futures-util = "0.3" thiserror = "2.0.14" serde = { version = "1.0.209", features = ["derive", "rc"] } serde_json = "1.0.127" +sha2 = "0.10" askama = "0.14" sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } reqwest = { version = "0.12", features = [ diff --git a/ROADMAP/fleet_platform/public_e2e_test_cluster.md b/ROADMAP/fleet_platform/public_e2e_test_cluster.md new file mode 100644 index 00000000..0619a3e1 --- /dev/null +++ b/ROADMAP/fleet_platform/public_e2e_test_cluster.md @@ -0,0 +1,221 @@ +# Public Fleet E2E environment + +## Goal + +Run public-facing Fleet regression tests directly on the production-adjacent +OKD cluster. Keep the environment definition, identities, context, and CI +workflow in a standalone private repository checked out under +`private_repos/`. The public Harmony repository contains only reusable Scores, +Refs, lifecycle code, and generic test harnesses. + +Build and publish run on the same CI runner. Deployment receives only +digest-pinned image references published to an OCI registry. + +## Private repository + +The private repository owns: + +```text +Cargo.toml +src/main.rs # Fleet app + compiled context catalog +.github/workflows/e2e.yml # or the equivalent CI system +README.md # operator setup and recovery +``` + +It is checked out beside Harmony under `private_repos/`, which the public +repository ignores. No customer or private project identifier belongs in a +tracked Harmony file, fixture, log sample, or test name. + +The private deploy binary defines its contexts as typed Rust values. Contexts +are compiled into the binary; loading additional context definitions at runtime +is deferred until a real requirement justifies that interface. + +```rust +let contexts = ContextCatalog::new([ + NamedContext { + name: context_name!("e2e"), + target: ContextSpec::Remote(RemoteContext { + profile: Profile::Prod, + registry: oci_registry!("registry.example.com"), + registry_project: oci_repository!("harmony/e2e"), + domain: domain!("e2e.example.com"), + image_pull_secret: k8s_name!("registry-pull"), + access: OpenBaoClusterAccess { + namespace: openbao_namespace!("example"), + url: http_url!("https://secrets.nationtech.io"), + role: OpenBaoRoleName::new("example-ci")?, + zitadel_url: http_url!("https://identity.example.com"), + zitadel_audience: OidcAudience::new("example")?, + }, + }), + }, +])?; +``` + +Concrete names and identity details stay private. + +### Context type design + +`ContextSpec` should encode the access-mode choice instead of representing it +with unrelated optional fields: + +```rust +pub enum ContextSpec { + Local(LocalContext), + Remote(RemoteContext), +} +``` + +`RemoteContext` carries required registry, domain, image-pull, and OpenBao +values directly. `LocalContext` carries only its k3d settings. This makes +impossible combinations unrepresentable and removes checks such as "exactly one +of autoprovision, k3d, or OpenBao must be set." + +Use existing domain types where they are already correct: + +- `Profile` remains an enum. It does not need a string macro. +- Kubernetes namespaces, Secret names, and other DNS-label resource names use + `K8sName` and a compile-time `k8s_name!` literal. +- IP and MAC literals continue using the existing Harmony macros. + +Add types only for distinct grammars or values that are easy to transpose: + +- `ContextName`: CLI selection key. It is not implicitly a Kubernetes name. +- `DomainName`: normalized DNS name without scheme, path, port, or wildcard. +- `HttpUrl`: HTTP(S)-only remote URL. The existing `Url` also permits local + folders and is too broad for OpenBao and Zitadel endpoints. +- `OciRegistry`: registry authority with optional port, but no repository path. +- `OciRepository`: validated OCI repository path. +- `OpenBaoNamespace`: a safe OpenBao path namespace, distinct from `K8sName`. +- `OpenBaoRoleName`: auth-role identifier, distinct from the namespace. +- `OidcAudience`: non-empty OIDC audience, kept semantically distinct even + though the protocol permits several string shapes. + +Do not add a macro for every newtype. Literal macros are justified for the +grammars where compile-time diagnostics materially help: DNS names, HTTP URLs, +OCI registries/repositories, Kubernetes names, and OpenBao path namespaces. +Opaque identifiers can use typed constructors. + +The runtime parser and literal macro must share one validator in +`harmony_types`; the macro must not copy the grammar. `harmony_macros` already +depends on `harmony_types`, so it can validate the literal during expansion and +emit the typed value. Each type needs accepted/rejected unit cases plus +compile-fail macro tests. + +### Application integration + +Change the lifecycle entry point from discovering `.harmony/contexts.toml` to +receiving the catalog: + +```rust +harmony_cli::app::app_main(FleetApp, contexts).await +``` + +`--context e2e` and `HARMONY_CONTEXT=e2e` select only from contexts compiled +into that deploy binary. The public Fleet binary can compile its local context; +the private binary compiles the remote E2E context. Both use the same exported +`FleetApp` and Scores. + +## CI trust bootstrap + +The CI runner receives one secret: + +```text +HARMONY_ZITADEL_KEY_JSON +``` + +It is the JSON key for a dedicated Zitadel machine identity. Harmony uses that +identity to obtain an OpenBao token from `secrets.nationtech.io`. `ConfigClient` +then loads the namespace-scoped kubeconfig and deployment secrets. The runner +holds no standing kubeconfig, OpenBao token, registry password, or application +secret. + +Required server-side setup: + +- [ ] Create a dedicated Zitadel machine identity for E2E CI. +- [ ] Grant only the role accepted by the E2E OpenBao auth role. +- [ ] Create an OpenBao policy restricted to the E2E config and secret paths. +- [ ] Store `ClusterAccess` with the namespace-scoped kubeconfig. +- [ ] Store registry credentials and deploy inputs under the E2E namespace. +- [ ] Add the machine key JSON as the CI repository's only secret. +- [ ] Compile the typed `e2e` context into the private deploy binary. +- [ ] Select it with `HARMONY_CONTEXT=e2e`. + +## Known implementation gap + +Cluster access already follows this flow. Image publication currently reads +`REGISTRY_USER` and `REGISTRY_TOKEN` directly from the environment. Before the +single-secret workflow is complete, registry authentication must be loaded +through the resolved `AppContext` and `ConfigClient` like the other secrets. + +The context should resolve once at the start of the combined build-and-publish +job. Build itself remains credential-free; publish consumes the registry +credential fetched from OpenBao. + +The current `harmony_app::context` TOML parser and `Option` context +definition must also be replaced by the typed catalog above. This is a public +application-layer change; concrete context values remain private. + +## Phase 1: cluster and identity + +- [ ] Add the context domain types and their shared validators to + `harmony_types`. +- [ ] Add only the justified literal macros to `harmony_macros`, including + compile-fail tests. +- [ ] Replace TOML discovery with a caller-supplied `ContextCatalog`. +- [ ] Export the canonical `FleetApp` composition so the private deploy binary + does not duplicate it. +- [ ] Create a dedicated E2E namespace or namespace prefix on the public OKD + cluster. +- [ ] Configure namespace-scoped RBAC for the OpenBao-brokered kubeconfig. +- [ ] Add public DNS and exact-host OKD Routes for E2E services. +- [ ] Verify the CI machine identity can read `ClusterAccess` and no unrelated + OpenBao path. +- [ ] Run a read-only context check from CI with only + `HARMONY_ZITADEL_KEY_JSON` present. + +## Phase 2: OCI publication + +- [ ] Move registry credentials behind `ConfigClient`. +- [ ] Create a disposable registry project or repository scope for E2E images. +- [ ] Run `build` and `publish` in one CI job. +- [ ] Verify the pushed manifest digest matches the digest returned by build. +- [ ] Pull and run that exact digest in a fresh namespace. +- [ ] Apply retention rules so E2E images expire automatically. + +The production publication path remains the same operation against the +production registry; only the context and credential policy differ. + +## Phase 3: NATS regression + +- [ ] Deploy the existing certificate and mTLS-capable NATS supercluster Scores + unchanged before modifying their APIs. +- [ ] Exercise public HTTPS/WebSocket and native TLS/mTLS endpoints through OKD + Routes. +- [ ] Assert cluster formation, authenticated traffic, SNI routing, reconnect + after pod restart, and certificate validation from outside the cluster. +- [ ] Record failures as regressions against the existing module before changing + Fleet composition. + +## Phase 4: Fleet lifecycle + +- [ ] Build and publish the operator and callout images in one job. +- [ ] Pass their digest-pinned references to `deploy`. +- [ ] Deploy Fleet through the private `e2e` context using production Scores and + typed Refs. +- [ ] Enroll an external test agent through the public NATS endpoint. +- [ ] Apply, update, and delete a Podman deployment and verify reflected state. +- [ ] Exercise Zitadel group changes and OpenBao secret access. +- [ ] Verify failed reconciliation recovers after the dependency is restored. +- [ ] Remove per-run resources and verify no desired-state or grant residue + remains. + +## Guardrails + +- Use a dedicated machine identity, OpenBao role, registry scope, and Kubernetes + RBAC grant for E2E. +- Keep every concrete private identifier in the private repository or OpenBao. +- Put a TTL label on per-run namespaces and clean abandoned runs automatically. +- Never print the machine key, OpenBao token, kubeconfig, or registry credential. +- Split CI into identity, registry, NATS, and Fleet stages so failures identify + one boundary. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c82feea2..7706f74e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -3,6 +3,7 @@ [Harmony Documentation](./README.md) - [Core Concepts](./concepts.md) +- [Typed Score References](./concepts/score-references.md) - [Getting Started Guide](./guides/getting-started.md) ## Use Cases @@ -33,6 +34,10 @@ - [Configuration](./concepts/configuration.md) +## Reference Designs + +- [Fleet Score References](./reference/fleet-score-references.md) + ## Architecture Decision Records - [ADR Overview](./adr/README.md) @@ -57,5 +62,11 @@ - [019 · Network Bond Setup](./adr/019-Network-bond-setup.md) - [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md) - [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md) + - [022 · Fleet Agent Upgrade Procedure](./adr/022-fleet-agent-upgrade.md) + - [023 · Deploy Architecture](./adr/023-deploy-architecture.md) + - [024 · Fleet Platform Capability Decomposition](./adr/024-fleet-platform-capability-decomposition.md) - [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md) - [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md) + - [026 · Application Lifecycle CLI](./adr/026-application-lifecycle-cli.md) + - [027 · Multi-Tenant Cloud Identity](./adr/027-multi-tenant-identity.md) + - [028 · Typed Score References](./adr/028-typed-score-references.md) diff --git a/docs/adr/028-typed-score-references.md b/docs/adr/028-typed-score-references.md new file mode 100644 index 00000000..cebcef1a --- /dev/null +++ b/docs/adr/028-typed-score-references.md @@ -0,0 +1,176 @@ +# Architecture Decision Record: Typed Score References + +Initial Author: Jean-Gabriel Gill-Couture + +Initial Date: 2026-07-15 + +Last Updated Date: 2026-07-15 + +## Status + +Proposed. Fleet is the first application. + +## Context + +ADR-023 establishes one Score per deployable component and requires deploy +crates to compose Scores instead of rebuilding their operations. It does not +define how one Score refers to desired state owned by another component. + +Today, deploy code often passes URLs, names, IDs, and credential locations as +strings. Consumers then parse those strings or reproduce the producer's naming +conventions. Rust cannot distinguish different string contracts, and a refactor +can leave valid code that fails only at runtime. + +Earlier composition design attempts treated these relationships as a dependency +graph or pipeline. That caused too strong coupling between each component's +lifecycle. Harmony needs a smaller contract: a consumer should know what it +references without depending on the producer Score or controlling when it runs. + +Infrastructure also does not converge once and remain stable. A referenced +component may not exist yet, may still be converging, or may become unavailable +later. The reference model must not imply otherwise. + +## Decision + +Scores will exchange concrete typed Refs for desired state owned by another +component: + +[![A producer Score passes a typed Ref to a dependent Score.](../diagrams/score-ref-composition.svg)](../diagrams/score-ref-composition.svg) + +The complete API is small: + +```rust +pub struct DbRef { + // Desired-state properties needed to connect. +} + +let db_score = SomeDbScore::new(...); +let db_ref = db_score.connection(); +let app_score = AppScore::new(...).with_db(&db_ref); +``` + +The producer derives a Ref from its desired state. Creating or passing the Ref +performs no I/O and does not require the producer to have run. + +The consumer depends on the Ref type, not the producer Score. The Ref contains +the typed identifiers needed to describe the relationship. It does not contain +a client, execute provider operations, or transfer ownership of the producer's +lifecycle. + +A Ref is eventually consistent. It does not prove that its target exists, has +converged, or remains available. The consumer's `Interpret`/`Topology` path must +handle that uncertainty. Where Kubernetes already understands the relationship, +the consumer should emit the native Kubernetes reference and let Kubernetes +reconcile it. + +Refs do not impose execution order. They do not define a dependency graph, +pipeline, scheduler, or continuous convergence mechanism. + +### Cyclic relationships + +A cyclic relationship uses the same operation in both directions: + +[![BrokerScore and AuthScore exchange BrokerRef and AuthRef before either Score runs.](../diagrams/score-ref-cycle.svg)](../diagrams/score-ref-cycle.svg) + +```rust +let broker_score = BrokerScore::new(...); +let broker_ref = broker_score.connection(); + +let auth_score = AuthScore::new(...).with_broker(&broker_ref); +let auth_ref = auth_score.endpoint(); + +let broker_score = broker_score.with_auth(&auth_ref); +``` + +Both Refs derive from desired state before either Score runs. Declaring the +cycle therefore needs no graph traversal or scheduler. + +### Values produced later + +A Ref may identify a value that will be generated later, such as an OIDC client +ID. Composition can pass the Ref before that value exists. This is the same +eventual-consistency rule, not a separate output mechanism. + +### Guarantees + +| Statement | Guarantee | +|---|---| +| Consumer accepts the intended Ref type | Rust compile time | +| Ref construction performs no I/O | Ref API contract | +| Target exists or has converged | Not guaranteed | +| Runtime systems are compatible or authorized | Not guaranteed(*) | +| Target remains available | Not guaranteed | + +> (*) The runtime compatibility or authorization is not guaranteed by the **Ref** itself. But it is usually Harmony's job to provide ref types in a way that makes them compatible. For example, a PostgreSQL score exposing a PostgreSQLRootAccountRef will have the responsibility to provision it. +> +> In other words, __correct provisioning of a Ref is the score's developer's job, not the ref's job.__ + +Changing a Ref's type-level contract produces compiler errors in affected +consumers. Rust cannot catch a provider returning incorrect live state, so +runtime validation and readiness remain necessary. + +## Rationale + +- Typed Refs replace string parsing and copied naming conventions. +- Scores share only the desired-state contract needed by the consumer. +- Refs can be composed before any component runs. +- Consumers remain responsible for infrastructure that is not yet ready or is + temporarily unavailable. +- Kubernetes keeps reconciling relationships it already understands. +- Harmony does not need a graph or pipeline abstraction to express a typed + relationship. + +## Consequences + +### Positive + +- Refactoring a relationship's typed contract reaches its consumers through the + compiler. +- Components remain independent of each other's Score implementations and + execution order. +- Production and E2E can compose the same Score APIs against different + Topologies. +- Values produced later need not become string dependencies in deploy code. + +### Negative + +- Each real relationship needs a concrete Ref type. +- Consumers must handle unresolved and unavailable targets. +- Types do not remove runtime readiness, compatibility, or authorization checks. +- Existing string-based wiring must be migrated as its owning Scores gain Refs. + +## Alternatives considered + +### Continue passing strings + +Rejected. Strings erase domain meaning and force consumers to know producer +naming and parsing rules. + +### Pass producer Scores to consumers + +Rejected. This couples component lifecycle and implementation when the consumer +only needs a desired-state reference. + +### Make execution return values the composition mechanism + +Rejected for this decision. It requires the producer to run before composition +can continue and turns execution order into the dependency model. + +### Introduce a generic graph or output registry + +Rejected. Typed Refs solve the current problem without either abstraction. + +## Out of scope + +ADR-028 does not define continuous convergence, retries, scheduling, graph +execution, Ref persistence, or a generic `Ref`. Those decisions require their +own evidence and designs. + +## References + +- [Typed Score references](../concepts/score-references.md): concept guide. +- [Fleet Score references](../reference/fleet-score-references.md): first application. +- [ADR-002](./002-hexagonal-architecture.md): domain and adapter boundaries. +- [ADR-015](./015-higher-order-topologies.md): compile-time Topology + composition. +- [ADR-023](./023-deploy-architecture.md): component Scores and deploy crates. diff --git a/docs/adr/README.md b/docs/adr/README.md index f8df20b3..1935b80b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -59,6 +59,7 @@ Every ADR follows this structure: | [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted | | [026](./026-application-lifecycle-cli.md) | Application Lifecycle CLI | Accepted | | [027](./027-multi-tenant-identity.md) | Multi-Tenant Cloud Identity | Accepted | +| [028](./028-typed-score-references.md) | Typed Score References | Proposed | ## Contributing diff --git a/docs/adr/drafts/024-architecture-review.md b/docs/adr/drafts/024-architecture-review.md index 34cf7b44..ab58292b 100644 --- a/docs/adr/drafts/024-architecture-review.md +++ b/docs/adr/drafts/024-architecture-review.md @@ -422,9 +422,8 @@ what vocabulary should the new fleet shape stay consistent with? apply via typed code. - **Implication for fleet:** chart-rendering helpers (`build_operator_deployment` et al.) are template-hydration - edges. They *should* be hidden from domain code. Today they're - `pub` — visible to consumers like `fleet_staging_install` who - reach in and grab `operator_secret(opts)`. That's adapter + edges. They *should* be hidden from domain code. Exposing them to + deploy callers instead of composing `FleetOperatorScore` is adapter leakage. ### §2.4 — Synthesis: principles for the redesign diff --git a/docs/concepts/score-references.md b/docs/concepts/score-references.md new file mode 100644 index 00000000..c6fb9b72 --- /dev/null +++ b/docs/concepts/score-references.md @@ -0,0 +1,125 @@ +# Typed Score references + +Harmony builds loosely dependent but strongly typed infrastructure components +using Refs. + +This means that a `ZitadelScore` does not directly depend on a +`PostgreSQLScore`. It can instead accept a `PostgreSQLRootAccountRef` in its +builder to know how to connect to PostgreSQL. This is safer than passing strings +around: the Ref has a type Rust can check, and consumers do not need to parse or +reconstruct the producer's naming conventions. + +The loose dependency matters. Previous design proposals treated composition as a +dependency graph or pipeline, which forced components to know too much about one +another. A Ref is only a typed reference to declarative desired state. It does +not give the consumer control over the producer's lifecycle. + +Most importantly, a Ref does not guarantee that its target exists or has +converged when it is passed. Like a Kubernetes reference, it is eventually +consistent. Consumers must tolerate the target being absent, starting, or +temporarily unavailable. + +Kubernetes gets this right: infrastructure is chaotic. A component can fail +after it becomes ready. Drives fail, power goes out, and networks split. The +long-term answer is continuous convergence, not an assumption that deployment +finishes infrastructure forever. + +The convergence mechanism is outside ADR-028. Choosing eventually consistent +Refs now lets Scores declare relationships without committing Harmony to a +scheduler, dependency engine, or reconciliation design. + +## The whole pattern + +[![SomeDbScore produces DbRef for AppScore.](../diagrams/score-ref-composition.svg)](../diagrams/score-ref-composition.svg) + +```rust +pub struct DbRef { + // Desired-state properties needed to connect. +} + +let db_score = SomeDbScore::new(...); +let some_db_ref = db_score.connection(); +let app_score = AppScore::new(...).with_db(&some_db_ref); +``` + +That is the mechanism. `connection()` derives `DbRef` from the database's desired +state. It performs no I/O and does not require `SomeDbScore` to have been +interpreted. + +`AppScore` knows `DbRef`. It does not need to know `SomeDbScore`, its provider, +or its readiness procedure. + +Compare that with copying a connection string: + +```rust +app.database_url("postgres://some-db.apps.svc:5432/app"); +``` + +That string duplicates database naming and connection knowledge in the consumer. +A rename or layout change can leave valid Rust that fails at runtime. A typed +Ref removes that string contract and makes type-level contract changes visible +to the compiler. + +## Cyclic relationships + +A cycle is the same pattern in both directions: + +[![BrokerScore and AuthScore exchange BrokerRef and AuthRef before either Score runs.](../diagrams/score-ref-cycle.svg)](../diagrams/score-ref-cycle.svg) + +```rust +let broker_score = BrokerScore::new(...); +let broker_ref = broker_score.connection(); + +let auth_score = AuthScore::new(...).with_broker(&broker_ref); +let auth_ref = auth_score.endpoint(); + +let broker_score = broker_score.with_auth(&auth_ref); +``` + +No Score has run. Each Ref comes from desired state, so the two relationships +can be declared without traversing a graph or resolving either resource. + +## Eventual consistency + +Composition code passes Refs; it does not wait for them to become ready. The +consumer's `Interpret`/`Topology` path handles an unresolved target in the way +appropriate for that component. It may: + +- emit a native Kubernetes reference and let Kubernetes reconcile it; +- wait or retry when the value is needed immediately; +- return a non-success outcome for the current interpretation attempt. + +ADR-028 does not choose among those behaviors or define when another attempt +runs. It only requires consumers not to treat possession of a Ref as proof of +availability. + +Refs may also identify values generated later, such as an OIDC client ID. The +same rule applies: passing the Ref does not claim that the value exists yet. + +## What types guarantee + +Concrete types such as `PostgreSQLRootAccountRef`, `OidcApplicationRef`, and +`NatsAccountRef` prevent unrelated references from being exchanged by accident. +They also remove string parsing and copied naming rules from consumers. + +Types do not prove that: + +- the referenced resource exists; +- the producer has converged; +- two live systems can communicate; +- credentials are authorized; +- the infrastructure will remain available. + +Those remain runtime concerns. + +## Authoring rule + +Use a Ref when one Score would otherwise copy a name, URL, ID, credential +location, or other desired-state identifier owned by another component. Use an +ordinary typed value when there is no such relationship. + +Refs express relationships. They do not impose execution order, create a +pipeline, or make Harmony responsible for continuous convergence. + +See [ADR-028](../adr/028-typed-score-references.md) for the decision and the +[Fleet reference](../reference/fleet-score-references.md) for the first application. diff --git a/docs/diagrams/fleet-identity-composition.svg b/docs/diagrams/fleet-identity-composition.svg new file mode 100644 index 00000000..6fb9101b --- /dev/null +++ b/docs/diagrams/fleet-identity-composition.svg @@ -0,0 +1,45 @@ + + PostgreSQL and Zitadel compose identity functionality through a typed Ref + PostgreSQLScore exports PostgreSQLRootAccountRef to ZitadelScore. Together they provide identity functionality. + + + + + + + + + Compose identity + A consumer receives a typed contract, not the producer Score. + + + SCORE + PostgreSQLScore + provisions PostgreSQL + + + + REF TYPE + PostgreSQLRootAccountRef + + + + SCORE + ZitadelScore + provisions Zitadel + + = + + IDENTITY + FUNCTIONALITY + composed + + + Score owns behavior + + Ref carries typed desired state + + diff --git a/docs/diagrams/fleet-messaging-composition.svg b/docs/diagrams/fleet-messaging-composition.svg new file mode 100644 index 00000000..0d265606 --- /dev/null +++ b/docs/diagrams/fleet-messaging-composition.svg @@ -0,0 +1,50 @@ + + NATS and its authentication callout compose authenticated messaging + NatsScore exports NatsAccountRef to NatsAuthCalloutScore. NatsAuthCalloutScore exports NatsAuthCalloutRef back to NatsScore. The composed functionality requires OidcProviderRef and exports NatsClientRef. + + + + + + + + + Compose authenticated messaging + Two Scores can exchange desired-state Refs without imposing execution order. + + + AUTHENTICATED MESSAGING FUNCTIONALITY + + REQUIRES + + OidcProviderRef + + + SCORE + NatsScore + provisions NATS + + + + NatsAccountRef + + + + NatsAuthCalloutRef + + + SCORE + NatsAuthCalloutScore + provisions the callout + + EXPORTS + + NatsClientRef + One capability for consumers + + The common region is the functionality; the blue boxes remain independent Scores. + + diff --git a/docs/diagrams/fleet-score-references.svg b/docs/diagrams/fleet-score-references.svg new file mode 100644 index 00000000..6fcd8009 --- /dev/null +++ b/docs/diagrams/fleet-score-references.svg @@ -0,0 +1,100 @@ + + Fleet reference implementation contract map + A three-column map lists reusable Fleet functionality, the concrete Ref types it produces, and the Score types that consume each Ref. + + + + Fleet reference implementation + Read each contract left to right. The map shows type compatibility, not execution order. + + FUNCTIONALITY + PRODUCES REF TYPE + CONSUMED BY SCORE TYPE + + + + + + Identity + PostgreSQLScore + + + ZitadelScore + + + OidcProviderRef + + OidcApplicationRef + + OidcMachineIdentityRef + + + OpenBaoScore + + NatsAuthCalloutScore + + FleetDeviceEnrollmentScore + + FleetOperatorScore + + FleetOperatorScore + + + + + + + + Messaging + NatsScore + NatsAuthCalloutScore + + + NatsClientRef + + FleetOperatorScore + + FleetDeviceScore + + + + + Secrets + OpenBaoScore + + + OpenBaoRef + + OpenBaoJwtAuthRef + + FleetOperatorScore + + FleetDeviceScore + + + + + + Device identity + FleetDeviceEnrollmentScore + + + DeviceIdentityRef + + FleetDeviceScore + + + + SCORE TYPE + owns behavior + + SomeRef + carries typed desired state, not readiness + + diff --git a/docs/diagrams/score-ref-composition.svg b/docs/diagrams/score-ref-composition.svg new file mode 100644 index 00000000..5a1717b8 --- /dev/null +++ b/docs/diagrams/score-ref-composition.svg @@ -0,0 +1,38 @@ + + A Score passes a typed Ref to a dependent Score + SomeDbScore produces DbRef from desired state. AppScore accepts DbRef without depending on SomeDbScore. + + + + + + + + + Score → Ref → dependent Score + The Ref is the complete relationship between otherwise independent components. + + PRODUCER + + SCORE + SomeDbScore + owns database behavior + + + + REF TYPE + DbRef + + + CONSUMER + + SCORE + AppScore + knows DbRef, not SomeDbScore + + Creating and passing DbRef performs no I/O and says nothing about readiness. + + diff --git a/docs/diagrams/score-ref-cycle.svg b/docs/diagrams/score-ref-cycle.svg new file mode 100644 index 00000000..7a25532d --- /dev/null +++ b/docs/diagrams/score-ref-cycle.svg @@ -0,0 +1,41 @@ + + Two Scores exchange typed Refs in a cyclic relationship + BrokerScore passes BrokerRef to AuthScore. AuthScore passes AuthRef to BrokerScore. Both Refs come from desired state before either Score runs. + + + + + + + + + A cycle is the same pattern twice + Both Refs come from desired state before either Score runs. + + + COMPOSED FUNCTIONALITY + + + SCORE + BrokerScore + owns the broker + + + + BrokerRef + + + + AuthRef + + + SCORE + AuthScore + owns authentication + + The cycle describes desired relationships; it does not require a graph scheduler. + + diff --git a/docs/guides/fleet-staging-install.md b/docs/guides/fleet-staging-install.md index e6004238..a13d3c1e 100644 --- a/docs/guides/fleet-staging-install.md +++ b/docs/guides/fleet-staging-install.md @@ -1,460 +1,105 @@ -# Fleet staging install on OKD +# Deploy Fleet to a remote cluster -End-to-end runbook for deploying the fleet stack (Zitadel + NATS + -auth callout + operator) on an OKD cluster, with a remote agent -connecting through the public WSS endpoint. Targets the staging -shape — single-instance NATS, public Zitadel + NATS WS Routes with -edge-TLS via cert-manager, env-only Secret config (no volume -mounts) so default `restricted-v2` SCC is enough. +Fleet uses the same lifecycle commands and Scores for local, staging, and +production clusters. The selected context changes the target and publication +behavior; there is no separate staging installer. -Time budget: ~30 min on a warm cluster, ~60 min cold. +## Prerequisites -## 0. Prereqs +- Docker with Buildx, Helm, and kubectl are on `PATH`. +- The remote cluster has cert-manager and the CloudNativePG operator. +- The image registry contains credentials for `REGISTRY_USER` and + `REGISTRY_TOKEN`. +- The deploy identity can obtain the cluster kubeconfig through OpenBao. +- Public DNS points the context domain at the cluster ingress. -- `oc` CLI logged in with cluster-admin (or at least - cluster-scoped privileges on the namespaces below — namespace - create, CRD apply, ClusterRole create). -- `podman` on your laptop, authenticated to the destination registry - (default `hub.nationtech.io/harmony` — `podman login` if needed). -- `helm` on PATH (used by Harmony's helm chart Scores). -- The staging cluster has: - - cert-manager installed and a `ClusterIssuer` ready for the - cluster's base domain (default name: `letsencrypt-prod` — - override with `--cluster-issuer` if yours differs). - - CNPG (cloudnative-pg) operator installed (Zitadel relies on it - for its Postgres cluster). - - DNS: the chosen `--base-domain` resolves to the OKD ingress - router. For `cb1.nationtech.io`, that means `*.cb1.nationtech.io` - or at least `sso-staging.cb1.nationtech.io` and - `nats-fleet-staging.cb1.nationtech.io` must point at the OKD - router VIP. If you're using the cluster's apps domain - (`apps.cb1.nationtech.io`), set `--base-domain` accordingly. -- Access to write a `[credentials]` TOML on whichever machine will - run the agent (your laptop is fine for the demo). +## Define the context -## 1. Build and push images +Remote contexts belong in the customer's private repository, not this public +repository. Create a context file like this: -The staging install pulls operator + auth-callout images from your -registry. The helper script builds both, tags them, and pushes: +```toml +[contexts.customer-prod] +profile = "prod" +registry = "hub.example.com" +project = "customer" +domain = "fleet.example.com" +image_pull_secret = "registry-pull" +openbao_namespace = "customer-prod" +openbao_url = "https://openbao.example.com" +zitadel_url = "https://sso.example.com" +openbao_role = "customer-prod-cd" +zitadel_audience = "openbao" +``` + +Set `HARMONY_ZITADEL_KEY_JSON` to the deploy service user's Zitadel key JSON. +The context uses that identity to obtain its OpenBao-brokered kubeconfig. + +## Provision callout credentials + +Before the first deploy, create the immutable +`fleet-system/fleet-callout-credentials` Secret. It must contain +`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass`. ```bash -cd /path/to/harmony -./fleet/scripts/build_and_push_images.sh +kubectl create namespace fleet-system --dry-run=client -o yaml | kubectl apply -f - +kubectl -n fleet-system create secret generic fleet-callout-credentials \ + --from-literal=issuer-nkey-seed="$ISSUER_NKEY_SEED" \ + --from-literal=issuer-public-key="$ISSUER_PUBLIC_KEY" \ + --from-literal=nats-auth-pass="$NATS_AUTH_PASS" \ + --dry-run=client -o yaml | kubectl apply -f - +kubectl -n fleet-system patch secret fleet-callout-credentials \ + --type=merge -p '{"immutable":true}' ``` -Defaults: `REGISTRY=hub.nationtech.io/harmony`, `IMAGE_TAG=dev`, -`PUSH=1`. Override with environment variables. Skip the push (e.g. -to inspect the images locally first) with `PUSH=0`. +The registry pull Secret named by `image_pull_secret` must also exist in each +namespace that pulls private images. -Output ends with the exact `--operator-image` / `--callout-image` -flags to paste into step 4. +## Deploy -**Verify:** +`ship` builds the operator and callout images, pushes their digest-pinned +references, then converges the Fleet Scores serially: ```bash -podman images | grep harmony # both refs present locally -podman pull hub.nationtech.io/harmony/harmony-fleet-operator:dev # registry confirmed +export REGISTRY_USER=... +export REGISTRY_TOKEN=... +export HARMONY_ZITADEL_KEY_JSON=... + +cargo run -p harmony-fleet-deploy -- \ + ship --context customer-prod --config path/to/contexts.toml ``` -## 2. Create namespaces +For CI, the stages can run independently. Pass the digest references printed by +`build` into `publish` and `deploy`: ```bash -oc new-project zitadel-staging -oc new-project fleet-staging +cargo run -p harmony-fleet-deploy -- build --context customer-prod --config path/to/contexts.toml +cargo run -p harmony-fleet-deploy -- publish --context customer-prod --config path/to/contexts.toml \ + --image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \ + --image callout=hub.example.com/customer/harmony-nats-callout@sha256:... +cargo run -p harmony-fleet-deploy -- deploy --context customer-prod --config path/to/contexts.toml \ + --image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \ + --image callout=hub.example.com/customer/harmony-nats-callout@sha256:... ``` -If `hub.nationtech.io` requires authentication, add the imagePullSecret -to both namespaces (each pod that pulls from the registry needs it): +`deploy` never builds or publishes images. It returns only after each Score's +readiness check passes, including the operator's authenticated NATS startup. + +## Verify ```bash -# adjust to whatever you have for hub.nationtech.io -oc -n fleet-staging secrets link default --for=pull -oc -n zitadel-staging secrets link default --for=pull +cargo run -p harmony-fleet-deploy -- status --context customer-prod --config path/to/contexts.toml +cargo run -p harmony-fleet-deploy -- logs --context customer-prod --config path/to/contexts.toml +kubectl -n fleet-system get deployments,pods +kubectl get deployments.fleet.nationtech.io,devices.fleet.nationtech.io ``` -(For Zitadel + Postgres the chart pulls from public registries, so -the secret is only strictly required in `fleet-staging` for the -operator + callout images. Linking both is safest.) - -## 3. Set KUBECONFIG and verify cluster context +For local validation before a remote deploy: ```bash -export KUBECONFIG=$ADMIN_KUBECONFIG -oc whoami -oc config current-context -oc get clusterversion # confirm OKD reachable + healthy +cargo run -p harmony-fleet-deploy -- ship --context local +HARMONY_FLEET_E2E=1 cargo test -p harmony-fleet-e2e --test ping -- --nocapture ``` -The install runs with this `KUBECONFIG`. **Double-check** before -running step 4 — Harmony's `K8sAnywhereTopology::from_env()` honors -this and there's no second confirmation prompt. - -## 4. Run `fleet_staging_install` - -```bash -cargo run --release -p example_fleet_staging_install -- \ - --base-domain cb1.nationtech.io \ - --operator-image hub.nationtech.io/harmony/harmony-fleet-operator:dev \ - --callout-image hub.nationtech.io/harmony/harmony-nats-callout:dev -``` - -Optional flags (defaults shown): - -``` - --cluster-issuer letsencrypt-prod - --fleet-namespace fleet-staging - --zitadel-namespace zitadel-staging - --nats-account FLEET - --zitadel-version v4.12.1 - --project-name fleet - --admin-role fleet-admin - --device-role device - --operator-username fleet-operator - --admin-username fleet-ops -``` - -Step-by-step the binary does: - -1. **Zitadel helm install** — Postgres (CNPG) + Zitadel chart into - `--zitadel-namespace`. Edge-TLS Route at `sso-staging.` - with cert-manager-driven certificate. -2. **Zitadel setup** — project, two roles (`fleet-admin`, `device`), - API app `nats`, and two machine users (`fleet-ops` for manual - admin work, `fleet-operator` for the operator pod). Both get - JSON keys cached at `~/.local/share/harmony/zitadel/client-config.json`. -3. **NATS install** — single-instance JetStream, `auth_callout` - block referencing the issuer NKey pubkey, WebSocket listener on - 8080. Edge-TLS Route at `nats-fleet-staging.`. -4. **Auth callout deployment** — env-only Secret config (no mounts), - wired to the same issuer + Zitadel project audience. -5. **Operator deployment** — single Secret holding the credentials - TOML (with the operator's JSON keyfile inlined). One env var, - `FLEET_OPERATOR_CREDENTIALS_TOML`, no volumes. - -The binary prints the URLs + project_id at the end. Save that block -— you'll need the project_id for the agent config. - -**Expected output tail:** - -``` -=== fleet-staging install complete === -Zitadel: https://sso-staging.cb1.nationtech.io/ -NATS WS public: wss://nats-fleet-staging.cb1.nationtech.io/ -NATS in-cluster: nats://fleet-nats.fleet-staging.svc.cluster.local:4222 -Operator: oc -n fleet-staging get deploy/harmony-fleet-operator -Auth callout: oc -n fleet-staging get deploy/fleet-callout -Project id: 371xxxxxxxxxxxxxxx -Admin user: fleet-ops (machine key in ~/.local/share/harmony/zitadel/client-config.json) -Operator user: fleet-operator (machine key embedded in operator's Secret) -``` - -## 5. Verify each layer - -### 5.1 Zitadel reachable, certificate provisioned - -```bash -# pod up -oc -n zitadel-staging get pods -# expect: zitadel-* Running, zitadel-pg-1/2 Running - -# Route + certificate (cert-manager creates the secret) -oc -n zitadel-staging get route -oc -n zitadel-staging get certificate - -# OIDC discovery from the public URL -curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer -# expect: "https://sso-staging.cb1.nationtech.io" -``` - -If `curl` fails with TLS errors, the cert-manager certificate isn't -ready yet. Watch its status: - -```bash -oc -n zitadel-staging describe certificate -oc -n cert-manager logs deploy/cert-manager --tail=50 -``` - -A `Ready` condition `True` + `secretName: zitadel-tls` populated -means the Route can serve HTTPS. - -### 5.2 NATS pod up, callout connected - -```bash -oc -n fleet-staging get pods -# expect: -# fleet-nats-0 2/2 Running (NATS + reloader sidecar) -# fleet-callout-... 1/1 Running - -oc -n fleet-staging logs deploy/fleet-callout --tail=30 | grep -E "starting|JWKS|listening" -# expect: -# starting harmony NATS auth callout -# JWKS refreshed count=2 -# auth callout service listening subject="$SYS.REQ.USER.AUTH" -``` - -If the callout pod CrashLoopBackOff: - -```bash -oc -n fleet-staging logs deploy/fleet-callout --previous --tail=30 -``` - -Most common: OIDC issuer URL mismatch. The callout's -`OIDC_ISSUER_URL` env must byte-equal what Zitadel emits as `iss` in -its discovery doc. Check both: - -```bash -oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_ISSUER_URL -# vs. -curl -s https://sso-staging.cb1.nationtech.io/.well-known/openid-configuration | jq .issuer -``` - -### 5.3 Operator authenticated and running - -```bash -oc -n fleet-staging get pods -l app.kubernetes.io/name=harmony-fleet-operator -oc -n fleet-staging logs deploy/harmony-fleet-operator --tail=30 -``` - -Look for, in order: - -``` -minted fresh Zitadel access token audience= -connected successfully server=4222 -NATS connected -KV bucket ready bucket=desired-state -starting Deployment controller -device-reconciler: watching device-info KV -aggregator: startup complete -``` - -If you see `Permissions Violation` errors, the callout's -`OIDC_AUDIENCE` (project_id at deploy time) doesn't match the -project_id in Zitadel today. Re-run step 4 — the live-query fix -in the Zitadel setup will refresh. - -### 5.4 NATS WSS reachable from outside the cluster - -```bash -curl -sSI https://nats-fleet-staging.cb1.nationtech.io/ | head -5 -``` - -Expect a 4xx (NATS doesn't speak HTTP, but the TLS handshake should -succeed and you'll get back a WebSocket-upgrade-related response). -A connection refused or TLS handshake error means the Route or -cert-manager is unhappy. - -### 5.5 CRDs registered - -```bash -oc get crd | grep fleet.nationtech.io -# expect: -# deployments.fleet.nationtech.io -# devices.fleet.nationtech.io -``` - -## 6. Connect a remote agent - -The fleet agent runs on the device (laptop, Pi, anywhere with -outbound HTTPS). It needs: - -- Its own Zitadel machine user with the `device` role grant. -- The JSON keyfile from that user. -- A `[credentials]` TOML pointing at the public Zitadel + the WSS - NATS URL. - -### 6.1 Mint a per-device machine user - -Use `oc port-forward` or a helper to call Zitadel's API. Easier -path: drop a quick Score that adds one machine user. For tonight, -do it from the Zitadel UI: - -1. Browse to `https://sso-staging.cb1.nationtech.io/ui/console/`, - log in as the human admin (password from Zitadel ConfigMap on - first install — see `docs/guides/fleet-zitadel-faq.md`). -2. Pick the `Default` org → `fleet` project → Roles → confirm - `device` exists. -3. Org → Users → Service Users → New: name `device-laptop-01`, - userName `device-laptop-01`. Save. -4. The user's "Personal Information" tab → Authorizations or - "Authorization" → "+New" — grant the `fleet` project's `device` - role to this user. -5. The user's "Keys" tab → "+New", type `JSON`, expiration future - date. **Download the keyfile JSON** — Zitadel only shows the - private half once. Save as `~/.local/share/harmony/fleet/agents/device-laptop-01.json`. - -### 6.2 Build the agent locally - -```bash -cargo build --release -p harmony-fleet-agent -ls -la target/release/harmony-fleet-agent -``` - -### 6.3 Render the agent's config TOML - -```bash -PROJECT_ID=$(oc -n fleet-staging exec deploy/fleet-callout -- printenv OIDC_AUDIENCE) -cat > /tmp/fleet-agent-config.toml <`, matching -the callout's `DEVICE_ID_PREFIX_STRIP=device-`. The Zitadel machine -user must literally be `device-laptop-01` for the JWT-bearer flow -to extract the right device id. - -### 6.4 Run the agent - -```bash -sudo mkdir -p /etc/fleet-agent -sudo cp ~/.local/share/harmony/fleet/agents/device-laptop-01.json \ - /etc/fleet-agent/zitadel-key.json -sudo chown $(id -u):$(id -g) /etc/fleet-agent/zitadel-key.json -sudo chmod 0400 /etc/fleet-agent/zitadel-key.json - -FLEET_AGENT_CONFIG=/tmp/fleet-agent-config.toml \ - RUST_LOG=info \ - ./target/release/harmony-fleet-agent -``` - -Watch the log: - -``` -fleet-agent-v0 starting device_id=device-laptop-01 -podman socket ready -inventory loaded hostname=... -connecting to NATS ["wss://nats-fleet-staging.cb1.nationtech.io"] -minted fresh Zitadel access token audience= -connected successfully server=... -NATS connected -fleet publisher ready -watching KV keys filter=device-laptop-01.> -``` - -If you hit `Permissions Violation` errors after `connected`: -- check `oc -n fleet-staging logs deploy/fleet-callout --tail=20` — - it'll show why the JWT was rejected (audience, role claim, - device_id format). - -### 6.5 Verify the operator created a Device CR - -```bash -oc get devices -# expect: -# NAME AGE -# device-laptop-01 Xs -oc describe device device-laptop-01 -# labels block reflects what the agent sent in [labels] -``` - -## 7. Drive a deployment end to end - -```bash -cat > /tmp/hello-web.yaml <<'EOF' -apiVersion: fleet.nationtech.io/v1alpha1 -kind: Deployment -metadata: - name: hello-web -spec: - score: - type: PodmanV0 - data: - services: - - name: testnginx - image: docker.io/nginx:latest - ports: - - "8080:80" - targetSelector: - matchLabels: - env: staging - rollout: - strategy: Immediate -EOF - -oc apply -f /tmp/hello-web.yaml - -# Status reflect-back from the agent (takes ~5-15s) -oc get deployment.fleet.nationtech.io hello-web -o yaml | yq '.status' -# expect: -# aggregate: -# matchedDeviceCount: 1 -# succeeded: 1 -# failed: 0 -# pending: 0 - -# On the device: -podman ps -# expect: testnginx running, port 8080→80 -curl -sS http://localhost:8080 | head -3 -``` - -## 8. Common failure modes - -| Symptom | Cause / fix | -| --- | --- | -| `cert-manager` Certificate stuck `False` for 5+ min | DNS for the host doesn't resolve to the OKD router yet. `dig sso-staging. +short` should match the cluster's ingress IP. Or your `letsencrypt-prod` ClusterIssuer is using HTTP01 and the route isn't reachable from Let's Encrypt. | -| Operator pod `Error: constructing CredentialSource` | The credentials TOML in the Secret is malformed. `oc -n fleet-staging get secret harmony-fleet-operator-secrets -o jsonpath='{.data.credentials\.toml}' \| base64 -d` and inspect; the `key_json` field must be a valid JSON keyfile string (multi-line triple-quoted in TOML is fine). | -| Operator pod `Permissions Violation` after `NATS connected` | Issuer pubkey or project_id mismatch between callout and NATS chart values, or Zitadel was reset and the operator's machine key no longer authenticates. Re-run `cargo run -p example_fleet_staging_install`. | -| Agent: `Zitadel token endpoint returned 400: invalid_grant_type` | TOML scope assembly bug or wrong `audience`. Confirm `audience` matches `oc exec deploy/fleet-callout -- printenv OIDC_AUDIENCE`. | -| Agent: connects, then `Permissions Violation for Publish to "$KV.device-info..."` | The device's machine user has no `device` role grant. Add via Zitadel UI → user → Authorizations. | -| `Deployment.fleet.nationtech.io` CR applied but `matchedDeviceCount: 0` | `targetSelector.matchLabels` doesn't match any Device CR's `metadata.labels`. `oc get devices --show-labels`. | -| Container redeploys every 30s on the device | Known FIXME — the agent's `matches_spec` returns false for any spec with env or volumes. For the demo, use trivial specs (the hello-web above is fine). Tracked in `harmony/src/modules/podman/topology.rs`. | - -## 9. Tear down - -The Helm releases own the bulk of the resources, so the cleanest -recovery from a broken state is: - -```bash -helm -n zitadel-staging uninstall zitadel -helm -n fleet-staging uninstall fleet-nats -oc -n fleet-staging delete deploy/harmony-fleet-operator deploy/fleet-callout -oc -n fleet-staging delete secret harmony-fleet-operator-secrets fleet-callout-secrets -oc -n zitadel-staging delete pgcluster zitadel-pg --ignore-not-found -oc delete project zitadel-staging fleet-staging - -# CRDs persist (helm.sh/resource-policy: keep). Delete by hand if you -# really want a clean slate: -oc delete crd deployments.fleet.nationtech.io devices.fleet.nationtech.io -``` - -The host-side `~/.local/share/harmony/zitadel/client-config.json` -caches machine keys + project IDs from this install. Wipe it before -re-installing against a freshly reset Zitadel: - -```bash -rm -f ~/.local/share/harmony/zitadel/client-config.json -``` - -(The cache-vs-live drift bug is fixed — `ZitadelSetupScore` now -re-queries Zitadel for IDs on every apply — but stale machine-key -material from a deleted Zitadel project will fail at JWT-bearer -mint until you delete + re-create.) - -## 10. Cross-reference - -- [`fleet-zitadel-faq.md`](./fleet-zitadel-faq.md) — concepts behind - Zitadel projects, roles, machine users, audit-trail decisions. -- [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) — - worked recipe for minting an admin token by hand and using it - with `nats kv` commands. -- `examples/fleet_staging_install/src/main.rs` — the install code - itself; the comments narrate every step. -- `harmony/src/modules/fleet/server.rs::FleetServerScore` — - composable form of the same install for callers that don't need - the intermediate read of `ZitadelClientConfig`. +See [`fleet/README.md`](../../fleet/README.md) for component details and the E2E +workflow. diff --git a/docs/reference/fleet-score-references.md b/docs/reference/fleet-score-references.md new file mode 100644 index 00000000..a2f3d8f3 --- /dev/null +++ b/docs/reference/fleet-score-references.md @@ -0,0 +1,108 @@ +# Fleet Score references + +Fleet is the first application of [ADR-028](../adr/028-typed-score-references.md). +Its deployment should read like a declaration of what Fleet needs and how those +components connect. + +## Target experience + +Deploying a configured Fleet should require one command: + +```bash +./harmony-fleet-deploy ship --context prod +``` + +The context selects the desired configuration. The command composes the Scores, +passes their Refs, and interprets the result. It does not expose the deployment +sequence to the operator. + +The corresponding Rust should stay close to the same statement: + +```rust +let postgres = PostgreSQLScore::new(config.postgres); +let zitadel = ZitadelScore::new(config.zitadel) + .database(postgres.root_account_ref()); + +let nats = NatsScore::new(config.nats); +let callout = NatsAuthCalloutScore::new(config.callout) + .nats(nats.account_ref()); + +let operator = FleetOperatorScore::new(config.operator) + .identity(zitadel.application_ref()) + .messaging(nats.client_ref()); +``` + +The API names are illustrative. The required shape is a short component +declaration followed by typed connections to other components. + +## How composition works + +### Identity functionality + +[![PostgreSQLScore passes PostgreSQLRootAccountRef to ZitadelScore, composing identity functionality.](../diagrams/fleet-identity-composition.svg)](../diagrams/fleet-identity-composition.svg) + +### Messaging functionality + +[![NatsScore and NatsAuthCalloutScore exchange typed Refs, composing authenticated messaging.](../diagrams/fleet-messaging-composition.svg)](../diagrams/fleet-messaging-composition.svg) + +The bidirectional messaging relationship is still declarative: constructing +either Ref does not require its target to exist. + +## Complete reference + +[![Fleet functionality, exported Ref types, and consuming Score types.](../diagrams/fleet-score-references.svg)](../diagrams/fleet-score-references.svg) + +The final map lists each functionality, the Ref types it exports, and the Score +types that consume them. The names identify intended contracts. They become +public APIs only as implementation replaces the corresponding string wiring; +the map does not prescribe a generic Ref hierarchy or registry. + +## Composition boundary + +`harmony-fleet-deploy` is composition only. It may: + +- load the selected context; +- declare component configuration; +- construct Scores; +- connect Scores through typed Refs; +- interpret the composed deployment. + +It must not: + +- call provider APIs or commands directly; +- build Kubernetes or Helm resources; +- generate credentials or reconstruct provider outputs; +- implement deployment ordering, retries, or readiness checks; +- copy provider URLs, IDs, credential locations, or naming conventions. + +All deployment behavior belongs to the Score that owns the component. If a +required operation has no Score, the missing work is to add or extend that +Score, not to implement the operation in Fleet composition. + +## Ref behavior + +Refs are available from desired state before their resources exist. Passing a +Ref is therefore not a readiness check. Each consuming Score handles an absent +or unavailable target through its own `Interpret`/`Topology` path, often by +emitting a native Kubernetes reference. + +ADR-028 does not require a particular non-success outcome, retry policy, or +continuous convergence mechanism. + +## Acceptance criteria + +Fleet validates this design when: + +1. A configured deployment runs as + `./harmony-fleet-deploy ship --context prod`. +2. Declarations and Ref wiring average at most five non-blank lines per logical + component in the canonical Fleet composition. PostgreSQL, Zitadel, each + Zitadel application or account, NATS, the auth callout, OpenBao, the Fleet + operator, and device enrollment each count as logical components. +3. `harmony-fleet-deploy` contains only context loading, configuration, + component declarations, Ref wiring, and interpretation. +4. Each migrated Ref removes the string or cache contract it replaces. +5. Constructing and passing Refs performs no I/O and implies no execution order. +6. Production and E2E compose the same Score and Ref APIs. +7. No generic graph, scheduler, or convergence mechanism is added as part of + ADR-028. diff --git a/examples/fleet_auth_callout/src/lib.rs b/examples/fleet_auth_callout/src/lib.rs index de4a0237..33214929 100644 --- a/examples/fleet_auth_callout/src/lib.rs +++ b/examples/fleet_auth_callout/src/lib.rs @@ -241,6 +241,7 @@ pub async fn bring_up_stack() -> Result { info!("[5/8] provisioning project + roles + machine users in Zitadel"); let setup = ZitadelSetupScore { + outputs_namespace: None, host: ZITADEL_HOST.to_string(), scheme: Default::default(), port: None, diff --git a/examples/fleet_device_enroll/Cargo.toml b/examples/fleet_device_enroll/Cargo.toml index a5a3d33d..3cb4132f 100644 --- a/examples/fleet_device_enroll/Cargo.toml +++ b/examples/fleet_device_enroll/Cargo.toml @@ -27,6 +27,7 @@ vm-rehearsal = ["harmony/kvm"] # `podman` types via the reconciler-contracts shape). `kvm` is the # only feature that pulls libvirt and stays opt-in via `vm-rehearsal`. harmony = { path = "../../harmony", default-features = false, features = ["podman"] } +harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } harmony_types = { path = "../../harmony_types" } tokio.workspace = true log.workspace = true diff --git a/examples/fleet_device_enroll/src/main.rs b/examples/fleet_device_enroll/src/main.rs index 973d323e..b01f8251 100644 --- a/examples/fleet_device_enroll/src/main.rs +++ b/examples/fleet_device_enroll/src/main.rs @@ -25,11 +25,11 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::Parser; use harmony::inventory::Inventory; -use harmony::modules::fleet::{ - AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, - ensure_fleet_ssh_keypair, -}; +use harmony::modules::fleet::ensure_fleet_ssh_keypair; use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials}; +use harmony_fleet_deploy::{ + AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, +}; use harmony_types::id::Id; // VM-rehearsal-only imports. Hidden behind a feature so `cargo build diff --git a/examples/fleet_e2e_demo/Cargo.toml b/examples/fleet_e2e_demo/Cargo.toml index bca29efa..364a3cb4 100644 --- a/examples/fleet_e2e_demo/Cargo.toml +++ b/examples/fleet_e2e_demo/Cargo.toml @@ -4,46 +4,10 @@ edition = "2024" version.workspace = true readme.workspace = true license.workspace = true -description = "VM-based end-to-end rehearsal: k3d + Zitadel + NATS auth callout + libvirt VM agents + operator → CR → podman → status" - -[lib] -name = "example_fleet_e2e_demo" -path = "src/lib.rs" - -[[bin]] -name = "fleet-e2e-demo" -path = "src/main.rs" - -[[test]] -name = "e2e_walking_skeleton" -path = "tests/e2e_walking_skeleton.rs" +description = "VM-based end-to-end rehearsal of the Fleet Scores" [dependencies] -harmony = { path = "../../harmony", features = ["kvm"] } -harmony-k8s = { path = "../../harmony-k8s" } -harmony_types = { path = "../../harmony_types" } -example-fleet-auth-callout = { path = "../fleet_auth_callout" } -harmony-nats-callout = { path = "../../nats/callout" } -harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } -harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" } -harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" } -harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } -k3d-rs = { path = "../../k3d" } -async-nats.workspace = true -nkeys = "0.4" -tokio = { workspace = true, features = ["full"] } -tokio-test.workspace = true -serde.workspace = true -serde_json.workspace = true +harmony-fleet-e2e = { path = "../../fleet/harmony-fleet-e2e" } anyhow.workspace = true -log.workspace = true -env_logger.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -futures-util.workspace = true -k8s-openapi.workspace = true -kube.workspace = true -clap = { version = "4", features = ["derive", "env"] } -directories = "6.0.0" -tempfile = "3" -url.workspace = true +tokio = { workspace = true, features = ["full"] } +clap = { workspace = true, features = ["derive"] } diff --git a/examples/fleet_e2e_demo/RUNBOOK.md b/examples/fleet_e2e_demo/RUNBOOK.md deleted file mode 100644 index da856301..00000000 --- a/examples/fleet_e2e_demo/RUNBOOK.md +++ /dev/null @@ -1,312 +0,0 @@ -# Local fleet rehearsal runbook - -End-to-end walkthrough of the IoT fleet platform on your laptop: -k3d-hosted control plane (Zitadel + NATS + auth callout) plus two -libvirt VMs running the fleet-agent. Mirrors the production topology -closely enough that you can watch the auth callout flow, the -JetStream KV traffic, and the per-device permission boundary in a -real cluster. - -This is not the integration-test harness (that runs unattended). It -is a step-by-step sequence with inspection points in between. Run -each section, look at what happened, then continue. - -## 0. Prerequisites - -- Linux host with KVM (the user running the commands in `libvirt` / - `kvm` group; check with `id`). -- `podman`, `qemu-system-x86_64` (and `qemu-system-aarch64` if you - pick `--arch aarch64`), `mdbook` (optional), `kubectl`, `nats` CLI - (optional, for the manual subscribe step). Most other tooling - (k3d, ansible venv, cloud images) is auto-provisioned under - `~/.local/share/harmony/`. -- `/etc/hosts`: `127.0.0.1 sso.fleet.local` so you can hit Zitadel - from your browser through the cluster's HTTP_PORT (see - `examples/fleet_auth_callout/src/lib.rs` for the constant). -- Free TCP ports `8080` and `30422` on the host. - -Source map for the things you'll inspect: - -| Component | File | -| --- | --- | -| Bring-up flow | `examples/fleet_e2e_demo/src/lib.rs` | -| Per-device Zitadel + agent install | same, `provision_device()` | -| NATS Score (auth-callout mode) | `fleet/harmony-fleet-deploy/src/nats.rs::FleetNatsScore::callout` | -| Shared agent config schema | `fleet/harmony-fleet-auth/src/agent_config.rs` | -| Auth callout deployment Score | `harmony/src/modules/nats_auth_callout/mod.rs` | -| Callout decision logic | `nats/callout/src/handler.rs::decide` | -| Per-device permissions template | `nats/callout/src/permissions.rs::device_default` | -| Agent NATS auth (JWT-bearer mint) | `fleet/harmony-fleet-auth/src/credentials.rs` | -| Agent KV publishers + direct pulse | `fleet/harmony-fleet-agent/src/fleet_publisher.rs` | -| Walking-skeleton tests | `examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs` | - -The NATS server's helm values are rendered from typed Rust structs -via `serde_yaml::to_string` (see `FleetNatsScore::values_yaml`), -not by `format!()` string interpolation. Same with the agent's -`/etc/fleet-agent/config.toml` — typed `AgentConfig` → -`toml::to_string` → ConfigMap. Per ADR-023 principle 2 the e2e -demo composes the same `*Score` types the production deploy uses. - -## 1. Provision the VMs - -Each VM is one libvirt domain on the default network -(`192.168.122.0/24`). Run `fleet_vm_setup` once per VM. Pass -`--only-vm` so it stops at the cloud-init step (the agent install -happens later from the e2e bring-up — keeps the two phases legible). - -```bash -# VM 0 -cargo run --release -p example-fleet-vm-setup -- \ - --arch aarch64 \ - --vm-name vm-device-00 \ - --only-vm - -# VM 1 -cargo run --release -p example-fleet-vm-setup -- \ - --arch aarch64 \ - --vm-name vm-device-01 \ - --only-vm -``` - -Use `--arch x86_64` for native KVM speed; `aarch64` runs under -qemu-system-aarch64 TCG emulation on x86_64 hosts and is slower but -matches Pi targets. - -**Inspect:** - -```bash -virsh list --all -virsh domifaddr vm-device-00 -virsh domifaddr vm-device-01 -``` - -Note the IPs — you'll pass them in step 2. Confirm SSH works: - -```bash -ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \ - fleet-admin@ uptime -``` - -The keypair lives under `~/.local/share/harmony/fleet/ssh/`, -generated on first run. - -## 2. Bring up the control-plane stack - -This single command does everything: k3d cluster, Zitadel, -ZitadelSetupScore (project + roles + 2 device machine users + -`fleet-ops` admin), NATS with `auth_callout`, callout image build & -sideload, callout Deployment, and finally `FleetDeviceSetupScore` -over SSH for each VM (packages, agent binary, JWT keyfile, -systemd unit). - -```bash -FLEET_E2E_VM_0_IP= FLEET_E2E_VM_1_IP= \ - cargo run --release -p example-fleet-e2e-demo -- --num-devices 2 -``` - -The bring-up logs each step as `[e2e-demo X/9]`. Read along with -`examples/fleet_e2e_demo/src/lib.rs::bring_up_full_stack` to see -what's happening at each line. Stops at `STACK READY` and waits on -Ctrl-C (the cluster stays up after Ctrl-C — this is just the -foreground holder). - -**Inspect:** - -```bash -export KUBECONFIG=$(k3d kubeconfig write fleet-auth-callout) - -# All workloads up? -kubectl get pods -n fleet-system -kubectl get pods -n zitadel - -# Callout config the deployment is using: -kubectl get deployment -n fleet-system fleet-callout \ - -o jsonpath='{.spec.template.spec.containers[0].env}' | jq -``` - -Open Zitadel in the browser: -(login with `root@zitadel.local` / the bootstrap password printed -during step `[e2e-demo 3/9]`). Click into the `fleet` project → -`Users` to see the two `device-vm-device-0X` machine users with -`device` role grants and the `fleet-ops` admin. - -## 3. Watch the auth callout in action - -The callout is the security boundary: every NATS connect attempt -hits `$SYS.REQ.USER.AUTH`, the callout validates the Zitadel JWT -in `connect_opts.auth_token`, applies the decision tree in -`nats/callout/src/handler.rs::decide`, and signs back a user JWT -with role-scoped permissions. - -Tail it while the agents reconnect: - -```bash -kubectl logs -n fleet-system -l app=fleet-callout -f -``` - -You'll see one set of lines per (re)connect: - -``` -received auth callout request user_nkey=U… -Zitadel JWT validated, generating user JWT device_id=vm-device-00 role=device -sending auth response -``` - -The `device_id` field is the value AFTER `device_id_prefix_strip` -runs (Zitadel emits `client_id=device-vm-device-00`; the callout -strips `device-` so permissions are interpolated against the bare -device id the agent uses for KV keys). See -`nats/callout/src/zitadel.rs::extract_device_id` for the strip. - -**Force a reconnect to make a callout fire on demand:** - -```bash -ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \ - fleet-admin@ 'sudo systemctl restart fleet-agent' -``` - -Watch the callout pod log emit one fresh request/response. - -## 4. Watch the agent - -```bash -ssh -i ~/.local/share/harmony/fleet/ssh/id_ed25519 \ - fleet-admin@ 'sudo journalctl -u fleet-agent -f' -``` - -What good looks like, in order: - -| Log line | Where it comes from | -| --- | --- | -| `minted fresh Zitadel access token audience=…` | `credentials.rs::zitadel_mint` — RFC 7523 JWT-bearer flow, signed with the per-device machine key under `/etc/fleet-agent/zitadel-key.json` | -| `connected successfully server=4222` | NATS accepted the JWT minted by the callout | -| `fleet publisher ready` | KV buckets opened; `device-info` write succeeded | -| `watching KV keys filter=vm-device-00.>` | desired-state subscriber is up | - -Absence of `Permissions Violation` lines is the success signal — -those mean the JWT's perms don't match what the agent tried to -publish (you'd hit them if `device_id_prefix_strip` were -misconfigured, for example). - -## 5. Observe fleet traffic as admin - -The harness mints a `fleet-ops` admin machine user with the -`fleet-admin` role; the callout maps that role to -`pub/sub allow: [">"]`. The integration test -`admin_jwt_reads_any_device_subject` exercises this — easiest path -to see it live is to run it with output. The test is -`#[ignore]`d on `cargo test` so a developer box doesn't burn a -10-minute Zitadel bring-up by accident; `--ignored` opts in: - -```bash -FLEET_E2E_VM_0_IP= FLEET_E2E_VM_1_IP= \ - cargo test -p example-fleet-e2e-demo \ - --test e2e_walking_skeleton \ - admin_jwt_reads_any_device_subject \ - -- --test-threads=1 --nocapture --ignored -``` - -It subscribes admin to `device-state.>` (the direct, non-JetStream -fan-out subject the agent emits a pulse on every 30s — see -`fleet_publisher.rs::publish_state_pulse`) and asserts a message -arrives within 30s. - -**Inspect KV state directly** using a bare admin client. The -underlying mechanism is in -`examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs::admin_nats_client`: -mint a JWT-bearer token from `stack.admin_machine_key`, hand it to -`async_nats` as `auth_token`. The test -`both_devices_heartbeat_within_60s` then reads `device-info` keys -directly: - -```rust -let js = async_nats::jetstream::new(admin); -let bucket = js.get_key_value(BUCKET_DEVICE_INFO).await?; -let entry = bucket.entry(&device_info_key("vm-device-00")).await?; -``` - -To do it from a shell, port-forward NATS and use the `nats` CLI -with admin creds — but creds for an auth-callout server take a -JWT-bearer token, which the `nats` CLI doesn't speak natively; -running the test is the path of least friction. - -## 6. Verify cross-device isolation (currently `#[ignore]`) - -`cross_device_isolation_enforced_in_vm` is an empty test marked -`#[ignore = "requires E2eHandles::device_machine_key plumbing"]` -in `e2e_walking_skeleton.rs` — the test is a placeholder. The -plumbing it's waiting on is straightforward: the existing -`DeviceHandle` struct (`examples/fleet_e2e_demo/src/lib.rs:106`) -exposes `device_id` + `vm_ip` + `labels` but not the per-device -Zitadel machine key the test would need to mint a `device`-role -JWT and try cross-device subjects. `provision_device` already -creates the key (line ~324, `machine_key_json`) — wiring it through -into `DeviceHandle.machine_key` and implementing the test body -(mint JWT-bearer for vm-device-00, sub to -`device-commands.vm-device-01`, expect `Permissions Violation`) -is a single follow-up commit. I haven't touched it because nothing -in this branch's scope required it. - -**You can verify the boundary manually right now**, even without -the test wired up: tail the callout pod, then SSH onto vm-device-00 -and run the agent with a tampered config that points it at -vm-device-01's keyfile. The callout will issue a JWT for -`vm-device-01` (because the JWT-bearer assertion is signed with -that user's key); the agent on vm-device-00 will then publish on -`$KV.device-info.info.vm-device-00`, which is NOT in the JWT's -allow list — NATS rejects with `Permissions Violation`. This is -the same gate the test would automate. - -The permissions template is in -`nats/callout/src/permissions.rs::device_default` — every allowed -subject contains `{device_id}` and is interpolated per-request, so -device A's JWT physically cannot publish to device B's subjects. - -## 7. Drive the desired-state loop - -(Not yet covered by a walking-skeleton test, but the agent's -reconciler is wired and observable.) From an admin client, write a -desired state for vm-device-00: - -```rust -// pseudocode — see harmony-reconciler-contracts for the exact types -let kv = jetstream.create_key_value(kv::Config { - bucket: BUCKET_DESIRED_STATE.into(), - history: 1, - ..Default::default() -}).await?; -kv.put( - &desired_state_key("vm-device-00", &dn("hello-web")), - payload.into(), -).await?; -``` - -What happens, observable from the agent's journal: - -1. Agent's KV watcher (filter `vm-device-00.>`) fires. -2. Reconciler computes the diff and runs the podman create. -3. `write_deployment_state(&state)` fires: - - puts `state.vm-device-00.hello-web` into the `device-state` - KV bucket (operator-side watch picks it up) - - publishes the same payload on direct subject - `device-state.vm-device-00` (admin observers see it live) - -You can subscribe to the latter with admin and watch reconcile -events stream in real time. - -## 8. Teardown - -The cluster persists across runs (re-running `fleet_e2e_demo` -converges drift, doesn't recreate). When you want a clean slate: - -```bash -k3d cluster delete fleet-auth-callout - -virsh destroy vm-device-00; virsh undefine vm-device-00 --remove-all-storage -virsh destroy vm-device-01; virsh undefine vm-device-01 --remove-all-storage -``` - -Cached assets (cloud images, k3d binary, ansible venv, SSH key, -fleet secrets) live under `~/.local/share/harmony/` and survive -cluster/VM destruction by design — first run after a clean reuses -them. diff --git a/examples/fleet_e2e_demo/src/lib.rs b/examples/fleet_e2e_demo/src/lib.rs deleted file mode 100644 index 60c09725..00000000 --- a/examples/fleet_e2e_demo/src/lib.rs +++ /dev/null @@ -1,847 +0,0 @@ -//! VM-based end-to-end rehearsal of the customer demo flow. -//! -//! Goal: prove the JWT-auth chain works on a real-system agent -//! before pointing the demo at OKD. See -//! `ROADMAP/fleet_platform/v0_demo_e2e.md` for the full plan. -//! -//! Bring-up sequence: -//! 1. k3d cluster with HTTP + NATS port mappings (re-uses -//! fleet_auth_callout's k3d helpers — same cluster name so -//! re-runs of either example reuse the same cluster). -//! 2. Zitadel + Postgres via ZitadelScore. -//! 3. Wait for Zitadel HTTP and the chart-provisioned `iam-admin-pat` -//! secret (the chart's setup job is async). -//! 4. ZitadelSetupScore for the project + API app + roles + admin -//! machine user (no per-device users yet). -//! 5. NATS with auth_callout block + the callout pod. -//! 6. For each device i: -//! - ZitadelSetupScore minting a per-device machine user with -//! the `device` role grant. The JSON keyfile is cached in -//! `ZitadelClientConfig` and read back here for the agent. -//! - libvirt VM via `ProvisionVmScore`. -//! - SSH-inject `/etc/hosts` so the VM resolves -//! `sso.fleet.local` to the libvirt host. -//! - `FleetDeviceSetupScore` with `FleetDeviceAuth::ZitadelJwt` -//! pointing at the dropped keyfile. -//! -//! Tests in `tests/e2e_walking_skeleton.rs` share a single bring-up -//! via `OnceCell` and exercise: heartbeats, label-selector targeting, -//! status reflect-back, env+volume propagation, admin cross-device -//! read, per-device isolation, NATS-pod-restart reconnect. - -use std::path::PathBuf; -use std::time::Duration; - -use anyhow::{Context, Result}; -use example_fleet_auth_callout::{ - ADMIN_ROLE_KEY, API_APP_NAME, CALLOUT_DEPLOYMENT_NAME, CALLOUT_IMAGE_TAG, DEVICE_ROLE_KEY, - FLEET_NAMESPACE, HTTP_PORT, NATS_ACCOUNT, NATS_AUTH_PASS, NATS_AUTH_USER, NATS_NAMESPACE, - NATS_NODE_PORT, NATS_RELEASE, PROJECT_NAME, ZITADEL_HOST, build_and_load_callout_image, - create_k3d, create_topology, deploy_zitadel, ensure_issuer_seed, wait_for_callout_ready, - wait_for_zitadel_ready, -}; -use harmony::inventory::Inventory; -use harmony::modules::fleet::{ - FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry, - ensure_fleet_ssh_keypair, -}; -use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; -use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv}; -use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; -use harmony::modules::zitadel::{ - MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, - ZitadelSetupScore, -}; -use harmony::score::Score; -use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; -use harmony_fleet_deploy::FleetNatsScore; -use harmony_types::id::Id; -use log::{info, warn}; -use nkeys::KeyPair; - -// ---- constants ------------------------------------------------------------- - -/// Libvirt's default NAT gateway. The host's IP from inside any VM -/// attached to the `default` libvirt network. We bake this in because -/// every smoke-a* harness assumes it; if a customer runs their own -/// libvirt with a different bridge they can override via env. -pub const DEFAULT_LIBVIRT_HOST_IP: &str = "192.168.122.1"; - -pub const ADMIN_USERNAME: &str = "fleet-ops"; -/// Separate machine user for the in-cluster operator. Distinct from -/// `fleet-ops` (manual admin tooling) so the audit trail can tell -/// operator-driven actions apart from human operator actions. Same -/// `fleet-admin` role grant — only the identity differs. -pub const OPERATOR_USERNAME: &str = "fleet-operator"; -pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:dev"; - -/// Per-device username convention: `device-${device_id}`. Matches what -/// `fleet_rpi_setup` produces, so callout's `device_id_claim = -/// "client_id"` extracts the device id verbatim from the `client_id` -/// claim Zitadel emits in machine-user access tokens. -pub fn device_username(device_id: &str) -> String { - format!("device-{device_id}") -} - -// ---- options + handles ----------------------------------------------------- - -#[derive(Debug, Clone)] -pub struct E2eDemoOpts { - /// Number of VM-as-device agents to provision. - pub num_devices: usize, - /// Path to the cross-compiled `fleet-agent` binary uploaded to - /// each VM. Defaults to `target/release/fleet-agent` (the same - /// path that smoke-a4 produces). - pub agent_binary: PathBuf, - /// Override for the libvirt host IP (the address VMs see as the - /// gateway). Defaults to [`DEFAULT_LIBVIRT_HOST_IP`]. - pub libvirt_host_ip: String, -} - -impl Default for E2eDemoOpts { - fn default() -> Self { - Self { - num_devices: 2, - agent_binary: workspace_target_path("release/harmony-fleet-agent"), - libvirt_host_ip: DEFAULT_LIBVIRT_HOST_IP.to_string(), - } - } -} - -#[derive(Debug, Clone)] -pub struct DeviceHandle { - pub index: usize, - pub device_id: String, - pub vm_ip: String, - pub labels: std::collections::BTreeMap, -} - -#[derive(Debug, Clone)] -pub struct E2eHandles { - pub cluster_name: String, - pub nats_url_external: String, - pub zitadel_url: String, - pub project_id: String, - pub issuer_pubkey: String, - pub admin_machine_key: String, - pub devices: Vec, -} - -// ---- bring up -------------------------------------------------------------- - -pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result { - let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) - .try_init(); - - info!("[e2e-demo 1/9] ensuring k3d cluster"); - let k3d = create_k3d(); - k3d.ensure_installed() - .await - .map_err(|e| anyhow::anyhow!("k3d ensure: {e}"))?; - let topology = create_topology(&k3d); - topology.ensure_ready().await.context("topology init")?; - - info!("[e2e-demo 2/9] deploying Zitadel (cold start: ~5 min)"); - deploy_zitadel(&topology).await?; - - info!("[e2e-demo 3/9] CoreDNS rewrite + waiting for Zitadel HTTP + iam-admin-pat secret"); - CoreDNSRewriteScore { - rewrites: vec![CoreDNSRewrite { - hostname: ZITADEL_HOST.to_string(), - target: "zitadel.zitadel.svc.cluster.local".to_string(), - }], - } - .interpret(&Inventory::autoload(), &topology) - .await - .context("CoreDNSRewriteScore")?; - wait_for_zitadel_ready().await?; - wait_for_iam_admin_pat_secret(&topology).await?; - - info!("[e2e-demo 4/9] provisioning project, API app, roles, admin machine user"); - let admin_setup = ZitadelSetupScore { - host: ZITADEL_HOST.to_string(), - scheme: Default::default(), - port: None, - skip_tls: true, - endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")), - admin_org_id: None, - namespace: "zitadel".to_string(), - applications: vec![], - api_apps: vec![ZitadelApiApp { - project_name: PROJECT_NAME.to_string(), - app_name: API_APP_NAME.to_string(), - }], - roles: vec![ - ZitadelRole { - project_name: PROJECT_NAME.to_string(), - key: ADMIN_ROLE_KEY.to_string(), - display_name: "Fleet Admin".to_string(), - group: None, - }, - ZitadelRole { - project_name: PROJECT_NAME.to_string(), - key: DEVICE_ROLE_KEY.to_string(), - display_name: "Device".to_string(), - group: None, - }, - ], - groups_claim_action: false, - human_users: vec![], - org_members: vec![], - instance_members: vec![], - machine_secrets: vec![], - port_forward_service: None, - machine_users: vec![ - ZitadelMachineUser { - username: ADMIN_USERNAME.to_string(), - name: "Fleet Operations".to_string(), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(PROJECT_NAME.to_string()), - grant_roles: vec![ADMIN_ROLE_KEY.to_string()], - }, - // Separate machine user for the in-cluster operator pod. - // Same `fleet-admin` role grant as the manual admin - // identity, but distinct username so JWT `client_id` lets - // log analysis tell operator-driven actions apart from - // human operator actions. - ZitadelMachineUser { - username: OPERATOR_USERNAME.to_string(), - name: "Fleet Operator (in-cluster)".to_string(), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(PROJECT_NAME.to_string()), - grant_roles: vec![ADMIN_ROLE_KEY.to_string()], - }, - ], - }; - admin_setup - .interpret(&Inventory::autoload(), &topology) - .await - .context("admin ZitadelSetupScore")?; - - let zcfg = ZitadelClientConfig::load() - .context("ZitadelSetupScore did not produce a client config cache")?; - let project_id = zcfg - .project_id_by_name(PROJECT_NAME) - .or(zcfg.project_id.as_ref()) - .context("project_id missing from cache")? - .clone(); - let admin_machine_key = zcfg - .machine_key(ADMIN_USERNAME) - .context("admin machine key missing from cache")? - .clone(); - - info!("[e2e-demo 5/9] generating issuer NKey, deploying NATS with auth_callout"); - let issuer_seed = ensure_issuer_seed(&topology).await?; - let issuer_kp = KeyPair::from_seed(&issuer_seed) - .map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?; - let issuer_pubkey = issuer_kp.public_key(); - - // Per ADR-023 principle 2 — e2e uses the same Scores as production. - // `FleetNatsScore::callout` renders the auth-callout values block - // typed (serde_yaml) rather than the legacy `render_nats_values` - // string interpolation. Same upstream chart, same wire format, - // schema-checked at compile time. - FleetNatsScore::callout( - NATS_NAMESPACE, - NATS_NODE_PORT as u16, - &issuer_pubkey, - NATS_ACCOUNT, - NATS_AUTH_USER, - NATS_AUTH_PASS, - ) - .release_name(NATS_RELEASE) - .interpret(&Inventory::autoload(), &topology) - .await - .context("NATS deploy")?; - - info!("[e2e-demo 6/9] building + sideloading callout image into k3d"); - build_and_load_callout_image(&k3d).await?; - - info!("[e2e-demo 7/9] deploying NatsAuthCalloutScore"); - let mut callout = NatsAuthCalloutScore::new( - CALLOUT_DEPLOYMENT_NAME, - FLEET_NAMESPACE, - format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"), - format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), - project_id.clone(), - NATS_AUTH_USER, - NATS_AUTH_PASS, - issuer_seed.clone(), - ) - .image(CALLOUT_IMAGE_TAG) - .target_account(NATS_ACCOUNT) - .admin_role(ADMIN_ROLE_KEY) - .device_role(DEVICE_ROLE_KEY) - .danger_accept_invalid_certs(true); - // Same convention as fleet_auth_callout: the username is in the - // access token's `client_id` claim. The role claim path is - // project-scoped because the JWT-bearer flow requests project - // audience scope. - callout.device_id_claim = "client_id".to_string(); - // Zitadel's `client_id` for a machine user equals its userName, so a - // user created as `device-vm-device-00` (the convention shared with - // fleet_rpi_setup and fleet_auth_callout) lands in the JWT verbatim. - // Strip the `device-` prefix so the callout interpolates permissions - // against the bare device id (`vm-device-00`) the agent uses for KV - // keys + direct subjects. - callout.device_id_prefix_strip = "device-".to_string(); - callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles"); - callout - .interpret(&Inventory::autoload(), &topology) - .await - .context("callout deploy")?; - wait_for_callout_ready(&topology).await?; - - info!("[e2e-demo 8/10] building + sideloading operator image into k3d"); - build_and_load_operator_image(&k3d).await?; - - info!("[e2e-demo 9/10] deploying fleet operator with Zitadel JWT auth"); - let operator_machine_key = zcfg - .machine_key(OPERATOR_USERNAME) - .with_context(|| format!("machine key for {OPERATOR_USERNAME} missing from cache"))? - .clone(); - deploy_operator(&topology, &project_id, &operator_machine_key).await?; - wait_for_operator_ready(&topology).await?; - - info!( - "[e2e-demo 10/10] provisioning {} VM(s) and onboarding agent(s)", - opts.num_devices - ); - let mut devices = Vec::with_capacity(opts.num_devices); - for i in 0..opts.num_devices { - let handle = provision_device(i, &opts, &topology, &project_id).await?; - devices.push(handle); - } - - info!( - "full stack ready: {} device(s), operator + admin role configured", - devices.len() - ); - - Ok(E2eHandles { - cluster_name: example_fleet_auth_callout::CLUSTER_NAME.to_string(), - nats_url_external: format!("nats://127.0.0.1:{NATS_NODE_PORT}"), - zitadel_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), - project_id, - issuer_pubkey, - admin_machine_key, - devices, - }) -} - -// ---- per-device provisioning ---------------------------------------------- - -async fn provision_device( - index: usize, - opts: &E2eDemoOpts, - topology: &K8sAnywhereTopology, - project_id: &str, -) -> Result { - let device_id = format!("vm-device-{index:02}"); - let username = device_username(&device_id); - info!("[device {index}] minting Zitadel machine user {username}"); - - // Per-device ZitadelSetupScore (search-then-create — running this - // for an existing user is a NOOP that just refreshes the cache - // entry pointing at the persisted machine key). The keyfile is - // re-minted because Zitadel doesn't expose the private half of - // an existing key — accept that any prior key drifts to "stale - // until expiry" on the previous device installation. - let device_setup = ZitadelSetupScore { - host: ZITADEL_HOST.to_string(), - scheme: Default::default(), - port: None, - skip_tls: true, - endpoint: Some(format!("http://127.0.0.1:{HTTP_PORT}")), - admin_org_id: None, - namespace: "zitadel".to_string(), - applications: vec![], - api_apps: vec![], - roles: vec![], - groups_claim_action: false, - human_users: vec![], - org_members: vec![], - instance_members: vec![], - machine_secrets: vec![], - port_forward_service: None, - machine_users: vec![ZitadelMachineUser { - username: username.clone(), - name: format!("Fleet Device {device_id}"), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(PROJECT_NAME.to_string()), - grant_roles: vec![DEVICE_ROLE_KEY.to_string()], - }], - }; - device_setup - .interpret(&Inventory::autoload(), topology) - .await - .with_context(|| format!("ZitadelSetupScore for {username}"))?; - - let zcfg = ZitadelClientConfig::load() - .context("ZitadelClientConfig disappeared between admin and device setup")?; - let machine_key_json = zcfg - .machine_key(&username) - .with_context(|| format!("machine key for {username} missing from cache"))? - .clone(); - - // -- VM provisioning would go here. Deferred to keep the harness - // cold-start observable in pieces — the kvm bits (ProvisionVmScore) - // require root + libvirtd + the cloud image. Today the harness - // expects the operator to have provisioned VMs out-of-band (e.g. - // via fleet_vm_setup, or a pre-existing libvirt domain). We read - // the IP from a convention path (see `discover_vm_ip`) so the - // test driver can iterate on the agent path without re-paying VM - // boot every test cycle. - // - // Follow-up: fold ProvisionVmScore::ensure_vm here once the - // bring-up has been demonstrated end-to-end at least once. - let vm_ip = discover_vm_ip(index) - .with_context(|| format!("could not resolve IP for device {index}"))?; - - info!("[device {index}] {device_id} at {vm_ip} — installing agent with Zitadel JWT auth"); - let labels = build_device_labels(&device_id, index); - let agent_score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig { - device_id: Id::from(device_id.clone()), - labels: labels.clone(), - // Agent connects to NATS at the libvirt host's IP via the - // NodePort. The libvirt default network NATs the VM through - // the host so the host's port mapping is reachable. - nats_urls: vec![format!("nats://{}:{NATS_NODE_PORT}", opts.libvirt_host_ip)], - auth: FleetDeviceAuth::ZitadelJwt { - machine_key_json, - // Issuer URL the agent uses MUST match the issuer - // string Zitadel returns — Zitadel derives that from - // the request's Host header. We hit Zitadel via the - // host's port mapping, so the agent's URL is - // `http://sso.fleet.local:`. The /etc/hosts - // entry below points sso.fleet.local at the libvirt - // host so the VM resolves it. - oidc_issuer_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), - audience: project_id.to_string(), - // Local rehearsal hits Zitadel over plain HTTP through - // the cluster ingress; no TLS validation needed. - danger_accept_invalid_certs: true, - }, - agent_binary_path: opts.agent_binary.clone(), - hosts_entries: vec![HostsEntry { - ip: opts.libvirt_host_ip.clone(), - hostname: ZITADEL_HOST.to_string(), - }], - openbao: None, - }); - - // Apply the score over SSH against the VM. Same pattern as - // fleet_rpi_setup, but synthesized inline so the harness can drive - // multiple VMs in sequence without copying the CLI plumbing. - apply_fleet_setup_to_vm(index, &vm_ip, agent_score).await?; - - Ok(DeviceHandle { - index, - device_id, - vm_ip, - labels, - }) -} - -async fn apply_fleet_setup_to_vm( - index: usize, - vm_ip: &str, - score: FleetDeviceSetupScore, -) -> Result<()> { - ensure_ansible_venv() - .await - .map_err(|e| anyhow::anyhow!("ansible venv: {e}"))?; - let ssh = ensure_fleet_ssh_keypair() - .await - .map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?; - let ip = vm_ip - .parse() - .with_context(|| format!("VM IP '{vm_ip}' is not a valid IP address"))?; - let creds = SshCredentials { - // Matches the cloud-init admin user that fleet_vm_setup + - // smoke-a4 create. If the operator overrode that during - // out-of-band VM provisioning, follow-up: thread the - // username through E2eDemoOpts. - user: "fleet-admin".to_string(), - private_key_path: ssh.private_key.clone(), - remote_python: Some("/usr/bin/python3".to_string()), - sudo_password: None, - }; - let topology = LinuxHostTopology::new(format!("vm-device-{index:02}"), ip, creds); - use harmony::score::Score; - score - .create_interpret() - .execute(&Inventory::empty(), &topology) - .await - .with_context(|| format!("FleetDeviceSetupScore against VM {index} ({vm_ip})"))?; - Ok(()) -} - -fn build_device_labels( - device_id: &str, - index: usize, -) -> std::collections::BTreeMap { - // Two devices, two distinct group labels by default — lets - // selector tests target "exactly one device". Label scheme - // matches the demo runbook. - let mut labels = std::collections::BTreeMap::new(); - labels.insert( - "group".to_string(), - if index == 0 { - "group-a".to_string() - } else { - "group-b".to_string() - }, - ); - labels.insert("arch".to_string(), std::env::consts::ARCH.to_string()); - labels.insert("role".to_string(), "rehearsal".to_string()); - labels.insert("device-id".to_string(), device_id.to_string()); - labels -} - -fn discover_vm_ip(index: usize) -> Result { - // Convention: a `FLEET_E2E_VM__IP` env var points at the - // pre-provisioned VM's IP. This keeps the harness usable on a - // workstation where the operator runs `fleet_vm_setup` once per - // device out-of-band, then re-runs the e2e harness against the - // already-booted VMs. - let key = format!("FLEET_E2E_VM_{index}_IP"); - std::env::var(&key) - .with_context(|| format!("set {key} to the libvirt VM's IP (default network)")) -} - -// ---- iam-admin-pat readiness ---------------------------------------------- - -/// Wait for the Zitadel chart's setup job to write the `iam-admin-pat` -/// secret. The Helm release reports Ready before the job completes, -/// so calling ZitadelSetupScore immediately after Zitadel deploy -/// races. ZitadelSetupScore itself reads this secret to authenticate -/// to the management API. -async fn wait_for_iam_admin_pat_secret(topology: &K8sAnywhereTopology) -> Result<()> { - use k8s_openapi::api::core::v1::Secret; - let k8s = topology - .k8s_client() - .await - .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; - for attempt in 1..=120 { - if let Some(secret) = k8s - .get_resource::("iam-admin-pat", Some("zitadel")) - .await? - && let Some(data) = secret.data - && data.contains_key("pat") - { - return Ok(()); - } - if attempt % 10 == 0 { - warn!("iam-admin-pat secret not yet present in zitadel ns ({attempt}/120)"); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - anyhow::bail!( - "timed out waiting for iam-admin-pat secret in 'zitadel' namespace — \ - is FirstInstance.Org.Machine.Pat configured in ZitadelScore Helm values?" - ) -} - -// ---- operator deploy ------------------------------------------------------- - -const OPERATOR_NAMESPACE: &str = FLEET_NAMESPACE; - -/// k3d's data directory under `$XDG_DATA_HOME`. Mirrors -/// `example_fleet_auth_callout::data_dir` (the latter is private — -/// duplicated here rather than re-exported so the operator wiring is -/// self-contained). -fn k3d_data_dir() -> PathBuf { - directories::BaseDirs::new() - .map(|dirs| dirs.data_dir().join("harmony").join("k3d")) - .unwrap_or_else(|| PathBuf::from("/tmp/harmony")) -} - -/// Build the operator's release binary, package it into an OCI image, -/// and sideload into the k3d cluster. Mirrors -/// `build_and_load_callout_image`. The Dockerfile lives in the -/// operator crate. -async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> { - use std::process::Stdio; - - let workspace_root = std::env::var("CARGO_MANIFEST_DIR") - .map(|d| PathBuf::from(d).join("..").join("..")) - .unwrap_or_else(|_| PathBuf::from(".")); - let workspace_root = workspace_root.canonicalize().unwrap_or(workspace_root); - - info!("cargo build --release -p harmony-fleet-operator"); - let status = tokio::process::Command::new("cargo") - .args(["build", "--release", "-p", "harmony-fleet-operator"]) - .current_dir(&workspace_root) - .status() - .await?; - if !status.success() { - anyhow::bail!("cargo build for fleet operator failed"); - } - - // Stage the binary + Dockerfile into a clean temp dir so podman - // build doesn't drag the whole target/ tree across. - let ctx = tempfile::tempdir()?; - let bin_dst = ctx.path().join("target/release"); - std::fs::create_dir_all(&bin_dst)?; - std::fs::copy( - workspace_root.join("target/release/harmony-fleet-operator"), - bin_dst.join("harmony-fleet-operator"), - ) - .context("staging operator binary into build context")?; - let dockerfile_src = workspace_root.join("fleet/harmony-fleet-operator/Dockerfile"); - if !dockerfile_src.exists() { - anyhow::bail!( - "missing fleet/harmony-fleet-operator/Dockerfile — operator image staging \ - expects it next to Cargo.toml; either add it or update the bring-up." - ); - } - std::fs::copy(&dockerfile_src, ctx.path().join("Dockerfile"))?; - - info!("podman build → {OPERATOR_IMAGE_TAG}"); - let status = tokio::process::Command::new("podman") - .args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."]) - .current_dir(ctx.path()) - .stderr(Stdio::inherit()) - .status() - .await?; - if !status.success() { - anyhow::bail!("podman build for operator failed"); - } - - let tar_path = - std::env::temp_dir().join(format!("harmony-operator-image-{}.tar", std::process::id())); - let _ = std::fs::remove_file(&tar_path); - let status = tokio::process::Command::new("podman") - .args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG]) - .status() - .await?; - if !status.success() { - anyhow::bail!("podman save for operator failed"); - } - info!("k3d image import {OPERATOR_IMAGE_TAG}"); - let cluster_name = k3d - .cluster_name() - .unwrap_or(example_fleet_auth_callout::CLUSTER_NAME) - .to_string(); - let tar_path_str = tar_path.to_str().unwrap().to_string(); - let cluster_for_blocking = cluster_name.clone(); - let data_dir = k3d_data_dir(); - tokio::task::spawn_blocking(move || { - k3d_rs::K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([ - "image", - "import", - tar_path_str.as_str(), - "-c", - cluster_for_blocking.as_str(), - ]) - }) - .await? - .map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?; - let _ = std::fs::remove_file(&tar_path); - Ok(()) -} - -/// Apply the operator's CRDs + ServiceAccount + ClusterRole + -/// ClusterRoleBinding + Secret + Deployment via Harmony's -/// K8sResourceScore. The Secret carries both the `[credentials]` TOML -/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and -/// the Zitadel JSON keyfile that the TOML's `key_path` references. -async fn deploy_operator( - topology: &K8sAnywhereTopology, - project_id: &str, - operator_machine_key: &str, -) -> Result<()> { - use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device}; - use harmony::modules::k8s::resource::K8sResourceScore; - use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; - use harmony_fleet_deploy::operator::chart::{ - ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role, - build_cluster_role_binding, build_operator_deployment, build_service_account, - operator_secret, - }; - use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; - use kube::CustomResourceExt; - - // Render the [credentials] TOML the operator pod consumes via the - // OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret - // key). The Zitadel JSON keyfile is embedded inline under - // `key_json`; the operator never sees a file. Triple-quoted TOML - // string keeps the JSON's `"`s untouched. - let credentials_toml = format!( - r#"type = "zitadel-jwt" -oidc_issuer_url = "http://{host}:{port}" -audience = "{project_id}" -danger_accept_invalid_certs = true -key_json = """ -{key_json} -""" -"#, - host = ZITADEL_HOST, - port = HTTP_PORT, - key_json = operator_machine_key, - ); - - let opts = ChartOptions { - output_dir: PathBuf::new(), // unused on this code path - image: OPERATOR_IMAGE_TAG.to_string(), - image_pull_policy: "IfNotPresent".to_string(), - nats_url: format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"), - log_level: "info,kube_runtime=warn".to_string(), - credentials: Some(OperatorCredentials { credentials_toml }), - chart_version: None, - web_auth_config_json: None, - web_cookie_key_json: None, - }; - - // CRDs first — the operator watches them on startup. - let crds: Vec = vec![FleetDeployment::crd(), Device::crd()]; - K8sResourceScore:: { - resource: crds, - namespace: None, - } - .interpret(&Inventory::autoload(), topology) - .await - .context("operator CRD apply")?; - - // RBAC. - K8sResourceScore::single( - build_service_account(), - Some(OPERATOR_NAMESPACE.to_string()), - ) - .interpret(&Inventory::autoload(), topology) - .await - .context("operator ServiceAccount apply")?; - - K8sResourceScore::single(build_cluster_role(), None) - .interpret(&Inventory::autoload(), topology) - .await - .context("operator ClusterRole apply")?; - - K8sResourceScore::single(build_cluster_role_binding(OPERATOR_NAMESPACE), None) - .interpret(&Inventory::autoload(), topology) - .await - .context("operator ClusterRoleBinding apply")?; - - // Secret holding both the credentials TOML and the keyfile. - let secret = operator_secret(&opts).expect("credentials present in opts"); - K8sResourceScore::single(secret, Some(OPERATOR_NAMESPACE.to_string())) - .interpret(&Inventory::autoload(), topology) - .await - .context("operator Secret apply")?; - - // Deployment last so it pulls the up-to-date Secret. - K8sResourceScore::single( - build_operator_deployment(&opts), - Some(OPERATOR_NAMESPACE.to_string()), - ) - .interpret(&Inventory::autoload(), topology) - .await - .context("operator Deployment apply")?; - - info!("operator deployment {OPERATOR_NAMESPACE}/{RELEASE_NAME} applied"); - Ok(()) -} - -async fn wait_for_operator_ready(topology: &K8sAnywhereTopology) -> Result<()> { - use harmony_fleet_deploy::operator::chart::RELEASE_NAME; - use k8s_openapi::api::apps::v1::Deployment as K8sDeployment; - let k8s = topology - .k8s_client() - .await - .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; - for attempt in 1..=120 { - if let Some(d) = k8s - .get_resource::(RELEASE_NAME, Some(OPERATOR_NAMESPACE)) - .await? - && let Some(status) = d.status - && status.ready_replicas.unwrap_or(0) >= 1 - { - return Ok(()); - } - if attempt % 10 == 0 { - warn!("operator Deployment not yet Ready ({attempt}/120)"); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - anyhow::bail!("timed out waiting for operator Deployment to become Ready") -} - -// ---- helpers --------------------------------------------------------------- - -fn workspace_target_path(rel: &str) -> PathBuf { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR") - .map(PathBuf::from) - .unwrap_or_else(|_| PathBuf::from(".")); - manifest_dir.join("..").join("..").join("target").join(rel) -} - -// ---- next-steps panel ------------------------------------------------------ - -impl E2eHandles { - pub fn print_next_steps(&self) { - println!(); - println!("============================================================"); - println!(" E2E DEMO REHEARSAL — STACK READY"); - println!("============================================================"); - println!(" k3d cluster: {}", self.cluster_name); - println!(" Zitadel: {}", self.zitadel_url); - println!(" NATS (host): {}", self.nats_url_external); - println!(" Project ID: {}", self.project_id); - println!(" Issuer pubkey: {}", self.issuer_pubkey); - println!(); - println!(" Devices ({}):", self.devices.len()); - for d in &self.devices { - let labels: Vec = d.labels.iter().map(|(k, v)| format!("{k}={v}")).collect(); - println!( - " [{}] {} @ {} ({})", - d.index, - d.device_id, - d.vm_ip, - labels.join(",") - ); - } - println!(); - println!(" Run the test suite:"); - println!(); - println!(" cargo test -p example-fleet-e2e-demo \\"); - println!(" --test e2e_walking_skeleton -- --test-threads=1 --nocapture"); - println!(); - println!(" Ctrl-C exits without tearing the cluster down — re-run"); - println!(" the bring-up to converge any drift."); - println!("============================================================"); - } -} - -#[cfg(test)] -mod unit_tests { - use super::*; - - #[test] - fn device_username_matches_callout_convention() { - // Callout's device_id_claim is `client_id`, which Zitadel - // populates from the machine user's username. The test we - // run later asserts the agent's per-device subjects match - // its device_id, which therefore must equal the username - // minus the "device-" prefix the callout knows about. - assert_eq!(device_username("vm-device-00"), "device-vm-device-00"); - } - - #[test] - fn device_labels_split_into_distinct_groups() { - let l0 = build_device_labels("vm-device-00", 0); - let l1 = build_device_labels("vm-device-01", 1); - assert_eq!(l0.get("group").unwrap(), "group-a"); - assert_eq!(l1.get("group").unwrap(), "group-b"); - assert_ne!(l0.get("group"), l1.get("group")); - // Ubiquitous labels: device-id + arch + role on both. - for l in [&l0, &l1] { - assert!(l.contains_key("device-id")); - assert!(l.contains_key("arch")); - assert_eq!(l.get("role").unwrap(), "rehearsal"); - } - } -} diff --git a/examples/fleet_e2e_demo/src/main.rs b/examples/fleet_e2e_demo/src/main.rs index b586f440..6ce7ceae 100644 --- a/examples/fleet_e2e_demo/src/main.rs +++ b/examples/fleet_e2e_demo/src/main.rs @@ -1,51 +1,22 @@ -//! `cargo run -p example-fleet-e2e-demo -- --num-devices 2 ...` -//! -//! Brings up the full E2E rehearsal stack: k3d + Zitadel + NATS auth -//! callout + per-device Zitadel machine users + (out-of-band) -//! libvirt VMs + agents authenticating via JWT-bearer. -//! -//! See `src/lib.rs` and `ROADMAP/fleet_platform/v0_demo_e2e.md`. - -use anyhow::{Context, Result}; +use anyhow::Result; use clap::Parser; -use example_fleet_e2e_demo::{DEFAULT_LIBVIRT_HOST_IP, E2eDemoOpts, bring_up_full_stack}; -use std::path::PathBuf; +use harmony_fleet_e2e::{AuthMode, VmStack, VmStackOptions}; -#[derive(Parser, Debug)] -#[command( - name = "fleet-e2e-demo", - about = "VM-based end-to-end rehearsal of the fleet platform demo flow" -)] +#[derive(Parser)] struct Cli { - /// Number of VM-as-device agents to bring up. Each one needs its - /// own libvirt domain (provisioned out-of-band today via - /// `fleet_vm_setup` — see `FLEET_E2E_VM__IP` env vars below). - #[arg(long, default_value_t = 2)] + #[arg(long, default_value_t = 1)] num_devices: usize, - /// Path to the cross-compiled `fleet-agent` binary uploaded to - /// each VM. Same binary that smoke-a4 produces. - #[arg(long, default_value = "target/release/harmony-fleet-agent")] - agent_binary: PathBuf, - /// Override for the libvirt host IP (the address VMs see as the - /// gateway). Defaults to the libvirt default network's gateway. - #[arg(long, default_value = DEFAULT_LIBVIRT_HOST_IP)] - libvirt_host_ip: String, } #[tokio::main] async fn main() -> Result<()> { let cli = Cli::parse(); - let handles = bring_up_full_stack(E2eDemoOpts { + let _stack = VmStack::bring_up(VmStackOptions { num_devices: cli.num_devices, - agent_binary: cli.agent_binary, - libvirt_host_ip: cli.libvirt_host_ip, + auth_mode: AuthMode::Callout, + ..Default::default() }) - .await - .context("bring_up_full_stack")?; - handles.print_next_steps(); - - println!(); - println!(" Press Ctrl-C to exit (cluster keeps running)."); + .await?; tokio::signal::ctrl_c().await?; Ok(()) } diff --git a/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs b/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs deleted file mode 100644 index 16084277..00000000 --- a/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! End-to-end walking-skeleton tests for the VM-based demo rehearsal. -//! -//! Shares one bring-up across the whole suite via `OnceCell`. Run -//! sequentially — they touch shared k3d + libvirt VM state. -//! -//! Pre-flight (manual, before `cargo test`): -//! -//! - libvirt + qemu installed; default network active. -//! - Two cloud-init Ubuntu VMs provisioned (e.g. via -//! `cargo run -p example_fleet_vm_setup`). Their IPs exported as -//! `FLEET_E2E_VM_0_IP` and `FLEET_E2E_VM_1_IP`. -//! - SSH keypair the VMs trust at `~/.ssh/id_ed25519` (or -//! override path; harness reads the standard pair). -//! -//! Run: -//! -//! ```bash -//! FLEET_E2E_VM_0_IP=192.168.122.42 \ -//! FLEET_E2E_VM_1_IP=192.168.122.43 \ -//! cargo test -p example-fleet-e2e-demo --test e2e_walking_skeleton \ -//! -- --test-threads=1 --nocapture -//! ``` - -use std::sync::Arc; -use std::time::Duration; - -use anyhow::{Context, Result}; -use async_nats::ConnectOptions; -use example_fleet_auth_callout::{mint_access_token, scopes_for_project}; -use example_fleet_e2e_demo::{E2eDemoOpts, E2eHandles, bring_up_full_stack}; -use futures_util::StreamExt; -use tokio::sync::OnceCell; - -static STACK: OnceCell> = OnceCell::const_new(); - -async fn shared_stack() -> Result> { - let cell = STACK - .get_or_try_init(|| async { - let h = bring_up_full_stack(E2eDemoOpts::default()).await?; - anyhow::Ok(Arc::new(h)) - }) - .await?; - Ok(cell.clone()) -} - -async fn admin_nats_client(stack: &E2eHandles) -> Result { - let token = mint_access_token( - &stack.zitadel_url, - &stack.admin_machine_key, - &scopes_for_project(&stack.project_id), - ) - .await - .context("mint admin Zitadel token")?; - ConnectOptions::with_token(token) - .connection_timeout(Duration::from_secs(5)) - .connect(&stack.nats_url_external) - .await - .map_err(|e| anyhow::anyhow!("admin connect: {e}")) -} - -// -- Test 1 ------------------------------------------------------------- - -/// Each provisioned VM publishes a DeviceInfo within the heartbeat -/// window. Reads from the `device-info` KV bucket via the admin -/// client (admin role can subscribe to anything). -#[tokio::test] -#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"] -async fn both_devices_heartbeat_within_60s() -> Result<()> { - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); - let stack = shared_stack().await?; - let admin = admin_nats_client(&stack).await?; - - let js = async_nats::jetstream::new(admin); - let bucket = js - .get_key_value(harmony_reconciler_contracts::BUCKET_DEVICE_INFO) - .await - .context("device-info bucket")?; - - let deadline = std::time::Instant::now() + Duration::from_secs(60); - let expected: std::collections::HashSet = - stack.devices.iter().map(|d| d.device_id.clone()).collect(); - let mut seen = std::collections::HashSet::new(); - - while std::time::Instant::now() < deadline && seen != expected { - for d in &stack.devices { - let key = harmony_reconciler_contracts::device_info_key(&d.device_id); - if let Some(_e) = bucket.entry(&key).await? { - seen.insert(d.device_id.clone()); - } - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - assert_eq!( - seen, expected, - "each provisioned device must publish DeviceInfo within 60s; saw {seen:?}" - ); - Ok(()) -} - -// -- Test 5 (admin cross-device read) ----------------------------------- - -/// The admin's Zitadel JWT carries `fleet-admin` role. Callout maps -/// that to `pub/sub allow: [">"]`, so subscribing to `device-state.>` -/// is admitted and observes every device's traffic. -#[tokio::test] -#[ignore = "requires libvirt VMs + k3d + Zitadel + NATS bring-up — see header"] -async fn admin_jwt_reads_any_device_subject() -> Result<()> { - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); - let stack = shared_stack().await?; - let admin = admin_nats_client(&stack).await?; - - let mut sub = admin.subscribe("device-state.>").await?; - admin.flush().await?; - - // Hold the subscription open long enough that any device's - // periodic state publication should land. We don't pump traffic - // ourselves — the agents themselves publish per-deployment state - // on every reconcile tick. If no traffic arrives in 30s it means - // either the agents aren't connected or they're not publishing, - // both of which are fatal for the demo. - let result = tokio::time::timeout(Duration::from_secs(30), sub.next()).await; - assert!( - result.is_ok() && result.as_ref().unwrap().is_some(), - "admin must observe at least one device-state.* message in 30s" - ); - Ok(()) -} - -// -- Test 6 (per-device isolation) --------------------------------------- - -/// A per-device JWT has subject permissions scoped to its own -/// `device-state.{device_id}` and `device-commands.{device_id}`. The -/// callout enforces this; subscribing to a sibling device's commands -/// must fail at NATS connect-time or at SUB-time. -/// -/// Skipped here because the per-device JWT minting helper (analogous -/// to `mint_access_token` but for a `device` role user) needs the -/// per-device machine key to be plumbed back from `bring_up_full_stack` -/// through `E2eHandles`. Follow-up commit adds -/// `E2eHandles::device_machine_key(idx)` so this test can be -/// implemented without re-running `ZitadelSetupScore` from the test -/// body. -#[tokio::test] -#[ignore = "requires E2eHandles::device_machine_key plumbing"] -async fn cross_device_isolation_enforced_in_vm() {} - -// -- Test 7 (load-bearing reconnect) ------------------------------------- - -/// Kill the NATS pod, wait for the new one to come up, verify both -/// agents reconnect with fresh JWTs and resume publishing within -/// 30 seconds. This is the test that validates the "never lose -/// connectivity to a device" guarantee under realistic disturbance. -/// -/// Skipped pending operator install in the harness — without the -/// operator the agents have no `desired-state` to publish status -/// against, so verifying "publishing resumed" needs a separate -/// signal. Follow-up commit observes the agents' periodic -/// heartbeat publication directly via the device-heartbeat KV. -#[tokio::test] -#[ignore = "requires NATS-pod-restart driver and heartbeat-presence assertion"] -async fn agent_recovers_from_nats_pod_restart() {} diff --git a/examples/fleet_load_test/Cargo.toml b/examples/fleet_load_test/Cargo.toml index 89bd3a8a..c98d8afa 100644 --- a/examples/fleet_load_test/Cargo.toml +++ b/examples/fleet_load_test/Cargo.toml @@ -9,7 +9,7 @@ name = "fleet_load_test" path = "src/main.rs" [dependencies] -harmony = { path = "../../harmony", default-features = false } +harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } async-nats = { workspace = true } chrono = { workspace = true } diff --git a/examples/fleet_load_test/src/main.rs b/examples/fleet_load_test/src/main.rs index 594f8685..fec77d19 100644 --- a/examples/fleet_load_test/src/main.rs +++ b/examples/fleet_load_test/src/main.rs @@ -28,13 +28,13 @@ use anyhow::{Context, Result}; use async_nats::jetstream::{self, kv}; use chrono::Utc; use clap::Parser; -use harmony::modules::fleet::operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy}; -use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore}; +use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy}; use harmony_reconciler_contracts::{ BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key, device_info_key, device_state_key, }; +use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore}; use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use kube::Client; diff --git a/examples/fleet_nats_install/src/main.rs b/examples/fleet_nats_install/src/main.rs index 8270abca..f0bfa708 100644 --- a/examples/fleet_nats_install/src/main.rs +++ b/examples/fleet_nats_install/src/main.rs @@ -1,5 +1,5 @@ //! Install a single-node NATS server into the cluster `KUBECONFIG` -//! points at, using harmony's `NatsBasicScore` + `K8sBareTopology`. +//! points at, using harmony's `NatsScore` + `K8sBareTopology`. //! //! This binary is the glue between the smoke harness (`smoke-a4.sh`) //! and the framework Score. Typical usage from a demo script: @@ -20,13 +20,13 @@ use anyhow::{Context, Result}; use clap::Parser; use harmony::inventory::Inventory; use harmony::modules::k8s::K8sBareTopology; -use harmony::modules::nats::NatsBasicScore; +use harmony::modules::nats::NatsScore; use harmony::score::Score; #[derive(Parser, Debug)] #[command( name = "fleet_nats_install", - about = "Install single-node NATS (JetStream) via NatsBasicScore" + about = "Install single-node NATS (JetStream) via NatsScore" )] struct Cli { /// Target namespace. Created if missing. @@ -45,7 +45,7 @@ struct Cli { /// NodePort when `--expose=node-port`. Must be in the cluster's /// nodeport range (default 30000-32767). Ignored otherwise. #[arg(long, default_value_t = 30422)] - node_port: i32, + node_port: u16, /// Override the NATS container image. #[arg(long)] image: Option, @@ -67,7 +67,7 @@ async fn main() -> Result<()> { .map_err(|e| anyhow::anyhow!(e)) .context("building K8sBareTopology from KUBECONFIG")?; - let mut score = NatsBasicScore::new(&cli.name, &cli.namespace); + let mut score = NatsScore::new(&cli.name, &cli.namespace); match cli.expose { ExposeMode::ClusterIp => {} ExposeMode::NodePort => score = score.node_port(cli.node_port), @@ -81,7 +81,7 @@ async fn main() -> Result<()> { let outcome = interpret .execute(&Inventory::empty(), &topology) .await - .map_err(|e| anyhow::anyhow!("execute NatsBasicScore: {e}"))?; + .map_err(|e| anyhow::anyhow!("execute NatsScore: {e}"))?; println!( "NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}", diff --git a/examples/fleet_rpi_setup/Cargo.toml b/examples/fleet_rpi_setup/Cargo.toml index 559b400f..c72ee3ec 100644 --- a/examples/fleet_rpi_setup/Cargo.toml +++ b/examples/fleet_rpi_setup/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" [dependencies] harmony = { path = "../../harmony" } +harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } harmony_cli = { path = "../../harmony_cli" } harmony_secret = { path = "../../harmony_secret" } harmony_types = { path = "../../harmony_types" } diff --git a/examples/fleet_rpi_setup/src/main.rs b/examples/fleet_rpi_setup/src/main.rs index 586db968..5e572dd9 100644 --- a/examples/fleet_rpi_setup/src/main.rs +++ b/examples/fleet_rpi_setup/src/main.rs @@ -37,8 +37,8 @@ use anyhow::{Context, Result}; use clap::Parser; use harmony::config::secret::SudoPassword; use harmony::inventory::Inventory; -use harmony::modules::fleet::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore}; use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec}; +use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore}; use harmony_secret::SecretManager; use harmony_types::id::Id; use log::info; diff --git a/examples/fleet_server_install/src/main.rs b/examples/fleet_server_install/src/main.rs index 9d1700e1..051b5cbb 100644 --- a/examples/fleet_server_install/src/main.rs +++ b/examples/fleet_server_install/src/main.rs @@ -21,7 +21,7 @@ //! //! Behaviour: //! - Installs single-node NATS (JetStream) into `--nats-namespace` -//! using `NatsBasicScore`, exposed per `--nats-expose`. +//! using `NatsScore`, exposed per `--nats-expose`. //! - Installs the operator chart into `--operator-namespace` via //! `FleetOperatorScore` (which renders the chart in a tempdir //! and helm-installs it). @@ -47,7 +47,7 @@ use anyhow::Result; use clap::Parser; use harmony::inventory::Inventory; -use harmony::modules::nats::NatsBasicScore; +use harmony::modules::nats::NatsScore; use harmony::modules::zitadel::ZitadelScore; use harmony::score::Score; use harmony::topology::K8sAnywhereTopology; @@ -72,7 +72,7 @@ struct Cli { nats_expose: NatsExpose, /// NodePort when `--nats-expose=node-port`. Ignored otherwise. #[arg(long, default_value_t = 30422)] - nats_node_port: i32, + nats_node_port: u16, /// Optional NATS image override (`repository:tag`). #[arg(long)] nats_image: Option, @@ -124,7 +124,7 @@ async fn main() -> Result<()> { let topology = K8sAnywhereTopology::from_env(); - let mut nats = NatsBasicScore::new(&cli.nats_name, &cli.nats_namespace); + let mut nats = NatsScore::new(&cli.nats_name, &cli.nats_namespace); match cli.nats_expose { NatsExpose::ClusterIp => {} NatsExpose::NodePort => nats = nats.node_port(cli.nats_node_port), @@ -135,17 +135,14 @@ async fn main() -> Result<()> { } // Point the operator at NATS via the in-cluster service DNS the - // NatsBasicScore install creates. ClusterIP and LoadBalancer both + // NatsScore install creates. ClusterIP and LoadBalancer both // expose the same `.:4222` for in-cluster // callers. - let nats_url = format!("nats://{}.{}:4222", cli.nats_name, cli.nats_namespace); - - let operator = FleetOperatorScore::new() + let operator = FleetOperatorScore::new(&cli.operator_image) .namespace(&cli.operator_namespace) .release_name(&cli.operator_release) - .image(&cli.operator_image) .image_pull_policy(&cli.operator_image_pull_policy) - .nats_url(&nats_url) + .messaging(&nats.client_ref()) .log_level(&cli.log_level); // FleetServerScore now takes NatsK8sScore (auth-callout-aware, diff --git a/examples/fleet_staging_deploy/Cargo.toml b/examples/fleet_staging_deploy/Cargo.toml deleted file mode 100644 index a6b4a96b..00000000 --- a/examples/fleet_staging_deploy/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "example-fleet-staging-deploy" -edition = "2024" -version.workspace = true -readme.workspace = true -license.workspace = true -description = "Deploy the fleet platform stack (Zitadel + NATS + auth callout) onto an OKD/Kubernetes cluster. Operator-side, run-once-per-customer." - -[lib] -name = "example_fleet_staging_deploy" -path = "src/lib.rs" - -[[bin]] -name = "fleet-staging-deploy" -path = "src/main.rs" - -[dependencies] -harmony = { path = "../../harmony" } -harmony-k8s = { path = "../../harmony-k8s" } -harmony_types = { path = "../../harmony_types" } -harmony-nats-callout = { path = "../../nats/callout" } -nkeys = "0.4" -async-nats.workspace = true -reqwest = { workspace = true } -tokio = { workspace = true, features = ["full"] } -serde.workspace = true -serde_json.workspace = true -anyhow.workspace = true -log.workspace = true -env_logger.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -clap = { version = "4", features = ["derive", "env"] } -k8s-openapi.workspace = true -kube.workspace = true -url.workspace = true diff --git a/examples/fleet_staging_deploy/src/lib.rs b/examples/fleet_staging_deploy/src/lib.rs deleted file mode 100644 index 59069767..00000000 --- a/examples/fleet_staging_deploy/src/lib.rs +++ /dev/null @@ -1,579 +0,0 @@ -//! Operator-side staging deploy harness. -//! -//! Runs once per customer instance against an OKD / Kubernetes cluster -//! to bring up the fleet platform's central services: -//! -//! 1. Zitadel + Postgres (HTTPS via OKD HAProxy ingress, edge TLS). -//! 2. The fleet project + roles (`fleet-admin`, `device`) + an API app -//! (so the project ID can be the JWT-bearer audience). -//! 3. NATS with `auth_callout` and a WSS ingress (so Pis on a customer -//! LAN connect through `wss://nats./`). -//! 4. The auth callout Deployment, configured to validate Zitadel JWTs -//! and emit per-device permissions on user JWTs to NATS. -//! -//! Everything keys off [`FleetDomainConfig::base_domain`] — -//! `zitadel.`, `nats.`, `api.` are the only -//! customer-visible hostnames. Pi-side onboarding (see -//! `examples/fleet_rpi_setup/`) consumes the Zitadel admin PAT plus -//! the project ID this harness prints, so the operator's flow is: -//! -//! ```text -//! cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io -//! ↓ prints PROJECT_ID, NATS WSS URL, instructions to extract iam-admin-pat -//! HARMONY_ZITADEL_ADMIN_PAT=$(kubectl -n zitadel get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d) \ -//! cargo run -p example-fleet-rpi-setup -- \ -//! --pi-host 192.168.1.42 \ -//! --bootstrap-token "$HARMONY_ZITADEL_ADMIN_PAT" \ -//! --zitadel-issuer-url https://zitadel.customer1.nationtech.io \ -//! --zitadel-project-id \ -//! --nats-url wss://nats.customer1.nationtech.io/ \ -//! --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent -//! ``` -//! -//! The harness is **idempotent** by design — re-running picks up -//! existing resources via the new helm-upgrade-by-default behavior + -//! ZitadelSetupScore's search-then-create flow + a persisted issuer -//! NKey in a K8s secret so user JWTs survive restarts. - -use std::time::Duration; - -use anyhow::{Context, Result}; -use harmony::inventory::Inventory; -use harmony::modules::nats::NatsHelmChartScore; -use harmony::modules::nats_auth_callout::{NatsAuthCalloutScore, render_auth_callout_block}; -use harmony::modules::zitadel::{ - ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelRole, - ZitadelScore, ZitadelSetupScore, -}; -use harmony::score::Score; -use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; -use log::info; -use nkeys::KeyPair; - -// ---- domain config --------------------------------------------------------- - -/// Single source of truth for all customer-visible hostnames. Every -/// `..` URL the staging deploy emits derives from -/// the one base domain — no hostnames are hardcoded so the same code -/// runs across customers / staging / canary instances. -#[derive(Debug, Clone)] -pub struct FleetDomainConfig { - /// e.g. `customer1.nationtech.io`. The deploy emits - /// `zitadel.`, `nats.`, `api.` against it. - pub base_domain: String, -} - -impl FleetDomainConfig { - pub fn new(base_domain: impl Into) -> Self { - Self { - base_domain: base_domain.into(), - } - } - pub fn zitadel_host(&self) -> String { - format!("zitadel.{}", self.base_domain) - } - pub fn nats_wss_host(&self) -> String { - format!("nats.{}", self.base_domain) - } - pub fn zitadel_issuer_url(&self) -> String { - format!("https://{}", self.zitadel_host()) - } - pub fn nats_wss_url(&self) -> String { - format!("wss://{}/", self.nats_wss_host()) - } -} - -// ---- naming + constants ---------------------------------------------------- - -pub const FLEET_NAMESPACE: &str = "fleet-staging"; -pub const ZITADEL_NAMESPACE: &str = "zitadel-staging"; -pub const NATS_RELEASE: &str = "fleet-nats"; -pub const CALLOUT_DEPLOYMENT_NAME: &str = "fleet-callout"; -pub const PROJECT_NAME: &str = "fleet"; -pub const API_APP_NAME: &str = "nats"; -pub const CLI_APP_NAME: &str = "harmony-cli"; -pub const ADMIN_ROLE_KEY: &str = "fleet-admin"; -pub const DEVICE_ROLE_KEY: &str = "device"; -pub const NATS_AUTH_USER: &str = "auth"; -pub const NATS_ACCOUNT: &str = "DEVICES"; -pub const NATS_SYSTEM_USER: &str = "sys-admin"; -pub const ISSUER_SEED_SECRET: &str = "callout-issuer-seed"; - -// ---- handles --------------------------------------------------------------- - -#[derive(Debug, Clone)] -pub struct StagingHandles { - pub domain: FleetDomainConfig, - pub project_id: String, - pub issuer_pubkey: String, - /// Tag of the callout image expected to exist in a registry the - /// cluster pulls from. The operator pushes it before running the - /// deploy; this field is just the name we put on the Deployment - /// for traceability. - pub callout_image: String, - /// OIDC client_id of the `harmony-cli` Device Code app — what the - /// `fleet_sso_login` CLI sends in its device-authorization request. - /// `None` if the app pre-existed without the cache picking it up - /// (re-running the staging deploy after `rm -rf - /// ~/.local/share/harmony/zitadel/`). - pub cli_client_id: Option, -} - -// ---- bring up -------------------------------------------------------------- - -pub struct StagingDeployOpts { - pub domain: FleetDomainConfig, - pub kubeconfig_context: Option, - /// Image reference the cluster will pull. Operator must have - /// pushed this beforehand (e.g. `quay.io/customer/harmony-nats-callout:demo`). - pub callout_image: String, - /// Per-NATS-account password for the callout's own NATS connection. - /// Stored in a K8s secret + listed in the chart's - /// `accounts..users` so the callout bypasses callout to - /// connect (otherwise it'd deadlock authenticating itself). - pub nats_auth_pass: String, - /// SYS account password (for `kubectl exec nats-box` debugging). - pub nats_system_pass: String, -} - -pub async fn bring_up_staging(opts: StagingDeployOpts) -> Result { - let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) - .try_init(); - - if let Some(ctx) = &opts.kubeconfig_context { - unsafe { - std::env::set_var("HARMONY_K8S_CONTEXT", ctx); - std::env::set_var("HARMONY_USE_LOCAL_K3D", "false"); - std::env::set_var("HARMONY_AUTOINSTALL", "false"); - } - } - let topology = K8sAnywhereTopology::from_env(); - topology.ensure_ready().await.context("topology init")?; - - info!( - "[1/5] deploying Zitadel at https://{}", - opts.domain.zitadel_host() - ); - deploy_zitadel(&opts.domain, &topology).await?; - - info!("[2/5] waiting for Zitadel HTTPS to respond"); - wait_for_zitadel_ready(&opts.domain).await?; - - info!("[3/5] provisioning project '{PROJECT_NAME}', api app, CLI device-code app, and roles"); - provision_zitadel_project(&opts.domain, &topology).await?; - let project_id = read_project_id()?; - let cli_client_id = read_cli_client_id(); - info!(" → project_id = {project_id}"); - if let Some(cid) = &cli_client_id { - info!(" → cli_client_id = {cid}"); - } else { - log::warn!( - " → cli_client_id missing from cache; CLI login won't work until you reset the local zitadel cache" - ); - } - - info!("[4/5] generating issuer NKey + deploying NATS with auth_callout + WSS ingress"); - let issuer_seed = ensure_issuer_seed(&topology).await?; - let issuer_kp = KeyPair::from_seed(&issuer_seed) - .map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?; - let issuer_pubkey = issuer_kp.public_key(); - - NatsHelmChartScore::new( - NATS_RELEASE.to_string(), - FLEET_NAMESPACE.to_string(), - render_nats_values( - &opts.domain, - &issuer_pubkey, - &opts.nats_auth_pass, - &opts.nats_system_pass, - ), - ) - .interpret(&Inventory::autoload(), &topology) - .await - .context("NATS deploy")?; - - info!( - "[5/5] deploying NatsAuthCalloutScore (image: {})", - opts.callout_image - ); - NatsAuthCalloutScore::new( - CALLOUT_DEPLOYMENT_NAME, - FLEET_NAMESPACE, - format!("nats://{NATS_RELEASE}.{FLEET_NAMESPACE}.svc.cluster.local:4222"), - opts.domain.zitadel_issuer_url(), - // The aud the callout validates against is the project ID — - // Zitadel emits it in access tokens minted via the - // project-id-audience scope. - project_id.clone(), - NATS_AUTH_USER, - opts.nats_auth_pass.clone(), - issuer_seed, - ) - .image(&opts.callout_image) - .target_account(NATS_ACCOUNT) - .admin_role(ADMIN_ROLE_KEY) - .device_role(DEVICE_ROLE_KEY) - .interpret(&Inventory::autoload(), &topology) - .await - .context("callout deploy")?; - - Ok(StagingHandles { - domain: opts.domain, - project_id, - issuer_pubkey, - callout_image: opts.callout_image, - cli_client_id, - }) -} - -fn read_cli_client_id() -> Option { - ZitadelClientConfig::load()? - .client_id(CLI_APP_NAME) - .cloned() -} - -async fn deploy_zitadel(domain: &FleetDomainConfig, topology: &K8sAnywhereTopology) -> Result<()> { - let z = ZitadelScore { - host: domain.zitadel_host(), - zitadel_version: "v4.12.1".to_string(), - // OKD HAProxy edge-terminates TLS for us, so the issuer URL - // is `https://zitadel.` (port 443 implied) — leave - // external_port at None so Zitadel's emitted issuer omits the - // port, matching what clients reach. - external_secure: true, - external_port: None, - namespace: ZITADEL_NAMESPACE.to_string(), - ..Default::default() - }; - z.interpret(&Inventory::autoload(), topology) - .await - .context("ZitadelScore")?; - Ok(()) -} - -async fn provision_zitadel_project( - domain: &FleetDomainConfig, - topology: &K8sAnywhereTopology, -) -> Result<()> { - let setup = ZitadelSetupScore { - host: domain.zitadel_host(), - // Direct HTTPS through OKD's HAProxy ingress — operator runs - // anywhere with kubeconfig + DNS access. Defaults give - // `https://` (port 443). - scheme: Default::default(), - port: None, - skip_tls: false, - endpoint: None, - admin_org_id: None, - namespace: ZITADEL_NAMESPACE.to_string(), - applications: vec![ZitadelApplication { - project_name: PROJECT_NAME.to_string(), - app_name: CLI_APP_NAME.to_string(), - // Device Code grant — the only browser-driven OIDC flow - // that fits a CLI tool: prints a verification URL + user - // code, polls for a token, no embedded web server / open - // listener required. - app_type: ZitadelAppType::DeviceCode, - }], - api_apps: vec![ZitadelApiApp { - project_name: PROJECT_NAME.to_string(), - app_name: API_APP_NAME.to_string(), - }], - roles: vec![ - ZitadelRole { - project_name: PROJECT_NAME.to_string(), - key: ADMIN_ROLE_KEY.to_string(), - display_name: "Fleet Admin".to_string(), - group: None, - }, - ZitadelRole { - project_name: PROJECT_NAME.to_string(), - key: DEVICE_ROLE_KEY.to_string(), - display_name: "Device".to_string(), - group: None, - }, - ], - // No machine users provisioned here — `fleet_rpi_setup` mints - // them on demand per device, so the staging deploy stays - // device-count-agnostic. - groups_claim_action: false, - machine_users: vec![], - ..Default::default() - }; - setup - .interpret(&Inventory::autoload(), topology) - .await - .context("ZitadelSetupScore")?; - Ok(()) -} - -fn read_project_id() -> Result { - let cfg = ZitadelClientConfig::load() - .context("ZitadelSetupScore did not produce a client config cache")?; - cfg.project_id_by_name(PROJECT_NAME) - .or(cfg.project_id.as_ref()) - .context("project_id missing from ZitadelClientConfig cache") - .cloned() -} - -/// Persist the callout's issuer NKey seed in a K8s secret so re-runs -/// of the staging deploy don't invalidate previously-issued user JWTs -/// already in flight on customer Pis. -async fn ensure_issuer_seed(topology: &K8sAnywhereTopology) -> Result { - use k8s_openapi::ByteString; - use k8s_openapi::api::core::v1::{Namespace, Secret}; - use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; - use std::collections::BTreeMap; - - let k8s = topology - .k8s_client() - .await - .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; - - if k8s - .get_resource::(FLEET_NAMESPACE, None) - .await? - .is_none() - { - let ns = Namespace { - metadata: ObjectMeta { - name: Some(FLEET_NAMESPACE.to_string()), - ..Default::default() - }, - ..Default::default() - }; - k8s.create(&ns, None).await.ok(); - } - - if let Some(existing) = k8s - .get_resource::(ISSUER_SEED_SECRET, Some(FLEET_NAMESPACE)) - .await? - && let Some(data) = existing.data - && let Some(seed_bytes) = data.get("seed") - { - let seed = String::from_utf8(seed_bytes.0.clone())?; - return Ok(seed.trim().to_string()); - } - - let seed = KeyPair::new_account() - .seed() - .map_err(|e| anyhow::anyhow!("nkey seed: {e}"))?; - let mut data = BTreeMap::new(); - data.insert("seed".to_string(), ByteString(seed.as_bytes().to_vec())); - let secret = Secret { - metadata: ObjectMeta { - name: Some(ISSUER_SEED_SECRET.to_string()), - namespace: Some(FLEET_NAMESPACE.to_string()), - ..Default::default() - }, - data: Some(data), - type_: Some("Opaque".to_string()), - ..Default::default() - }; - k8s.create(&secret, Some(FLEET_NAMESPACE)).await.ok(); - Ok(seed) -} - -// ---- NATS values ----------------------------------------------------------- - -/// Render NATS Helm values for an OKD-flavored deployment with WSS -/// ingress + auth callout + JetStream. -/// -/// **Why WSS rather than plain NATS-on-TLS:** OKD's default ingress -/// controller (HAProxy) is HTTP-aware and edge-terminates TLS. NATS -/// over WebSocket goes through that ingress unchanged; native NATS -/// TCP would require a TCP loadbalancer service or a passthrough -/// Route, both of which are extra infra the customer's cluster may -/// not have. WSS is also the default async-nats client transport on -/// `wss://...` URLs — no special agent code needed. -pub fn render_nats_values( - domain: &FleetDomainConfig, - issuer_pubkey: &str, - nats_auth_pass: &str, - nats_system_pass: &str, -) -> String { - let auth_callout = render_auth_callout_block(issuer_pubkey, NATS_AUTH_USER, NATS_ACCOUNT); - let auth_callout_indented = auth_callout - .lines() - .enumerate() - .map(|(i, l)| { - if i == 0 { - l.to_string() - } else { - format!(" {l}") - } - }) - .collect::>() - .join("\n"); - format!( - r#"fullnameOverride: {nats_release} -config: - cluster: - enabled: false - jetstream: - enabled: true - fileStorage: - enabled: true - size: 5Gi - websocket: - enabled: true - port: 8443 - ingress: - enabled: true - className: openshift-default - pathType: Prefix - hosts: - - {nats_wss_host} - annotations: - # OKD HAProxy edge-terminates TLS — the chart's default Route - # generation needs `route.openshift.io/termination: edge` so - # the Route's TLS block is "edge", matching the cluster's wildcard - # cert behavior. Switch to `reencrypt` if you need TLS all the - # way to the NATS pod. - route.openshift.io/termination: edge - haproxy.router.openshift.io/timeout: "1h" - merge: - {auth_callout_indented} - accounts: - {nats_account}: - jetstream: enabled - users: - - user: "{auth_user}" - password: "{auth_pass}" - SYS: - users: - - user: "{sys_user}" - password: "{sys_pass}" - system_account: SYS -service: - ports: - nats: - enabled: true -"#, - nats_release = NATS_RELEASE, - nats_wss_host = domain.nats_wss_host(), - nats_account = NATS_ACCOUNT, - auth_user = NATS_AUTH_USER, - auth_pass = nats_auth_pass, - sys_user = NATS_SYSTEM_USER, - sys_pass = nats_system_pass, - ) -} - -// ---- readiness ------------------------------------------------------------- - -async fn wait_for_zitadel_ready(domain: &FleetDomainConfig) -> Result<()> { - let issuer = domain.zitadel_issuer_url(); - let well_known = format!("{issuer}/.well-known/openid-configuration"); - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(5)) - .build()?; - for attempt in 1..=180 { - match client.get(&well_known).send().await { - Ok(r) if r.status().is_success() => return Ok(()), - Ok(r) if attempt % 30 == 0 => { - info!("Zitadel HTTPS {} (attempt {attempt}/180)", r.status()); - } - Err(e) if attempt % 30 == 0 => { - info!("Zitadel unreachable: {e} (attempt {attempt}/180)"); - } - _ => {} - } - tokio::time::sleep(Duration::from_secs(2)).await; - } - anyhow::bail!("timed out waiting for Zitadel at {well_known}") -} - -// ---- helpful printout ------------------------------------------------------ - -impl StagingHandles { - /// Print the operator's "what to do next" panel after a successful - /// staging deploy. Pasted at the end of the binary's run. - pub fn print_next_steps(&self) { - let zitadel = self.domain.zitadel_issuer_url(); - let nats = self.domain.nats_wss_url(); - println!(); - println!("============================================================"); - println!(" STAGING DEPLOY COMPLETE"); - println!("============================================================"); - println!(" Base domain: {}", self.domain.base_domain); - println!(" Zitadel: {zitadel}"); - println!(" NATS (WSS): {nats}"); - println!(" Project ID: {}", self.project_id); - println!(" Callout image: {}", self.callout_image); - println!(" Issuer pubkey: {}", self.issuer_pubkey); - if let Some(cid) = &self.cli_client_id { - println!(" CLI client_id: {cid}"); - println!(); - println!(" CLI SSO login (developer-side):"); - println!(); - println!(" cargo run -p example-fleet-sso-login -- \\"); - println!(" --base-domain {} \\", self.domain.base_domain); - println!(" --client-id {cid}"); - } - println!(); - println!(" Onboard a Pi:"); - println!(); - println!(" PAT=$(kubectl -n zitadel get secret iam-admin-pat \\"); - println!(" -o jsonpath='{{.data.pat}}' | base64 -d)"); - println!(); - println!(" cargo run -p example-fleet-rpi-setup -- \\"); - println!(" --pi-host \\"); - println!(" --bootstrap-token \"$PAT\" \\"); - println!(" --zitadel-issuer-url {zitadel} \\"); - println!(" --zitadel-project-id {} \\", self.project_id); - println!(" --nats-url {nats} \\"); - println!(" --agent-binary "); - println!(); - println!("============================================================"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn domain_config_derives_hostnames() { - let d = FleetDomainConfig::new("customer1.nationtech.io"); - assert_eq!(d.zitadel_host(), "zitadel.customer1.nationtech.io"); - assert_eq!(d.nats_wss_host(), "nats.customer1.nationtech.io"); - assert_eq!( - d.zitadel_issuer_url(), - "https://zitadel.customer1.nationtech.io" - ); - assert_eq!(d.nats_wss_url(), "wss://nats.customer1.nationtech.io/"); - } - - #[test] - fn nats_values_render_includes_wss_ingress_and_auth_callout() { - let d = FleetDomainConfig::new("acme.io"); - let yaml = render_nats_values(&d, "ABCDEF", "auth-pass", "sys-pass"); - // WSS plumbing. - assert!(yaml.contains("websocket:")); - assert!(yaml.contains("port: 8443")); - assert!(yaml.contains("nats.acme.io")); - // OKD edge-TLS annotations. - assert!(yaml.contains("openshift-default")); - assert!(yaml.contains("route.openshift.io/termination: edge")); - // Auth callout wired through with the issuer pubkey. - assert!(yaml.contains("auth_callout")); - assert!(yaml.contains("issuer: ABCDEF")); - assert!(yaml.contains("auth_users: [ auth ]")); - assert!(yaml.contains("system_account: SYS")); - // Account user. - assert!(yaml.contains("password: \"auth-pass\"")); - } - - #[test] - fn nats_values_inline_account_block_under_merge() { - // Prevent regressions where the auth_callout block leaks - // outside the `merge:` indentation level — chart expects it - // under config.merge. - let d = FleetDomainConfig::new("x.io"); - let yaml = render_nats_values(&d, "K", "p", "s"); - let idx_merge = yaml.find("\n merge:\n").expect("merge block present"); - let idx_callout = yaml.find("auth_callout:").expect("auth_callout present"); - assert!(idx_callout > idx_merge, "auth_callout must follow merge:"); - } -} diff --git a/examples/fleet_staging_deploy/src/main.rs b/examples/fleet_staging_deploy/src/main.rs deleted file mode 100644 index bd9e7b2f..00000000 --- a/examples/fleet_staging_deploy/src/main.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! `cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io ...` -//! -//! Operator-side, run-once-per-customer-instance harness. Brings up -//! the central fleet platform services (Zitadel + NATS + auth callout) -//! against an OKD/K8s cluster pointed to by `KUBECONFIG`. Prints the -//! exact follow-up command the operator runs against a Pi to onboard -//! the first device. -//! -//! See `src/lib.rs` for the architectural notes. - -use anyhow::{Context, Result}; -use clap::Parser; -use example_fleet_staging_deploy::{FleetDomainConfig, StagingDeployOpts, bring_up_staging}; - -#[derive(Parser, Debug)] -#[command( - name = "fleet-staging-deploy", - about = "Deploy Zitadel + NATS + auth callout onto an OKD cluster" -)] -struct Cli { - /// Base DNS domain. All cluster-visible services derive from this: - /// `zitadel.`, `nats.`. The customer's wildcard cert / - /// CoreDNS / DNS provider must already point this at the cluster. - #[arg(long, env = "FLEET_BASE_DOMAIN")] - base_domain: String, - /// kubeconfig context to deploy against. Defaults to the - /// kubeconfig's current-context. Set this when your kubeconfig - /// has multiple contexts and you don't want to rely on the - /// global current. - #[arg(long, env = "FLEET_KUBE_CONTEXT")] - kube_context: Option, - /// Container image reference for the harmony-nats-callout binary. - /// The cluster pulls this; operator must have pushed it before - /// running the deploy. Defaults to a quay.io path that the - /// customer should override per their registry. - #[arg( - long, - env = "FLEET_CALLOUT_IMAGE", - default_value = "quay.io/nationtech/harmony-nats-callout:demo" - )] - callout_image: String, - /// Password for the NATS service-account user the callout uses on - /// its own NATS connection. Stored in a K8s secret + listed in - /// the chart's `accounts.DEVICES.users` (which bypass callout — - /// otherwise the callout would deadlock authenticating itself). - #[arg(long, env = "FLEET_NATS_AUTH_PASS")] - nats_auth_pass: String, - /// Password for the NATS SYS account (used for nats-box debugging - /// inside the cluster). - #[arg(long, env = "FLEET_NATS_SYSTEM_PASS")] - nats_system_pass: String, -} - -#[tokio::main] -async fn main() -> Result<()> { - let cli = Cli::parse(); - let domain = FleetDomainConfig::new(cli.base_domain); - - let handles = bring_up_staging(StagingDeployOpts { - domain, - kubeconfig_context: cli.kube_context, - callout_image: cli.callout_image, - nats_auth_pass: cli.nats_auth_pass, - nats_system_pass: cli.nats_system_pass, - }) - .await - .context("staging deploy")?; - - handles.print_next_steps(); - Ok(()) -} diff --git a/examples/fleet_staging_install/Cargo.toml b/examples/fleet_staging_install/Cargo.toml deleted file mode 100644 index f438eafb..00000000 --- a/examples/fleet_staging_install/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "example_fleet_staging_install" -edition = "2024" -version.workspace = true -readme.workspace = true -license.workspace = true -description = "Production-shape fleet install: Zitadel + NATS + auth callout + operator on OKD" - -[[bin]] -name = "fleet_staging_install" -path = "src/main.rs" - -[dependencies] -harmony = { path = "../../harmony" } -harmony_cli = { path = "../../harmony_cli" } -harmony_config = { path = "../../harmony_config" } -harmony_secret = { path = "../../harmony_secret" } -harmony-k8s = { path = "../../harmony-k8s" } -harmony-nats-callout = { path = "../../nats/callout" } -harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } -nkeys = "0.4" -rand = "0.9" -base64.workspace = true -anyhow.workspace = true -clap = { version = "4", features = ["derive", "env"] } -tokio.workspace = true -tracing = { workspace = true } -serde = { workspace = true } -schemars = "0.8" diff --git a/examples/fleet_staging_install/src/main.rs b/examples/fleet_staging_install/src/main.rs deleted file mode 100644 index 3215b3f9..00000000 --- a/examples/fleet_staging_install/src/main.rs +++ /dev/null @@ -1,417 +0,0 @@ -//! Production-shape fleet install for OKD (or any cluster with the same -//! capabilities): Zitadel SSO + NATS (auth-callout) + operator + OpenBao, -//! composed from Scores. -//! -//! Tunables come from [`ConfigClient`] (`HARMONY_CONFIG_FleetStagingConfig` -//! env JSON → OpenBao → interactive prompt), not a bespoke CLI. The only -//! flags are `harmony_cli`'s: `--filter`/`--list`/`-y` select which workload -//! Scores to (re)deploy — e.g. `--filter FleetOperatorScore` bumps the -//! operator without touching NATS or the callout. -//! -//! Zitadel + OpenBao are an idempotent bootstrap: ZitadelSetupScore mints the -//! `project_id` + `fleet-operator` machine key that the callout and operator -//! Scores consume, so it must converge (and cache to disk) before they're -//! built. That data flow is why those two can't sit in the filterable batch. - -use std::sync::Arc; - -use anyhow::{Context, Result}; -use clap::Parser; -use harmony::inventory::Inventory; -use harmony::modules::nats::capability::NatsCluster; -use harmony::modules::nats::score_nats_k8s::{AuthCalloutCfg, NatsK8sScore, WebSocketRouteCfg}; -use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; -use harmony::modules::openbao::{ - OpenbaoInstance, OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, - cached_root_token, -}; -use harmony::modules::zitadel::{ - MachineKeyType, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, - ZitadelMachineUser, ZitadelRole, ZitadelScore, ZitadelSetupScore, -}; -use harmony::score::Score; -use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; -use harmony_config::{Config, ConfigClient, StoreSource}; -use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, OperatorCredentials}; -use harmony_k8s::KubernetesDistribution; -use harmony_secret::{OpenbaoSecretStore, OpenbaoStoreOptions}; -use nkeys::KeyPair; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use tracing::info; - -/// Non-secret install tunables. `base_domain` drives every public hostname; -/// the image refs and `*-stg.` hosts have no safe default, so an empty -/// value is rejected at startup. Everything else defaults to the staging -/// conventions. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)] -#[serde(default)] -struct FleetStagingConfig { - base_domain: String, - operator_image: String, - callout_image: String, - cluster_issuer: String, - fleet_namespace: String, - zitadel_namespace: String, - nats_account: String, - zitadel_version: String, - project_name: String, - admin_role: String, - device_role: String, - operator_username: String, - admin_username: String, -} - -impl Default for FleetStagingConfig { - fn default() -> Self { - Self { - base_domain: String::new(), - operator_image: String::new(), - callout_image: String::new(), - cluster_issuer: "letsencrypt-prod".to_string(), - fleet_namespace: "fleet-staging".to_string(), - zitadel_namespace: "zitadel-staging".to_string(), - nats_account: "FLEET".to_string(), - zitadel_version: "v4.12.1".to_string(), - project_name: "fleet".to_string(), - admin_role: "fleet-admin".to_string(), - device_role: "device".to_string(), - operator_username: "fleet-operator".to_string(), - admin_username: "fleet-ops".to_string(), - } - } -} - -#[tokio::main] -async fn main() -> Result<()> { - harmony_cli::cli_logger::init(); - let args = harmony_cli::Args::parse(); - - let cfg: FleetStagingConfig = ConfigClient::for_namespace("harmony") - .await - .get_or_prompt() - .await - .context("loading FleetStagingConfig")?; - anyhow::ensure!( - !cfg.base_domain.is_empty() - && !cfg.operator_image.is_empty() - && !cfg.callout_image.is_empty(), - "base_domain, operator_image and callout_image must be set" - ); - - let topology = K8sAnywhereTopology::from_env(); - topology.ensure_ready().await?; - - let zitadel_host = format!("sso-stg.{}", cfg.base_domain); - let nats_ws_host = format!("nats-fleet-stg.{}", cfg.base_domain); - let secrets_host = format!("secrets-stg.{}", cfg.base_domain); - let nats_release = "fleet-nats"; - let cli_app_name = "harmony-cli"; - - // ---- Bootstrap (idempotent): Zitadel mints the project + operator key --- - ZitadelScore { - host: zitadel_host.clone(), - zitadel_version: cfg.zitadel_version.clone(), - external_secure: true, - external_port: None, - namespace: cfg.zitadel_namespace.clone(), - cluster_issuer: cfg.cluster_issuer.clone(), - ..Default::default() - } - .interpret(&Inventory::empty(), &topology) - .await - .context("Zitadel helm install")?; - - ZitadelSetupScore { - host: zitadel_host.clone(), - scheme: Default::default(), - port: None, - skip_tls: false, - endpoint: None, - admin_org_id: None, - namespace: cfg.zitadel_namespace.clone(), - // Devices read per-deployment secrets gated by group membership; - // the Action surfaces role keys as the `groups` claim OpenBao - // binds against (ADR-025). - groups_claim_action: true, - human_users: vec![], - org_members: vec![], - instance_members: vec![], - machine_secrets: vec![], - port_forward_service: None, - // Device-code OIDC app for human admin login from - // `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel - // generates is read back below and printed for `--admin-oidc-client-id`. - applications: vec![ZitadelApplication { - project_name: cfg.project_name.clone(), - app_name: cli_app_name.to_string(), - app_type: ZitadelAppType::DeviceCode, - }], - api_apps: vec![ZitadelApiApp { - project_name: cfg.project_name.clone(), - app_name: "nats".to_string(), - }], - roles: vec![ - ZitadelRole { - project_name: cfg.project_name.clone(), - key: cfg.admin_role.clone(), - display_name: "Fleet Admin".to_string(), - group: None, - }, - ZitadelRole { - project_name: cfg.project_name.clone(), - key: cfg.device_role.clone(), - display_name: "Device".to_string(), - group: None, - }, - ], - machine_users: vec![ - ZitadelMachineUser { - username: cfg.admin_username.clone(), - name: "Fleet Operations".to_string(), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(cfg.project_name.clone()), - grant_roles: vec![cfg.admin_role.clone()], - }, - ZitadelMachineUser { - username: cfg.operator_username.clone(), - name: "Fleet Operator (in-cluster)".to_string(), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(cfg.project_name.clone()), - grant_roles: vec![cfg.admin_role.clone()], - }, - ], - } - .interpret(&Inventory::empty(), &topology) - .await - .context("Zitadel setup (project + roles + machine users)")?; - - // Read back the project_id + operator key + device-code client_id. - let zcfg = ZitadelClientConfig::load() - .context("ZitadelSetupScore did not produce a client config cache")?; - let project_id = zcfg - .project_id_by_name(&cfg.project_name) - .or(zcfg.project_id.as_ref()) - .context("project_id missing from cache after setup")? - .clone(); - let operator_machine_key = zcfg - .machine_key(&cfg.operator_username) - .with_context(|| { - format!( - "machine key for {} missing from cache", - cfg.operator_username - ) - })? - .clone(); - let cli_client_id = zcfg - .client_id(cli_app_name) - .with_context(|| format!("OIDC client_id for app '{cli_app_name}' missing from cache"))? - .clone(); - - // ---- OpenBao: deploy + policy, co-located in the fleet namespace -------- - // The operator's credentials are seeded here so a later - // `harmony-fleet-deploy --from-tag ` upgrades the operator alone. - let openbao = OpenbaoInstance { - namespace: cfg.fleet_namespace.clone(), - release: "openbao".to_string(), - }; - OpenbaoScore { - instance: openbao.clone(), - host: secrets_host.clone(), - openshift: true, - tls_issuer: Some(cfg.cluster_issuer.clone()), - } - .interpret(&Inventory::empty(), &topology) - .await - .context("OpenBao deploy")?; - OpenbaoSetupScore { - instance: openbao.clone(), - policies: vec![OpenbaoPolicy { - name: "fleet-deployer".to_string(), - hcl: r#"path "secret/data/harmony/*" { capabilities = ["read"] } -path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"# - .to_string(), - }], - // Device login (ADR-025): batch tokens so re-login is free, 1h - // TTL as the group-membership convergence bound, groups bound - // from the Action-flattened `groups` claim. - jwt_auth: Some(OpenbaoJwtAuth { - oidc_discovery_url: format!("https://{zitadel_host}"), - jwt_validation_pubkeys: String::new(), - bound_issuer: format!("https://{zitadel_host}"), - role_name: "fleet-device".to_string(), - bound_audiences: project_id.clone(), - user_claim: "sub".to_string(), - groups_claim: "groups".to_string(), - token_type: "batch".to_string(), - ttl: "1h".to_string(), - max_ttl: "1h".to_string(), - }), - ..Default::default() - } - .interpret(&Inventory::empty(), &topology) - .await - .context("OpenBao setup")?; - - // ---- Workload Scores: filterable via `harmony_cli::Args` ---------------- - // The callout signs user JWTs with this account NKey; NATS is configured - // with the matching pubkey via the auth_callout block in its helm values. - let issuer_kp = KeyPair::new_account(); - let issuer_seed = issuer_kp - .seed() - .map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?; - let nats_auth_user = "auth"; - let nats_auth_pass = generate_alphanum(24); - let nats_url = format!( - "nats://{nats_release}.{}.svc.cluster.local:4222", - cfg.fleet_namespace - ); - - let nats = NatsK8sScore { - distribution: KubernetesDistribution::OpenshiftFamily, - cluster: NatsCluster { - namespace: cfg.fleet_namespace.clone(), - // `domain` and the static-string fields below are only read in the - // supercluster path (gateway Some), which staging doesn't take. - domain: cfg.base_domain.clone(), - replicas: 1, - name: nats_release.to_string(), - gateway_advertise: String::new(), - dns_name: nats_ws_host.clone(), - supercluster_ca_secret_name: "fleet-nats-supercluster-ca", - tls_cert_name: "fleet-nats-tls", - jetstream_enabled: "true", - }, - peers: None, - ca_bundle: None, - gateway: None, - auth_callout: Some(AuthCalloutCfg { - issuer_pubkey: issuer_kp.public_key(), - auth_user: nats_auth_user.to_string(), - auth_pass: nats_auth_pass.clone(), - account: cfg.nats_account.clone(), - }), - websocket: Some(WebSocketRouteCfg { - host: nats_ws_host.clone(), - cluster_issuer: cfg.cluster_issuer.clone(), - }), - }; - - let mut callout = NatsAuthCalloutScore::new( - "fleet-callout", - &cfg.fleet_namespace, - nats_url.clone(), - format!("https://{zitadel_host}"), - project_id.clone(), - nats_auth_user, - &nats_auth_pass, - &issuer_seed, - ) - .image(&cfg.callout_image) - .target_account(&cfg.nats_account) - .admin_role(&cfg.admin_role) - .device_role(&cfg.device_role) - .danger_accept_invalid_certs(false); - callout.device_id_claim = "client_id".to_string(); - callout.device_id_prefix_strip = "device-".to_string(); - callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles"); - - let credentials = OperatorCredentials::zitadel_jwt( - &format!("https://{zitadel_host}"), - &project_id, - &operator_machine_key, - ); - let mut operator = FleetOperatorScore::new() - .namespace(&cfg.fleet_namespace) - .release_name("harmony-fleet-operator") - .image(&cfg.operator_image) - .image_pull_policy("Always") - .nats_url(nats_url.clone()) - .log_level("info,kube_runtime=warn"); - operator.credentials = Some(credentials.clone()); - - let scores: Vec>> = - vec![Box::new(nats), Box::new(callout), Box::new(operator)]; - harmony_cli::run(Inventory::empty(), topology.clone(), scores, Some(args)) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - - // ---- Seed operator credentials as FleetDeploySecrets -------------------- - // Reached via port-forward with the cached root token, so it doesn't wait - // on the public route/cert. No kubeconfig — CD callers use their own context. - let k8s = topology - .k8s_client() - .await - .map_err(|e| anyhow::anyhow!(e))?; - let pf = k8s - .port_forward(&openbao.pod(), &openbao.namespace, 8200, 8200) - .await - .context("port-forward to OpenBao")?; - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - let store = OpenbaoSecretStore::new(OpenbaoStoreOptions { - base_url: format!("http://127.0.0.1:{}", pf.port()), - kv_mount: "secret".to_string(), - auth_mount: "token".to_string(), - skip_tls: true, - token: Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?), - username: None, - password: None, - zitadel_sso_url: None, - zitadel_client_id: None, - jwt_role: None, - jwt_auth_mount: None, - zitadel_jwt_bearer: None, - }) - .await - .context("OpenBao client")?; - ConfigClient::new(vec![ - Arc::new(StoreSource::new("harmony".to_string(), store)) - as Arc, - ]) - .set(&FleetDeploySecrets { - operator_credentials_toml: credentials.credentials_toml.clone(), - kubeconfig: None, - operator_cookie_key_b64: generate_cookie_key_b64(), - }) - .await - .context("seed FleetDeploySecrets")?; - - info!("=== fleet-staging install complete ==="); - info!( - "Zitadel: https://{zitadel_host}/ (admin user {})", - cfg.admin_username - ); - info!("NATS WS public: wss://{nats_ws_host}/"); - info!("OpenBao: https://{secrets_host}/"); - info!("Project id: {project_id}"); - info!("SSO client_id: {cli_client_id} (app '{cli_app_name}', device-code grant)"); - info!( - "Enroll a device: fleet_device_enroll --target ssh://@ \ - --issuer-url https://{zitadel_host} --audience {project_id} \ - --nats-url wss://{nats_ws_host} --admin-oidc-client-id {cli_client_id} \ - --agent-binary " - ); - - Ok(()) -} - -fn generate_alphanum(len: usize) -> String { - use rand::Rng; - const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - let mut rng = rand::rng(); - (0..len) - .map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char) - .collect() -} - -/// Standard-base64 of 64 random bytes — the operator's session-cookie -/// signing key. Generated per install and seeded as a secret. -fn generate_cookie_key_b64() -> String { - use base64::Engine; - use rand::Rng; - let mut bytes = [0u8; 64]; - rand::rng().fill(&mut bytes); - base64::engine::general_purpose::STANDARD.encode(bytes) -} diff --git a/examples/fleet_vm_setup/Cargo.toml b/examples/fleet_vm_setup/Cargo.toml index 1f495e17..636d8fc3 100644 --- a/examples/fleet_vm_setup/Cargo.toml +++ b/examples/fleet_vm_setup/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" [dependencies] harmony = { path = "../../harmony", features = ["kvm"] } +harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } harmony_types = { path = "../../harmony_types" } tokio.workspace = true log.workspace = true diff --git a/examples/fleet_vm_setup/src/main.rs b/examples/fleet_vm_setup/src/main.rs index ef5d365c..5ef1f31a 100644 --- a/examples/fleet_vm_setup/src/main.rs +++ b/examples/fleet_vm_setup/src/main.rs @@ -12,13 +12,13 @@ use anyhow::{Context, Result}; use clap::Parser; use harmony::inventory::Inventory; use harmony::modules::fleet::{ - FleetDeviceSetupConfig, FleetDeviceSetupScore, ProvisionVmScore, - check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair, + ProvisionVmScore, check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair, }; use harmony::modules::kvm::KvmVirtualMachineHost; use harmony::modules::kvm::config::init_executor; use harmony::modules::linux::{LinuxHostTopology, SshCredentials}; use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig}; +use harmony_fleet_deploy::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore}; use harmony_types::id::Id; use std::path::PathBuf; @@ -213,7 +213,7 @@ async fn main() -> Result<()> { nats_urls: vec![cli.nats_url.clone()], // VM smoke harness keeps shared-creds for v0; the customer- // facing Pi flow uses Zitadel JWT (see fleet_rpi_setup). - auth: harmony::modules::fleet::FleetDeviceAuth::TomlShared { + auth: FleetDeviceAuth::TomlShared { nats_user: cli.nats_user.clone(), nats_pass: cli.nats_pass.clone(), }, diff --git a/examples/harmony_apply_deployment/Cargo.toml b/examples/harmony_apply_deployment/Cargo.toml index 0b21ca79..d220aa02 100644 --- a/examples/harmony_apply_deployment/Cargo.toml +++ b/examples/harmony_apply_deployment/Cargo.toml @@ -9,7 +9,8 @@ name = "harmony_apply_deployment" path = "src/main.rs" [dependencies] -harmony = { path = "../../harmony", default-features = false, features = ["podman"] } +harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" } +harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } kube = { workspace = true, features = ["runtime", "derive"] } k8s-openapi = { workspace = true } serde_json.workspace = true diff --git a/examples/harmony_apply_deployment/src/main.rs b/examples/harmony_apply_deployment/src/main.rs index 1b3dd259..f5c9b713 100644 --- a/examples/harmony_apply_deployment/src/main.rs +++ b/examples/harmony_apply_deployment/src/main.rs @@ -12,8 +12,8 @@ //! (not `iot_`-anything), in line with the review call to position //! the operator as a generic fleet/reconcile tool. //! -//! The CRD types live in `harmony::modules::fleet::operator`; the score types -//! live in `harmony::modules::podman` (PodmanV0 being the first +//! The CRD types live in `harmony_fleet_operator`; the score types +//! live in `harmony_reconciler_contracts` (PodmanV0 being the first //! reconciler variant — future variants drop in alongside). //! //! Typical demo-driver usage: @@ -38,11 +38,10 @@ use anyhow::{Context, Result}; use clap::Parser; -use harmony::modules::fleet::operator::crd::{ - Deployment, DeploymentSpec, Rollout, RolloutStrategy, +use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy}; +use harmony_reconciler_contracts::{ + EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, VolumeMount, }; -use harmony::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore}; -use harmony::topology::{EnvVar, RestartPolicy, VolumeMount}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use kube::Client; use kube::api::{Api, DeleteParams, Patch, PatchParams}; @@ -143,7 +142,7 @@ async fn main() -> Result<()> { Ok(()) } -/// Mirrors `harmony::topology::RestartPolicy` so we can keep the CLI +/// Mirrors the contract `RestartPolicy` so we can keep the CLI /// schema stable even if the underlying enum gains variants. #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] enum CliRestart { diff --git a/examples/harmony_sso/src/main.rs b/examples/harmony_sso/src/main.rs index 71318940..dc81b795 100644 --- a/examples/harmony_sso/src/main.rs +++ b/examples/harmony_sso/src/main.rs @@ -249,6 +249,7 @@ async fn main() -> anyhow::Result<()> { host: OPENBAO_HOST.to_string(), openshift: false, tls_issuer: None, + node_port: None, } .interpret(&Inventory::autoload(), &topology) .await diff --git a/examples/openbao/src/main.rs b/examples/openbao/src/main.rs index 79197bbb..28f51fa6 100644 --- a/examples/openbao/src/main.rs +++ b/examples/openbao/src/main.rs @@ -94,6 +94,7 @@ async fn main() -> Result<()> { host: cfg.host.clone(), openshift: cfg.openshift, tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()), + node_port: None, }; // JWT auth composes in only when both issuer and audience are set; it @@ -128,6 +129,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"# policies, users: vec![], jwt_auth, + oidc_application: None, }; let scores: Vec>> = vec![Box::new(deploy), Box::new(setup)]; diff --git a/fleet/README.md b/fleet/README.md index 38c3436d..7bec5b15 100644 --- a/fleet/README.md +++ b/fleet/README.md @@ -7,7 +7,7 @@ IoT / decentralized-edge orchestration for harmony. A fleet stack is: | **Operator** | [`harmony-fleet-operator`](harmony-fleet-operator/) | Watches `Deployment` CRs, writes desired state into NATS JetStream KV, aggregates device state back into CR status. Runtime binary; no `harmony` dep. | | **Agent** | [`harmony-fleet-agent`](harmony-fleet-agent/) | One per device. Watches the desired-state KV, drives the local runtime (podman today), publishes heartbeats + per-deployment state, answers `device-commands.*` request/reply. | | **Auth** | [`harmony-fleet-auth`](harmony-fleet-auth/) | Shared NATS credential plumbing — `TomlShared` (dev) and `ZitadelJwt` (prod with auth-callout). | -| **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. Imports `harmony` and exposes one `*Score` per component (`FleetOperatorScore`, `FleetAgentScore`, `FleetNatsScore`, `FleetServerScore`). Both the production CLI and the e2e harness compose these — see [ADR-023](../docs/adr/023-deploy-architecture.md). | +| **Deploy** | [`harmony-fleet-deploy`](harmony-fleet-deploy/) | The canonical deploy crate. It composes independent component Scores through typed references; there is no Fleet-wide aggregate Score. See [ADR-023](../docs/adr/023-deploy-architecture.md) and [ADR-028](../docs/adr/028-typed-score-references.md). | | **E2E harness** | [`harmony-fleet-e2e`](harmony-fleet-e2e/) | Brings the stack up in a fresh k3d namespace and runs integration tests against it. | The on-the-wire types both ends agree on (KV bucket names, key formats, command-protocol payloads) live in [`../harmony-reconciler-contracts`](../harmony-reconciler-contracts/). @@ -16,6 +16,20 @@ The on-the-wire types both ends agree on (KV bucket names, key formats, command- `FleetOperatorScore`, `FleetAgentScore`, etc. are real Rust types with capability-bound `Topology` parameters. Production deploys, the e2e harness, and any future control-plane tool all compose the **same** Scores; the only thing that changes is the `Topology` instance. **No handrolled YAML or imperative manifest factories anywhere.** Read [ADR-023](../docs/adr/023-deploy-architecture.md) before adding deploy logic. +## Local Fleet + +The local context creates or reuses the `harmony` k3d cluster, builds and imports +the Fleet images, and deploys PostgreSQL, Zitadel, NATS with auth callout, +OpenBao, and the Fleet operator: + +```bash +cargo run -p harmony-fleet-deploy -- ship --context local +``` + +The command returns only after the operator has authenticated through Zitadel +and initialized its NATS JetStream state. The credentials Score creates an +immutable local Kubernetes Secret on the first run and reuses it thereafter. + --- ## Quickstart — run the e2e ping test @@ -31,10 +45,10 @@ What it does, in order: 1. Ensures a `fleet-e2e` k3d cluster exists (creates one if not). NodePort `30423` on the host forwards to NATS inside the cluster. 2. Builds `harmony-fleet-agent` in release mode, packages it into `localhost/harmony-fleet-agent:e2e`, and sideloads the image into the k3d cluster's containerd store. 3. Mints a per-bring-up namespace `e2e-` and prunes any leftover `e2e-*` namespaces from prior runs (NodePort `30423` is cluster-scoped, so a stuck `Terminating` namespace would block the new bring-up — the prune waits up to 90 s for full cleanup before proceeding). -4. Deploys NATS via `FleetNatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service). +4. Deploys NATS via `NatsScore` (helm chart, JetStream on, static admin/device users, NodePort Service). 5. Waits for NATS to be reachable from the host on `nats://localhost:30423` (admin/e2e-admin). 6. Deploys one `FleetAgentScore { target: Pod }` — runs with `runtime_enabled = false` so it skips podman and only runs the command-server + heartbeat loop. -7. Waits for the agent Deployment to be Ready. +7. `FleetAgentScore` waits for the agent Deployment to be Ready. 8. The test publishes `device-commands..ping` via `FleetCommandsClient::ping` and asserts the agent replies with `{ device_id, agent_version, uptime_s }`. Cold first run: ~80 s (release build of the agent dominates). Warm: ~25 s. @@ -102,14 +116,17 @@ k3d cluster delete fleet-e2e ## Production deploys -`harmony-fleet-deploy` puts the published operator chart on a real cluster (OKD, vanilla k8s, anywhere `K8sAnywhereTopology` can reach) — the `harmony apply` / CD path. It loads `FleetDeploySecrets` from config (Env → OpenBao) and runs one `FleetOperatorScore`; auth is Zitadel-SSO-only. The full bring-up stack (`FleetNatsScore` + `FleetAgentScore` + …) is composed by the e2e harness directly over the same lib Scores, not by this binary. +Production contexts live in the customer's private repository. They provide a +registry, project, domain, image-pull Secret, and OpenBao-brokered kubeconfig. +The cluster must also contain the immutable +`fleet-system/fleet-callout-credentials` Secret with +`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass` keys. The deploy +validates this Secret before configuring NATS. ```bash -# Deploy a released tag (version parsed from it in Rust): cargo run -p harmony-fleet-deploy -- \ - --filter FleetOperatorScore \ - --from-tag harmony-fleet-operator-v0.0.2 \ - --namespace fleet-system --yes + ship --context customer-prod \ + --config path/to/contexts.toml ``` See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap. @@ -132,22 +149,12 @@ cargo run -p harmony-fleet-operator --features web-frontend -- serve-web --mock --- -## Existing manual rehearsal — `examples/fleet_e2e_demo` +## VM rehearsal -`examples/fleet_e2e_demo` brings up a *fuller* stack than the e2e harness — real Zitadel, the auth-callout, libvirt VM agents over SSH — at the cost of a 5-min cold start. It's the manual rehearsal flow; not what you want during the dev loop. See the example's [`RUNBOOK.md`](../examples/fleet_e2e_demo/RUNBOOK.md). +`examples/fleet_e2e_demo` is a small executable over +`harmony-fleet-e2e::VmStack`. It runs the same callout, OpenBao, operator, and +device enrollment Scores as the maintained VM tests: -The harness and the rehearsal will converge: the [follow-up PR](#whats-next) lifts `FleetCalloutScore` + a mock-OIDC fixture into `harmony-fleet-deploy`, at which point the harness can run the full production auth path in ~30 s instead of 5 min, and `fleet_e2e_demo` thins down to a caller over the same Scores. - ---- - -## What's next - -This branch lands the deploy-architecture cleanup (ADR-023), the per-component Scores, and the ping path. Slated immediately after: - -1. **Zitadel + auth callout in `harmony-fleet-deploy`.** New `FleetCalloutScore` (preset over `NatsAuthCalloutScore`) plus an in-cluster mock-OIDC fixture so the e2e harness can exercise the real auth-callout code path without paying Zitadel's 5-min cold-start cost. The harness's `AuthMode::Callout` variant is already on the public API for this. -2. **Operator pod in the e2e harness.** `FleetOperatorScore` is already in the deploy crate; wiring it into the harness gives integration tests against the actual `Deployment` / `Device` reconcile loops. -3. **`Verb::Logs` and `Verb::Exec`** — the next two verbs on the `device-commands.*` protocol. Same harness, same TDD shape as `ping`. -4. **CRD types out of `harmony` core.** `harmony::modules::fleet::operator::crd` is the last fleet-deploy thing still living in `harmony`. The `ReconcileScore` payload coupling is the only blocker. -5. **Smoke-test contract.** ADR-023 principle 4 — every Score blocks on a smoke test before `deploy` returns success. Today the e2e suite plays that role; the trait/companion shape lands once it's been validated in practice. - -See [`PLAN_requests_over_nats.md`](PLAN_requests_over_nats.md) for the full TDD-style plan this branch implements. +```bash +cargo run -p example-fleet-e2e-demo -- --num-devices 2 +``` diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 70dcb1c9..6b73ad20 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -8,13 +8,13 @@ rust-version = "1.85" harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony_secret = { path = "../../harmony_secret" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } -harmony = { path = "../../harmony", default-features = false, features = ["podman"] } async-nats = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2.workspace = true tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } @@ -22,3 +22,4 @@ anyhow = { workspace = true } clap = { workspace = true } toml = { workspace = true } thiserror = { workspace = true } +podman-api = "0.9" diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 31886430..26fad5ff 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -1,6 +1,7 @@ mod command_server; mod config; mod fleet_publisher; +mod podman; mod reconciler; use std::sync::Arc; @@ -20,12 +21,9 @@ use harmony_reconciler_contracts::{ BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter, }; -use harmony::inventory::Inventory; -use harmony::modules::podman::PodmanTopology; -use harmony::topology::Topology; - use crate::command_server::CommandServer; use crate::fleet_publisher::FleetPublisher; +use crate::podman::PodmanRuntime; use crate::reconciler::Reconciler; /// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch @@ -146,9 +144,11 @@ async fn publish_heartbeat_loop(fleet: Arc) { /// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise. /// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state /// on heartbeat but not always send it over the wire -fn local_inventory(inventory: &Inventory) -> InventorySnapshot { +fn local_inventory() -> InventorySnapshot { InventorySnapshot { - hostname: inventory.location.name.clone(), + hostname: std::fs::read_to_string("/proc/sys/kernel/hostname") + .map(|hostname| hostname.trim().to_string()) + .unwrap_or_else(|_| "localhost".to_string()), arch: std::env::consts::ARCH.to_string(), os: std::env::consts::OS.to_string(), kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease") @@ -206,7 +206,7 @@ async fn main() -> Result<()> { // reconciler depends on the topology. let topology = if cfg.agent.runtime_enabled { let t = Arc::new( - PodmanTopology::from_default_socket() + PodmanRuntime::from_default_socket() .map_err(|e| anyhow::anyhow!("failed to open podman socket: {e}"))?, ); t.ensure_ready().await.context("podman socket not ready")?; @@ -220,9 +220,8 @@ async fn main() -> Result<()> { None }; - let inventory = Arc::new(Inventory::from_localhost()); - tracing::info!(hostname = %inventory.location.name, "inventory loaded"); - let inventory_snapshot = local_inventory(&inventory); + let inventory_snapshot = local_inventory(); + tracing::info!(hostname = %inventory_snapshot.hostname, "inventory loaded"); let creds = credential_source_from_config(&cfg.credentials) .context("building NATS credential source")?; @@ -291,7 +290,6 @@ async fn main() -> Result<()> { Arc::new(Reconciler::new( device_id.clone(), t.clone(), - inventory.clone(), Some(fleet.clone()), secrets, )) diff --git a/fleet/harmony-fleet-agent/src/podman.rs b/fleet/harmony-fleet-agent/src/podman.rs new file mode 100644 index 00000000..de0efc29 --- /dev/null +++ b/fleet/harmony-fleet-agent/src/podman.rs @@ -0,0 +1,302 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Result, anyhow, bail}; +use futures_util::StreamExt; +use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy, VolumeMount}; +use podman_api::Podman; +use podman_api::models::{ContainerMount, PortMapping}; +use podman_api::opts::{ + ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts, + ContainerRestartPolicy, ContainerStopOpts, PullOpts, +}; +use sha2::{Digest, Sha256}; + +const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment"; +const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256"; +const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by"; +const MANAGED_BY_VALUE: &str = "harmony"; +const STOP_TIMEOUT: Duration = Duration::from_secs(300); + +pub struct PodmanRuntime { + podman: Podman, +} + +impl PodmanRuntime { + pub fn from_default_socket() -> Result { + Self::from_unix_socket(default_user_socket()) + } + + pub fn from_unix_socket(socket: PathBuf) -> Result { + Ok(Self { + podman: Podman::unix(&socket), + }) + } + + pub async fn ensure_ready(&self) -> Result<()> { + self.podman + .info() + .await + .map_err(|error| anyhow!("podman socket unreachable: {error}"))?; + Ok(()) + } + + pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> { + for service in &score.services { + self.ensure_service_running(service, deployment).await?; + } + let desired = score + .services + .iter() + .map(|service| service.name.as_str()) + .collect::>(); + for container in self.deployment_containers(deployment).await? { + let name = container_name(&container); + if !desired.contains(name.as_str()) { + self.remove_service(&name).await?; + } + } + Ok(()) + } + + pub async fn remove_deployment(&self, deployment: &str) -> Result<()> { + for container in self.deployment_containers(deployment).await? { + self.remove_service(&container_name(&container)).await?; + } + Ok(()) + } + + pub async fn remove_service(&self, name: &str) -> Result<()> { + let opts = ContainerStopOpts::builder() + .timeout(STOP_TIMEOUT.as_secs() as usize) + .build(); + let container = self.podman.containers().get(name); + if container.exists().await.unwrap_or(false) { + let _ = container.stop(&opts).await; + } + self.remove_container(name).await + } + + async fn ensure_service_running( + &self, + service: &PodmanService, + deployment: &str, + ) -> Result<()> { + let existing = self.get_by_name(&service.name).await?; + if let Some(existing) = existing.as_ref() { + if matches_spec(existing, service) { + if existing.state.as_deref() == Some("running") { + return Ok(()); + } + let id = existing.id.clone().unwrap_or_else(|| service.name.clone()); + self.podman.containers().get(id).start(None).await?; + return Ok(()); + } + self.remove_container(&service.name).await?; + } + + self.ensure_image_present(&service.image).await?; + let labels = HashMap::from([ + (MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()), + (DEPLOYMENT_LABEL.to_string(), deployment.to_string()), + (SPEC_LABEL.to_string(), spec_hash(service)?), + ]); + let ports = service + .ports + .iter() + .map(|port| parse_port_mapping(port)) + .collect::>>()?; + let env: HashMap = service + .env + .iter() + .map(|env| (env.name.clone(), env.value.clone())) + .collect(); + let mounts = service + .volumes + .iter() + .map(volume_to_mount) + .collect::>(); + let mut builder = ContainerCreateOpts::builder() + .name(&service.name) + .image(&service.image) + .labels(labels) + .portmappings(ports) + .env(env) + .restart_policy(map_restart_policy(service.restart_policy)); + if !mounts.is_empty() { + builder = builder.mounts(mounts); + } + let created = self.podman.containers().create(&builder.build()).await?; + self.podman.containers().get(created.id).start(None).await?; + Ok(()) + } + + async fn get_by_name(&self, name: &str) -> Result> { + let opts = ContainerListOpts::builder() + .all(true) + .filter([ContainerListFilter::Name(name.to_string())]) + .build(); + Ok(self + .podman + .containers() + .list(&opts) + .await? + .into_iter() + .next()) + } + + async fn deployment_containers( + &self, + deployment: &str, + ) -> Result> { + let opts = ContainerListOpts::builder() + .all(true) + .filter([ContainerListFilter::LabelKeyVal( + DEPLOYMENT_LABEL.to_string(), + deployment.to_string(), + )]) + .build(); + Ok(self.podman.containers().list(&opts).await?) + } + + async fn ensure_image_present(&self, image: &str) -> Result<()> { + let images = self.podman.images(); + if images.get(image).exists().await? { + return Ok(()); + } + let mut pull = images.pull(&PullOpts::builder().reference(image).build()); + while let Some(event) = pull.next().await { + if let Some(error) = event?.error { + bail!("podman pull {image} failed: {error}"); + } + } + Ok(()) + } + + async fn remove_container(&self, name: &str) -> Result<()> { + let opts = ContainerDeleteOpts::builder().force(true).build(); + match self.podman.containers().get(name).delete(&opts).await { + Ok(()) => Ok(()), + Err(error) if is_not_found(&error) => Ok(()), + Err(error) => Err(error.into()), + } + } +} + +fn default_user_socket() -> PathBuf { + if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") { + return PathBuf::from(format!("{dir}/podman/podman.sock")); + } + let uid = std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find_map(|line| line.strip_prefix("Uid:")) + .and_then(|value| value.split_whitespace().next()) + .and_then(|value| value.parse::().ok()) + }) + .unwrap_or(0); + PathBuf::from(format!("/run/user/{uid}/podman/podman.sock")) +} + +fn parse_port_mapping(raw: &str) -> Result { + let (host, container) = raw + .split_once(':') + .ok_or_else(|| anyhow!("invalid port mapping '{raw}' - expected 'HOST:CONTAINER'"))?; + Ok(PortMapping { + host_port: Some( + host.parse() + .map_err(|_| anyhow!("invalid host port '{host}' in mapping '{raw}'"))?, + ), + container_port: Some( + container + .parse() + .map_err(|_| anyhow!("invalid container port '{container}' in mapping '{raw}'"))?, + ), + host_ip: None, + protocol: None, + range: None, + }) +} + +fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanService) -> bool { + observed + .labels + .as_ref() + .and_then(|labels| labels.get(SPEC_LABEL)) + .is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected)) +} + +fn spec_hash(service: &PodmanService) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(service)?) + )) +} + +fn container_name(container: &podman_api::models::ListContainer) -> String { + container + .names + .as_ref() + .and_then(|names| names.first()) + .cloned() + .or_else(|| container.id.clone()) + .unwrap_or_default() +} + +fn volume_to_mount(volume: &VolumeMount) -> ContainerMount { + ContainerMount { + _type: Some("bind".to_string()), + source: Some(volume.host_path.clone()), + destination: Some(volume.container_path.clone()), + options: Some(vec![if volume.read_only { "ro" } else { "rw" }.to_string()]), + uid_mappings: None, + gid_mappings: None, + } +} + +fn map_restart_policy(policy: RestartPolicy) -> ContainerRestartPolicy { + match policy { + RestartPolicy::No => ContainerRestartPolicy::No, + RestartPolicy::UnlessStopped => ContainerRestartPolicy::UnlessStopped, + RestartPolicy::OnFailure => ContainerRestartPolicy::OnFailure, + RestartPolicy::Always => ContainerRestartPolicy::Always, + } +} + +fn is_not_found(error: &podman_api::Error) -> bool { + error + .to_string() + .to_lowercase() + .contains("no such container") +} + +#[cfg(test)] +mod tests { + use super::*; + use harmony_reconciler_contracts::EnvVar; + + fn service() -> PodmanService { + PodmanService { + name: "web".into(), + image: "nginx:latest".into(), + ports: vec!["8080:80".into()], + env: vec![EnvVar::new("MODE", "prod")], + secret_env: vec![], + volumes: vec![], + restart_policy: RestartPolicy::Always, + } + } + + #[test] + fn spec_hash_is_stable_and_covers_desired_state() { + let original = service(); + assert_eq!(spec_hash(&original).unwrap(), spec_hash(&original).unwrap()); + + let mut changed = original; + changed.env[0].value = "dev".into(); + assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap()); + } +} diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 228f8470..40455d53 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -4,15 +4,15 @@ use std::time::Duration; use anyhow::Result; use chrono::Utc; -use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase}; +use harmony_reconciler_contracts::{ + DeploymentName, DeploymentState, EnvVar, Id, Phase, PodmanV0Score, ReconcileScore, +}; use tokio::sync::Mutex; -use harmony::inventory::Inventory; -use harmony::modules::podman::{EnvVar, PodmanTopology, PodmanV0Score, ReconcileScore}; -use harmony::score::Score; use harmony_secret::SecretStore; use crate::fleet_publisher::FleetPublisher; +use crate::podman::PodmanRuntime; /// Where deployment secrets come from (ADR-025). `prefix` scopes the /// KV reads to `//` — it must match the @@ -25,19 +25,21 @@ pub struct SecretSource { /// Cache key → last-seen state, populated by `apply` and consulted by the /// 30-second periodic tick and the delete path. -struct CachedEntry { - /// Serialized score JSON. Used for string-compare idempotency per - /// ROADMAP §5.5 — cheaper and more deterministic than a hash. - serialized: String, - /// Parsed score. Cached so the periodic reconcile tick and delete - /// handlers don't have to re-parse the JSON. - score: PodmanV0Score, +#[derive(Clone)] +enum CachedEntry { + Apply { + serialized: String, + score: PodmanV0Score, + resolved: bool, + }, + Remove { + score: Option, + }, } pub struct Reconciler { device_id: Id, - topology: Arc, - inventory: Arc, + runtime: Arc, /// Keyed by NATS KV key (`.`). A single entry per /// KV key — in v0 there is no fan-out from one key to many scores. state: Mutex>, @@ -58,15 +60,13 @@ pub struct Reconciler { impl Reconciler { pub fn new( device_id: Id, - topology: Arc, - inventory: Arc, + runtime: Arc, fleet: Option>, secrets: Option, ) -> Self { Self { device_id, - topology, - inventory, + runtime, state: Mutex::new(HashMap::new()), deployments: Mutex::new(HashMap::new()), fleet, @@ -75,10 +75,9 @@ impl Reconciler { } /// Materialize every `secret_env` reference into a plain env var. - /// Resolution happens once per spec change, never on the periodic - /// tick: secret values bind at apply time (rotation = spec bump or - /// re-apply), which keeps the secret store off the 30s hot path — - /// fleet-scale login load is the ADR-025 constraint. + /// Resolution is retried until it succeeds, then the resolved score is + /// reused until the spec changes. This keeps the secret store off the + /// periodic hot path for converged deployments. async fn resolve_secrets( &self, deployment: Option<&DeploymentName>, @@ -150,13 +149,7 @@ impl Reconciler { /// entry. Idempotent: a delete for a never-applied deployment is /// a no-op in memory and a harmless tombstone write on the wire. async fn drop_phase(&self, deployment: &DeploymentName) { - let was_known = { - let mut phases = self.deployments.lock().await; - phases.remove(deployment).is_some() - }; - if !was_known { - return; - } + self.deployments.lock().await.remove(deployment); if let Some(publisher) = &self.fleet { publisher.delete_deployment_state(deployment).await; } @@ -182,30 +175,32 @@ impl Reconciler { { let state = self.state.lock().await; - if let Some(existing) = state.get(key) { - if existing.serialized == serialized { + if let Some(CachedEntry::Apply { + serialized: existing, + .. + }) = state.get(key) + { + if existing == &serialized { tracing::debug!(key, "score unchanged — noop"); return Ok(()); } } } + self.state.lock().await.insert( + key.to_string(), + CachedEntry::Apply { + serialized, + score: incoming, + resolved: false, + }, + ); + if let Some(name) = &deployment { self.apply_phase(name, Phase::Pending, None).await; } - let resolved = match self.resolve_secrets(deployment.as_ref(), incoming).await { - Ok(score) => score, - Err(e) => { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) - .await; - } - return Err(e); - } - }; - - match self.run_score(key, &resolved).await { + match self.reconcile_cached(key).await { Ok(()) => { if let Some(name) = &deployment { self.apply_phase(name, Phase::Running, None).await; @@ -219,82 +214,56 @@ impl Reconciler { return Err(e); } } - - let mut state = self.state.lock().await; - // Cache the *resolved* score: the periodic tick re-runs it - // as-is, so converged deployments never re-hit the secret - // store (see `resolve_secrets`). - state.insert( - key.to_string(), - CachedEntry { - serialized, - score: resolved, - }, - ); Ok(()) } - /// Handle a Delete/Purge event. Stops and removes every container - /// referenced by the last cached score for this key. Idempotent: if we - /// never saw a Put for this key (agent restart after delete), logs and - /// returns ok. + /// Handle a Delete/Purge event. The tombstone remains cached until all + /// deployment containers are removed, including when no score survived a + /// restart. pub async fn remove(&self, key: &str) -> Result<()> { - let deployment = deployment_from_key(key); let mut state = self.state.lock().await; - let Some(entry) = state.remove(key) else { - tracing::info!(key, "delete for unknown key — nothing to remove"); - if let Some(name) = &deployment { - self.drop_phase(name).await; - } - return Ok(()); + let score = match state.get(key) { + Some(CachedEntry::Apply { score, .. }) => Some(score.clone()), + Some(CachedEntry::Remove { score }) => score.clone(), + None => None, }; + state.insert(key.to_string(), CachedEntry::Remove { score }); drop(state); - - use harmony::topology::ContainerRuntime; - for service in &entry.score.services { - if let Err(e) = self.topology.remove_service(&service.name).await { - tracing::warn!( - key, - service = %service.name, - error = %e, - "failed to remove container" - ); - } else { - tracing::info!(key, service = %service.name, "removed container"); - } - } - if let Some(name) = &deployment { - self.drop_phase(name).await; - } - Ok(()) + self.remove_cached(key).await } /// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of /// event-driven PLEG. Agent polls podman every 30s as ground truth; - /// KV watch events are accelerators." Re-runs each cached score against - /// podman-api; the underlying `ensure_service_running` is idempotent - /// so a converged state produces no log noise. + /// KV watch events are accelerators." Retries cached applies and removals + /// against podman-api. pub async fn tick(&self) -> Result<()> { - let snapshot: Vec<(String, PodmanV0Score)> = { - let state = self.state.lock().await; - state - .iter() - .map(|(k, v)| (k.clone(), v.score.clone())) - .collect() - }; - for (key, score) in snapshot { + let keys: Vec = self.state.lock().await.keys().cloned().collect(); + for key in keys { let deployment = deployment_from_key(&key); - match self.run_score(&key, &score).await { + let removing = matches!( + self.state.lock().await.get(&key), + Some(CachedEntry::Remove { .. }) + ); + let result = if removing { + self.remove_cached(&key).await + } else { + self.reconcile_cached(&key).await + }; + match result { Ok(()) => { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Running, None).await; + if !removing { + if let Some(name) = &deployment { + self.apply_phase(name, Phase::Running, None).await; + } } } Err(e) => { tracing::warn!(key, error = %e, "periodic reconcile failed"); - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) - .await; + if !removing { + if let Some(name) = &deployment { + self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) + .await; + } } } } @@ -313,13 +282,78 @@ impl Reconciler { } } + async fn reconcile_cached(&self, key: &str) -> Result<()> { + let Some(CachedEntry::Apply { + serialized, + mut score, + resolved, + }) = self.state.lock().await.get(key).cloned() + else { + return Ok(()); + }; + + if !resolved { + score = self + .resolve_secrets(deployment_from_key(key).as_ref(), score) + .await?; + let mut state = self.state.lock().await; + let Some(CachedEntry::Apply { + serialized: current, + score: cached_score, + resolved, + }) = state.get_mut(key) + else { + return Ok(()); + }; + if current != &serialized { + return Ok(()); + } + *cached_score = score.clone(); + *resolved = true; + } + + self.run_score(key, &score).await + } + + async fn remove_cached(&self, key: &str) -> Result<()> { + let Some(CachedEntry::Remove { score }) = self.state.lock().await.get(key).cloned() else { + return Ok(()); + }; + let deployment = deployment_from_key(key); + if let Some(name) = &deployment { + self.runtime.remove_deployment(name.as_str()).await?; + } else if let Some(score) = score { + for service in &score.services { + self.runtime.remove_service(&service.name).await?; + } + } + + let removed = { + let mut state = self.state.lock().await; + if matches!(state.get(key), Some(CachedEntry::Remove { .. })) { + state.remove(key); + true + } else { + false + } + }; + if removed { + if let Some(name) = &deployment { + self.drop_phase(name).await; + } + } + Ok(()) + } + async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> { - let interpret = Score::::create_interpret(score); - let outcome = interpret - .execute(&self.inventory, &self.topology) + let deployment = deployment_from_key(key) + .map(|name| name.to_string()) + .unwrap_or_else(|| key.to_string()); + self.runtime + .reconcile(&deployment, score) .await - .map_err(|e| anyhow::anyhow!("PodmanV0Score interpret failed for {key}: {e}"))?; - tracing::info!(key, outcome = ?outcome, "reconciled"); + .map_err(|e| anyhow::anyhow!("PodmanV0Score reconcile failed for {key}: {e}"))?; + tracing::info!(key, services = score.services.len(), "reconciled"); Ok(()) } } @@ -350,26 +384,19 @@ mod tests { //! `drop_phase` directly with an inert topology (no real podman //! socket) and a `None` FleetPublisher. use super::*; - use harmony::inventory::Inventory; - use harmony::modules::podman::PodmanTopology; use std::path::PathBuf; + use std::sync::Mutex as StdMutex; + use std::sync::atomic::{AtomicUsize, Ordering}; fn reconciler() -> Reconciler { reconciler_with_secrets(None) } fn reconciler_with_secrets(secrets: Option) -> Reconciler { - let topology = Arc::new( - PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(), + let runtime = Arc::new( + PodmanRuntime::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(), ); - let inventory = Arc::new(Inventory::empty()); - Reconciler::new( - Id::from("test-device".to_string()), - topology, - inventory, - None, - secrets, - ) + Reconciler::new(Id::from("test-device".to_string()), runtime, None, secrets) } fn dn(s: &str) -> DeploymentName { @@ -403,8 +430,40 @@ mod tests { } } + #[derive(Debug, Default)] + struct FlakyStore { + calls: AtomicUsize, + value: StdMutex>, + } + + #[async_trait::async_trait] + impl SecretStore for FlakyStore { + async fn get_raw( + &self, + _namespace: &str, + _key: &str, + ) -> Result, harmony_secret::SecretStoreError> { + self.calls.fetch_add(1, Ordering::Relaxed); + self.value + .lock() + .unwrap() + .as_ref() + .map(|value| value.as_bytes().to_vec()) + .ok_or_else(|| harmony_secret::SecretStoreError::Store("temporarily denied".into())) + } + + async fn set_raw( + &self, + _namespace: &str, + _key: &str, + _value: &[u8], + ) -> Result<(), harmony_secret::SecretStoreError> { + unimplemented!("tests never write") + } + } + fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score { - use harmony::modules::podman::{PodmanService, SecretEnvVar}; + use harmony_reconciler_contracts::{PodmanService, SecretEnvVar}; PodmanV0Score { services: vec![PodmanService { name: "web-svc".to_string(), @@ -479,6 +538,73 @@ mod tests { assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed)); } + #[tokio::test] + async fn tick_retries_secret_resolution_then_reuses_resolved_score() { + let store = Arc::new(FlakyStore::default()); + let r = reconciler_with_secrets(Some(SecretSource { + store: store.clone(), + prefix: "fleet-test".to_string(), + })); + let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score( + "DB_PASSWORD", + "db_password", + ))) + .unwrap(); + + r.apply("test-device.web", &payload) + .await + .expect_err("first secret read fails"); + *store.value.lock().unwrap() = Some("hunter2".to_string()); + + r.tick().await.unwrap(); + r.tick().await.unwrap(); + + assert_eq!(store.calls.load(Ordering::Relaxed), 2); + let state = r.state.lock().await; + let Some(CachedEntry::Apply { + score, resolved, .. + }) = state.get("test-device.web") + else { + panic!("apply retry state missing"); + }; + assert!(*resolved); + assert_eq!( + score.services[0].env[0], + EnvVar::new("DB_PASSWORD", "hunter2") + ); + } + + #[tokio::test] + async fn failed_removal_is_retried_and_later_put_cancels_it() { + let r = reconciler(); + + r.remove("test-device.web") + .await + .expect_err("unreachable Podman keeps the tombstone pending"); + assert!(matches!( + r.state.lock().await.get("test-device.web"), + Some(CachedEntry::Remove { score: None }) + )); + + r.tick().await.unwrap(); + assert!(matches!( + r.state.lock().await.get("test-device.web"), + Some(CachedEntry::Remove { .. }) + )); + + let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(PodmanV0Score { + services: vec![], + })) + .unwrap(); + r.apply("test-device.web", &payload) + .await + .expect_err("unreachable Podman keeps the new apply pending"); + assert!(matches!( + r.state.lock().await.get("test-device.web"), + Some(CachedEntry::Apply { .. }) + )); + } + #[tokio::test] async fn plain_scores_resolve_without_store() { // No secret_env → no store needed; resolution is a no-op. diff --git a/fleet/harmony-fleet-auth/src/lib.rs b/fleet/harmony-fleet-auth/src/lib.rs index 2d0b53e6..dad5191f 100644 --- a/fleet/harmony-fleet-auth/src/lib.rs +++ b/fleet/harmony-fleet-auth/src/lib.rs @@ -24,9 +24,11 @@ mod agent_config; mod config; mod credentials; +mod observation; pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config}; pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR}; +pub use observation::AgentObservation; use std::sync::Arc; diff --git a/fleet/harmony-fleet-deploy/src/companion/agent.rs b/fleet/harmony-fleet-auth/src/observation.rs similarity index 98% rename from fleet/harmony-fleet-deploy/src/companion/agent.rs rename to fleet/harmony-fleet-auth/src/observation.rs index 67cd852e..5aeeb246 100644 --- a/fleet/harmony-fleet-deploy/src/companion/agent.rs +++ b/fleet/harmony-fleet-auth/src/observation.rs @@ -12,7 +12,7 @@ //! Per ADR-023 P7 this is a companion struct, not a new method on //! `Score` / `Interpret`. The base trait surface stays unchanged. -use harmony_fleet_auth::AgentConfig; +use crate::AgentConfig; use harmony_reconciler_contracts::{ BUCKET_DESIRED_STATE, DeploymentName, Id, desired_state_key, desired_state_watch_filter, }; @@ -104,7 +104,7 @@ impl AgentObservation { #[cfg(test)] mod tests { use super::*; - use harmony_fleet_auth::{AgentSection, CredentialsSection, NatsSection}; + use crate::{AgentSection, CredentialsSection, NatsSection}; use std::collections::BTreeMap; fn cfg(device_id: &str, runtime_enabled: bool) -> AgentConfig { diff --git a/fleet/harmony-fleet-deploy/Cargo.toml b/fleet/harmony-fleet-deploy/Cargo.toml index 130d4bd0..ce2007f3 100644 --- a/fleet/harmony-fleet-deploy/Cargo.toml +++ b/fleet/harmony-fleet-deploy/Cargo.toml @@ -5,6 +5,7 @@ version.workspace = true readme.workspace = true license.workspace = true description = "Deploy-side Scores for the fleet stack: operator, agent, NATS, callout. Imports harmony for the Score-Topology-Interpret machinery; consumed by the production CLI, the e2e harness, and future control-plane tooling." +default-run = "harmony-fleet-deploy" [lib] path = "src/lib.rs" @@ -15,20 +16,15 @@ path = "src/lib.rs" name = "harmony-fleet-deploy" path = "src/main.rs" -# `harmony-fleet-publish --from-tag ` builds + publishes the -# operator's image + chart for a release. -[[bin]] -name = "harmony-fleet-publish" -path = "src/bin/harmony-fleet-publish.rs" - [dependencies] harmony = { path = "../../harmony", features = ["podman"] } harmony_cli = { path = "../../harmony_cli" } +harmony_app = { path = "../../harmony_app" } harmony_config = { path = "../../harmony_config" } harmony_types = { path = "../../harmony_types" } harmony_macros = { path = "../../harmony_macros" } harmony-fleet-auth = { path = "../harmony-fleet-auth" } -harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony-fleet-operator = { path = "../harmony-fleet-operator" } harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } anyhow = { workspace = true } @@ -40,12 +36,15 @@ kube = { workspace = true, features = ["runtime", "derive"] } log = { workspace = true } env_logger = { workspace = true } non-blank-string-rs = "1" +inquire.workspace = true schemars = "0.8" serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } +similar.workspace = true tempfile = "3" thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } toml = { workspace = true } tracing = { workspace = true } +url.workspace = true diff --git a/fleet/harmony-fleet-deploy/src/agent.rs b/fleet/harmony-fleet-deploy/src/agent.rs index a8ac29d8..9a9f6974 100644 --- a/fleet/harmony-fleet-deploy/src/agent.rs +++ b/fleet/harmony-fleet-deploy/src/agent.rs @@ -2,9 +2,8 @@ //! instance. //! //! v1 supports the in-cluster Pod form factor (the e2e harness and -//! same-cluster production deploys). The VM/SSH/RPi path stays in -//! `harmony::modules::fleet::FleetDeviceSetupScore` until a follow-up -//! migrates it into this crate. +//! same-cluster production deploys). [`crate::FleetDeviceSetupScore`] +//! owns the VM/SSH/RPi form factor. //! //! Composition: the Pod target Score interprets to two //! [`K8sResourceScore`] applies in sequence — a `ConfigMap` carrying @@ -16,6 +15,7 @@ //! compile error — that's the point. use std::collections::BTreeMap; +use std::time::Duration; use async_trait::async_trait; use harmony::data::Version; @@ -217,6 +217,24 @@ impl Interpret for FleetAgentInterpret { source: e, })?; + topology + .k8s_client() + .await + .map_err(InterpretError::new)? + .wait_until_deployment_ready( + &FleetAgentScore::resource_name(&device_id), + Some(&self.score.namespace), + Some(Duration::from_secs(120)), + ) + .await + .map_err(|error| { + InterpretError::new(format!( + "agent deployment {}/{} not ready: {error}", + self.score.namespace, + FleetAgentScore::resource_name(&device_id) + )) + })?; + Ok(Outcome::success_with_details( outcome.message, vec![ diff --git a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-publish.rs b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-publish.rs deleted file mode 100644 index 8d4adefb..00000000 --- a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-publish.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! `harmony-fleet-publish` — build + publish the operator image + chart -//! at a version. `docker` / `helm` must be on PATH and logged in to the -//! registry (CI's login actions; dev's manual login). - -use anyhow::{Result, bail}; -use clap::Parser; -use harmony_fleet_deploy::release::{release_operator, version_from_tag}; - -#[derive(Parser, Debug)] -#[command( - name = "harmony-fleet-publish", - about = "Build + publish the operator image + chart at a version" -)] -struct Cli { - /// The image + chart version, e.g. `0.1.0` or `0.0.0-dev.1730000000` - /// (must be valid semver — helm rejects anything else). Wins over - /// `--from-tag`; the laptop/dev path. - #[arg(long)] - version: Option, - - /// Parse the version out of a release tag like - /// `harmony-fleet-operator-v0.1.0`. Defaults to `$GITHUB_REF_NAME` - /// so the release workflow passes nothing. Used when `--version` - /// is unset. - #[arg(long, env = "GITHUB_REF_NAME")] - from_tag: Option, - - #[arg(long, default_value = "hub.nationtech.io")] - registry: String, - - #[arg(long, default_value = "harmony")] - project: String, - - /// Build + package only; skip both pushes (local k3d smoke-test). - #[arg(long)] - no_push: bool, -} - -impl Cli { - fn version(&self) -> Result { - match (&self.version, &self.from_tag) { - (Some(v), _) => Ok(v.clone()), - (None, Some(tag)) => version_from_tag(tag), - (None, None) => bail!("set --version (e.g. 0.1.0) or --from-tag"), - } - } -} - -fn main() -> Result<()> { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); - let cli = Cli::parse(); - let version = cli.version()?; - release_operator(&version, &cli.registry, &cli.project, !cli.no_push) -} diff --git a/fleet/harmony-fleet-deploy/src/companion/mod.rs b/fleet/harmony-fleet-deploy/src/companion/mod.rs deleted file mode 100644 index 4a0e47f2..00000000 --- a/fleet/harmony-fleet-deploy/src/companion/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Score companions — observe deployed-Score behavior without -//! widening the `Score` / `Interpret` trait surface (ADR-023 P7). -//! -//! A companion takes a Score (or, when the runtime contract lives -//! elsewhere, the typed config the Score emits) and exposes a -//! derived view a caller cares about — what the agent will watch on -//! NATS, which CRDs the operator owns, what the readiness probe -//! checks. Companions never call into the framework or topology; -//! they're pure functions over data the Score already exposes. -//! -//! Today: one companion, [`AgentObservation`], used by the e2e -//! harness to derive the device-scoped KV watch filter from a typed -//! [`harmony_fleet_auth::AgentConfig`]. Future companions for the -//! operator's CRD ownership + the readiness/smoke-test contract -//! land alongside this module. - -pub mod agent; - -pub use agent::AgentObservation; diff --git a/harmony/src/modules/fleet/setup_score.rs b/fleet/harmony-fleet-deploy/src/device_setup.rs similarity index 97% rename from harmony/src/modules/fleet/setup_score.rs rename to fleet/harmony-fleet-deploy/src/device_setup.rs index ab689343..57fdcfec 100644 --- a/harmony/src/modules/fleet/setup_score.rs +++ b/fleet/harmony-fleet-deploy/src/device_setup.rs @@ -11,19 +11,17 @@ use harmony_types::id::Id; use log::{debug, info, warn}; use serde::{Deserialize, Serialize}; -use crate::data::Version; -use crate::domain::interpret::{ - Interpret, InterpretError, InterpretName, InterpretStatus, Outcome, -}; -use crate::domain::inventory::Inventory; -use crate::domain::topology::{ +use harmony::data::Version; +use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; +use harmony::inventory::Inventory; +use harmony::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig}; +use harmony::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials}; +use harmony::score::Score; +use harmony::topology::{ FileDelivery, FileFetcher, FileSource, FileSpec, HostReachable, LinuxHostConfiguration, PackageInstaller, SystemdManager, SystemdScope, SystemdUnitSpec, Topology, UnixUserManager, UserSpec, }; -use crate::modules::zitadel::admin_auth::{self, ADMIN_API_SCOPES, DeviceCodeFlowConfig}; -use crate::modules::zitadel::setup::{ZitadelScheme, ZitadelSetupScore, mint_device_credentials}; -use crate::score::Score; /// User-visible configuration for the setup Score. Everything a customer /// needs to tell us to bring a device into the fleet. @@ -760,6 +758,23 @@ impl Interpret for FleetDeviceSetupInte "✅ running unchanged" }; + let mut active = false; + for _ in 0..30 { + if SystemdManager::systemd_unit_active(topology, "fleet-agent", SystemdScope::System) + .await + .map_err(wrap)? + { + active = true; + break; + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + if !active { + return Err(InterpretError::new( + "fleet-agent.service did not become active within 30s".to_string(), + )); + } + // Build human-readable recap. Style matches OKDAddNodeScore: // one fact per line, "Key: value" prefix, all the operationally // useful state in a single Outcome details block so a CLI run @@ -811,7 +826,7 @@ impl Interpret for FleetDeviceSetupInte } } -fn wrap(e: crate::executors::ExecutorError) -> InterpretError { +fn wrap(e: harmony::executors::ExecutorError) -> InterpretError { InterpretError::new(e.to_string()) } diff --git a/fleet/harmony-fleet-deploy/src/lib.rs b/fleet/harmony-fleet-deploy/src/lib.rs index 445fb3e4..46f5a3df 100644 --- a/fleet/harmony-fleet-deploy/src/lib.rs +++ b/fleet/harmony-fleet-deploy/src/lib.rs @@ -7,38 +7,16 @@ //! depend on this crate — they're the targets being deployed, not //! the deployer. //! -//! Coverage today: -//! -//! - [`operator`] — `FleetOperatorScore` (moved out of -//! `harmony::modules::fleet::operator`), helm chart generator, -//! CRD apply via [`operator::install_crds`]. -//! - [`nats`] — `FleetNatsScore` (preset over `NatsHelmChartScore` -//! with the values the fleet stack needs). -//! - [`agent`] — `FleetAgentScore` (Pod-target only in v1; the -//! VM/SSH target is `FleetDeviceSetupScore` in `harmony` core, -//! slated to move here in a follow-up). -//! -//! Out of scope for now: -//! -//! - **Callout** (mock-OIDC fixture + `NatsAuthCalloutScore` wiring) — -//! PR 1.5. -//! - **Smoke-test contract**. ADR-023 principle 4 says `deploy` -//! returns only after a smoke test passes; the trait/struct shape -//! for that contract is open. The e2e harness's `ping` test plays -//! the role of the smoke test for now. +//! The binary composes independent component Scores through typed references; +//! it does not own provider operations or a Fleet-wide aggregate Score. pub mod agent; -pub mod companion; -pub mod nats; +mod device_setup; pub mod operator; -pub mod release; -pub mod secrets; -pub mod server; pub use agent::{FleetAgentScore, PodTarget}; -pub use companion::AgentObservation; -pub use nats::{FleetNatsScore, UserPassCredentials}; -pub use operator::{FleetOperatorScore, OperatorCredentials, PublishedChart}; -pub use release::{release_operator, version_from_tag}; -pub use secrets::{FleetDeployConfig, FleetDeploySecrets}; -pub use server::FleetServerScore; +pub use device_setup::{ + AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, + HostsEntry, merge_hosts_file, +}; +pub use operator::{FleetOperatorScore, OperatorCredentials}; diff --git a/fleet/harmony-fleet-deploy/src/main.rs b/fleet/harmony-fleet-deploy/src/main.rs index 0eb9ff95..439ae776 100644 --- a/fleet/harmony-fleet-deploy/src/main.rs +++ b/fleet/harmony-fleet-deploy/src/main.rs @@ -1,149 +1,153 @@ -//! `harmony-fleet-deploy` — deploy the published fleet operator chart -//! (`harmony apply`). Loads its secrets from config and runs one -//! [`FleetOperatorScore`] against [`K8sAnywhereTopology`]. -//! -//! The full dev/e2e stack (NATS + agent, user/pass) is composed by the -//! e2e harness directly over the `harmony_fleet_deploy` Scores — not -//! here. This binary is the prod/CD path: Zitadel-SSO-only, no -//! handrolled manifests. - -use std::io::Write; - -use anyhow::{Context, Result, bail}; -use clap::Parser; -use harmony::inventory::Inventory; +use anyhow::Result; +use async_trait::async_trait; +use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService}; +use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; +use harmony::modules::openbao::{OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore}; +use harmony::modules::postgresql::K8sPostgreSQLScore; +use harmony::modules::zitadel::{ZitadelAppType, ZitadelScore, ZitadelSetupScore}; +use harmony::score::Score; use harmony::topology::K8sAnywhereTopology; -use harmony_cli::Args as HarmonyCliArgs; -use harmony_config::ConfigClient; -use harmony_fleet_deploy::{ - FleetDeployConfig, FleetDeploySecrets, FleetOperatorScore, version_from_tag, -}; -use harmony_zitadel_auth::{OperatorCookieKey, ZitadelAuthConfig}; -use tracing::info; +use harmony_app::{AppContext, AppError, AppIdentity, HarmonyApp, ImageRefs, ImageSpec, Profile}; +use harmony_fleet_deploy::FleetOperatorScore; -#[derive(Parser, Debug)] -#[command( - name = "harmony-fleet-deploy", - about = "Deploy the published harmony fleet operator chart" -)] -struct CliConfig { - /// Override the k8s namespace from config (e.g. `fleet-staging`). - #[arg(long, env = "HARMONY_FLEET_NAMESPACE")] - namespace: Option, +const PROJECT: &str = "fleet"; +const ADMIN_ROLE: &str = "fleet-admin"; +const DEVICE_ROLE: &str = "device"; +const OPERATOR_APP: &str = "harmony-fleet-operator"; +const OPERATOR_USER: &str = "fleet-operator"; +const NATS_ACCOUNT: &str = "FLEET"; - /// Release tag to deploy (e.g. `harmony-fleet-operator-v0.0.2`); the - /// version is parsed from it in Rust so the workflow passes a tag and - /// never parses one in YAML. Wins over `--operator-chart-version`. - #[arg(long, env = "HARMONY_FLEET_OPERATOR_TAG")] - from_tag: Option, +struct FleetApp; - /// Bare chart version, for the laptop path. - #[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_VERSION")] - operator_chart_version: Option, - - /// Override the OCI chart registry from config. - #[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_REGISTRY")] - operator_chart_registry: Option, - - /// Override the OCI chart project from config. - #[arg(long, env = "HARMONY_FLEET_OPERATOR_CHART_PROJECT")] - operator_chart_project: Option, - - /// Config namespace `FleetDeploySecrets` and `FleetDeployConfig` resolve under (Env → OpenBao). - #[arg( - long, - env = "HARMONY_CONFIG_NAMESPACE", - default_value = "fleet-staging" - )] - config_namespace: String, - - #[command(flatten)] - harmony_cli: HarmonyCliArgs, -} - -impl CliConfig { - fn chart_version(&self) -> Result { - match (&self.from_tag, &self.operator_chart_version) { - (Some(tag), _) => version_from_tag(tag), - (None, Some(v)) => Ok(v.clone()), - (None, None) => bail!("set --from-tag or --operator-chart-version"), +#[async_trait] +impl HarmonyApp for FleetApp { + fn identity(&self) -> AppIdentity { + AppIdentity { + name: "fleet".to_string(), + namespace: "fleet-system".to_string(), } } + + fn images(&self, ctx: &AppContext) -> Result, AppError> { + Ok(vec![ + ImageSpec { + name: "operator".to_string(), + image: ctx.image("harmony-fleet-operator")?, + context: ".".into(), + dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(), + build_args: Vec::new(), + }, + ImageSpec { + name: "callout".to_string(), + image: ctx.image("harmony-nats-callout")?, + context: ".".into(), + dockerfile: "nats/callout/Dockerfile".into(), + build_args: Vec::new(), + }, + ]) + } + + async fn scores( + &self, + ctx: &AppContext, + images: &ImageRefs, + ) -> Result>>, AppError> { + let namespace = "fleet-system"; + let zitadel_namespace = "zitadel"; + let openbao_namespace = "openbao"; + let image_pull_secret = ctx.image_pull_secret()?; + + let postgres = K8sPostgreSQLScore::new(zitadel_namespace).cluster_name("zitadel-pg"); + let database = postgres.root_account_ref(); + + let mut zitadel = ZitadelScore::new( + ctx.service_host("zitadel", zitadel_namespace)?, + zitadel_namespace, + ) + .database(database); + if ctx.profile() == Profile::Local { + // Q : should that be handled by harmony internally? + zitadel = zitadel.http(Some(8080)); + } + let provider = zitadel.provider_ref(); + + let identity = ZitadelSetupScore::for_provider(&provider, zitadel_namespace, namespace) + .application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode) + .api_application(PROJECT, "nats") + .role(PROJECT, ADMIN_ROLE, "Fleet Admin") + .role(PROJECT, DEVICE_ROLE, "Device") + .machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE]) + .port_forward("zitadel") + .groups_claim(); + let application = identity.application_ref(OPERATOR_APP); + let operator_identity = identity.machine_identity_ref(OPERATOR_USER); + + let credentials = match ctx.profile() { + // Q: this looks weird. Why do we need to generate creds locally but not in prod? I + // would think that in both cases creds should be "ensured". Created if not existing. + // Maybe in prod it should use harmony config with a remote openbao and store them there + // (as we plan on using external secrets asap) + Profile::Local => NatsAuthCalloutCredentialsScore::generated, + Profile::Prod => NatsAuthCalloutCredentialsScore::existing, + }(namespace, "fleet-callout-credentials"); + let service = match ctx.profile() { + // FIXME this should definitely not surface up to the end user deployment crate. This is + // topology dependent and is the topology's job (local k3d vs regular cluster) to decide + // how to expose a port and then the score ref should be appropriate + Profile::Local => NatsService::NodePort(30422), + Profile::Prod => NatsService::LoadBalancer, + }; + // I feel like this should not be a standalone nats score but rather be configuration passed + // to the main nats score that is installing the nats cluster + let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT) + .with_jetstream_size("2Gi"); + let account = nats.account_ref(); + let callout = + NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth") + .credentials(&credentials.credentials_ref()) + .with_oidc(&provider, &application) + .image(images.require("callout")?) + .image_pull_secret(image_pull_secret) + .admin_role(ADMIN_ROLE) + .device_role(DEVICE_ROLE) + .device_id_claim("client_id"); + let nats = nats.with_auth_callout(&callout.auth_callout_ref()); + + let mut openbao = OpenbaoScore::new( + openbao_namespace, + "openbao", + ctx.service_host("openbao", openbao_namespace)?, + ); + if ctx.profile() == Profile::Prod { + openbao = openbao.tls("letsencrypt-prod"); + } + let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application( + &provider, + &application, + OpenbaoJwtAuth::oidc("fleet-device"), + ); + + let operator = FleetOperatorScore::new(images.require("operator")?) + .namespace(namespace) + .image_pull_secret(image_pull_secret) + .messaging(&nats.client_ref()) + .identity(&provider, &application, &operator_identity); + + Ok(vec![ + Box::new(postgres), + Box::new(zitadel), + Box::new(credentials), + Box::new(nats), + Box::new(identity), + Box::new(callout), + Box::new(openbao), + Box::new(openbao_setup), + Box::new(operator), + ]) + } } #[tokio::main] async fn main() -> Result<()> { - harmony_cli::cli_logger::init(); - let cli = CliConfig::parse(); - let version = cli.chart_version()?; - - let config_client = ConfigClient::for_namespace(&cli.config_namespace).await; - - let secrets: FleetDeploySecrets = config_client - .get() - .await - .context("loading FleetDeploySecrets (set HARMONY_CONFIG_FleetDeploySecrets or OpenBao)")?; - - let config: FleetDeployConfig = config_client - .get_or_prompt() - .await - .context("loading FleetDeployConfig")?; - - // Point KUBECONFIG at the scoped deployer credential before the - // topology reads it, so the runner pod needs no standing permissions. - // Held to end of scope so the tempfile outlives the deploy. - let _kubeconfig = match &secrets.kubeconfig { - Some(kubeconfig) => { - let mut f = tempfile::NamedTempFile::new().context("kubeconfig tempfile")?; - f.write_all(kubeconfig.as_bytes()) - .context("writing kubeconfig")?; - // SAFETY: single-threaded startup, before any topology reads it. - unsafe { std::env::set_var("KUBECONFIG", f.path()) }; - Some(f) - } - None => None, - }; - - let namespace = cli.namespace.unwrap_or(config.namespace); - let registry = cli - .operator_chart_registry - .unwrap_or(config.operator_chart_registry); - let project = cli - .operator_chart_project - .unwrap_or(config.operator_chart_project); - - // Coherent with the other staging hosts (sso-stg., secrets-stg.). - let ui_host = format!("fleet-stg.{}", config.base_domain); - - // Dashboard SSO config: hosts derive from base_domain; client_id + - // audiences come from config, cookie key from secrets. Baked into the - // operator Secret for the pod's ConfigClient to read. - let web_auth = ZitadelAuthConfig { - zitadel_base: format!("https://sso-stg.{}", config.base_domain), - base_url: format!("https://{ui_host}"), - client_id: config.operator_oidc_client_id, - scope: "openid profile email".to_string(), - trusted_audiences: config.operator_trusted_audiences, - logout_redirect_uri: format!("https://{ui_host}/"), - }; - let cookie = OperatorCookieKey { - cookie_key_b64: secrets.operator_cookie_key_b64, - }; - - let operator = FleetOperatorScore::new() - .namespace(namespace) - .nats_url(config.nats_url) - .credentials(secrets.operator_credentials_toml) - .published_chart(registry, project, version) - .ingress(ui_host, Some(config.cluster_issuer)) - .web_auth(web_auth, cookie); - - harmony_cli::run( - Inventory::autoload(), - K8sAnywhereTopology::from_env(), - vec![Box::new(operator)], - Some(cli.harmony_cli), - ) - .await - .map_err(|e| anyhow::anyhow!("{e}")) + harmony_cli::app::app_main(FleetApp).await } diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 419b3976..bdb888b5 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -16,6 +16,7 @@ //! operator Deployment into a helm chart the cluster runs itself. use std::collections::BTreeMap; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; @@ -25,7 +26,8 @@ use k8s_openapi::api::apps::v1::{ }; use k8s_openapi::api::core::v1::{ Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile, - Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, + Secret, SecretKeySelector, SecretVolumeSource, SecurityContext, Service, ServiceAccount, + ServicePort, ServiceSpec, Volume, VolumeMount, }; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject}; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; @@ -36,8 +38,9 @@ use kube::api::ObjectMeta; use serde::Serialize; use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; -use harmony::modules::fleet::operator::crd::{Deployment, Device}; +use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; +use harmony_fleet_operator::{Deployment, Device}; /// Inputs for chart generation. Default values are aimed at a /// local-dev k3d install; override via the `chart` subcommand flags. @@ -77,56 +80,24 @@ pub struct ChartOptions { /// JSON of the dashboard's `OperatorCookieKey`, stored under /// [`ENV_WEB_COOKIE_KEY`]. pub web_cookie_key_json: Option, + pub identity: Option, + pub identity_version: Option, + pub image_pull_secret: Option, } -/// What the operator pod needs to authenticate to NATS via the auth -/// callout: a TOML snippet matching the agent's `[credentials]` -/// table. -/// -/// **Single Secret key**: the credentials TOML is the only payload. -/// The Zitadel JSON keyfile content is embedded inline inside the -/// TOML as the `key_json` field (multi-line triple-quoted string). -/// One env-var-from-Secret on the operator Pod -/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume -/// mounts, OKD restricted-v2 SCC compatible. +#[derive(Debug, Clone, Serialize)] +pub struct OperatorIdentityRefs { + pub provider: OidcProviderRef, + pub application: OidcApplicationRef, + pub machine: OidcMachineIdentityRef, +} + +/// Static NATS credentials for deployments without typed OIDC identity refs. #[derive(Debug, Clone, Serialize)] pub struct OperatorCredentials { - /// TOML payload, e.g. - /// ```text - /// type = "zitadel-jwt" - /// oidc_issuer_url = "http://sso.fleet.local:8080" - /// audience = "" - /// key_json = '''{"type":"serviceaccount","keyId":"...","key":"-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----\n","userId":"..."}''' - /// ``` - /// - /// `key_json` **must** use TOML *literal* multi-line strings - /// (`'''...'''`), NOT *basic* multi-line strings (`"""..."""`). - /// Basic strings interpret backslash escapes — every `\n` in the - /// embedded RSA private key would expand to a literal 0x0A before - /// JSON parsing, and JSON rejects raw control chars inside strings - /// ("control character … found while parsing a string"). Literal - /// strings preserve `\n` as the two chars `\\` + `n`, the JSON - /// parser then interprets that as a string escape and decodes the - /// multi-line PEM correctly. pub credentials_toml: String, } -impl OperatorCredentials { - /// Build the auth-callout credentials from a Zitadel machine key. - /// `key_json` goes in a TOML *literal* string — see the field docs - /// for why basic strings corrupt the embedded PEM. - pub fn zitadel_jwt(oidc_issuer_url: &str, audience: &str, key_json: &str) -> Self { - Self { - credentials_toml: format!( - "type = \"zitadel-jwt\"\n\ - oidc_issuer_url = \"{oidc_issuer_url}\"\n\ - audience = \"{audience}\"\n\ - key_json = '''{key_json}'''\n" - ), - } - } -} - impl Default for ChartOptions { fn default() -> Self { Self { @@ -143,6 +114,9 @@ impl Default for ChartOptions { chart_version: None, web_auth_config_json: None, web_cookie_key_json: None, + identity: None, + identity_version: None, + image_pull_secret: None, } } } @@ -401,12 +375,32 @@ fn operator_service() -> Service { } } +pub(crate) fn config_hash(opts: &ChartOptions) -> String { + let mut secret_hash = DefaultHasher::new(); + opts.credentials + .as_ref() + .map(|credentials| &credentials.credentials_toml) + .hash(&mut secret_hash); + opts.web_auth_config_json.hash(&mut secret_hash); + opts.web_cookie_key_json.hash(&mut secret_hash); + opts.identity + .as_ref() + .map(|identity| identity.machine.secret_name()) + .hash(&mut secret_hash); + opts.identity_version.hash(&mut secret_hash); + format!("{:x}", secret_hash.finish()) +} + fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { let mut match_labels = BTreeMap::new(); match_labels.insert( "app.kubernetes.io/name".to_string(), RELEASE_NAME.to_string(), ); + let pod_annotations = BTreeMap::from([( + "harmony.nationtech.io/config-hash".to_string(), + config_hash(opts), + )]); let mut env = vec![ EnvVar { @@ -420,6 +414,32 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { ..Default::default() }, ]; + if let Some(identity) = &opts.identity { + env.extend([ + EnvVar { + name: "FLEET_OPERATOR_OIDC_ISSUER".to_string(), + value: Some(identity.provider.issuer().to_string()), + ..Default::default() + }, + EnvVar { + name: "FLEET_OPERATOR_OIDC_AUDIENCE".to_string(), + value_from: Some(EnvVarSource { + config_map_key_ref: Some(k8s_openapi::api::core::v1::ConfigMapKeySelector { + name: identity.application.config_map_name().to_string(), + key: identity.application.project_id_key().to_string(), + optional: Some(false), + }), + ..Default::default() + }), + ..Default::default() + }, + EnvVar { + name: "FLEET_OPERATOR_OIDC_KEY_PATH".to_string(), + value: Some("/var/run/secrets/fleet/key.json".to_string()), + ..Default::default() + }, + ]); + } // The whole `[credentials]` TOML payload — including the JSON // keyfile inlined under `key_json` — travels as a single env @@ -490,18 +510,47 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(match_labels), + annotations: Some(pod_annotations), ..Default::default() }), spec: Some(PodSpec { service_account_name: Some(SERVICE_ACCOUNT.to_string()), + image_pull_secrets: opts.image_pull_secret.as_ref().map(|name| { + vec![k8s_openapi::api::core::v1::LocalObjectReference { + name: name.clone(), + }] + }), containers: vec![Container { name: "operator".to_string(), image: Some(opts.image.clone()), image_pull_policy: Some(opts.image_pull_policy.clone()), env: Some(env), + volume_mounts: opts.identity.as_ref().map(|_| { + vec![VolumeMount { + name: "operator-identity".to_string(), + mount_path: "/var/run/secrets/fleet".to_string(), + read_only: Some(true), + ..Default::default() + }] + }), security_context: Some(container_security_context()), ..Default::default() }], + volumes: opts.identity.as_ref().map(|identity| { + vec![Volume { + name: "operator-identity".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(identity.machine.secret_name().to_string()), + items: Some(vec![k8s_openapi::api::core::v1::KeyToPath { + key: identity.machine.key_json_key().to_string(), + path: "key.json".to_string(), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }] + }), ..Default::default() }), }, @@ -511,27 +560,6 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { } } -// Re-export the manifest builders so the e2e bring-up can apply the -// operator inline (Score-style) without re-implementing the manifests. -// -// The SA + Deployment helpers return namespace-neutral manifests; -// callers inject the target namespace through `K8sResourceScore::single`. -// The CRB takes the SA's namespace as an explicit argument because the -// CRB subject must reference a concrete namespace — there's no -// kube-side "current namespace" for cluster-scoped resources. -pub fn build_service_account() -> ServiceAccount { - service_account() -} -pub fn build_cluster_role() -> ClusterRole { - cluster_role() -} -pub fn build_cluster_role_binding(subject_namespace: &str) -> ClusterRoleBinding { - cluster_role_binding(subject_namespace) -} -pub fn build_operator_deployment(opts: &ChartOptions) -> K8sDeployment { - operator_deployment(opts) -} - /// Minimum-privilege container security context. /// /// - `runAsNonRoot: true` — a compromised operator pod with diff --git a/fleet/harmony-fleet-deploy/src/operator/install.rs b/fleet/harmony-fleet-deploy/src/operator/install.rs deleted file mode 100644 index 29b9b7af..00000000 --- a/fleet/harmony-fleet-deploy/src/operator/install.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Install the operator's CRDs into a target Kubernetes cluster via -//! a harmony Score. -//! -//! Moved from `fleet/harmony-fleet-operator/src/install.rs` per -//! ADR-023: this is deploy-side logic, so it lives in the deploy -//! crate, not in the operator runtime binary. The runtime binary's -//! `Install` subcommand is gone; users invoke `harmony-fleet-deploy -//! operator-install-crds` (or compose [`install_crds`] from another -//! deploy flow) instead. - -use harmony::inventory::Inventory; -use harmony::modules::fleet::operator::crd::{Deployment, Device}; -use harmony::modules::k8s::K8sBareTopology; -use harmony::modules::k8s::resource::K8sResourceScore; -use harmony::score::Score; -use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; -use kube::CustomResourceExt; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum InstallCrdsError { - #[error("building K8sBareTopology from KUBECONFIG: {0}")] - Topology(String), - #[error("executing K8sResourceScore for Deployment + Device CRDs: {0}")] - Interpret(String), -} - -/// Apply the operator's CRDs to whatever cluster `KUBECONFIG` points -/// at. Returns once the apply call completes — does **not** wait for -/// the apiserver to mark each CRD `Established`; the caller does that -/// (e.g. with `kubectl wait --for=condition=Established`) if it -/// cares. -pub async fn install_crds() -> Result<(), InstallCrdsError> { - let topology = K8sBareTopology::from_kubeconfig("harmony-fleet-operator-install") - .await - .map_err(InstallCrdsError::Topology)?; - let inventory = Inventory::empty(); - - let crds: Vec = vec![Deployment::crd(), Device::crd()]; - let score = K8sResourceScore:: { - resource: crds, - namespace: None, - }; - - let interpret = Score::::create_interpret(&score); - let outcome = interpret - .execute(&inventory, &topology) - .await - .map_err(|e| InstallCrdsError::Interpret(e.to_string()))?; - - tracing::info!(?outcome, "CRDs installed"); - Ok(()) -} diff --git a/fleet/harmony-fleet-deploy/src/operator/mod.rs b/fleet/harmony-fleet-deploy/src/operator/mod.rs index 3d5ee390..26bda7a6 100644 --- a/fleet/harmony-fleet-deploy/src/operator/mod.rs +++ b/fleet/harmony-fleet-deploy/src/operator/mod.rs @@ -3,17 +3,12 @@ //! //! Lives in `harmony-fleet-deploy` per ADR-023: deploy logic for any //! fleet component sits in this crate, not in `harmony` core. The -//! operator's CRD types ([`harmony::modules::fleet::operator::crd`]) +//! operator's CRD types ([`harmony_fleet_operator::crd`]) //! stay in `harmony` for the moment — a follow-up PR finishes lifting //! them out — but the *Score* + chart generator are home here. -pub mod chart; -pub mod install; +pub(crate) mod chart; pub mod score; -pub use chart::{ - ChartOptions, OperatorCredentials, RELEASE_NAME, SECRET_KEY_CREDENTIALS_TOML, SECRET_NAME, - build_chart, operator_secret, -}; -pub use install::install_crds; -pub use score::{FleetOperatorScore, PublishedChart}; +pub use chart::OperatorCredentials; +pub use score::FleetOperatorScore; diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index c60eaee0..10440095 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -18,12 +18,12 @@ //! use harmony::inventory::Inventory; //! //! let topology = K8sBareTopology::from_kubeconfig("fleet-operator").await?; -//! let score = FleetOperatorScore::new() -//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev"); +//! let score = FleetOperatorScore::new("hub.nationtech.io/harmony/harmony-fleet-operator:dev") +//! .messaging(&nats.client_ref()); //! score.create_interpret().execute(&Inventory::empty(), &topology).await?; //! ``` -use std::str::FromStr; +use std::{str::FromStr, time::Duration}; use async_trait::async_trait; use harmony_types::id::Id; @@ -36,22 +36,17 @@ use harmony::inventory::Inventory; use harmony::modules::helm::chart::{HelmChartScore, NonBlankString}; use harmony::modules::k8s::ingress::K8sIngressScore; use harmony::modules::k8s::resource::K8sResourceScore; +use harmony::modules::nats::NatsClientRef; +use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; use harmony::score::Score; use harmony::topology::{HelmCommand, K8sclient, Topology}; +use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret}; +use kube::api::ListParams; use crate::operator::chart; -use crate::operator::chart::{ChartOptions, OperatorCredentials, build_chart, operator_secret}; - -/// The already-published OCI chart to install (the CD `harmony apply` -/// path). When set, the operator installs -/// `oci://{registry}/{project}/harmony-fleet-operator-chart:{version}` and the -/// score's `image` is ignored (the image is baked into the chart). -#[derive(Debug, Clone, Serialize)] -pub struct PublishedChart { - pub registry: String, - pub project: String, - pub version: String, -} +use crate::operator::chart::{ + ChartOptions, OperatorCredentials, OperatorIdentityRefs, build_chart, operator_secret, +}; /// Declarative install of the harmony fleet operator. Construct via /// [`new`](Self::new), tune with the builder-style methods, hand to @@ -66,12 +61,9 @@ pub struct FleetOperatorScore { pub release_name: String, pub image: String, pub image_pull_policy: String, - pub nats_url: String, + nats: Option, pub log_level: String, pub credentials: Option, - /// `None` renders + installs the chart from local source (dev/e2e); - /// `Some` installs the published OCI chart at a pinned version (CD). - pub published_chart: Option, /// `Some(host)` exposes the UI via an Ingress after install (CD); /// `None` leaves it cluster-internal (dev/e2e harnesses). pub operator_ui_host: Option, @@ -82,6 +74,8 @@ pub struct FleetOperatorScore { /// for the pod's `ConfigClient` to read. `None` leaves the dashboard /// unauthenticated (dev/e2e). pub web_auth: Option, + pub identity: Option, + pub image_pull_secret: Option, } /// The dashboard's auth inputs the operator reads via `ConfigClient`. @@ -94,22 +88,23 @@ pub struct WebAuth { impl FleetOperatorScore { /// Build a score targeting the `fleet-system` namespace with the /// dev-default image and NATS URL. Use the builders to override. - pub fn new() -> Self { + pub fn new(image: impl Into) -> Self { let defaults = ChartOptions::default(); Self { // FleetOperatorScore's own default; the chart itself is // namespace-neutral. Callers override via `.namespace(..)`. namespace: "fleet-system".to_string(), release_name: "harmony-fleet-operator".to_string(), - image: defaults.image, + image: image.into(), image_pull_policy: defaults.image_pull_policy, - nats_url: defaults.nats_url, + nats: None, log_level: defaults.log_level, credentials: defaults.credentials, - published_chart: None, operator_ui_host: None, cluster_issuer: None, web_auth: None, + identity: None, + image_pull_secret: None, } } @@ -133,22 +128,6 @@ impl FleetOperatorScore { self } - /// Install the published OCI chart at `version` instead of rendering - /// one from local source (the CD `harmony apply` path). - pub fn published_chart( - mut self, - registry: impl Into, - project: impl Into, - version: impl Into, - ) -> Self { - self.published_chart = Some(PublishedChart { - registry: registry.into(), - project: project.into(), - version: version.into(), - }); - self - } - pub fn namespace(mut self, ns: impl Into) -> Self { self.namespace = ns.into(); self @@ -159,18 +138,32 @@ impl FleetOperatorScore { self } - pub fn image(mut self, image: impl Into) -> Self { - self.image = image.into(); - self - } - pub fn image_pull_policy(mut self, policy: impl Into) -> Self { self.image_pull_policy = policy.into(); self } - pub fn nats_url(mut self, url: impl Into) -> Self { - self.nats_url = url.into(); + pub fn messaging(mut self, nats: &NatsClientRef) -> Self { + self.nats = Some(nats.clone()); + self + } + + pub fn identity( + mut self, + provider: &OidcProviderRef, + application: &OidcApplicationRef, + machine: &OidcMachineIdentityRef, + ) -> Self { + self.identity = Some(OperatorIdentityRefs { + provider: provider.clone(), + application: application.clone(), + machine: machine.clone(), + }); + self + } + + pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self { + self.image_pull_secret = secret.map(str::to_string); self } @@ -190,12 +183,6 @@ impl FleetOperatorScore { } } -impl Default for FleetOperatorScore { - fn default() -> Self { - Self::new() - } -} - impl Score for FleetOperatorScore { fn create_interpret(&self) -> Box> { Box::new(FleetOperatorInterpret { @@ -213,6 +200,51 @@ pub struct FleetOperatorInterpret { score: FleetOperatorScore, } +async fn smoke_test_operator( + namespace: &str, + expected_config_hash: &str, + topology: &T, +) -> Result<(), InterpretError> { + let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; + tokio::time::timeout(Duration::from_secs(180), async { + loop { + let pods = k8s + .list_resources::( + Some(namespace), + Some( + ListParams::default() + .labels(&format!("app.kubernetes.io/name={}", chart::RELEASE_NAME)), + ), + ) + .await; + if let Ok(pods) = pods { + for pod in pods.items { + let has_expected_config = pod + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get("harmony.nationtech.io/config-hash") + }) + .is_some_and(|hash| hash == expected_config_hash); + if let Some(name) = pod.metadata.name + && has_expected_config + && k8s + .pod_logs(namespace, &name, Some(100)) + .await + .is_ok_and(|logs| logs.contains("KV bucket ready")) + { + return; + } + } + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string())) +} + #[async_trait] impl Interpret for FleetOperatorInterpret { async fn execute( @@ -220,6 +252,74 @@ impl Interpret for FleetOperatorInterp inventory: &Inventory, topology: &T, ) -> Result { + let nats_url = self + .score + .nats + .as_ref() + .ok_or_else(|| InterpretError::new("NatsClientRef is required".to_string()))? + .url(); + if let Some(identity) = &self.score.identity + && (identity.application.namespace() != self.score.namespace + || identity.machine.namespace() != self.score.namespace) + { + return Err(InterpretError::new(format!( + "operator identity outputs must be in namespace '{}'", + self.score.namespace + ))); + } + let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; + k8s.ensure_namespace(&self.score.namespace) + .await + .map_err(|e| { + InterpretError::new(format!( + "failed to ensure operator namespace '{}': {e}", + self.score.namespace + )) + })?; + let identity_version = if let Some(identity) = &self.score.identity { + Some( + tokio::time::timeout(Duration::from_secs(180), async { + loop { + let application = k8s + .get_resource::( + identity.application.config_map_name(), + Some(identity.application.namespace()), + ) + .await + .ok() + .flatten(); + let machine = k8s + .get_resource::( + identity.machine.secret_name(), + Some(identity.machine.namespace()), + ) + .await + .ok() + .flatten(); + if let (Some(application), Some(machine)) = (application, machine) + && let Some(project_id) = application + .data + .as_ref() + .and_then(|data| data.get(identity.application.project_id_key())) + { + return format!( + "{}:{}", + project_id, + machine.metadata.resource_version.unwrap_or_default() + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| { + InterpretError::new("timed out waiting for operator identity refs".to_string()) + })?, + ) + } else { + None + }; + let credentials = self.score.credentials.clone(); // Apply the credentials Secret BEFORE the helm install (the // chart's Deployment references it via secretKeyRef). Applied // directly, not via the chart — it's environment-specific. The @@ -236,11 +336,23 @@ impl Interpret for FleetOperatorInterp ), None => (None, None), }; - if let Some(creds) = &self.score.credentials + let expected_config_hash = chart::config_hash(&ChartOptions { + credentials: credentials.clone(), + web_auth_config_json: web_auth_config_json.clone(), + web_cookie_key_json: web_cookie_key_json.clone(), + identity: self.score.identity.clone(), + identity_version: identity_version.clone(), + image_pull_secret: self.score.image_pull_secret.clone(), + ..ChartOptions::default() + }); + if let Some(creds) = &credentials && let Some(secret) = operator_secret(&ChartOptions { credentials: Some(creds.clone()), - web_auth_config_json, - web_cookie_key_json, + web_auth_config_json: web_auth_config_json.clone(), + web_cookie_key_json: web_cookie_key_json.clone(), + identity: self.score.identity.clone(), + identity_version: identity_version.clone(), + image_pull_secret: self.score.image_pull_secret.clone(), ..ChartOptions::default() }) { @@ -254,69 +366,57 @@ impl Interpret for FleetOperatorInterp .await?; } - // Install through HelmChartScore from either the published OCI - // chart (CD) or a freshly rendered local chart (dev/e2e). Each - // branch runs its own install so the local tempdir stays alive - // across it. - let helm_outcome = if let Some(p) = &self.score.published_chart { - let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, chart::CHART_NAME); - info!( - "Installing helm release '{}' from published chart {chart_ref} version {}", - self.score.release_name, p.version - ); - HelmChartScore { - namespace: Some(non_blank(&self.score.namespace, "namespace")?), - release_name: non_blank(&self.score.release_name, "release_name")?, - chart_name: non_blank(&chart_ref, "chart_name")?, - chart_version: Some(non_blank(&p.version, "chart_version")?), - values_overrides: None, - values_yaml: None, - create_namespace: true, - install_only: false, - force_conflicts: false, - repository: None, - } - .interpret(inventory, topology) - .await? - } else { - let tmp = tempfile::tempdir() - .map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?; - let chart_path = build_chart(&ChartOptions { - output_dir: tmp.path().to_path_buf(), - image: self.score.image.clone(), - image_pull_policy: self.score.image_pull_policy.clone(), - nats_url: self.score.nats_url.clone(), - log_level: self.score.log_level.clone(), - credentials: self.score.credentials.clone(), - chart_version: None, - // The auth Secret is applied separately above; the - // rendered chart only needs the Deployment env wiring. - web_auth_config_json: None, - web_cookie_key_json: None, - }) - .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?; - let chart_path_str = chart_path.to_str().ok_or_else(|| { - InterpretError::new("operator chart path is not utf-8".to_string()) - })?; - info!( - "Installing helm release '{}' from rendered chart {}", - self.score.release_name, chart_path_str - ); - HelmChartScore { - namespace: Some(non_blank(&self.score.namespace, "namespace")?), - release_name: non_blank(&self.score.release_name, "release_name")?, - chart_name: non_blank(chart_path_str, "chart_name")?, - chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?), - values_overrides: None, - values_yaml: None, - create_namespace: true, - install_only: false, - force_conflicts: false, - repository: None, - } - .interpret(inventory, topology) - .await? - }; + let tmp = tempfile::tempdir() + .map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?; + let chart_path = build_chart(&ChartOptions { + output_dir: tmp.path().to_path_buf(), + image: self.score.image.clone(), + image_pull_policy: self.score.image_pull_policy.clone(), + nats_url: nats_url.to_string(), + log_level: self.score.log_level.clone(), + credentials: credentials.clone(), + chart_version: None, + web_auth_config_json: web_auth_config_json.clone(), + web_cookie_key_json: web_cookie_key_json.clone(), + identity: self.score.identity.clone(), + identity_version, + image_pull_secret: self.score.image_pull_secret.clone(), + }) + .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?; + let chart_path_str = chart_path + .to_str() + .ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?; + let helm_outcome = HelmChartScore { + namespace: Some(non_blank(&self.score.namespace, "namespace")?), + release_name: non_blank(&self.score.release_name, "release_name")?, + chart_name: non_blank(chart_path_str, "chart_name")?, + chart_version: Some(non_blank(env!("CARGO_PKG_VERSION"), "chart_version")?), + values_overrides: None, + values_yaml: None, + create_namespace: false, + install_only: false, + force_conflicts: false, + repository: None, + } + .interpret(inventory, topology) + .await?; + + k8s.wait_until_deployment_ready( + &self.score.release_name, + Some(&self.score.namespace), + Some(Duration::from_secs(180)), + ) + .await + .map_err(|e| { + InterpretError::new(format!( + "operator deployment {}/{} not ready: {e}", + self.score.namespace, self.score.release_name + )) + })?; + + if credentials.is_some() || self.score.identity.is_some() { + smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?; + } // Expose the UI. Applied after the chart so the backing Service // (shipped in the chart) exists. Skipped when no host is set — @@ -324,7 +424,7 @@ impl Interpret for FleetOperatorInterp let mut details = vec![ format!("operator namespace: {}", self.score.namespace), format!("operator release: {}", self.score.release_name), - format!("operator NATS URL: {}", self.score.nats_url), + format!("operator NATS URL: {nats_url}"), ]; if let Some(host) = &self.score.operator_ui_host { let to_fqdn = |s: &str| { @@ -383,7 +483,7 @@ mod tests { #[test] fn defaults_target_fleet_system() { - let s = FleetOperatorScore::new(); + let s = FleetOperatorScore::new("operator:dev"); assert_eq!(s.namespace, "fleet-system"); assert_eq!(s.release_name, "harmony-fleet-operator"); assert_eq!(s.image_pull_policy, "IfNotPresent"); @@ -391,27 +491,19 @@ mod tests { #[test] fn builders_compose() { - let s = FleetOperatorScore::new() + let nats = harmony::modules::nats::NatsScore::new("nats", "default").client_ref(); + let s = FleetOperatorScore::new("hub.example/op:v1") .namespace("custom-ns") - .image("hub.example/op:v1") .image_pull_policy("Always") - .nats_url("nats://nats:4222") + .messaging(&nats) .log_level("debug"); assert_eq!(s.namespace, "custom-ns"); assert_eq!(s.image, "hub.example/op:v1"); assert_eq!(s.image_pull_policy, "Always"); - assert_eq!(s.nats_url, "nats://nats:4222"); + assert_eq!( + s.nats.unwrap().url(), + "nats://nats.default.svc.cluster.local:4222" + ); assert_eq!(s.log_level, "debug"); } - - #[test] - fn credentials_set_on_published_chart_path() { - let s = FleetOperatorScore::new() - .credentials("type = \"zitadel-jwt\"") - .published_chart("hub.example", "proj", "1.2.3"); - let creds = s.credentials.expect("credentials wired"); - assert_eq!(creds.credentials_toml, "type = \"zitadel-jwt\""); - // The published-chart interpret applies this Secret before install. - assert!(s.published_chart.is_some()); - } } diff --git a/fleet/harmony-fleet-deploy/src/release.rs b/fleet/harmony-fleet-deploy/src/release.rs deleted file mode 100644 index 5cf766dd..00000000 --- a/fleet/harmony-fleet-deploy/src/release.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Build + publish the operator's image + chart for a tagged release. -//! -//! A release is five commands — docker build/push, hydrate the chart, -//! helm package/push — plus parsing the version out of the git tag. -//! Kept as a plain function with `anyhow` (this is binary glue, not -//! library API). When a second component needs releasing, lift the -//! per-component constants below into a small spec then — not before. - -use std::path::{Path, PathBuf}; -use std::process::Command; - -use anyhow::{Context, Result, bail}; - -use crate::operator::chart::{CHART_NAME, ChartOptions, build_chart}; - -const TAG_PREFIX: &str = "harmony-fleet-operator-"; -const IMAGE_NAME: &str = "harmony-fleet-operator"; -const DOCKERFILE: &str = "fleet/harmony-fleet-operator/Dockerfile"; - -/// Version from a release tag: `harmony-fleet-operator-v0.1.0` → `0.1.0`. -/// One version drives both the image tag and the chart version. -pub fn version_from_tag(tag: &str) -> Result { - let rest = tag - .strip_prefix(TAG_PREFIX) - .with_context(|| format!("tag '{tag}' is not a '{TAG_PREFIX}*' release tag"))?; - let version = rest.strip_prefix('v').unwrap_or(rest); - if version.is_empty() { - bail!("tag '{tag}' has no version after the prefix"); - } - Ok(version.to_string()) -} - -/// Build + push the operator image and helm chart at `version`. With -/// `push = false`, builds + packages only (a local k3d smoke-test). -/// `docker` and `helm` must be on PATH and logged in to the registry. -pub fn release_operator(version: &str, registry: &str, project: &str, push: bool) -> Result<()> { - let image = format!("{registry}/{project}/{IMAGE_NAME}:{version}"); - let oci_repo = format!("oci://{registry}/{project}"); - - log::info!("docker build {image}"); - run("docker", &["build", "-f", DOCKERFILE, "-t", &image, "."])?; - if push { - log::info!("docker push {image}"); - run("docker", &["push", &image])?; - } - - let tmp = tempfile::tempdir().context("chart tempdir")?; - let chart_dir = build_chart(&ChartOptions { - output_dir: tmp.path().to_path_buf(), - image: image.clone(), - chart_version: Some(version.to_string()), - ..ChartOptions::default() - }) - .map_err(|e| anyhow::anyhow!("hydrate operator chart: {e}"))?; - - let tgz = helm_package(&chart_dir, tmp.path())?; - if push { - log::info!("helm push {} {oci_repo}", tgz.display()); - run("helm", &["push", path_str(&tgz)?, &oci_repo])?; - } - - log::info!("released image={image} chart={oci_repo}/{CHART_NAME}:{version} pushed={push}"); - Ok(()) -} - -fn run(bin: &str, args: &[&str]) -> Result<()> { - let status = Command::new(bin) - .args(args) - .status() - .with_context(|| format!("spawn {bin} (is it installed and in PATH?)"))?; - if !status.success() { - bail!("`{bin} {}` failed ({status})", args.join(" ")); - } - Ok(()) -} - -fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result { - let output = Command::new("helm") - .args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?]) - .output() - .context("spawn helm package")?; - if !output.status.success() { - bail!("helm package failed ({})", output.status); - } - // helm prints "…saved it to: "; the last token is the path. - String::from_utf8_lossy(&output.stdout) - .split_whitespace() - .last() - .map(PathBuf::from) - .context("helm package printed no path") -} - -fn path_str(p: &Path) -> Result<&str> { - p.to_str() - .with_context(|| format!("path not utf-8: {}", p.display())) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn version_strips_prefix_and_leading_v() { - assert_eq!( - version_from_tag("harmony-fleet-operator-v0.1.0").unwrap(), - "0.1.0" - ); - assert_eq!( - version_from_tag("harmony-fleet-operator-2.3.4").unwrap(), - "2.3.4" - ); - } - - #[test] - fn rejects_foreign_or_empty_tags() { - assert!(version_from_tag("harmony-fleet-agent-v0.1.0").is_err()); - assert!(version_from_tag("harmony-fleet-operator-").is_err()); - } -} diff --git a/fleet/harmony-fleet-deploy/src/secrets.rs b/fleet/harmony-fleet-deploy/src/secrets.rs deleted file mode 100644 index 74768364..00000000 --- a/fleet/harmony-fleet-deploy/src/secrets.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Secrets and config for the published-chart (CD) operator deploy, via -//! [`harmony_config::ConfigClient`]. SSO-only by construction: no -//! user/pass field exists, so dev-only user/pass auth can't reach a prod -//! deploy. Resolved EnvSource → OpenBao, so the in-cluster runner pulls -//! them at job time with no plaintext on the pod. - -use harmony_config::Config; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -/// `#[config(secret)]` keeps every field out of cleartext SQLite and -/// masks it in logs. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)] -#[config(secret)] -pub struct FleetDeploySecrets { - /// Operator's zitadel-jwt `[credentials]` TOML — its NATS auth. The - /// CD path's only credential; there is no user/pass fallback. - pub operator_credentials_toml: String, - - /// Scoped fleet-deployer credential, fetched at job time so the - /// runner pod holds no standing permissions. `None` uses the ambient - /// `KUBECONFIG`. - #[serde(default)] - pub kubeconfig: Option, - - /// Operator dashboard session-cookie signing key (standard-base64 of - /// ≥64 bytes). `#[serde(default)]` so older seeds without it still - /// load — but the dashboard won't authenticate until it's set. - #[serde(default)] - pub operator_cookie_key_b64: String, -} - -/// Non-secret deploy config: k8s namespaces + chart coords. Loaded via -/// `ConfigClient::for_namespace("fleet-staging")` alongside `FleetDeploySecrets`. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)] -pub struct FleetDeployConfig { - /// K8s namespace where the operator, NATS, and callout live. - pub namespace: String, - - /// Full NATS URL the operator connects to (e.g. `nats://fleet-nats.fleet-staging:4222`). - pub nats_url: String, - - /// K8s namespace where Zitadel lives (for operator UI SSO). - pub zitadel_namespace: String, - - /// K8s namespace where OpenBao lives (for operator secret fetching). - pub openbao_namespace: String, - - /// OCI chart registry (e.g. `hub.nationtech.io`). - pub operator_chart_registry: String, - - /// OCI chart project (e.g. `harmony`). - pub operator_chart_project: String, - - /// Public base domain. The operator UI host is derived as - /// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`. - pub base_domain: String, - - /// cert-manager `ClusterIssuer` for the operator UI's TLS cert. - pub cluster_issuer: String, - - /// Zitadel OIDC app the operator dashboard authenticates against. - #[serde(default)] - pub operator_oidc_client_id: String, - - /// Token audiences the dashboard accepts (Zitadel project/app ids). - #[serde(default)] - pub operator_trusted_audiences: Vec, -} - -impl Default for FleetDeployConfig { - fn default() -> Self { - Self { - namespace: "fleet-staging".to_string(), - nats_url: "nats://fleet-nats.fleet-staging:4222".to_string(), - zitadel_namespace: "zitadel-staging".to_string(), - openbao_namespace: "openbao-staging".to_string(), - operator_chart_registry: "hub.nationtech.io".to_string(), - operator_chart_project: "harmony".to_string(), - base_domain: "cb1.nationtech.io".to_string(), - cluster_issuer: "letsencrypt-prod".to_string(), - operator_oidc_client_id: String::new(), - operator_trusted_audiences: Vec::new(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use harmony_config::ConfigClass; - - #[test] - fn secret_class_and_optional_kubeconfig() { - let s: FleetDeploySecrets = - serde_json::from_str(r#"{ "operator_credentials_toml": "x" }"#).unwrap(); - assert!(s.kubeconfig.is_none()); - // Secrets must never land in cleartext SQLite. - assert_eq!(FleetDeploySecrets::CLASS, ConfigClass::Secret); - } - - #[test] - fn config_class_is_not_secret() { - assert_eq!(FleetDeployConfig::CLASS, ConfigClass::Standard); - } - - #[test] - fn config_defaults() { - let c = FleetDeployConfig::default(); - assert_eq!(c.namespace, "fleet-staging"); - assert_eq!(c.zitadel_namespace, "zitadel-staging"); - assert_eq!(c.openbao_namespace, "openbao-staging"); - assert_eq!(c.base_domain, "cb1.nationtech.io"); - assert_eq!(c.cluster_issuer, "letsencrypt-prod"); - } -} diff --git a/fleet/harmony-fleet-deploy/src/server.rs b/fleet/harmony-fleet-deploy/src/server.rs deleted file mode 100644 index 02a75b7d..00000000 --- a/fleet/harmony-fleet-deploy/src/server.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! [`FleetServerScore`] — one-shot install of harmony's fleet -//! server-side stack into a Kubernetes cluster. -//! -//! Composes: -//! 1. [`NatsBasicScore`](crate::modules::nats::NatsBasicScore) — single-node -//! NATS with JetStream, in-cluster. -//! 2. [`FleetOperatorScore`](super::FleetOperatorScore) — operator -//! Deployment + CRDs (Deployment / Device) + RBAC. -//! 3. (Optional) [`ZitadelScore`] — central OIDC identity provider -//! (Postgres + Zitadel helm chart). Set `identity = Some(...)` to -//! install; `None` skips it. Adds the `K8sclient + PostgreSQL` -//! capability requirements to the topology either way (they're -//! static trait bounds; pick a topology that supplies them, e.g. -//! [`K8sAnywhereTopology`](crate::topology::K8sAnywhereTopology)). -//! -//! Argo is the next follow-up — same pattern, optional field, hangs -//! off the same sequential chain in [`Interpret::execute`]. -//! -//! NATS goes first deliberately: the operator pod's `NATS_URL` -//! environment points at the in-cluster NATS service, so installing -//! the operator first leads to a few minutes of `connect_with_retry` -//! noise in the operator log until NATS comes up. -//! -//! Idempotency: each inner Score is itself idempotent (NatsBasicScore -//! delegates to the upstream `nats/nats` chart via HelmChartScore, -//! FleetOperatorScore pins `chart_version`, ZitadelScore relies on -//! HelmChartScore + CNPG idempotency). Running this Score twice is -//! safe and produces no churn. -//! -//! Typical usage: -//! -//! ```ignore -//! use harmony::modules::fleet::{FleetOperatorScore, FleetServerScore}; -//! use harmony::modules::nats::NatsBasicScore; -//! use harmony::modules::zitadel::ZitadelScore; -//! use harmony::score::Score; -//! use harmony::inventory::Inventory; -//! use harmony::topology::K8sAnywhereTopology; -//! -//! let topology = K8sAnywhereTopology::from_env(); -//! let nats = NatsBasicScore::new("fleet-nats", "fleet-system").load_balancer(); -//! let operator = FleetOperatorScore::new() -//! .image("hub.nationtech.io/harmony/harmony-fleet-operator:dev"); -//! let mut score = FleetServerScore::new(nats, operator); -//! score.identity = Some(ZitadelScore { -//! host: "zitadel.localhost".into(), -//! external_secure: false, -//! ..Default::default() -//! }); -//! score.create_interpret().execute(&Inventory::empty(), &topology).await?; -//! ``` - -use async_trait::async_trait; -use harmony::data::Version; -use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; -use harmony::inventory::Inventory; -use harmony::modules::nats::capability::Nats; -use harmony::modules::nats::score_nats_k8s::NatsK8sScore; -use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; -use harmony::modules::postgresql::capability::PostgreSQL; -use harmony::modules::zitadel::{ZitadelScore, ZitadelSetupScore}; -use harmony::score::Score; -use harmony::topology::{HelmCommand, K8sclient, TlsRouter, Topology}; -use harmony_types::id::Id; -use serde::Serialize; - -use crate::operator::FleetOperatorScore; - -/// Install the fleet server-side stack on Kubernetes: NATS (via -/// `NatsK8sScore`, OKD-aware Routes), operator, optionally Zitadel -/// (helm + post-install identity setup), optionally the auth callout. -/// -/// Execution order is deliberate: -/// -/// 1. `identity` (Zitadel helm chart + cert-manager Route) FIRST -/// so the Zitadel API is reachable when `identity_setup` runs. -/// 2. `identity_setup` provisions project + roles + machine users -/// via the Zitadel management API. Produces the keys the -/// operator needs to authenticate. -/// 3. `nats` brings up the NATS pod. When the score's -/// `auth_callout` field is `Some`, the rendered helm values -/// include the matching `authorization.auth_callout` block -/// pointing at the (still-missing) callout pod's issuer. -/// 4. `auth_callout` deploys the callout pod itself. Once it's -/// Ready, NATS auth handshakes start succeeding. -/// 5. `operator` last. By this point Zitadel, NATS, and the -/// callout are all up; the operator's pod boots, mints its -/// Zitadel token, connects through the callout, watches its -/// CRDs. -#[derive(Debug, Clone, Serialize)] -pub struct FleetServerScore { - pub nats: NatsK8sScore, - pub operator: FleetOperatorScore, - /// Helm install of Zitadel + Postgres. Required when the rest - /// of the chain wants Zitadel-backed auth. - pub identity: Option, - /// Post-install Zitadel API calls — project, roles, machine - /// users, JWT keys. Without this the operator has no machine - /// user to authenticate as. - pub identity_setup: Option, - /// Auth callout deployment. Pair with - /// `nats.auth_callout = Some(...)` so the rendered NATS helm - /// values delegate to this callout's issuer pubkey. - pub auth_callout: Option, - // TODO(argo): pub gitops: Option, -} - -impl FleetServerScore { - pub fn new(nats: NatsK8sScore, operator: FleetOperatorScore) -> Self { - Self { - nats, - operator, - identity: None, - identity_setup: None, - auth_callout: None, - } - } -} - -impl Score - for FleetServerScore -{ - fn create_interpret(&self) -> Box> { - Box::new(FleetServerInterpret { - score: self.clone(), - }) - } - - fn name(&self) -> String { - "FleetServerScore".to_string() - } -} - -#[derive(Debug)] -pub struct FleetServerInterpret { - score: FleetServerScore, -} - -#[async_trait] -impl Interpret - for FleetServerInterpret -{ - async fn execute( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - // Use Score::interpret (not create_interpret().execute()) so each - // inner score fires its own InterpretExecutionStarted/Finished - // events — that's what gives the cli_logger the per-score - // progress lines and what surfaces the inner outcomes' details - // in the cli_reporter end-of-run summary. - - // 1. Identity: Zitadel helm chart needs to be up before the - // setup score can talk to its API. - if let Some(identity) = &self.score.identity { - identity.interpret(inventory, topology).await?; - } - // 2. Identity setup: project + roles + machine users via the - // Zitadel management API. Output (machine keys) feeds the - // operator's credentials at apply time. The operator - // Score has those bytes already (callers wire them in via - // `OperatorCredentials.credentials_toml` before passing - // `FleetServerScore` to the framework). - if let Some(setup) = &self.score.identity_setup { - setup.interpret(inventory, topology).await?; - } - // 3. NATS: helm-install NATS with whatever auth_callout - // config was baked into `nats` already. - self.score.nats.interpret(inventory, topology).await?; - // 4. Auth callout deployment. - if let Some(callout) = &self.score.auth_callout { - callout.interpret(inventory, topology).await?; - } - // 5. Operator last so it doesn't waste reconnect attempts - // while NATS / callout are still coming up. The pod's own - // `connect_with_retry` handles the small remaining race. - self.score.operator.interpret(inventory, topology).await?; - // TODO(argo): chain GitOps install here. - - // Hand the cli_reporter a few next-step pointers so the - // post-install summary at the bottom of the run is genuinely - // useful. - let mut details = vec![ - format!( - "NATS in-cluster URL: nats://{}.{}:4222", - self.score.nats.cluster.name, self.score.nats.cluster.namespace - ), - format!( - "Operator deployment: kubectl -n {} get deploy", - self.score.operator.namespace - ), - "CRDs: kubectl get crd | grep fleet.nationtech.io".to_string(), - ]; - if let Some(ws) = &self.score.nats.websocket { - details.push(format!("NATS WS public URL: wss://{}/", ws.host)); - } - if let Some(identity) = &self.score.identity { - let scheme = if identity.external_secure { - "https" - } else { - "http" - }; - details.push(format!( - "Zitadel: {scheme}://{}/ (kubectl -n zitadel get all)", - identity.host - )); - } - if self.score.auth_callout.is_some() { - details - .push("Auth callout: kubectl -n fleet-system get deploy/fleet-callout".to_string()); - } - Ok(Outcome::success_with_details( - "Fleet server stack installed".to_string(), - details, - )) - } - - fn get_name(&self) -> InterpretName { - InterpretName::Custom("FleetServerInterpret") - } - - fn get_version(&self) -> Version { - Version::from("0.1.0").expect("static version literal") - } - - fn get_status(&self) -> InterpretStatus { - InterpretStatus::QUEUED - } - - fn get_children(&self) -> Vec { - vec![] - } -} diff --git a/fleet/harmony-fleet-e2e/Cargo.toml b/fleet/harmony-fleet-e2e/Cargo.toml index c1c36666..987b173b 100644 --- a/fleet/harmony-fleet-e2e/Cargo.toml +++ b/fleet/harmony-fleet-e2e/Cargo.toml @@ -51,6 +51,7 @@ harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-deploy = { path = "../harmony-fleet-deploy" } harmony-fleet-operator = { path = "../harmony-fleet-operator" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony_secret = { path = "../../harmony_secret" } harmony-k8s = { path = "../../harmony-k8s" } harmony_types = { path = "../../harmony_types" } k3d-rs = { path = "../../k3d" } diff --git a/fleet/harmony-fleet-e2e/src/callout.rs b/fleet/harmony-fleet-e2e/src/callout.rs index a841f4e5..e4d5f99b 100644 --- a/fleet/harmony-fleet-e2e/src/callout.rs +++ b/fleet/harmony-fleet-e2e/src/callout.rs @@ -24,22 +24,20 @@ use std::time::Duration; use anyhow::{Context, Result}; use harmony::inventory::Inventory; use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; -use harmony::modules::k8s::resource::K8sResourceScore; +use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService}; use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; use harmony::modules::openbao::{ OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token, }; +use harmony::modules::postgresql::K8sPostgreSQLScore; use harmony::modules::zitadel::{ - MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, - ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials, + OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef, ZitadelAppType, + ZitadelClientConfig, ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials, }; use harmony::score::Score; use harmony::topology::K8sAnywhereTopology; -use harmony_fleet_deploy::FleetNatsScore; use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode}; -use k8s_openapi::api::core::v1::{Secret, Service, ServicePort, ServiceSpec}; -use kube::api::ObjectMeta; -use nkeys::KeyPair; +use k8s_openapi::api::core::v1::Secret; use serde::Deserialize; use crate::images::CALLOUT_IMAGE_TAG; @@ -70,9 +68,9 @@ pub const ADMIN_USERNAME: &str = "e2e-admin"; const NATS_ACCOUNT: &str = "FLEET"; const NATS_AUTH_USER: &str = "auth"; +const OPERATOR_APP: &str = "harmony-fleet-operator"; // Dev-only static credential for the callout's own NATS user, same // posture as the user-pass mode creds in `stack.rs`. -const NATS_AUTH_PASS: &str = "e2e-callout-auth"; const CALLOUT_DEPLOYMENT: &str = "fleet-callout"; /// Handles the callout bring-up leaves behind for tests. @@ -87,6 +85,9 @@ pub struct CalloutHandles { /// Zitadel IAM admin PAT (from the chart's `iam-admin-pat` /// Secret) — used to enroll device machine users. pub admin_pat: String, + pub provider: OidcProviderRef, + pub application: OidcApplicationRef, + pub operator_identity: OidcMachineIdentityRef, pub openbao: OpenbaoInstance, pub openbao_root_token: String, } @@ -95,72 +96,34 @@ pub fn issuer_url() -> String { format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}") } -/// NodePort Service in front of Zitadel with in-cluster port == -/// nodePort == [`ZITADEL_PORT`] (targetPort = the chart's 8080), so -/// the issuer URL's port number is valid both inside and outside the -/// cluster. -fn zitadel_nodeport_service() -> Service { - let selector = std::collections::BTreeMap::from([ - ("app.kubernetes.io/name".to_string(), "zitadel".to_string()), - ( - "app.kubernetes.io/instance".to_string(), - "zitadel".to_string(), - ), - ]); - Service { - metadata: ObjectMeta { - name: Some("zitadel-e2e".to_string()), - namespace: Some(ZITADEL_NAMESPACE.to_string()), - ..Default::default() - }, - spec: Some(ServiceSpec { - type_: Some("NodePort".to_string()), - selector: Some(selector), - ports: Some(vec![ServicePort { - name: Some("http".to_string()), - port: ZITADEL_PORT as i32, - target_port: Some( - k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(8080), - ), - node_port: Some(ZITADEL_PORT as i32), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - } -} - /// Deploy Zitadel + project/roles/admin-user, returning everything /// except the OpenBao handles (filled by [`deploy_openbao`]). pub async fn bring_up_zitadel( topology: &K8sAnywhereTopology, kube_client: &kube::Client, + outputs_namespace: &str, ) -> Result { - ZitadelScore { - host: ZITADEL_HOST.to_string(), - zitadel_version: "v4.12.1".to_string(), - external_secure: false, - external_port: Some(ZITADEL_PORT as u32), - namespace: ZITADEL_NAMESPACE.to_string(), - ..Default::default() - } - .interpret(&Inventory::autoload(), topology) - .await - .map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?; + let postgres = K8sPostgreSQLScore::new(ZITADEL_NAMESPACE).cluster_name("zitadel-pg"); + let database = postgres.root_account_ref(); + postgres + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel PostgreSQL deploy: {e}"))?; - K8sResourceScore::single( - zitadel_nodeport_service(), - Some(ZITADEL_NAMESPACE.to_string()), - ) - .interpret(&Inventory::autoload(), topology) - .await - .map_err(|e| anyhow::anyhow!("Zitadel NodePort service: {e}"))?; + let zitadel = ZitadelScore::new(ZITADEL_HOST, ZITADEL_NAMESPACE) + .http(Some(ZITADEL_PORT.into())) + .database(database) + .node_port(ZITADEL_PORT); + let provider = zitadel.provider_ref(); + zitadel + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?; CoreDNSRewriteScore { rewrites: vec![CoreDNSRewrite { hostname: ZITADEL_HOST.to_string(), - target: format!("zitadel-e2e.{ZITADEL_NAMESPACE}.svc.cluster.local"), + target: format!("zitadel-nodeport.{ZITADEL_NAMESPACE}.svc.cluster.local"), }], } .interpret(&Inventory::autoload(), topology) @@ -169,46 +132,26 @@ pub async fn bring_up_zitadel( wait_for_zitadel_ready().await?; - let roles = [ADMIN_ROLE, DEVICE_ROLE] + let mut identity = + ZitadelSetupScore::for_provider(&provider, ZITADEL_NAMESPACE, outputs_namespace) + .application(PROJECT_NAME, OPERATOR_APP, ZitadelAppType::DeviceCode) + .api_application(PROJECT_NAME, "nats") + .machine_identity(PROJECT_NAME, ADMIN_USERNAME, "E2E Admin", [ADMIN_ROLE]) + .groups_claim(); + identity.endpoint = Some(format!("http://127.0.0.1:{ZITADEL_PORT}")); + identity.skip_tls = true; + for role in [ADMIN_ROLE, DEVICE_ROLE] .into_iter() .chain(GROUP_ROLES.iter().copied()) - .map(|key| ZitadelRole { - project_name: PROJECT_NAME.to_string(), - key: key.to_string(), - display_name: key.to_string(), - group: None, - }) - .collect(); - - ZitadelSetupScore { - machine_users: vec![ZitadelMachineUser { - username: ADMIN_USERNAME.to_string(), - name: "E2E Admin".to_string(), - create_pat: false, - machine_key: Some(MachineKeyType::Json), - project_name: Some(PROJECT_NAME.to_string()), - grant_roles: vec![ADMIN_ROLE.to_string()], - }], - roles, - // The flattening Action turns project role keys into the - // `groups` array claim OpenBao binds against (ADR-025) — this - // bring-up is its live verification. - groups_claim_action: true, - human_users: vec![], - org_members: vec![], - instance_members: vec![], - machine_secrets: vec![], - port_forward_service: None, - applications: vec![], - api_apps: vec![ZitadelApiApp { - project_name: PROJECT_NAME.to_string(), - app_name: "nats".to_string(), - }], - ..zitadel_connection() + { + identity = identity.role(PROJECT_NAME, role, role); } - .interpret(&Inventory::autoload(), topology) - .await - .map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?; + let application = identity.application_ref(OPERATOR_APP); + let operator_identity = identity.machine_identity_ref(ADMIN_USERNAME); + identity + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?; let zcfg = ZitadelClientConfig::load() .context("ZitadelSetupScore did not produce a client config cache")?; @@ -228,6 +171,9 @@ pub async fn bring_up_zitadel( project_id, admin_machine_key, admin_pat, + provider, + application, + operator_identity, // Placeholder until deploy_openbao fills them in. openbao: OpenbaoInstance { namespace: String::new(), @@ -281,7 +227,7 @@ pub async fn enroll_device( } /// Deploy NATS in callout mode + the callout service itself. -/// Returns the `FleetNatsScore` so the caller can derive the +/// Returns the `NatsScore` so the caller can derive the /// in-cluster URL the same way the user-pass path does. pub async fn deploy_nats_callout( topology: &K8sAnywhereTopology, @@ -289,44 +235,38 @@ pub async fn deploy_nats_callout( release: &str, node_port: u16, handles: &CalloutHandles, -) -> Result { - let issuer_kp = KeyPair::new_account(); - let issuer_seed = issuer_kp - .seed() - .map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?; - - let nats = FleetNatsScore::callout( - namespace.to_string(), - node_port, - issuer_kp.public_key(), +) -> Result { + let credentials = NatsAuthCalloutCredentialsScore::generated( + namespace, + format!("{CALLOUT_DEPLOYMENT}-credentials"), + ); + let credentials_ref = credentials.credentials_ref(); + credentials + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("callout credentials deploy: {e}"))?; + let nats = NatsScore::callout_account( + release, + namespace, + NatsService::NodePort(node_port), NATS_ACCOUNT, + ); + let callout = NatsAuthCalloutScore::for_account( + CALLOUT_DEPLOYMENT, + namespace, + &nats.account_ref(), NATS_AUTH_USER, - NATS_AUTH_PASS, ) - .release_name(release); + .credentials(&credentials_ref) + .with_oidc(&handles.provider, &handles.application) + .image(CALLOUT_IMAGE_TAG) + .admin_role(ADMIN_ROLE) + .device_role(DEVICE_ROLE) + .device_id_claim("client_id"); + let nats = nats.with_auth_callout(&callout.auth_callout_ref()); nats.interpret(&Inventory::autoload(), topology) .await .map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?; - - let mut callout = NatsAuthCalloutScore::new( - CALLOUT_DEPLOYMENT, - namespace, - nats.in_cluster_url(), - handles.issuer_url.clone(), - handles.project_id.clone(), - NATS_AUTH_USER, - NATS_AUTH_PASS, - issuer_seed, - ) - .image(CALLOUT_IMAGE_TAG) - .target_account(NATS_ACCOUNT) - .admin_role(ADMIN_ROLE) - .device_role(DEVICE_ROLE); - // Machine-user access tokens carry the userName in `client_id`; - // usernames ARE device ids here (see `enroll_device`). - callout.device_id_claim = "client_id".to_string(); - callout.device_id_prefix_strip = String::new(); - callout.roles_claim = format!("urn:zitadel:iam:org:project:{}:roles", handles.project_id); callout .interpret(&Inventory::autoload(), topology) .await @@ -335,10 +275,6 @@ pub async fn deploy_nats_callout( Ok(nats) } -pub fn callout_deployment_name() -> &'static str { - CALLOUT_DEPLOYMENT -} - /// Deploy OpenBao into the bring-up namespace: JWT auth bound to the /// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging /// shape) plus a NodePort Service so VMs reach it through the libvirt @@ -352,70 +288,21 @@ pub async fn deploy_openbao( namespace: namespace.to_string(), release: OPENBAO_RELEASE.to_string(), }; - OpenbaoScore { - instance: instance.clone(), - host: "openbao-e2e.local".to_string(), - openshift: false, - tls_issuer: None, - } - .interpret(&Inventory::autoload(), topology) - .await - .map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?; - - OpenbaoSetupScore { - instance: instance.clone(), - kv_mount: KV_MOUNT.to_string(), - jwt_auth: Some(OpenbaoJwtAuth { - oidc_discovery_url: handles.issuer_url.clone(), - jwt_validation_pubkeys: String::new(), - bound_issuer: handles.issuer_url.clone(), - role_name: "fleet-device".to_string(), - bound_audiences: handles.project_id.clone(), - user_claim: "sub".to_string(), - groups_claim: "groups".to_string(), - token_type: "batch".to_string(), - ttl: "1h".to_string(), - max_ttl: "1h".to_string(), - }), - ..Default::default() - } - .interpret(&Inventory::autoload(), topology) - .await - .map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?; - - // NodePort for VM access. Same selector shape as the chart's own - // ClusterIP Service. - let selector = std::collections::BTreeMap::from([ - ("app.kubernetes.io/name".to_string(), "openbao".to_string()), - ( - "app.kubernetes.io/instance".to_string(), - OPENBAO_RELEASE.to_string(), - ), - ("component".to_string(), "server".to_string()), - ]); - let svc = Service { - metadata: ObjectMeta { - name: Some("openbao-nodeport".to_string()), - namespace: Some(namespace.to_string()), - ..Default::default() - }, - spec: Some(ServiceSpec { - type_: Some("NodePort".to_string()), - selector: Some(selector), - ports: Some(vec![ServicePort { - name: Some("http".to_string()), - port: 8200, - node_port: Some(OPENBAO_NODE_PORT as i32), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - }; - K8sResourceScore::single(svc, Some(namespace.to_string())) + OpenbaoScore::new(namespace, OPENBAO_RELEASE, "openbao-e2e.local") + .node_port(OPENBAO_NODE_PORT) .interpret(&Inventory::autoload(), topology) .await - .map_err(|e| anyhow::anyhow!("OpenBao NodePort service: {e}"))?; + .map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?; + + OpenbaoSetupScore::new(instance.clone()) + .with_oidc_application( + &handles.provider, + &handles.application, + OpenbaoJwtAuth::oidc("fleet-device"), + ) + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?; handles.openbao = instance.clone(); handles.openbao_root_token = diff --git a/fleet/harmony-fleet-e2e/src/kv_admin.rs b/fleet/harmony-fleet-e2e/src/kv_admin.rs index eca29dd6..fb586a7c 100644 --- a/fleet/harmony-fleet-e2e/src/kv_admin.rs +++ b/fleet/harmony-fleet-e2e/src/kv_admin.rs @@ -17,11 +17,11 @@ use std::time::{Duration, Instant}; use async_nats::Client; use async_nats::jetstream::{self, kv}; -use harmony::modules::podman::{PodmanV0Score, ReconcileScore}; use harmony_reconciler_contracts::{ BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase, desired_state_key, device_state_key, }; +use harmony_reconciler_contracts::{PodmanV0Score, ReconcileScore}; use thiserror::Error; /// One handle per test, opens the buckets the test reads/writes. diff --git a/fleet/harmony-fleet-e2e/src/lib.rs b/fleet/harmony-fleet-e2e/src/lib.rs index 76d25f6d..f7386f8a 100644 --- a/fleet/harmony-fleet-e2e/src/lib.rs +++ b/fleet/harmony-fleet-e2e/src/lib.rs @@ -1,6 +1,6 @@ //! In-cluster end-to-end harness for the harmony fleet stack. //! -//! Brings up the fleet stack ([`FleetNatsScore`] + +//! Brings up the fleet stack ([`NatsScore`] + //! [`FleetAgentScore`]) in a fresh per-bring-up k3d namespace and //! hands the test a connected NATS admin client + the device ids the //! agents are running under. Bring-up is shared across tests in the @@ -10,7 +10,7 @@ //! Per ADR-023, the harness composes the same `*Score` types the //! production CLI uses. It does **not** build manifests itself. //! -//! [`FleetNatsScore`]: harmony_fleet_deploy::FleetNatsScore +//! [`NatsScore`]: harmony::modules::nats::NatsScore //! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore pub mod callout; diff --git a/fleet/harmony-fleet-e2e/src/namespace.rs b/fleet/harmony-fleet-e2e/src/namespace.rs index a517fc73..295ae373 100644 --- a/fleet/harmony-fleet-e2e/src/namespace.rs +++ b/fleet/harmony-fleet-e2e/src/namespace.rs @@ -14,9 +14,13 @@ use std::sync::Arc; use anyhow::{Context, Result}; +use harmony::inventory::Inventory; +use harmony::modules::k8s::namespace::K8sNamespaceScore; +use harmony::score::Score; +use harmony::topology::K8sAnywhereTopology; use k8s_openapi::api::core::v1::Namespace; use kube::Client; -use kube::api::{Api, ObjectMeta, PostParams}; +use kube::api::Api; /// Env var that, when set to `"1"`, prevents the guard from deleting /// the namespace on drop. Convenience for debugging hung tests. @@ -43,24 +47,15 @@ impl std::fmt::Debug for NamespaceGuard { } impl NamespaceGuard { - pub(crate) async fn create(client: Arc, namespace: String) -> Result { - let api: Api = Api::all((*client).clone()); - let ns = Namespace { - metadata: ObjectMeta { - name: Some(namespace.clone()), - labels: Some( - [ - ("harmony.io/managed-by".to_string(), "fleet-e2e".to_string()), - ("harmony.io/scope".to_string(), "ephemeral".to_string()), - ] - .into_iter() - .collect(), - ), - ..Default::default() - }, - ..Default::default() - }; - api.create(&PostParams::default(), &ns) + pub(crate) async fn create( + client: Arc, + topology: &K8sAnywhereTopology, + namespace: String, + ) -> Result { + K8sNamespaceScore::new(&namespace) + .label("harmony.io/managed-by", "fleet-e2e") + .label("harmony.io/scope", "ephemeral") + .interpret(&Inventory::autoload(), topology) .await .with_context(|| format!("creating namespace {namespace}"))?; let keep = matches!( diff --git a/fleet/harmony-fleet-e2e/src/stack.rs b/fleet/harmony-fleet-e2e/src/stack.rs index a4fff157..f974874a 100644 --- a/fleet/harmony-fleet-e2e/src/stack.rs +++ b/fleet/harmony-fleet-e2e/src/stack.rs @@ -4,7 +4,7 @@ //! same `*Score` types production uses. The only thing this module //! owns is the *plumbing* around them: ensure a k3d cluster exists, //! sideload the agent image into it, mint a per-bring-up namespace, -//! compose [`FleetNatsScore`] + [`FleetAgentScore`] against the test +//! compose [`NatsScore`] + [`FleetAgentScore`] against the test //! Topology, wait for pods to be Ready, and hand the test an admin //! NATS client. //! @@ -24,15 +24,14 @@ use std::sync::Arc; use std::time::Duration; use harmony::inventory::Inventory; -use harmony::modules::fleet::operator::crd::Deployment as FleetDeployment; +use harmony::modules::nats::{NatsScore, NatsService, UserPassCredentials}; use harmony::score::Score; use harmony::topology::K8sAnywhereTopology; use harmony_fleet_deploy::agent::PodTarget; -use harmony_fleet_deploy::nats::UserPassCredentials; use harmony_fleet_deploy::operator::OperatorCredentials; -use harmony_fleet_deploy::{FleetAgentScore, FleetNatsScore, FleetOperatorScore}; +use harmony_fleet_deploy::{FleetAgentScore, FleetOperatorScore}; +use harmony_fleet_operator::Deployment as FleetDeployment; use k3d_rs::{K3d, PortMapping}; -use k8s_openapi::api::apps::v1::Deployment; use kube::Client; use kube::api::{Api, DeleteParams, Patch, PatchParams}; use thiserror::Error; @@ -182,14 +181,10 @@ impl Stack { /// use [`shared_stack`]. pub async fn bring_up(opts: StackOptions) -> Result { let callout_mode = opts.auth_mode == AuthMode::Callout; - // Static user-pass material is what the Pod agent + operator - // paths inject today; behind the callout they'd be rejected - // at connect. The VM harness brings callout-authenticated - // agents instead. - if callout_mode && (opts.num_devices > 0 || opts.deploy_operator) { + // Pod agents still use static credentials; callout-mode devices are VMs. + if callout_mode && opts.num_devices > 0 { return Err(BringUpError::Callout(anyhow::anyhow!( - "callout mode supports infra-only bring-up (VM agents); \ - Pod agents / operator still authenticate with static creds" + "callout mode uses VM agents; Pod agents still authenticate with static creds" ))); } @@ -236,12 +231,18 @@ impl Stack { prune_stale_operator_cluster_resources(&kube_client).await; } + let namespace = unique_namespace(); + let guard = NamespaceGuard::create(kube_client.clone(), &topology, namespace.clone()) + .await + .map_err(BringUpError::Namespace)?; + tracing::info!(namespace = %namespace, "namespace created"); + // Zitadel + project/roles/admin identity. Sits outside the // per-bring-up namespace (stable `zitadel` ns) so the // multi-minute first install amortizes across runs. let mut callout_handles = if callout_mode { Some( - crate::callout::bring_up_zitadel(&topology, &kube_client) + crate::callout::bring_up_zitadel(&topology, &kube_client, &namespace) .await .map_err(BringUpError::Callout)?, ) @@ -249,15 +250,9 @@ impl Stack { None }; - let namespace = unique_namespace(); - let guard = NamespaceGuard::create(kube_client.clone(), namespace.clone()) - .await - .map_err(BringUpError::Namespace)?; - tracing::info!(namespace = %namespace, "namespace created"); - // Deploy NATS via the production Score. The harness owns no // bespoke nats-server.conf, no NodePort Service — that's all - // inside `FleetNatsScore`. NATS readiness is gated by the + // inside `NatsScore`. NATS readiness is gated by the // host-side `connect_nats_admin` retry loop further down (the // upstream NATS chart deploys a StatefulSet, not a Deployment, // so a generic Deployment-poll wouldn't match; the agent's @@ -267,19 +262,20 @@ impl Stack { // deploy crate — keeping them out of `harmony-fleet-deploy` // means production builds can't accidentally ship them. let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}"); - let (nats_client, nats_in_cluster_url) = match callout_handles.as_mut() { + let (nats_client, nats_ref) = match callout_handles.as_mut() { None => { - let nats = FleetNatsScore::user_pass( + let nats = NatsScore::user_pass( + NATS_RELEASE, namespace.clone(), - NATS_NODE_PORT, + NatsService::NodePort(NATS_NODE_PORT), + "APP", UserPassCredentials { admin_user: NATS_ADMIN_USER.to_string(), admin_pass: NATS_ADMIN_PASS.to_string(), device_user: NATS_DEVICE_USER.to_string(), device_pass: NATS_DEVICE_PASS.to_string(), }, - ) - .release_name(NATS_RELEASE); + ); nats.interpret(&Inventory::autoload(), &topology) .await .map_err(BringUpError::NatsDeploy)?; @@ -288,7 +284,7 @@ impl Stack { // up, can't connect, hits CrashLoopBackOff for a // while, and the agent Deployment-Ready wait // spuriously times out. - (connect_nats_admin(&nats_url).await?, nats.in_cluster_url()) + (connect_nats_admin(&nats_url).await?, nats.client_ref()) } Some(handles) => { let nats = crate::callout::deploy_nats_callout( @@ -300,47 +296,46 @@ impl Stack { ) .await .map_err(BringUpError::Callout)?; - wait_for_deployment( - &kube_client, - &namespace, - crate::callout::callout_deployment_name(), - ) - .await?; let client = crate::callout::connect_nats_admin_sso(&nats_url, handles) .await .map_err(BringUpError::Callout)?; crate::callout::deploy_openbao(&topology, &namespace, handles) .await .map_err(BringUpError::Callout)?; - (client, nats.in_cluster_url()) + (client, nats.client_ref()) } }; if opts.deploy_operator { - let mut operator = FleetOperatorScore::new() + let mut operator = FleetOperatorScore::new(opts.operator_image.clone()) .namespace(namespace.clone()) - .image(opts.operator_image.clone()) - .nats_url(nats_in_cluster_url.clone()) + .messaging(&nats_ref) .log_level(opts.log_level.clone()); - operator.credentials = Some(OperatorCredentials { - credentials_toml: format!( - r#"type = "toml-shared" - nats_user = "{NATS_ADMIN_USER}" - nats_pass = "{NATS_ADMIN_PASS}" - "# - ), - }); + if let Some(handles) = callout_handles.as_ref() { + operator = operator.identity( + &handles.provider, + &handles.application, + &handles.operator_identity, + ); + } else { + operator.credentials = Some(OperatorCredentials { + credentials_toml: format!( + r#"type = "toml-shared" + nats_user = "{NATS_ADMIN_USER}" + nats_pass = "{NATS_ADMIN_PASS}" + "# + ), + }); + } operator .interpret(&Inventory::autoload(), &topology) .await .map_err(BringUpError::OperatorDeploy)?; - - wait_for_deployment(&kube_client, &namespace, "harmony-fleet-operator").await?; } // Deploy one agent Pod per requested device, all sharing the - // in-cluster NATS DNS the FleetNatsScore exposes. + // in-cluster NATS DNS the NatsScore exposes. let device_ids: Vec = if opts.deploy_agent { (0..opts.num_devices) .map(|i| format!("vm-device-{i:02}-{}", &namespace["e2e-".len()..])) @@ -356,7 +351,7 @@ impl Stack { PodTarget::user_pass( device_id.clone(), opts.agent_image.clone(), - nats_in_cluster_url.clone(), + nats_ref.url().to_string(), NATS_DEVICE_USER, NATS_DEVICE_PASS, ) @@ -367,14 +362,6 @@ impl Stack { .await .map_err(BringUpError::AgentDeploy)?; } - for device_id in &device_ids { - wait_for_deployment( - &kube_client, - &namespace, - &FleetAgentScore::resource_name(device_id), - ) - .await?; - } } Ok(Self { @@ -591,46 +578,6 @@ async fn prune_stale_namespaces(client: &Client) { } } -async fn wait_for_deployment( - client: &Client, - namespace: &str, - name: &str, -) -> Result<(), BringUpError> { - // Temporary readiness gate. ADR-023 principle 4 promises that a - // future smoke-test contract on the Score itself will replace this - // — `deploy` will only return after the agent is up. Today the - // Score returns when the manifest is applied; we poll for Ready - // here so each test can rely on the pods being reachable. - let api: Api = Api::namespaced(client.clone(), namespace); - let deadline = std::time::Instant::now() + Duration::from_secs(120); - while std::time::Instant::now() < deadline { - match api.get(name).await { - Ok(d) => { - if let Some(status) = d.status - && status.ready_replicas.unwrap_or(0) >= 1 - { - tracing::info!(namespace = %namespace, deployment = %name, "deployment ready"); - return Ok(()); - } - } - Err(kube::Error::Api(e)) if e.code == 404 => {} - Err(e) => { - return Err(BringUpError::WaitTimeout { - namespace: namespace.to_string(), - name: name.to_string(), - detail: format!("get deployment failed: {e}"), - }); - } - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - Err(BringUpError::WaitTimeout { - namespace: namespace.to_string(), - name: name.to_string(), - detail: "timed out after 120s".to_string(), - }) -} - async fn connect_nats_admin(url: &str) -> Result { // NATS helm chart cold-start is ~30-60s on a freshly-pulled // image. Keep retrying for 2 minutes before giving up so the diff --git a/fleet/harmony-fleet-e2e/src/vm/device.rs b/fleet/harmony-fleet-e2e/src/vm/device.rs index 0cd0a690..ca2c4564 100644 --- a/fleet/harmony-fleet-e2e/src/vm/device.rs +++ b/fleet/harmony-fleet-e2e/src/vm/device.rs @@ -23,15 +23,17 @@ use std::sync::Arc; use harmony::inventory::Inventory; use harmony::modules::fleet::{ - DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, - HARMONY_FLEET_POOL_NAME, HostsEntry, ProvisionVmScore, ensure_fleet_ssh_keypair, - ensure_harmony_fleet_pool, read_public_key, + HARMONY_FLEET_POOL_NAME, ProvisionVmScore, ensure_fleet_ssh_keypair, ensure_harmony_fleet_pool, + read_public_key, }; use harmony::modules::kvm::KvmVirtualMachineHost; use harmony::modules::kvm::config::init_executor; use harmony::modules::linux::{LinuxHostTopology, SshCommandOutput, SshCredentials, ssh_exec}; use harmony::score::Score; use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig}; +use harmony_fleet_deploy::{ + DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry, +}; use harmony_types::id::Id; use harmony_types::net::IpAddress; use thiserror::Error; diff --git a/fleet/harmony-fleet-e2e/src/vm/mod.rs b/fleet/harmony-fleet-e2e/src/vm/mod.rs index 28fd03df..d612cea9 100644 --- a/fleet/harmony-fleet-e2e/src/vm/mod.rs +++ b/fleet/harmony-fleet-e2e/src/vm/mod.rs @@ -3,7 +3,7 @@ //! NATS the Pod harness deploys. //! //! Per ADR-023 P2 the harness composes the same production Scores -//! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `FleetNatsScore`) +//! (`ProvisionVmScore`, `FleetDeviceSetupScore`, `NatsScore`) //! — the only thing the test layer owns is the test-fixture wiring //! (per-binary `OnceCell`, drop-time VM cleanup, KV admin helpers). //! diff --git a/fleet/harmony-fleet-e2e/src/vm/stack.rs b/fleet/harmony-fleet-e2e/src/vm/stack.rs index 033d179a..bc4b064d 100644 --- a/fleet/harmony-fleet-e2e/src/vm/stack.rs +++ b/fleet/harmony-fleet-e2e/src/vm/stack.rs @@ -26,14 +26,14 @@ use tokio::process::Command; use tokio::sync::OnceCell; use uuid::Uuid; -use harmony::modules::fleet::{DeviceOpenbao, FleetDeviceAuth, HostsEntry}; +use harmony_fleet_deploy::{DeviceOpenbao, FleetDeviceAuth, HostsEntry}; use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack}; use crate::vm::agent_build::{AgentBuildError, build_agent_for}; use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions}; use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip}; -/// NATS device user the [`crate::stack`] `FleetNatsScore` provisions +/// NATS device user the [`crate::stack`] `NatsScore` provisions /// in user-pass mode. Mirrors `stack::NATS_DEVICE_USER` (which is /// private to that module) so VM agents authenticate against the /// same user-pass account. @@ -300,14 +300,8 @@ impl VmStack { /// Wait until every agent in the stack answers `Verb::Ping`. /// - /// `FleetDeviceSetupScore` returns the moment `systemctl - /// start fleet-agent` exits — *seconds* before the agent's NATS - /// subscription has actually propagated. Pinging in that window - /// hits `no_responders` immediately (NATS knows there's no - /// subscriber and short-circuits). Tests that want the agent - /// reachable must wait through that window; this helper does it - /// by retrying ping with a 3 s per-attempt timeout up to - /// `per_device_budget`. + /// Deployment readiness verifies the systemd unit. This assertion also + /// covers the first OIDC exchange, NATS connection, and command round-trip. pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> { // Fresh connection on the caller's runtime — see // `Stack::connect_admin` for why the shared client can't be diff --git a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs index 98b56602..c5058878 100644 --- a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs +++ b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs @@ -5,15 +5,14 @@ //! live Zitadel, and tests mint their own device tokens. use harmony::inventory::Inventory; -use harmony::modules::fleet::secret_access::DeploymentSecretGrants; use harmony::modules::openbao::{ - OpenBaoDeploymentSecretGrants, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, - OpenbaoSetupScore, cached_root_token, + OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token, }; use harmony::score::Score; use harmony::topology::{K8sAnywhereTopology, K8sclient}; use harmony_fleet_e2e::{StackOptions, shared_stack}; -use harmony_reconciler_contracts::DeploymentName; +use harmony_reconciler_contracts::{DeploymentName, DeploymentSecretGrants}; +use harmony_secret::OpenBaoDeploymentSecretGrants; use serde_json::json; const E2E_ENV: &str = "HARMONY_FLEET_E2E"; @@ -169,6 +168,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result { host: "openbao-groups-test.local".to_string(), openshift: false, tls_issuer: None, + node_port: None, }), Box::new(OpenbaoSetupScore { instance: instance.clone(), @@ -187,6 +187,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result { ttl: "1h".to_string(), max_ttl: "1h".to_string(), }), + oidc_application: None, }), ]; for score in scores { diff --git a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs index 7b05102a..4a376326 100644 --- a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs +++ b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs @@ -89,6 +89,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"# host: "openbao-policy-test.local".to_string(), openshift: false, tls_issuer: None, + node_port: None, }), Box::new(OpenbaoSetupScore { instance: instance.clone(), @@ -103,6 +104,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"# policies: vec![], }], jwt_auth: None, + oidc_application: None, }), ]; diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index 2dcb6b22..f8b9383c 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -1,8 +1,8 @@ -use harmony::modules::fleet::operator::crd::{ +use harmony_fleet_e2e::{StackOptions, shared_stack}; +use harmony_fleet_operator::crd::{ Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore, Rollout, RolloutStrategy, }; -use harmony_fleet_e2e::{StackOptions, shared_stack}; use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use kube::Client; diff --git a/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs b/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs index a26e2fd7..f94f80a2 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs @@ -13,10 +13,9 @@ use std::time::Duration; -use harmony::modules::podman::{PodmanService, PodmanV0Score}; -use harmony::topology::RestartPolicy; use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack}; use harmony_reconciler_contracts::{DeploymentName, Phase}; +use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy}; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; diff --git a/fleet/harmony-fleet-e2e/tests/vm_isolation.rs b/fleet/harmony-fleet-e2e/tests/vm_isolation.rs index ac546498..595df316 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_isolation.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_isolation.rs @@ -19,12 +19,11 @@ use std::collections::BTreeMap; use std::time::Duration; -use harmony::modules::podman::{PodmanService, PodmanV0Score}; -use harmony::topology::RestartPolicy; +use harmony_fleet_auth::AgentObservation; use harmony_fleet_auth::{AgentConfig, AgentSection, CredentialsSection, NatsSection}; -use harmony_fleet_deploy::companion::AgentObservation; use harmony_fleet_e2e::{AdminKv, PhaseExpectation, VmStackOptions, shared_vm_stack}; use harmony_reconciler_contracts::{DeploymentName, Id, desired_state_key}; +use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy}; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; diff --git a/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs b/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs index b5eabfb3..bb2ec2e9 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs @@ -17,15 +17,16 @@ use std::time::Duration; -use harmony::modules::fleet::secret_access::DeploymentSecretGrants; -use harmony::modules::openbao::OpenBaoDeploymentSecretGrants; -use harmony::modules::podman::{EnvVar, PodmanService, PodmanV0Score, SecretEnvVar}; -use harmony::topology::{K8sAnywhereTopology, K8sclient, RestartPolicy}; +use harmony::topology::{K8sAnywhereTopology, K8sclient}; use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX}; use harmony_fleet_e2e::{ AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack, }; use harmony_reconciler_contracts::{DeploymentName, Phase}; +use harmony_reconciler_contracts::{ + DeploymentSecretGrants, EnvVar, PodmanService, PodmanV0Score, RestartPolicy, SecretEnvVar, +}; +use harmony_secret::OpenBaoDeploymentSecretGrants; use serde_json::json; const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; diff --git a/fleet/harmony-fleet-operator/Cargo.toml b/fleet/harmony-fleet-operator/Cargo.toml index 90dc0edb..a6d98e45 100644 --- a/fleet/harmony-fleet-operator/Cargo.toml +++ b/fleet/harmony-fleet-operator/Cargo.toml @@ -14,9 +14,9 @@ default = [] web-frontend = ["dep:axum", "dep:axum-extra", "dep:maud", "dep:tokio-stream", "harmony_zitadel_auth/axum"] [dependencies] -harmony = { path = "../../harmony", features = ["podman"] } harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony_secret = { path = "../../harmony_secret" } harmony_config = { path = "../../harmony_config" } harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } toml = { workspace = true } diff --git a/fleet/harmony-fleet-operator/src/access.rs b/fleet/harmony-fleet-operator/src/access.rs new file mode 100644 index 00000000..7a81bb7f --- /dev/null +++ b/fleet/harmony-fleet-operator/src/access.rs @@ -0,0 +1,110 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::sync::Mutex; + +use async_trait::async_trait; +use harmony_reconciler_contracts::{ + DeploymentName, DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError, +}; + +#[derive(Debug, Default)] +pub struct StaticDeviceGroups { + map: HashMap>, +} + +impl StaticDeviceGroups { + pub fn parse(spec: &str) -> Self { + let map = spec + .split(';') + .filter_map(|entry| { + let (device, groups) = entry.split_once('=')?; + let device = device.trim(); + (!device.is_empty()).then(|| { + ( + device.to_string(), + groups + .split('|') + .map(str::trim) + .filter(|group| !group.is_empty()) + .map(str::to_string) + .collect(), + ) + }) + }) + .collect(); + Self { map } + } +} + +#[async_trait] +impl DeviceGroupSource for StaticDeviceGroups { + async fn device_groups(&self) -> Result>, GroupSourceError> { + Ok(self.map.clone()) + } +} + +#[derive(Debug, Default)] +pub struct InMemoryDeploymentSecretGrants { + grants: Mutex>>, +} + +impl InMemoryDeploymentSecretGrants { + pub fn new() -> Self { + Self::default() + } + + pub fn groups_for(&self, deployment: &DeploymentName) -> Vec { + self.grants + .lock() + .unwrap() + .get(deployment) + .map(|groups| groups.iter().cloned().collect()) + .unwrap_or_default() + } +} + +#[async_trait] +impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants { + async fn set_deployment_groups( + &self, + grants: &[(DeploymentName, Vec)], + ) -> Result<(), SecretAccessError> { + let mut current = self.grants.lock().unwrap(); + for (deployment, groups) in grants { + if groups.is_empty() { + current.remove(deployment); + } else { + current.insert(deployment.clone(), groups.iter().cloned().collect()); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn static_groups_parse_membership() { + let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk"); + let groups = source.device_groups().await.unwrap(); + assert_eq!(groups.len(), 2); + assert_eq!(groups["pi-01"].len(), 2); + } + + #[tokio::test] + async fn grants_replace_and_clear_membership() { + let grants = InMemoryDeploymentSecretGrants::new(); + let deployment = DeploymentName::try_new("web").unwrap(); + grants + .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()])]) + .await + .unwrap(); + assert_eq!(grants.groups_for(&deployment), vec!["edge-a"]); + grants + .set_deployment_groups(&[(deployment.clone(), vec![])]) + .await + .unwrap(); + assert!(grants.groups_for(&deployment).is_empty()); + } +} diff --git a/fleet/harmony-fleet-operator/src/controller.rs b/fleet/harmony-fleet-operator/src/controller.rs index 2f54180a..39b18658 100644 --- a/fleet/harmony-fleet-operator/src/controller.rs +++ b/fleet/harmony-fleet-operator/src/controller.rs @@ -33,7 +33,7 @@ use kube::runtime::finalizer::{Event as FinalizerEvent, finalizer}; use kube::runtime::watcher::Config as WatcherConfig; use kube::{Api, Client, ResourceExt}; -use harmony::modules::fleet::operator::Deployment; +use harmony_fleet_operator::Deployment; const FINALIZER: &str = "fleet.nationtech.io/finalizer"; diff --git a/harmony/src/modules/fleet/operator/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs similarity index 98% rename from harmony/src/modules/fleet/operator/crd.rs rename to fleet/harmony-fleet-operator/src/crd.rs index badcf893..70b5f80b 100644 --- a/harmony/src/modules/fleet/operator/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -4,7 +4,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -pub use crate::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore}; +pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore}; /// Deployment intent. Targets devices by label selector — identical /// to the pattern K8s itself uses for DaemonSet nodeSelector, Service diff --git a/fleet/harmony-fleet-operator/src/device_reconciler.rs b/fleet/harmony-fleet-operator/src/device_reconciler.rs index 617f9f5d..6f2dba9b 100644 --- a/fleet/harmony-fleet-operator/src/device_reconciler.rs +++ b/fleet/harmony-fleet-operator/src/device_reconciler.rs @@ -18,7 +18,7 @@ use kube::Client; use kube::api::{Api, DeleteParams, Patch, PatchParams}; use std::collections::BTreeMap; -use harmony::modules::fleet::operator::{Device, DeviceSpec}; +use crate::crd::{Device, DeviceSpec}; const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler"; diff --git a/fleet/harmony-fleet-operator/src/device_status.rs b/fleet/harmony-fleet-operator/src/device_status.rs index 052c9e0b..a09e7b2a 100644 --- a/fleet/harmony-fleet-operator/src/device_status.rs +++ b/fleet/harmony-fleet-operator/src/device_status.rs @@ -24,7 +24,7 @@ use kube::{Client, ResourceExt}; use serde_json::json; use tokio::sync::Mutex; -use harmony::modules::fleet::operator::{Device, Reachability}; +use crate::crd::{Device, Reachability}; /// A device with no heartbeat within this window is `Stale`. Agents /// ping every 30 s, so this tolerates ~2 missed pings. diff --git a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs index 41c7cffb..6dd18b4a 100644 --- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs +++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs @@ -33,11 +33,9 @@ use kube::{Client, ResourceExt}; use serde_json::json; use tokio::sync::Mutex; -use harmony::modules::fleet::groups::DeviceGroupSource; -use harmony::modules::fleet::operator::{ - AggregateLastError, Deployment, DeploymentAggregate, Device, -}; -use harmony::modules::fleet::secret_access::DeploymentSecretGrants; +use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource}; + +use crate::crd::{AggregateLastError, Deployment, DeploymentAggregate, Device}; use tracing::warn; const PATCH_TICK: Duration = Duration::from_secs(1); @@ -355,38 +353,20 @@ async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow: Ok(()) } -/// Push the allowed-groups grant for `deployments` to the secrets -/// backend in one batch. A deployment absent from the cache (deleted, or -/// no `allowedGroups` declared) resolves to the empty set = revocation. -/// Non-fatal: a backend hiccup logs and is retried on the next -/// deployment change rather than tearing down reconciliation. Skipped -/// entirely when no backend is configured (dev without OpenBao). -async fn sync_deployment_grants( - state: &SharedFleetState, - grants: Option<&Arc>, - deployments: impl IntoIterator, +async fn sync_grant_batch( + grants: &Arc, + batch: &[(DeploymentName, Vec)], ) { - let Some(grants) = grants else { return }; - let batch: Vec<(DeploymentName, Vec)> = { - let guard = state.lock().await; - deployments - .into_iter() - .map(|dn| { - let groups = guard - .deployments - .values() - .find(|c| c.deployment_name == dn) - .and_then(|c| c.allowed_groups.clone()) - .unwrap_or_default(); - (dn, groups) - }) - .collect() - }; - if batch.is_empty() { - return; - } - if let Err(e) = grants.set_deployment_groups(&batch).await { - warn!(error = %e, "aggregator: secret grant sync failed"); + let mut delay = Duration::from_secs(1); + loop { + match grants.set_deployment_groups(batch).await { + Ok(()) => return, + Err(error) => { + warn!(%error, ?delay, "aggregator: secret grant sync failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(30)); + } + } } } @@ -513,6 +493,19 @@ async fn on_deployment_upsert( score_json: score_json.clone(), }; + // Keep the deployment out of schedulable state until its secret grant + // exists; device and group watchers share the cache below. + if let Some(grants) = grants { + sync_grant_batch( + grants, + &[( + deployment_name.clone(), + cached.allowed_groups.clone().unwrap_or_default(), + )], + ) + .await; + } + let (new_targets, previous_targets) = { let mut guard = state.lock().await; let new_targets = matched_devices(&cached, &guard); @@ -528,10 +521,6 @@ async fn on_deployment_upsert( (new_targets, previous) }; - // Grant before workload signal: never tell a device to run a - // deployment whose secrets it cannot yet read. - sync_deployment_grants(state, grants, std::iter::once(deployment_name.clone())).await; - reconcile_kv( desired, &deployment_name, @@ -555,6 +544,10 @@ async fn on_deployment_delete( return; }; + if let Some(grants) = grants { + sync_grant_batch(grants, &[(deployment_name.clone(), Vec::new())]).await; + } + let previous = { let mut guard = state.lock().await; guard.deployments.remove(&key); @@ -575,9 +568,6 @@ async fn on_deployment_delete( tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed"); } } - - // Gone from the cache, so the grant resolves to empty = revocation. - sync_deployment_grants(state, grants, std::iter::once(deployment_name)).await; } // --------------------------------------------------------------------------- @@ -1041,19 +1031,19 @@ mod tests { #[tokio::test] async fn sync_pushes_declared_groups_per_deployment() { - use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; - - let mut s = FleetState::default(); - let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a", "edge-b"]); - let db = cached("ns", "db", "zone", "lab"); // no allowedGroups → no grant - s.deployments.insert(web.key.clone(), web); - s.deployments.insert(db.key.clone(), db); - let state: SharedFleetState = Arc::new(Mutex::new(s)); + use crate::access::InMemoryDeploymentSecretGrants; let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); let grants: Arc = inmem.clone(); - sync_deployment_grants(&state, Some(&grants), [dn("web"), dn("db")]).await; + sync_grant_batch( + &grants, + &[ + (dn("web"), vec!["edge-a".into(), "edge-b".into()]), + (dn("db"), vec![]), + ], + ) + .await; assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]); assert!(inmem.groups_for(&dn("db")).is_empty()); @@ -1061,7 +1051,7 @@ mod tests { #[tokio::test] async fn sync_revokes_deleted_deployment() { - use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; + use crate::access::InMemoryDeploymentSecretGrants; let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); let grants: Arc = inmem.clone(); @@ -1070,13 +1060,38 @@ mod tests { .await .unwrap(); - // Deployment no longer cached (CR deleted) → grant resolves empty. - let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default())); - sync_deployment_grants(&state, Some(&grants), std::iter::once(dn("web"))).await; + sync_grant_batch(&grants, &[(dn("web"), vec![])]).await; assert!(inmem.groups_for(&dn("web")).is_empty()); } + #[tokio::test] + async fn grant_sync_retries_before_returning() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FlakyGrants(AtomicUsize); + + #[async_trait::async_trait] + impl DeploymentSecretGrants for FlakyGrants { + async fn set_deployment_groups( + &self, + _grants: &[(DeploymentName, Vec)], + ) -> Result<(), harmony_reconciler_contracts::SecretAccessError> { + if self.0.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(harmony_reconciler_contracts::SecretAccessError::Backend( + "temporary failure".into(), + )); + } + Ok(()) + } + } + + let flaky = Arc::new(FlakyGrants(AtomicUsize::new(0))); + let grants: Arc = flaky.clone(); + sync_grant_batch(&grants, &[(dn("web"), vec!["edge-a".into()])]).await; + assert_eq!(flaky.0.load(Ordering::SeqCst), 2); + } + #[test] fn membership_change_reschedules_device() { let mut s = FleetState::default(); diff --git a/fleet/harmony-fleet-operator/src/lib.rs b/fleet/harmony-fleet-operator/src/lib.rs index da2f81d6..ddadb0c2 100644 --- a/fleet/harmony-fleet-operator/src/lib.rs +++ b/fleet/harmony-fleet-operator/src/lib.rs @@ -6,13 +6,17 @@ //! documentation generators, alternative entry points — can drive //! the reconcilers without `main.rs`'s bootstrap. //! -//! CRD type definitions (`Deployment`, `Device`, `DeploymentSpec`, -//! `ScorePayload`, …) live in -//! [`harmony::modules::fleet::operator`] so that -//! `harmony::modules::fleet::FleetOperatorScore` can reach them -//! without a workspace dependency cycle. Import them from there. +//! CRD type definitions live in this runtime crate and are reused by +//! `harmony-fleet-deploy` when installing the operator. +pub mod access; pub mod commands; +pub mod crd; pub mod device_reconciler; pub mod device_status; pub mod fleet_aggregator; + +pub use crd::{ + AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device, + DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy, +}; diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index cca8608b..599cb33e 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -5,11 +5,11 @@ mod frontend; #[cfg(feature = "web-frontend")] mod service; -use harmony::modules::fleet::groups::{DeviceGroupSource, StaticDeviceGroups}; -use harmony::modules::fleet::secret_access::DeploymentSecretGrants; -use harmony::modules::openbao::OpenBaoDeploymentSecretGrants; -use harmony::modules::zitadel::ZitadelDeviceGroups; +use harmony_fleet_operator::access::StaticDeviceGroups; use harmony_fleet_operator::{device_reconciler, fleet_aggregator}; +use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource}; +use harmony_secret::OpenBaoDeploymentSecretGrants; +use harmony_zitadel_auth::ZitadelDeviceGroups; use anyhow::{Context, Result}; use async_nats::jetstream; @@ -20,7 +20,6 @@ use harmony_fleet_auth::{ }; use harmony_reconciler_contracts::BUCKET_DESIRED_STATE; use kube::Client; -#[cfg(feature = "web-frontend")] use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -62,6 +61,15 @@ struct Cli { )] credentials_toml: String, + #[arg(long, env = "FLEET_OPERATOR_OIDC_ISSUER", global = true)] + oidc_issuer: Option, + + #[arg(long, env = "FLEET_OPERATOR_OIDC_AUDIENCE", global = true)] + oidc_audience: Option, + + #[arg(long, env = "FLEET_OPERATOR_OIDC_KEY_PATH", global = true)] + oidc_key_path: Option, + #[arg(long, env = "OPENBAO_URL", global = true)] openbao_url: Option, @@ -108,12 +116,28 @@ async fn main() -> Result<()> { .init(); let cli = Cli::parse(); + let credentials_toml = if cli.credentials_toml.is_empty() { + match (&cli.oidc_issuer, &cli.oidc_audience, &cli.oidc_key_path) { + (Some(issuer), Some(audience), Some(key_path)) => { + toml::to_string(&CredentialsSection::ZitadelJwt { + key_path: key_path.clone(), + key_json: None, + oidc_issuer_url: issuer.clone(), + audience: audience.clone(), + danger_accept_invalid_certs: false, + })? + } + _ => String::new(), + } + } else { + cli.credentials_toml.clone() + }; match cli.command.unwrap_or(Command::Run) { Command::Run => { run( &cli.nats_url, &cli.kv_bucket, - &cli.credentials_toml, + &credentials_toml, &cli.openbao_url, &cli.openbao_token, ) diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index 5e19aef4..611140a5 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -17,11 +17,11 @@ use kube::api::{Api, ListParams, Patch, PatchParams}; use kube::{Client, ResourceExt}; use serde_json::json; -use harmony::modules::fleet::operator::{ +use harmony_fleet_operator::crd::{ Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability, }; -use harmony::modules::podman::ReconcileScore; use harmony_fleet_operator::fleet_aggregator::selector_matches; +use harmony_reconciler_contracts::ReconcileScore; use super::{ Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail, @@ -390,7 +390,7 @@ impl FleetService for RealFleetService { #[cfg(test)] mod tests { use super::*; - use harmony::modules::podman::{PodmanService, PodmanV0Score}; + use harmony_reconciler_contracts::{PodmanService, PodmanV0Score}; fn liveness(r: Reachability) -> DeviceLiveness { DeviceLiveness { diff --git a/harmony-k8s/src/resources.rs b/harmony-k8s/src/resources.rs index de377bc7..f320ecc7 100644 --- a/harmony-k8s/src/resources.rs +++ b/harmony-k8s/src/resources.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; use k8s_openapi::api::{ - apps::v1::Deployment, - core::v1::{Namespace, Node, ServiceAccount}, + apps::v1::{Deployment, StatefulSet}, + core::v1::{Namespace, Node, Pod, ServiceAccount}, }; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::api::ApiResource; @@ -268,10 +268,84 @@ impl K8sClient { }; let timeout = timeout.unwrap_or(Duration::from_secs(120)); let establish = await_condition(api, name, conditions::is_deployment_completed()); - tokio::time::timeout(timeout, establish) - .await - .map(|_| ()) - .map_err(|_| "Timed out waiting for deployment".to_string()) + match tokio::time::timeout(timeout, establish).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(error)) => Err(format!("Failed waiting for deployment {name}: {error}")), + Err(_) => Err(format!( + "Timed out after {}s waiting for deployment {name}", + timeout.as_secs() + )), + } + } + + pub async fn wait_until_statefulset_ready( + &self, + name: &str, + namespace: Option<&str>, + timeout: Option, + ) -> Result<(), String> { + let timeout = timeout.unwrap_or(Duration::from_secs(120)); + match tokio::time::timeout(timeout, async { + loop { + match self.get_resource::(name, namespace).await { + Ok(Some(statefulset)) + if statefulset.status.as_ref().is_some_and(|status| { + status.ready_replicas.unwrap_or_default() > 0 + }) => + { + return Ok(()); + } + Ok(_) => {} + Err(error) => { + return Err(format!("Failed waiting for statefulset {name}: {error}")); + } + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + { + Ok(result) => result, + Err(_) => Err(format!( + "Timed out after {}s waiting for statefulset {name}", + timeout.as_secs() + )), + } + } + + pub async fn wait_until_pod_running( + &self, + name: &str, + namespace: Option<&str>, + timeout: Option, + ) -> Result<(), String> { + let timeout = timeout.unwrap_or(Duration::from_secs(120)); + match tokio::time::timeout(timeout, async { + loop { + match self.get_resource::(name, namespace).await { + Ok(Some(pod)) + if pod + .status + .as_ref() + .and_then(|status| status.phase.as_deref()) + == Some("Running") => + { + return Ok(()); + } + Ok(_) => {} + Err(error) => return Err(format!("Failed waiting for pod {name}: {error}")), + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + { + Ok(result) => result, + Err(_) => Err(format!( + "Timed out after {}s waiting for pod {name}", + timeout.as_secs() + )), + } } /// Gets a single named resource, using the correct API scope for `K`. diff --git a/harmony-reconciler-contracts/Cargo.toml b/harmony-reconciler-contracts/Cargo.toml index 8eed0c83..54da5ca1 100644 --- a/harmony-reconciler-contracts/Cargo.toml +++ b/harmony-reconciler-contracts/Cargo.toml @@ -14,6 +14,7 @@ license.workspace = true # module. None of those consumers should pay for the others' deps. [dependencies] +async-trait.workspace = true chrono = { workspace = true, features = ["serde"] } harmony_types = { path = "../harmony_types" } schemars = "0.8.22" diff --git a/harmony-reconciler-contracts/src/access.rs b/harmony-reconciler-contracts/src/access.rs new file mode 100644 index 00000000..dc120117 --- /dev/null +++ b/harmony-reconciler-contracts/src/access.rs @@ -0,0 +1,30 @@ +use std::collections::{HashMap, HashSet}; + +use async_trait::async_trait; + +use crate::DeploymentName; + +#[derive(Debug, thiserror::Error)] +pub enum GroupSourceError { + #[error("group source error: {0}")] + Source(String), +} + +#[async_trait] +pub trait DeviceGroupSource: Send + Sync { + async fn device_groups(&self) -> Result>, GroupSourceError>; +} + +#[derive(Debug, thiserror::Error)] +pub enum SecretAccessError { + #[error("secret access backend error: {0}")] + Backend(String), +} + +#[async_trait] +pub trait DeploymentSecretGrants: Send + Sync { + async fn set_deployment_groups( + &self, + grants: &[(DeploymentName, Vec)], + ) -> Result<(), SecretAccessError>; +} diff --git a/harmony-reconciler-contracts/src/kv.rs b/harmony-reconciler-contracts/src/kv.rs index 8ea10302..c494c89a 100644 --- a/harmony-reconciler-contracts/src/kv.rs +++ b/harmony-reconciler-contracts/src/kv.rs @@ -11,7 +11,7 @@ use crate::fleet::DeploymentName; /// Operator-written bucket. One entry per `(device, deployment)` pair. /// Values are the JSON-serialized Score envelope — today -/// `harmony::modules::podman::ReconcileScore`, tomorrow any variant of +/// `crate::ReconcileScore`, tomorrow any variant of /// a polymorphic `Score` enum the framework ships. pub const BUCKET_DESIRED_STATE: &str = "desired-state"; diff --git a/harmony-reconciler-contracts/src/lib.rs b/harmony-reconciler-contracts/src/lib.rs index e1c6cba2..65ba8cb2 100644 --- a/harmony-reconciler-contracts/src/lib.rs +++ b/harmony-reconciler-contracts/src/lib.rs @@ -9,18 +9,22 @@ //! //! This crate holds the wire-format bits both sides must agree on: //! NATS bucket names, KV key formats, and the typed payloads -//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`). The Score -//! types themselves live in their respective harmony modules. +//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`, and Podman +//! desired state). //! //! **Deliberately lean** — no tokio, no async-nats, no harmony. //! The on-device agent build pulls it in alongside a minimal //! async-nats client; the operator pulls it alongside kube-rs. +pub mod access; pub mod commands; pub mod fleet; pub mod kv; +pub mod podman; pub mod status; +pub use access::{DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError}; + pub use commands::{ CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB, HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject, @@ -34,6 +38,9 @@ pub use kv::{ desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key, device_state_key, }; +pub use podman::{ + EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount, +}; pub use status::{InventorySnapshot, Phase}; // Re-exports so consumers (agent, operator) don't need a direct diff --git a/harmony-reconciler-contracts/src/podman.rs b/harmony-reconciler-contracts/src/podman.rs new file mode 100644 index 00000000..32c55d98 --- /dev/null +++ b/harmony-reconciler-contracts/src/podman.rs @@ -0,0 +1,208 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvVar { + pub name: String, + pub value: String, +} + +impl EnvVar { + pub fn new(name: impl Into, value: impl Into) -> Self { + Self { + name: name.into(), + value: value.into(), + } + } +} + +impl From<(String, String)> for EnvVar { + fn from((name, value): (String, String)) -> Self { + Self { name, value } + } +} + +impl From<(&str, &str)> for EnvVar { + fn from((name, value): (&str, &str)) -> Self { + Self::new(name, value) + } +} + +#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)] +pub struct VolumeMount { + pub host_path: String, + pub container_path: String, + #[serde(default)] + pub read_only: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum RestartPolicy { + No, + #[default] + UnlessStopped, + OnFailure, + Always, +} + +impl RestartPolicy { + pub fn as_str(&self) -> &'static str { + match self { + Self::No => "no", + Self::UnlessStopped => "unless-stopped", + Self::OnFailure => "on-failure", + Self::Always => "always", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct SecretEnvVar { + pub name: String, + pub secret: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct PodmanService { + pub name: String, + pub image: String, + pub ports: Vec, + #[serde(default)] + pub env: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secret_env: Vec, + #[serde(default)] + pub volumes: Vec, + #[serde(default)] + pub restart_policy: RestartPolicy, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct PodmanV0Score { + pub services: Vec, +} + +impl PodmanV0Score { + pub fn deployment_label(&self) -> String { + self.services + .iter() + .map(|service| service.name.as_str()) + .collect::>() + .join(",") + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "data")] +pub enum ReconcileScore { + PodmanV0(PodmanV0Score), +} + +#[cfg(test)] +mod tests { + use super::*; + + fn service(name: &str, image: &str) -> PodmanService { + PodmanService { + name: name.into(), + image: image.into(), + ports: vec![], + env: vec![], + secret_env: vec![], + volumes: vec![], + restart_policy: RestartPolicy::default(), + } + } + + #[test] + fn score_wire_format_and_roundtrip_are_stable() { + let score = ReconcileScore::PodmanV0(PodmanV0Score { + services: vec![PodmanService { + ports: vec!["8080:80".into()], + env: vec![EnvVar::new("LOG_LEVEL", "info")], + volumes: vec![VolumeMount { + host_path: "/var/lib/web/data".into(), + container_path: "/data".into(), + read_only: false, + }], + restart_policy: RestartPolicy::Always, + ..service("web", "nginx:latest") + }], + }); + let json = serde_json::to_string(&score).unwrap(); + assert!(json.contains("\"type\":\"PodmanV0\"")); + assert!(json.contains("\"data\"")); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + score + ); + } + + #[test] + fn legacy_service_defaults_are_stable() { + let legacy = r#"{"type":"PodmanV0","data":{"services":[{"name":"web","image":"nginx","ports":["8080:80"]}]}}"#; + let ReconcileScore::PodmanV0(score) = serde_json::from_str(legacy).unwrap(); + let service = &score.services[0]; + assert!(service.env.is_empty()); + assert!(service.secret_env.is_empty()); + assert!(service.volumes.is_empty()); + assert_eq!(service.restart_policy, RestartPolicy::UnlessStopped); + } + + #[test] + fn empty_secret_env_remains_omitted() { + let json = serde_json::to_string(&service("web", "nginx")).unwrap(); + assert!(!json.contains("secret_env")); + + let with_secret = PodmanService { + secret_env: vec![SecretEnvVar { + name: "DB_PASSWORD".into(), + secret: "db_password".into(), + }], + ..service("api", "myapp") + }; + assert_eq!( + serde_json::from_str::(&serde_json::to_string(&with_secret).unwrap()) + .unwrap(), + with_secret + ); + } + + #[test] + fn restart_policy_wire_names_are_stable() { + assert_eq!( + serde_json::to_string(&RestartPolicy::UnlessStopped).unwrap(), + "\"unless-stopped\"" + ); + assert_eq!( + serde_json::to_string(&RestartPolicy::OnFailure).unwrap(), + "\"on-failure\"" + ); + } + + #[test] + fn env_order_is_preserved() { + let service = PodmanService { + env: vec![EnvVar::new("B", "2"), EnvVar::new("A", "1")], + ..service("api", "myapp") + }; + let roundtrip: PodmanService = + serde_json::from_str(&serde_json::to_string(&service).unwrap()).unwrap(); + assert_eq!(roundtrip.env, service.env); + } + + #[test] + fn env_schema_is_not_a_tuple() { + let schema = serde_json::to_value(schemars::schema_for!(PodmanService)).unwrap(); + assert!(schema.pointer("/properties/env/items").unwrap().is_object()); + } + + #[test] + fn deployment_label_joins_service_names() { + let score = PodmanV0Score { + services: vec![service("web", "nginx"), service("api", "myapp")], + }; + assert_eq!(score.deployment_label(), "web,api"); + } +} diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 7ad95cbb..048ff3e4 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -6,13 +6,9 @@ readme.workspace = true license.workspace = true [features] -# `podman` lights up the podman module AND the fleet operator's -# CRD types (which embed `ReconcileScore` from the podman module). -# Without it the operator CRD fails to compile, so it's part of -# the default build. default = ["podman"] kvm = ["dep:virt"] -podman = ["dep:podman-api"] +podman = [] testing = [] [dependencies] @@ -99,7 +95,7 @@ inquire.workspace = true brocade = { path = "../brocade" } option-ext = "0.2.0" rand.workspace = true -podman-api = { version = "0.9", optional = true } +nkeys = "0.4" sysinfo = "0.30" # virt (libvirt bindings) uses std::os::unix and only compiles on Unix targets. diff --git a/harmony/src/domain/topology/container_runtime.rs b/harmony/src/domain/topology/container_runtime.rs index a5f7a79f..0744a4c4 100644 --- a/harmony/src/domain/topology/container_runtime.rs +++ b/harmony/src/domain/topology/container_runtime.rs @@ -1,9 +1,10 @@ use async_trait::async_trait; -use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::executors::ExecutorError; +pub use harmony_reconciler_contracts::{EnvVar, RestartPolicy, VolumeMount}; + /// Capability: run containers on a node-level runtime. /// /// Models the subset of container-runtime operations Harmony Scores need in @@ -38,41 +39,6 @@ pub trait ContainerRuntime: Send + Sync { async fn list_managed_services(&self) -> Result, ExecutorError>; } -/// A single environment variable, name/value pair. -/// -/// Modeled as a struct rather than `(String, String)` because k8s CRD -/// validation rejects tuple schemas (`items: [, ]`) — -/// schemars generates that for tuples and apiextensions returns -/// `Forbidden: items must be a schema object and not an array`. The -/// struct shape produces `items: { type: object, properties: { name, -/// value } }`, which validates cleanly. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -pub struct EnvVar { - pub name: String, - pub value: String, -} - -impl EnvVar { - pub fn new(name: impl Into, value: impl Into) -> Self { - Self { - name: name.into(), - value: value.into(), - } - } -} - -impl From<(String, String)> for EnvVar { - fn from((name, value): (String, String)) -> Self { - Self { name, value } - } -} - -impl From<(&str, &str)> for EnvVar { - fn from((name, value): (&str, &str)) -> Self { - Self::new(name, value) - } -} - /// Desired state for a single container. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ContainerSpec { @@ -109,51 +75,6 @@ impl ContainerSpec { pub const MANAGED_BY_VALUE: &'static str = "harmony"; } -/// A single host-path → container-path bind mount. Bind mounts are the only -/// volume kind supported in v0 — they cover ~95% of compose use cases and -/// don't depend on a runtime-managed volume namespace. -#[derive(Debug, Clone, PartialEq, JsonSchema, Eq, Serialize, Deserialize)] -pub struct VolumeMount { - /// Absolute path on the host. - pub host_path: String, - /// Absolute path inside the container. - pub container_path: String, - /// Mount as read-only. Defaults to false (read-write) to match - /// docker-compose's default. - #[serde(default)] - pub read_only: bool, -} - -/// Restart policy for a managed container. Names follow podman/docker -/// conventions so docker-compose translation is mechanical. -#[derive(Debug, Clone, Copy, PartialEq, JsonSchema, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -pub enum RestartPolicy { - /// Don't restart on exit. - No, - /// Restart unless the user explicitly stopped the container. - /// Docker-compose's default for long-running services and what most - /// fleet workloads want. - #[default] - UnlessStopped, - /// Restart only if the container exits with a non-zero status. - OnFailure, - /// Always restart, even on clean exits and after host reboot. - Always, -} - -impl RestartPolicy { - /// Canonical string podman + docker accept on the CLI / in their APIs. - pub fn as_str(&self) -> &'static str { - match self { - RestartPolicy::No => "no", - RestartPolicy::UnlessStopped => "unless-stopped", - RestartPolicy::OnFailure => "on-failure", - RestartPolicy::Always => "always", - } - } -} - /// Observed state of a container on the runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ContainerState { diff --git a/harmony/src/domain/topology/host_configuration.rs b/harmony/src/domain/topology/host_configuration.rs index d63c0300..1dc29b35 100644 --- a/harmony/src/domain/topology/host_configuration.rs +++ b/harmony/src/domain/topology/host_configuration.rs @@ -97,6 +97,12 @@ pub trait SystemdManager: Send + Sync { scope: SystemdScope, ) -> Result; + async fn systemd_unit_active( + &self, + name: &str, + scope: SystemdScope, + ) -> Result; + /// Enable+start a user-scoped unit (e.g. `podman.socket` under /// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has /// already been called for the user. diff --git a/harmony/src/modules/fleet/groups.rs b/harmony/src/modules/fleet/groups.rs index f5a03939..2b645848 100644 --- a/harmony/src/modules/fleet/groups.rs +++ b/harmony/src/modules/fleet/groups.rs @@ -1,81 +1 @@ -//! Device group membership: the admin-managed identity input that gates -//! scheduling and secret access (ADR-025). -//! -//! Groups are the security boundary — they must come from identity the -//! device cannot self-assert (Zitadel role grants), never from -//! device-reported labels. The operator polls a [`DeviceGroupSource`] -//! and feeds the snapshot into scheduling. - -use std::collections::{HashMap, HashSet}; - -use async_trait::async_trait; - -#[derive(Debug, thiserror::Error)] -pub enum GroupSourceError { - #[error("group source error: {0}")] - Source(String), -} - -/// Complete device → groups view. Snapshot-based: membership changes -/// rarely and per-device, so a periodic full fetch beats per-device -/// queries. -#[async_trait] -pub trait DeviceGroupSource: Send + Sync { - async fn device_groups(&self) -> Result>, GroupSourceError>; -} - -/// Fixed membership parsed from a `dev=g1|g2;dev2=g3` string — dev -/// setups and e2e, where no Zitadel exists. -#[derive(Debug, Default)] -pub struct StaticDeviceGroups { - map: HashMap>, -} - -impl StaticDeviceGroups { - pub fn parse(spec: &str) -> Self { - let map = spec - .split(';') - .filter_map(|entry| { - let (device, groups) = entry.split_once('=')?; - let device = device.trim(); - if device.is_empty() { - return None; - } - Some(( - device.to_string(), - groups - .split('|') - .map(str::trim) - .filter(|g| !g.is_empty()) - .map(str::to_string) - .collect(), - )) - }) - .collect(); - Self { map } - } -} - -#[async_trait] -impl DeviceGroupSource for StaticDeviceGroups { - async fn device_groups(&self) -> Result>, GroupSourceError> { - Ok(self.map.clone()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn parses_devices_and_groups() { - let src = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk"); - let map = src.device_groups().await.unwrap(); - assert_eq!(map.len(), 2); - assert_eq!( - map["pi-01"], - HashSet::from(["edge-a".to_string(), "edge-b".to_string()]) - ); - assert_eq!(map["pi-02"], HashSet::from(["edge-b".to_string()])); - } -} +pub use harmony_reconciler_contracts::{DeviceGroupSource, GroupSourceError}; diff --git a/harmony/src/modules/fleet/mod.rs b/harmony/src/modules/fleet/mod.rs index a0fd2a13..8a595466 100644 --- a/harmony/src/modules/fleet/mod.rs +++ b/harmony/src/modules/fleet/mod.rs @@ -1,12 +1,5 @@ //! Harmony-side Scores for fleet device onboarding. //! -//! Today this module exposes [`FleetDeviceSetupScore`] — a customer -//! runs it against a freshly-booted device (Pi, VM, bare-metal node -//! later) to install podman, place the `fleet-agent` binary, drop -//! the TOML config, and bring up the agent under systemd. Re-running -//! with a changed config (different labels, new NATS URL, new -//! credentials) is how a device is moved between fleet partitions. -//! //! The operator + agent crates live outside `harmony/` under //! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`. //! What belongs here is the harmony-framework side: the Scores a @@ -22,10 +15,8 @@ pub mod assets; pub mod groups; #[cfg(all(feature = "kvm", unix))] pub mod libvirt_pool; -pub mod operator; pub mod preflight; pub mod secret_access; -mod setup_score; #[cfg(all(feature = "kvm", unix))] mod vm_score; @@ -42,9 +33,5 @@ pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony // `harmony-fleet-deploy` per ADR-023 — see that crate for the // composite + per-component Scores. pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch}; -pub use setup_score::{ - AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, - HostsEntry, merge_hosts_file, -}; #[cfg(all(feature = "kvm", unix))] pub use vm_score::ProvisionVmScore; diff --git a/harmony/src/modules/fleet/operator/mod.rs b/harmony/src/modules/fleet/operator/mod.rs deleted file mode 100644 index 0c89a456..00000000 --- a/harmony/src/modules/fleet/operator/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Operator CRD types (`Deployment`, `Device`, …). -//! -//! Per ADR-023, the deploy-side bits (`FleetOperatorScore`, the helm -//! chart generator, `install_crds`) moved to -//! `fleet/harmony-fleet-deploy/src/operator/`. Only the CRD type -//! definitions remain here — they're consumed at runtime by the -//! operator binary's reconcile loop. Lifting the CRDs out of -//! `harmony` core is in scope for a follow-up PR (they currently -//! couple to `crate::modules::podman` via the `ReconcileScore` payload -//! field, which needs to move alongside them). - -pub mod crd; - -pub use crd::{ - AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device, - DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy, -}; diff --git a/harmony/src/modules/fleet/secret_access.rs b/harmony/src/modules/fleet/secret_access.rs index bf835c15..ee4e6870 100644 --- a/harmony/src/modules/fleet/secret_access.rs +++ b/harmony/src/modules/fleet/secret_access.rs @@ -1,132 +1 @@ -//! Deployment → secret access control: the domain boundary the fleet -//! operator drives when a `Deployment`'s allowed groups change. -//! -//! Per ADR-025, groups are the security boundary: a deployment declares -//! which device groups may read its secrets, and devices prove group -//! membership through their signed identity at login. How a grant maps -//! onto a backend's primitive (an OpenBao policy attached to an external -//! group, …) is the *implementation's* concern; callers speak only -//! deployments and group names. That separation is what lets the -//! operator be unit-tested against [`InMemoryDeploymentSecretGrants`] -//! with no live secret store — the production backend is -//! [`crate::modules::openbao::OpenBaoDeploymentSecretGrants`]. - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Mutex; - -use async_trait::async_trait; -use harmony_reconciler_contracts::DeploymentName; - -#[derive(Debug, thiserror::Error)] -pub enum SecretAccessError { - #[error("secret access backend error: {0}")] - Backend(String), -} - -/// Sets which device groups may read each deployment's secrets. -#[async_trait] -pub trait DeploymentSecretGrants: Send + Sync { - /// Set the *complete* set of groups allowed to read each listed - /// deployment's secrets. Absolute and idempotent: afterwards exactly - /// the given groups hold the grant — others are revoked, an empty - /// list revokes everywhere. Deployments not listed are untouched. - async fn set_deployment_groups( - &self, - grants: &[(DeploymentName, Vec)], - ) -> Result<(), SecretAccessError>; -} - -/// In-memory [`DeploymentSecretGrants`] for tests: records the latest -/// group set per deployment so a test can assert the operator synced the -/// right grants on each flow, with nothing deployed. -#[derive(Debug, Default)] -pub struct InMemoryDeploymentSecretGrants { - grants: Mutex>>, -} - -impl InMemoryDeploymentSecretGrants { - pub fn new() -> Self { - Self::default() - } - - /// Current groups for `deployment`, sorted. Empty when no group has - /// access (never granted, or revoked). - pub fn groups_for(&self, deployment: &DeploymentName) -> Vec { - self.grants - .lock() - .unwrap() - .get(deployment) - .map(|s| s.iter().cloned().collect()) - .unwrap_or_default() - } -} - -#[async_trait] -impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants { - async fn set_deployment_groups( - &self, - grants: &[(DeploymentName, Vec)], - ) -> Result<(), SecretAccessError> { - let mut store = self.grants.lock().unwrap(); - for (deployment, groups) in grants { - if groups.is_empty() { - store.remove(deployment); - } else { - store.insert(deployment.clone(), groups.iter().cloned().collect()); - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn dn(s: &str) -> DeploymentName { - DeploymentName::try_new(s).expect("valid test name") - } - - #[tokio::test] - async fn sets_overwrites_and_clears_grants() { - let grants = InMemoryDeploymentSecretGrants::new(); - let web = dn("web"); - - grants - .set_deployment_groups(&[(web.clone(), vec!["edge-a".into(), "edge-b".into()])]) - .await - .unwrap(); - assert_eq!(grants.groups_for(&web), vec!["edge-a", "edge-b"]); - - // Absolute, not additive: a smaller set revokes the rest. - grants - .set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])]) - .await - .unwrap(); - assert_eq!(grants.groups_for(&web), vec!["edge-a"]); - - // Empty revokes everywhere. - grants - .set_deployment_groups(&[(web.clone(), vec![])]) - .await - .unwrap(); - assert!(grants.groups_for(&web).is_empty()); - } - - #[tokio::test] - async fn batch_touches_only_listed_deployments() { - let grants = InMemoryDeploymentSecretGrants::new(); - let web = dn("web"); - let db = dn("db"); - grants - .set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])]) - .await - .unwrap(); - grants - .set_deployment_groups(&[(db.clone(), vec!["edge-b".into()])]) - .await - .unwrap(); - assert_eq!(grants.groups_for(&web), vec!["edge-a"]); - assert_eq!(grants.groups_for(&db), vec!["edge-b"]); - } -} +pub use harmony_reconciler_contracts::{DeploymentSecretGrants, SecretAccessError}; diff --git a/harmony/src/modules/k8s/namespace.rs b/harmony/src/modules/k8s/namespace.rs index aee87e31..5880c92f 100644 --- a/harmony/src/modules/k8s/namespace.rs +++ b/harmony/src/modules/k8s/namespace.rs @@ -1,46 +1,53 @@ +use std::collections::BTreeMap; + use k8s_openapi::api::core::v1::Namespace; -use non_blank_string_rs::NonBlankString; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use serde::Serialize; -use serde_json::json; use crate::{ interpret::Interpret, + modules::k8s::resource::K8sResourceScore, score::Score, topology::{K8sclient, Topology}, }; #[derive(Debug, Clone, Serialize)] pub struct K8sNamespaceScore { - pub name: Option, + name: String, + labels: BTreeMap, +} + +impl K8sNamespaceScore { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + labels: BTreeMap::new(), + } + } + + pub fn label(mut self, key: impl Into, value: impl Into) -> Self { + self.labels.insert(key.into(), value.into()); + self + } } impl Score for K8sNamespaceScore { fn create_interpret(&self) -> Box> { - let name = match &self.name { - Some(name) => name, - None => todo!( - "Return NoOp interpret when no namespace specified or something that makes sense" - ), - }; - let _namespace: Namespace = serde_json::from_value(json!( - { - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": name, + K8sResourceScore::single( + Namespace { + metadata: ObjectMeta { + name: Some(self.name.clone()), + labels: Some(self.labels.clone()), + ..Default::default() }, - } - )) - .unwrap(); - todo!( - "We currently only support namespaced ressources (see Scope = NamespaceResourceScope)" - ); - // Box::new(K8sResourceInterpret { - // score: K8sResourceScore::single(namespace.clone()), - // }) + ..Default::default() + }, + None, + ) + .create_interpret() } fn name(&self) -> String { - "K8sNamespaceScore".to_string() + format!("K8sNamespaceScore({})", self.name) } } diff --git a/harmony/src/modules/linux/ansible_configurator.rs b/harmony/src/modules/linux/ansible_configurator.rs index 75cde1ed..12c5ba3e 100644 --- a/harmony/src/modules/linux/ansible_configurator.rs +++ b/harmony/src/modules/linux/ansible_configurator.rs @@ -357,6 +357,22 @@ impl AnsibleHostConfigurator { .await } + pub async fn systemd_unit_active( + &self, + conn: &AnsibleConnection<'_>, + name: &str, + scope: SystemdScope, + ) -> Result { + let (command, user) = match scope { + SystemdScope::System => (format!("systemctl is-active --quiet {name}"), None), + SystemdScope::User(user) => ( + format!("systemctl --user is-active --quiet {name}"), + Some(user.0), + ), + }; + Ok(host_exec(conn, &command, user.as_deref()).await?.rc == 0) + } + pub async fn ensure_linger( &self, conn: &AnsibleConnection<'_>, diff --git a/harmony/src/modules/linux/topology.rs b/harmony/src/modules/linux/topology.rs index 64fc717e..95ae6a8f 100644 --- a/harmony/src/modules/linux/topology.rs +++ b/harmony/src/modules/linux/topology.rs @@ -222,6 +222,16 @@ impl SystemdManager for LinuxHostTopology { .await } + async fn systemd_unit_active( + &self, + name: &str, + scope: SystemdScope, + ) -> Result { + self.configurator + .systemd_unit_active(&self.connection(), name, scope) + .await + } + async fn ensure_user_unit_active( &self, user: &str, @@ -315,6 +325,16 @@ impl SystemdManager for LinuxLocalhostTopology { .await } + async fn systemd_unit_active( + &self, + name: &str, + scope: SystemdScope, + ) -> Result { + self.configurator + .systemd_unit_active(&self.connection(), name, scope) + .await + } + async fn ensure_user_unit_active( &self, user: &str, diff --git a/harmony/src/modules/nats/auth_callout_credentials.rs b/harmony/src/modules/nats/auth_callout_credentials.rs new file mode 100644 index 00000000..ef2a20a2 --- /dev/null +++ b/harmony/src/modules/nats/auth_callout_credentials.rs @@ -0,0 +1,203 @@ +use std::collections::BTreeMap; + +use async_trait::async_trait; +use harmony_types::id::Id; +use k8s_openapi::{ByteString, api::core::v1::Secret}; +use kube::api::ObjectMeta; +use nkeys::KeyPair; +use rand::{Rng, distr::Alphanumeric}; +use serde::Serialize; + +use crate::data::Version; +use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; +use crate::inventory::Inventory; +use crate::modules::k8s::resource::K8sResourceScore; +use crate::score::Score; +use crate::topology::{K8sclient, Topology}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +pub struct NatsAuthCalloutCredentialsRef { + namespace: String, + secret_name: String, +} + +impl NatsAuthCalloutCredentialsRef { + pub fn namespace(&self) -> &str { + &self.namespace + } + + pub fn secret_name(&self) -> &str { + &self.secret_name + } + + pub fn seed_key(&self) -> &'static str { + "issuer-nkey-seed" + } + + pub fn issuer_key(&self) -> &'static str { + "issuer-public-key" + } + + pub fn password_key(&self) -> &'static str { + "nats-auth-pass" + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct NatsAuthCalloutCredentialsScore { + namespace: String, + secret_name: String, + generate: bool, +} + +impl NatsAuthCalloutCredentialsScore { + pub fn generated(namespace: impl Into, secret_name: impl Into) -> Self { + Self { + namespace: namespace.into(), + secret_name: secret_name.into(), + generate: true, + } + } + + pub fn existing(namespace: impl Into, secret_name: impl Into) -> Self { + Self { + namespace: namespace.into(), + secret_name: secret_name.into(), + generate: false, + } + } + + pub fn credentials_ref(&self) -> NatsAuthCalloutCredentialsRef { + NatsAuthCalloutCredentialsRef { + namespace: self.namespace.clone(), + secret_name: self.secret_name.clone(), + } + } +} + +impl Score for NatsAuthCalloutCredentialsScore { + fn name(&self) -> String { + format!("NatsAuthCalloutCredentialsScore({})", self.secret_name) + } + + fn create_interpret(&self) -> Box> { + Box::new(NatsAuthCalloutCredentialsInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug)] +struct NatsAuthCalloutCredentialsInterpret { + score: NatsAuthCalloutCredentialsScore, +} + +#[async_trait] +impl Interpret for NatsAuthCalloutCredentialsInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; + k8s.ensure_namespace(&self.score.namespace) + .await + .map_err(|error| InterpretError::new(error.to_string()))?; + if let Some(secret) = k8s + .get_resource::(&self.score.secret_name, Some(&self.score.namespace)) + .await + .map_err(|error| InterpretError::new(error.to_string()))? + { + let complete = secret.immutable == Some(true) + && secret.data.as_ref().is_some_and(|data| { + let credentials = self.score.credentials_ref(); + [ + credentials.seed_key(), + credentials.issuer_key(), + credentials.password_key(), + ] + .iter() + .all(|key| data.contains_key(*key)) + }); + if !complete { + return Err(InterpretError::new(format!( + "NATS auth callout Secret {}/{} must be immutable and contain all credential keys", + self.score.namespace, self.score.secret_name + ))); + } + return Ok(Outcome::noop(format!( + "NATS auth callout credentials {}/{} already exist", + self.score.namespace, self.score.secret_name + ))); + } + if !self.score.generate { + return Ok(Outcome::new( + InterpretStatus::BLOCKED, + format!( + "NATS auth callout credentials {}/{} do not exist", + self.score.namespace, self.score.secret_name + ), + Vec::new(), + )); + } + + let key = KeyPair::new_account(); + let credentials = self.score.credentials_ref(); + let data = BTreeMap::from([ + ( + credentials.seed_key().to_string(), + ByteString( + key.seed() + .map_err(|error| InterpretError::new(error.to_string()))? + .as_bytes() + .to_vec(), + ), + ), + ( + credentials.issuer_key().to_string(), + ByteString(key.public_key().as_bytes().to_vec()), + ), + ( + credentials.password_key().to_string(), + ByteString( + rand::rng() + .sample_iter(&Alphanumeric) + .take(32) + .collect::>(), + ), + ), + ]); + K8sResourceScore::single( + Secret { + metadata: ObjectMeta { + name: Some(self.score.secret_name.clone()), + namespace: Some(self.score.namespace.clone()), + ..Default::default() + }, + data: Some(data), + type_: Some("Opaque".to_string()), + immutable: Some(true), + ..Default::default() + }, + Some(self.score.namespace.clone()), + ) + .interpret(inventory, topology) + .await + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("NatsAuthCalloutCredentialsInterpret") + } + + fn get_version(&self) -> Version { + Version::from("0.1.0").expect("static version") + } + + fn get_status(&self) -> InterpretStatus { + InterpretStatus::QUEUED + } + + fn get_children(&self) -> Vec { + Vec::new() + } +} diff --git a/harmony/src/modules/nats/helm_chart.rs b/harmony/src/modules/nats/helm_chart.rs index 29624f76..737bbb98 100644 --- a/harmony/src/modules/nats/helm_chart.rs +++ b/harmony/src/modules/nats/helm_chart.rs @@ -4,7 +4,7 @@ //! for how a NATS pod / STS is actually built: probes, resource //! shapes, RBAC, stateful-set options, JetStream storage volumes, //! clustering, TLS, gateways, leaf nodes. Every high-level NATS -//! Score — `NatsBasicScore` for single-node, `NatsK8sScore` for +//! Score — `NatsScore` for single-node, `NatsK8sScore` for //! supercluster — delegates here. Differences between shapes are //! expressed as `values_yaml`, not as parallel resource constructors. //! @@ -44,7 +44,7 @@ const REPO_NAME: &str = "nats"; /// repository and leaves `values_yaml` as the one parameter. /// /// Callers should almost never construct this directly — build a -/// high-level preset (`NatsBasicScore`, `NatsK8sScore`) instead. +/// high-level preset (`NatsScore`, `NatsK8sScore`) instead. /// The type is `pub` so those presets across different files can /// share a single definition. #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/nats/mod.rs b/harmony/src/modules/nats/mod.rs index 04a42fd6..f1f9d5c2 100644 --- a/harmony/src/modules/nats/mod.rs +++ b/harmony/src/modules/nats/mod.rs @@ -1,10 +1,88 @@ +mod auth_callout_credentials; pub mod capability; pub mod decentralized; pub mod helm_chart; pub mod pki; -pub mod score_nats_basic; +pub mod score_nats; pub mod score_nats_k8s; pub mod score_nats_supercluster; +pub use auth_callout_credentials::{ + NatsAuthCalloutCredentialsRef, NatsAuthCalloutCredentialsScore, +}; pub use helm_chart::NatsHelmChartScore; -pub use score_nats_basic::{NatsBasicScore, NatsServiceType}; +pub use score_nats::{NatsAuth, NatsScore, NatsScoreError, NatsService, UserPassCredentials}; + +/// Desired NATS account location consumed by an auth callout. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct NatsAccountRef { + url: String, + account: String, +} + +impl NatsAccountRef { + pub(crate) fn new(url: impl Into, account: impl Into) -> Self { + Self { + url: url.into(), + account: account.into(), + } + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn account(&self) -> &str { + &self.account + } +} + +/// Desired auth-callout settings consumed by a NATS server. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct NatsAuthCalloutRef { + auth_user: String, + account: String, + credentials: NatsAuthCalloutCredentialsRef, +} + +impl NatsAuthCalloutRef { + pub(crate) fn new( + auth_user: String, + account: String, + credentials: NatsAuthCalloutCredentialsRef, + ) -> Self { + Self { + auth_user, + account, + credentials, + } + } + + pub fn auth_user(&self) -> &str { + &self.auth_user + } + + pub fn account(&self) -> &str { + &self.account + } + + pub fn credentials(&self) -> &NatsAuthCalloutCredentialsRef { + &self.credentials + } +} + +/// Desired NATS endpoint consumed by application clients. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct NatsClientRef { + url: String, +} + +impl NatsClientRef { + pub(crate) fn new(url: impl Into) -> Self { + Self { url: url.into() } + } + + pub fn url(&self) -> &str { + &self.url + } +} diff --git a/fleet/harmony-fleet-deploy/src/nats.rs b/harmony/src/modules/nats/score_nats.rs similarity index 52% rename from fleet/harmony-fleet-deploy/src/nats.rs rename to harmony/src/modules/nats/score_nats.rs index 459e1f7d..3fc9e762 100644 --- a/fleet/harmony-fleet-deploy/src/nats.rs +++ b/harmony/src/modules/nats/score_nats.rs @@ -1,48 +1,30 @@ -//! `FleetNatsScore` — fleet-tuned preset over [`NatsHelmChartScore`]. -//! -//! The fleet stack needs NATS with: -//! -//! - JetStream enabled with file storage (KV buckets for desired- -//! state, device-info, device-state, device-heartbeat live here). -//! - An `APP` account holding the harmony users; a `SYS` account -//! for system traffic. In production an auth-callout sits in front -//! of `APP`; in dev / e2e two static users (admin + device) live -//! directly on `APP` so a test harness can connect without a -//! callout. -//! - A NodePort `Service` so off-cluster tooling (the e2e test -//! process, an operator running outside the cluster, …) can talk -//! to NATS on the host. -//! -//! The values yaml is rendered from typed Rust structs via -//! `serde_yaml::to_string`, not by hand-formatted multiline strings. -//! Strings with YAML-special characters (`"`, `\`, leading `*`/`&`, -//! …) escape correctly; the schema and the resulting bytes track -//! each other by type. -//! -//! `FleetNatsScore` itself is a thin newtype around -//! `NatsHelmChartScore` — the upstream production chart is the -//! single source of truth for *how* a NATS pod is built; this Score -//! owns *which values* the fleet stack feeds in. +//! Single-node NATS with typed accounts, JetStream, and optional auth callout. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, time::Duration}; use async_trait::async_trait; -use harmony::data::Version; -use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; -use harmony::inventory::Inventory; -use harmony::modules::nats::NatsHelmChartScore; -use harmony::score::Score; -use harmony::topology::{HelmCommand, Topology}; use harmony_types::id::Id; use serde::Serialize; +use crate::data::Version; +use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; +use crate::inventory::Inventory; +use crate::modules::nats::{ + NatsAccountRef, NatsAuthCalloutCredentialsRef, NatsAuthCalloutRef, NatsClientRef, + NatsHelmChartScore, +}; +use crate::score::Score; +use crate::topology::{HelmCommand, K8sclient, Topology}; + /// Authentication mode the deployed NATS server enforces. #[derive(Debug, Clone, Serialize)] -pub enum FleetNatsAuth { +pub enum NatsAuth { + None, /// Static username/password pair on the `APP` account — admin /// (full pub/sub) + device (limited). Dev/e2e path; production - /// uses [`FleetNatsAuth::Callout`]. + /// uses [`NatsAuth::Callout`]. UserPass { + account: String, admin_user: String, admin_pass: String, device_user: String, @@ -57,7 +39,6 @@ pub enum FleetNatsAuth { /// Issuer NKey public key — the callout's signing key. Must /// match the seed the callout pod's `AuthCalloutService` uses /// or every signed JWT is rejected by the NATS server. - issuer_pubkey: String, /// Account on which the callout's static user lives and on /// which every callout-signed user JWT is scoped. account: String, @@ -65,12 +46,16 @@ pub enum FleetNatsAuth { /// the server knows about; everything else flows through the /// callout's JWT minting). auth_user: String, - auth_pass: String, + credentials: NatsAuthCalloutCredentialsRef, + }, + /// Account declaration used while composing the callout cycle. + CalloutPending { + account: String, }, } /// Caller-supplied user/password pairs for the `APP` account. -/// Kept as a single struct so the [`FleetNatsScore::user_pass`] +/// Kept as a single struct so the [`NatsScore::user_pass`] /// signature doesn't grow a chain of four positional strings — /// and so callers can't transpose admin/device pairs by accident. #[derive(Debug, Clone, Serialize)] @@ -81,76 +66,97 @@ pub struct UserPassCredentials { pub device_pass: String, } -/// Fleet-tuned NATS deployment. Built as a preset over -/// [`NatsHelmChartScore`]. +/// Single-node NATS deployment over [`NatsHelmChartScore`]. #[derive(Debug, Clone, Serialize)] -pub struct FleetNatsScore { +pub struct NatsScore { pub namespace: String, pub release_name: String, - pub auth: FleetNatsAuth, - /// Host-side NodePort the chart's `Service` publishes for - /// off-cluster tooling. Set high (30000-32767 default range) so - /// it doesn't collide with anything else; pinned so re-runs of - /// the harness keep a stable URL. - pub node_port: u16, - /// JetStream `fileStorage.size` value (Kubernetes resource + pub auth: NatsAuth, + pub service: NatsService, + /// JetStream `fileStore.pvc.size` value (Kubernetes resource /// string, e.g. `2Gi`). Sized to fit the desired-state + /// per-device KV buckets a small fleet uses. pub jetstream_size: String, + pub image: Option, } -impl FleetNatsScore { +#[derive(Debug, Clone, Copy, Serialize)] +pub enum NatsService { + ClusterIp, + NodePort(u16), + LoadBalancer, +} + +impl NatsScore { + pub fn new(release_name: impl Into, namespace: impl Into) -> Self { + Self { + namespace: namespace.into(), + release_name: release_name.into(), + auth: NatsAuth::None, + service: NatsService::ClusterIp, + jetstream_size: "10Gi".to_string(), + image: None, + } + } + /// Two static users on the `APP` account, no callout. Caller /// owns the credentials — the deploy crate does not ship dev /// defaults so the e2e harness and production code can't drift /// into sharing them by accident. pub fn user_pass( + release_name: impl Into, namespace: impl Into, - node_port: u16, + service: NatsService, + account: impl Into, creds: UserPassCredentials, ) -> Self { Self { namespace: namespace.into(), - release_name: "fleet-nats".to_string(), - auth: FleetNatsAuth::UserPass { + release_name: release_name.into(), + auth: NatsAuth::UserPass { + account: account.into(), admin_user: creds.admin_user, admin_pass: creds.admin_pass, device_user: creds.device_user, device_pass: creds.device_pass, }, - node_port, - jetstream_size: "2Gi".to_string(), + service, + jetstream_size: "10Gi".to_string(), + image: None, } } - /// Production / e2e-demo preset: auth-callout enforced. The - /// callout service must already be deployed (typically via - /// `harmony::modules::nats_auth_callout::NatsAuthCalloutScore`) - /// with the matching issuer NKey + account + auth-user pair. - pub fn callout( + /// Declare the NATS half of an auth-callout cycle before the callout exists. + pub fn callout_account( + release_name: impl Into, namespace: impl Into, - node_port: u16, - issuer_pubkey: impl Into, + service: NatsService, account: impl Into, - auth_user: impl Into, - auth_pass: impl Into, ) -> Self { Self { namespace: namespace.into(), - release_name: "fleet-nats".to_string(), - auth: FleetNatsAuth::Callout { - issuer_pubkey: issuer_pubkey.into(), + release_name: release_name.into(), + auth: NatsAuth::CalloutPending { account: account.into(), - auth_user: auth_user.into(), - auth_pass: auth_pass.into(), }, - node_port, - jetstream_size: "2Gi".to_string(), + service, + jetstream_size: "10Gi".to_string(), + image: None, } } - pub fn release_name(mut self, name: impl Into) -> Self { - self.release_name = name.into(); + pub fn node_port(mut self, port: u16) -> Self { + self.service = NatsService::NodePort(port); + self + } + + pub fn load_balancer(mut self) -> Self { + self.service = NatsService::LoadBalancer; + self + } + + pub fn image(mut self, image: impl Into) -> Self { + self.image = Some(image.into()); self } @@ -159,6 +165,29 @@ impl FleetNatsScore { self } + pub fn with_auth_callout(mut self, callout: &NatsAuthCalloutRef) -> Self { + assert_eq!(callout.credentials().namespace(), self.namespace); + self.auth = NatsAuth::Callout { + account: callout.account().to_string(), + auth_user: callout.auth_user().to_string(), + credentials: callout.credentials().clone(), + }; + self + } + + pub fn account_ref(&self) -> NatsAccountRef { + let account = match &self.auth { + NatsAuth::None => panic!("NATS account must be declared before exporting a Ref"), + NatsAuth::UserPass { account, .. } => account, + NatsAuth::Callout { account, .. } | NatsAuth::CalloutPending { account } => account, + }; + NatsAccountRef::new(self.in_cluster_url(), account) + } + + pub fn client_ref(&self) -> NatsClientRef { + NatsClientRef::new(self.in_cluster_url()) + } + /// Helper for callers (operator, agent, e2e harness) that need the /// in-cluster DNS name of this NATS deployment. pub fn in_cluster_url(&self) -> String { @@ -171,34 +200,73 @@ impl FleetNatsScore { /// Render the helm values YAML the chart consumes. /// /// The output is generated by `serde_yaml::to_string` over the - /// [`FleetNatsValues`] tree — not by string interpolation. Any + /// typed values tree, not by string interpolation. Any /// value the caller supplies (passwords, release name, …) is /// escaped per the YAML spec. - pub fn values_yaml(&self) -> Result { + pub fn values_yaml(&self) -> Result { let values = build_values( &self.release_name, - self.node_port, + self.service, &self.jetstream_size, &self.auth, - ); - serde_yaml::to_string(&values).map_err(FleetNatsError::RenderValues) + self.image.as_deref(), + )?; + serde_yaml::to_string(&values).map_err(NatsScoreError::RenderValues) } } #[derive(Debug, thiserror::Error)] -pub enum FleetNatsError { - #[error("rendering fleet NATS helm values yaml: {0}")] +pub enum NatsScoreError { + #[error("rendering NATS helm values yaml: {0}")] RenderValues(#[from] serde_yaml::Error), + #[error("auth callout reference has not been attached")] + MissingAuthCallout, } // ---- typed values tree ----------------------------------------------------- #[derive(Debug, Serialize)] -struct FleetNatsValues { +struct NatsValues { #[serde(rename = "fullnameOverride")] fullname_override: String, config: NatsValuesConfig, - service: NatsValuesService, + #[serde(skip_serializing_if = "Option::is_none")] + service: Option, + #[serde(skip_serializing_if = "Option::is_none")] + container: Option, +} + +#[derive(Debug, Serialize)] +struct NatsValuesContainer { + #[serde(skip_serializing_if = "Option::is_none")] + image: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + env: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct NatsValuesImage { + repository: String, + #[serde(skip_serializing_if = "Option::is_none")] + tag: Option, +} + +#[derive(Debug, Serialize)] +struct NatsValuesEnv { + #[serde(rename = "valueFrom")] + value_from: NatsValuesEnvSource, +} + +#[derive(Debug, Serialize)] +struct NatsValuesEnvSource { + #[serde(rename = "secretKeyRef")] + secret_key_ref: NatsValuesSecretKeyRef, +} + +#[derive(Debug, Serialize)] +struct NatsValuesSecretKeyRef { + name: String, + key: String, } #[derive(Debug, Serialize)] @@ -216,12 +284,18 @@ struct NatsValuesCluster { #[derive(Debug, Serialize)] struct NatsValuesJetstream { enabled: bool, - #[serde(rename = "fileStorage")] - file_storage: NatsValuesJetstreamFileStorage, + #[serde(rename = "fileStore")] + file_store: NatsValuesJetstreamFileStore, } #[derive(Debug, Serialize)] -struct NatsValuesJetstreamFileStorage { +struct NatsValuesJetstreamFileStore { + enabled: bool, + pvc: NatsValuesJetstreamPvc, +} + +#[derive(Debug, Serialize)] +struct NatsValuesJetstreamPvc { enabled: bool, size: String, } @@ -234,7 +308,6 @@ struct NatsValuesConfigMerge { #[serde(skip_serializing_if = "Option::is_none")] authorization: Option, accounts: BTreeMap, - system_account: String, } #[derive(Debug, Serialize)] @@ -300,12 +373,15 @@ struct NatsValuesServicePort { fn build_values( release_name: &str, - node_port: u16, + service: NatsService, jetstream_size: &str, - auth: &FleetNatsAuth, -) -> FleetNatsValues { - let (authorization, accounts) = match auth { - FleetNatsAuth::UserPass { + auth: &NatsAuth, + image: Option<&str>, +) -> Result { + let (authorization, accounts, container) = match auth { + NatsAuth::None => (None, BTreeMap::new(), None), + NatsAuth::UserPass { + account, admin_user, admin_pass, device_user, @@ -313,7 +389,7 @@ fn build_values( } => { let mut accounts = BTreeMap::new(); accounts.insert( - "APP".to_string(), + account.clone(), NatsValuesAccount { jetstream: Some("enabled".to_string()), users: vec![ @@ -328,23 +404,12 @@ fn build_values( ], }, ); - accounts.insert( - "SYS".to_string(), - NatsValuesAccount { - jetstream: None, - users: vec![NatsValuesUser { - user: "sys".to_string(), - password: "sys".to_string(), - }], - }, - ); - (None, accounts) + (None, accounts, None) } - FleetNatsAuth::Callout { - issuer_pubkey, + NatsAuth::Callout { account, auth_user, - auth_pass, + credentials, } => { let mut accounts = BTreeMap::new(); accounts.insert( @@ -353,93 +418,151 @@ fn build_values( jetstream: Some("enabled".to_string()), users: vec![NatsValuesUser { user: auth_user.clone(), - password: auth_pass.clone(), - }], - }, - ); - accounts.insert( - "SYS".to_string(), - NatsValuesAccount { - jetstream: None, - users: vec![NatsValuesUser { - user: "sys".to_string(), - password: "sys".to_string(), + password: "<< $NATS_AUTH_PASS >>".to_string(), }], }, ); let authorization = NatsValuesAuthorization { auth_callout: NatsValuesAuthCallout { - issuer: issuer_pubkey.clone(), + issuer: "<< $NATS_ISSUER_PUBLIC_KEY >>".to_string(), auth_users: vec![auth_user.clone()], account: account.clone(), }, }; - (Some(authorization), accounts) + let secret_env = |key: &str| NatsValuesEnv { + value_from: NatsValuesEnvSource { + secret_key_ref: NatsValuesSecretKeyRef { + name: credentials.secret_name().to_string(), + key: key.to_string(), + }, + }, + }; + ( + Some(authorization), + accounts, + Some(NatsValuesContainer { + image: None, + env: BTreeMap::from([ + ( + "NATS_AUTH_PASS".to_string(), + secret_env(credentials.password_key()), + ), + ( + "NATS_ISSUER_PUBLIC_KEY".to_string(), + secret_env(credentials.issuer_key()), + ), + ]), + }), + ) } + NatsAuth::CalloutPending { .. } => return Err(NatsScoreError::MissingAuthCallout), }; - FleetNatsValues { + Ok(NatsValues { fullname_override: release_name.to_string(), config: NatsValuesConfig { cluster: NatsValuesCluster { enabled: false }, jetstream: NatsValuesJetstream { enabled: true, - file_storage: NatsValuesJetstreamFileStorage { + file_store: NatsValuesJetstreamFileStore { enabled: true, - size: jetstream_size.to_string(), + pvc: NatsValuesJetstreamPvc { + enabled: true, + size: jetstream_size.to_string(), + }, }, }, merge: NatsValuesConfigMerge { authorization, accounts, - system_account: "SYS".to_string(), }, }, - service: NatsValuesService { - merge: NatsValuesServiceMerge { - spec: NatsValuesServiceSpec { - service_type: "NodePort".to_string(), - ports: vec![ - NatsValuesServicePort { + service: match service { + NatsService::ClusterIp => None, + NatsService::NodePort(node_port) => Some(NatsValuesService { + merge: NatsValuesServiceMerge { + spec: NatsValuesServiceSpec { + service_type: "NodePort".to_string(), + ports: vec![ + NatsValuesServicePort { + app_protocol: "tcp".to_string(), + name: "nats".to_string(), + port: 4222, + target_port: "nats".to_string(), + node_port: Some(node_port), + }, + NatsValuesServicePort { + app_protocol: "http".to_string(), + name: "monitor".to_string(), + port: 8222, + target_port: "monitor".to_string(), + node_port: None, + }, + ], + }, + }, + }), + NatsService::LoadBalancer => Some(NatsValuesService { + merge: NatsValuesServiceMerge { + spec: NatsValuesServiceSpec { + service_type: "LoadBalancer".to_string(), + ports: vec![NatsValuesServicePort { app_protocol: "tcp".to_string(), name: "nats".to_string(), port: 4222, target_port: "nats".to_string(), - node_port: Some(node_port), - }, - NatsValuesServicePort { - app_protocol: "http".to_string(), - name: "monitor".to_string(), - port: 8222, - target_port: "monitor".to_string(), node_port: None, - }, - ], + }], + }, }, - }, + }), + }, + container: match (container, image) { + (Some(mut container), Some(image)) => { + container.image = Some(split_image_ref(image)); + Some(container) + } + (None, Some(image)) => Some(NatsValuesContainer { + image: Some(split_image_ref(image)), + env: BTreeMap::new(), + }), + (container, None) => container, + }, + }) +} + +fn split_image_ref(image: &str) -> NatsValuesImage { + match image.rsplit_once(':') { + Some((repository, tag)) if !tag.contains('/') => NatsValuesImage { + repository: repository.to_string(), + tag: Some(tag.to_string()), + }, + _ => NatsValuesImage { + repository: image.to_string(), + tag: None, }, } } -impl Score for FleetNatsScore { +impl Score for NatsScore { fn create_interpret(&self) -> Box> { - Box::new(FleetNatsInterpret { + Box::new(NatsInterpret { score: self.clone(), }) } fn name(&self) -> String { - format!("FleetNatsScore({})", self.release_name) + format!("NatsScore({})", self.release_name) } } #[derive(Debug)] -struct FleetNatsInterpret { - score: FleetNatsScore, +struct NatsInterpret { + score: NatsScore, } #[async_trait] -impl Interpret for FleetNatsInterpret { +impl Interpret for NatsInterpret { async fn execute( &self, inventory: &Inventory, @@ -448,17 +571,29 @@ impl Interpret for FleetNatsInterpret { let values_yaml = self .score .values_yaml() - .map_err(|e| InterpretError::new(format!("fleet nats values: {e}")))?; + .map_err(|e| InterpretError::new(format!("NATS values: {e}")))?; let helm = NatsHelmChartScore::new( self.score.release_name.clone(), self.score.namespace.clone(), values_yaml, ); - helm.interpret(inventory, topology).await + let outcome = helm.interpret(inventory, topology).await?; + topology + .k8s_client() + .await + .map_err(InterpretError::new)? + .wait_until_statefulset_ready( + &self.score.release_name, + Some(&self.score.namespace), + Some(Duration::from_secs(180)), + ) + .await + .map_err(InterpretError::new)?; + Ok(outcome) } fn get_name(&self) -> InterpretName { - InterpretName::Custom("FleetNatsInterpret") + InterpretName::Custom("NatsInterpret") } fn get_version(&self) -> Version { @@ -494,9 +629,19 @@ mod tests { } } + fn user_pass(creds: UserPassCredentials) -> NatsScore { + NatsScore::user_pass( + "fleet-nats", + "fleet-system", + NatsService::NodePort(30423), + "APP", + creds, + ) + } + #[test] fn user_pass_renders_both_users_into_app_account() { - let score = FleetNatsScore::user_pass("fleet-system", 30423, test_creds()); + let score = user_pass(test_creds()); let yaml = score.values_yaml().expect("user-pass renders"); let v = parse(&yaml); assert_eq!( @@ -523,10 +668,6 @@ mod tests { v["service"]["merge"]["spec"]["ports"][0]["name"], serde_yaml::Value::String("nats".into()) ); - assert_eq!( - v["config"]["merge"]["system_account"], - serde_yaml::Value::String("SYS".into()) - ); } #[test] @@ -541,7 +682,7 @@ mod tests { device_user: "device".to_string(), device_pass: r"back\slash & * !".to_string(), }; - let score = FleetNatsScore::user_pass("fleet-system", 30423, creds); + let score = user_pass(creds); let yaml = score.values_yaml().expect("renders"); let v = parse(&yaml); assert_eq!( @@ -556,8 +697,13 @@ mod tests { #[test] fn in_cluster_url_uses_release_namespace_pair() { - let score = FleetNatsScore::user_pass("e2e-abc12345", 30423, test_creds()) - .release_name("fleet-nats"); + let score = NatsScore::user_pass( + "fleet-nats", + "e2e-abc12345", + NatsService::NodePort(30423), + "APP", + test_creds(), + ); assert_eq!( score.in_cluster_url(), "nats://fleet-nats.e2e-abc12345.svc.cluster.local:4222" @@ -565,41 +711,58 @@ mod tests { } #[test] - fn callout_mode_emits_authorization_block_and_single_app_user() { - let score = FleetNatsScore::callout( + fn typed_callout_cycle_exposes_account_and_client_refs() { + let nats = NatsScore::callout_account( + "local-nats", "fleet-system", - 30422, - "ABCDEF1234567890", + NatsService::NodePort(30422), "DEVICES", + ); + let account = nats.account_ref(); + assert_eq!( + account.url(), + "nats://local-nats.fleet-system.svc.cluster.local:4222" + ); + assert_eq!(account.account(), "DEVICES"); + assert!(matches!( + nats.values_yaml(), + Err(NatsScoreError::MissingAuthCallout) + )); + + let credentials = crate::modules::nats::NatsAuthCalloutCredentialsScore::existing( + "fleet-system", + "callout", + ); + let callout = crate::modules::nats_auth_callout::NatsAuthCalloutScore::for_account( + "callout", + "fleet-system", + &account, "auth", - "auth-pass-123", - ); - let yaml = score.values_yaml().expect("renders"); - let v = parse(&yaml); - // Callout block is emitted under config.merge.authorization + ) + .credentials(&credentials.credentials_ref()); + let nats = nats.with_auth_callout(&callout.auth_callout_ref()); + let values = parse(&nats.values_yaml().expect("renders")); assert_eq!( - v["config"]["merge"]["authorization"]["auth_callout"]["issuer"], - serde_yaml::Value::String("ABCDEF1234567890".into()) - ); - assert_eq!( - v["config"]["merge"]["authorization"]["auth_callout"]["auth_users"][0], - serde_yaml::Value::String("auth".into()) - ); - assert_eq!( - v["config"]["merge"]["authorization"]["auth_callout"]["account"], + values["config"]["merge"]["authorization"]["auth_callout"]["account"], serde_yaml::Value::String("DEVICES".into()) ); - // Only one user lives on the DEVICES account — the callout - // mints everyone else's JWT. - let users = &v["config"]["merge"]["accounts"]["DEVICES"]["users"]; - assert!(users.is_sequence(), "users is a sequence"); - assert_eq!(users.as_sequence().unwrap().len(), 1); - assert_eq!(users[0]["user"], serde_yaml::Value::String("auth".into())); + assert_eq!( + values["config"]["merge"]["authorization"]["auth_callout"]["issuer"], + serde_yaml::Value::String("<< $NATS_ISSUER_PUBLIC_KEY >>".into()) + ); + assert_eq!( + values["config"]["merge"]["accounts"]["DEVICES"]["users"] + .as_sequence() + .unwrap() + .len(), + 1 + ); + assert_eq!(nats.client_ref().url(), account.url()); } #[test] fn user_pass_mode_omits_authorization_block() { - let score = FleetNatsScore::user_pass("fleet-system", 30423, test_creds()); + let score = user_pass(test_creds()); let yaml = score.values_yaml().expect("renders"); let v = parse(&yaml); // No `authorization` field at all when we're not in callout @@ -611,20 +774,13 @@ mod tests { } #[test] - fn sys_account_omits_jetstream_field() { - let score = FleetNatsScore::user_pass("fleet-system", 30423, test_creds()); + fn account_enables_jetstream() { + let score = user_pass(test_creds()); let yaml = score.values_yaml().expect("renders"); let v = parse(&yaml); - // APP has `jetstream: enabled`, SYS must not have a jetstream key. assert_eq!( v["config"]["merge"]["accounts"]["APP"]["jetstream"], serde_yaml::Value::String("enabled".into()) ); - assert!( - v["config"]["merge"]["accounts"]["SYS"] - .get("jetstream") - .is_none() - || v["config"]["merge"]["accounts"]["SYS"]["jetstream"] == serde_yaml::Value::Null - ); } } diff --git a/harmony/src/modules/nats/score_nats_basic.rs b/harmony/src/modules/nats/score_nats_basic.rs deleted file mode 100644 index 368d02a5..00000000 --- a/harmony/src/modules/nats/score_nats_basic.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Single-node NATS — high-level preset over [`NatsHelmChartScore`]. -//! -//! The shape this Score covers: one NATS server pod in a cluster, -//! JetStream on by default, exposed via ClusterIP / NodePort / -//! LoadBalancer. No TLS, no clustering, no auth. For any of those, -//! graduate to `NatsK8sScore` (supercluster + TLS + gateways). -//! -//! Everything concrete — probes, resource limits, statefulset -//! options — comes from the upstream `nats/nats` helm chart. -//! This Score just picks the chart values that select a minimal -//! single-node install. -//! -//! Typical usage: -//! -//! ```ignore -//! use harmony::modules::k8s::K8sBareTopology; -//! use harmony::modules::nats::NatsBasicScore; -//! use harmony::score::Score; -//! use harmony::inventory::Inventory; -//! -//! let topology = K8sBareTopology::from_kubeconfig("nats-install").await?; -//! let score = NatsBasicScore::new("fleet-nats", "fleet-system").load_balancer(); -//! score.create_interpret().execute(&Inventory::empty(), &topology).await?; -//! ``` - -use async_trait::async_trait; -use harmony_types::id::Id; -use serde::Serialize; - -use crate::data::Version; -use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; -use crate::inventory::Inventory; -use crate::modules::nats::helm_chart::NatsHelmChartScore; -use crate::score::Score; -use crate::topology::{HelmCommand, Topology}; - -/// How the NATS client port is exposed. -#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -pub enum NatsServiceType { - /// In-cluster only. Caller reaches NATS via - /// `..svc.cluster.local:4222`. - ClusterIp, - /// NodePort on the given host port — must fall in the cluster's - /// configured service-node-port range (default 30000-32767). - NodePort(i32), - /// LoadBalancer service. On k3d this uses the built-in - /// `klipper-lb`, which pairs naturally with - /// `k3d cluster create -p PORT:PORT@loadbalancer`. - LoadBalancer, -} - -/// Declarative single-node NATS. Construct via [`new`], tune with -/// the builder-style methods, hand to a topology that implements -/// [`HelmCommand`]. -#[derive(Debug, Clone, Serialize)] -pub struct NatsBasicScore { - release_name: String, - namespace: String, - jetstream: bool, - service_type: NatsServiceType, - /// Optional image override (`repository:tag` or full ref). - /// `None` = use the chart's default image. - image: Option, -} - -impl NatsBasicScore { - /// Build a single-node NATS score with JetStream on and - /// ClusterIP exposure. Use the builder methods to change the - /// exposure or image. - pub fn new(release_name: impl Into, namespace: impl Into) -> Self { - Self { - release_name: release_name.into(), - namespace: namespace.into(), - jetstream: true, - service_type: NatsServiceType::ClusterIp, - image: None, - } - } - - pub fn jetstream(mut self, enabled: bool) -> Self { - self.jetstream = enabled; - self - } - - pub fn node_port(mut self, port: i32) -> Self { - self.service_type = NatsServiceType::NodePort(port); - self - } - - pub fn load_balancer(mut self) -> Self { - self.service_type = NatsServiceType::LoadBalancer; - self - } - - pub fn image(mut self, image: impl Into) -> Self { - self.image = Some(image.into()); - self - } - - /// Render the chart values for this preset. Public so tests + - /// downstream tools (e.g. `helm template` diffs) can inspect - /// exactly what the Score will install. - pub fn render_values(&self) -> String { - let mut y = String::new(); - y.push_str(&format!("fullnameOverride: {}\n", self.release_name)); - y.push_str("replicaCount: 1\n"); - y.push_str("config:\n"); - y.push_str(" cluster:\n"); - y.push_str(" enabled: false\n"); - y.push_str(" jetstream:\n"); - y.push_str(&format!(" enabled: {}\n", self.jetstream)); - if self.jetstream { - y.push_str(" fileStorage:\n"); - y.push_str(" enabled: true\n"); - y.push_str(" size: 10Gi\n"); - } - match self.service_type { - NatsServiceType::ClusterIp => { - // Chart default. No overrides needed. - } - NatsServiceType::NodePort(port) => { - y.push_str("service:\n"); - y.push_str(" merge:\n"); - y.push_str(" spec:\n"); - y.push_str(" type: NodePort\n"); - y.push_str(" ports:\n"); - y.push_str(" nats:\n"); - y.push_str(" merge:\n"); - y.push_str(&format!(" nodePort: {port}\n")); - } - NatsServiceType::LoadBalancer => { - y.push_str("service:\n"); - y.push_str(" merge:\n"); - y.push_str(" spec:\n"); - y.push_str(" type: LoadBalancer\n"); - } - } - if let Some(img) = &self.image { - let (repo, tag) = split_image_ref(img); - y.push_str("container:\n"); - y.push_str(" image:\n"); - y.push_str(&format!(" repository: {repo}\n")); - if let Some(tag) = tag { - y.push_str(&format!(" tag: {tag}\n")); - } - } - y - } - - /// Name accessors — used by downstream presets + tests that - /// need to reference what this Score will name its resources. - pub fn release_name(&self) -> &str { - &self.release_name - } - pub fn namespace(&self) -> &str { - &self.namespace - } -} - -fn split_image_ref(image: &str) -> (String, Option) { - // Split on the *last* colon that isn't part of a registry port - // (`registry.io:5000/foo:v1`). Good enough for the shapes we - // see in practice (`nats:2.10-alpine`, `ghcr.io/nats-io/nats:v2`). - match image.rsplit_once(':') { - Some((r, t)) if !t.contains('/') => (r.to_string(), Some(t.to_string())), - _ => (image.to_string(), None), - } -} - -impl Score for NatsBasicScore { - fn create_interpret(&self) -> Box> { - Box::new(NatsBasicInterpret { - score: self.clone(), - }) - } - - fn name(&self) -> String { - "NatsBasicScore".to_string() - } -} - -#[derive(Debug)] -pub struct NatsBasicInterpret { - score: NatsBasicScore, -} - -#[async_trait] -impl Interpret for NatsBasicInterpret { - async fn execute( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - let values_yaml = self.score.render_values(); - NatsHelmChartScore::new(&self.score.release_name, &self.score.namespace, values_yaml) - .create_interpret() - .execute(inventory, topology) - .await - } - - fn get_name(&self) -> InterpretName { - InterpretName::Custom("NatsBasicInterpret") - } - - fn get_version(&self) -> Version { - Version::from("0.1.0").expect("static version literal") - } - - fn get_status(&self) -> InterpretStatus { - InterpretStatus::QUEUED - } - - fn get_children(&self) -> Vec { - vec![] - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn defaults_are_clusterip_jetstream_on() { - let s = NatsBasicScore::new("n", "ns"); - assert_eq!(s.service_type, NatsServiceType::ClusterIp); - assert!(s.jetstream); - assert!(s.image.is_none()); - } - - #[test] - fn render_values_includes_fullname_and_replica() { - let y = NatsBasicScore::new("fleet-nats", "fleet-system").render_values(); - assert!(y.contains("fullnameOverride: fleet-nats")); - assert!(y.contains("replicaCount: 1")); - // cluster.enabled stays false for a single-node shape. - assert!(y.contains("cluster:\n enabled: false")); - } - - #[test] - fn render_values_enables_jetstream_with_storage_by_default() { - let y = NatsBasicScore::new("n", "ns").render_values(); - assert!(y.contains("jetstream:\n enabled: true")); - assert!(y.contains("fileStorage:\n enabled: true")); - } - - #[test] - fn render_values_omits_storage_when_jetstream_off() { - let y = NatsBasicScore::new("n", "ns") - .jetstream(false) - .render_values(); - assert!(y.contains("jetstream:\n enabled: false")); - assert!(!y.contains("fileStorage")); - } - - #[test] - fn render_values_node_port_patches_service_and_port() { - let y = NatsBasicScore::new("n", "ns") - .node_port(30222) - .render_values(); - assert!(y.contains("type: NodePort")); - assert!(y.contains("nodePort: 30222")); - } - - #[test] - fn render_values_load_balancer_sets_service_type() { - let y = NatsBasicScore::new("n", "ns") - .load_balancer() - .render_values(); - assert!(y.contains("type: LoadBalancer")); - // LoadBalancer doesn't specify a nodePort — let kube assign. - assert!(!y.contains("nodePort:")); - } - - #[test] - fn render_values_clusterip_has_no_service_block() { - let y = NatsBasicScore::new("n", "ns").render_values(); - assert!(!y.contains("service:")); - } - - #[test] - fn render_values_image_override_splits_repo_and_tag() { - let y = NatsBasicScore::new("n", "ns") - .image("registry.io/custom/nats:2.10-alpine") - .render_values(); - assert!(y.contains("repository: registry.io/custom/nats")); - assert!(y.contains("tag: 2.10-alpine")); - } - - #[test] - fn render_values_image_without_tag_omits_tag_line() { - let y = NatsBasicScore::new("n", "ns") - .image("my.internal/nats-no-tag") - .render_values(); - assert!(y.contains("repository: my.internal/nats-no-tag")); - assert!(!y.contains("tag:")); - } - - #[test] - fn setters_return_self_for_chaining() { - let s = NatsBasicScore::new("n", "ns") - .jetstream(true) - .load_balancer() - .image("nats:latest"); - assert_eq!(s.release_name(), "n"); - assert_eq!(s.namespace(), "ns"); - } -} diff --git a/harmony/src/modules/nats/score_nats_k8s.rs b/harmony/src/modules/nats/score_nats_k8s.rs index 387ee887..0accf072 100644 --- a/harmony/src/modules/nats/score_nats_k8s.rs +++ b/harmony/src/modules/nats/score_nats_k8s.rs @@ -61,6 +61,7 @@ pub struct AuthCalloutCfg { /// it would deadlock authenticating itself). pub auth_user: String, /// Password for that bypass user. + #[serde(skip_serializing)] pub auth_pass: String, /// Account name the callout puts authenticated users into. MUST /// match the callout binary's `TARGET_ACCOUNT`. diff --git a/harmony/src/modules/nats_auth_callout/mod.rs b/harmony/src/modules/nats_auth_callout/mod.rs index beec054f..d053c169 100644 --- a/harmony/src/modules/nats_auth_callout/mod.rs +++ b/harmony/src/modules/nats_auth_callout/mod.rs @@ -34,6 +34,7 @@ //! pod security, image override, etc.). use std::collections::BTreeMap; +use std::hash::{DefaultHasher, Hash, Hasher}; use async_trait::async_trait; use k8s_openapi::ByteString; @@ -42,11 +43,14 @@ use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use serde::Serialize; use serde_json::json; +use std::time::Duration; use crate::data::Version; use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; use crate::inventory::Inventory; use crate::modules::k8s::resource::K8sResourceScore; +use crate::modules::nats::{NatsAccountRef, NatsAuthCalloutCredentialsRef, NatsAuthCalloutRef}; +use crate::modules::zitadel::{OidcApplicationRef, OidcProviderRef}; use crate::score::Score; use crate::topology::{K8sclient, Topology}; use harmony_types::id::Id; @@ -83,12 +87,18 @@ pub struct NatsAuthCalloutScore { /// would deadlock authenticating itself). pub nats_auth_user: String, /// Password for the callout's NATS connection. Stored in a K8s Secret. + #[serde(skip_serializing)] pub nats_auth_pass: String, /// NKey account seed used to sign user JWTs. The corresponding /// public key MUST be configured as `auth_callout.issuer` on the /// NATS server; otherwise NATS will reject every response we sign. /// Stored in a K8s Secret. + #[serde(skip_serializing)] pub issuer_nkey_seed: String, + /// Public key corresponding to `issuer_nkey_seed`. Set for typed + /// composition so the NATS Score cannot drift from this callout. + pub issuer_public_key: Option, + pub credentials: Option, /// OIDC issuer URL (e.g. `http://zitadel.zitadel.svc.cluster.local:8080`). pub oidc_issuer_url: String, /// Expected `aud` claim in inbound user JWTs. @@ -102,6 +112,8 @@ pub struct NatsAuthCalloutScore { pub device_id_prefix_strip: String, /// JSON path to the roles claim. pub roles_claim: String, + /// Kubernetes source for generated application values in typed composition. + pub oidc_application: Option, /// Role name granting admin permissions. pub admin_role: String, /// Role name granting per-device permissions. @@ -110,9 +122,20 @@ pub struct NatsAuthCalloutScore { /// for local dev — Zitadel-on-k3d typically uses HTTP, but in /// development with a self-signed Zitadel cert this is the escape hatch). pub danger_accept_invalid_certs: bool, + pub image_pull_secret: Option, } impl NatsAuthCalloutScore { + pub fn for_account( + name: impl Into, + namespace: impl Into, + nats: &NatsAccountRef, + auth_user: impl Into, + ) -> Self { + Self::new(name, namespace, nats.url(), "", "", auth_user, "", "") + .target_account(nats.account()) + } + /// Sane defaults; required fields are passed positionally. // TODO(clippy): refactor to a typed options struct #[allow(clippy::too_many_arguments)] @@ -135,22 +158,97 @@ impl NatsAuthCalloutScore { nats_auth_user: nats_auth_user.into(), nats_auth_pass: nats_auth_pass.into(), issuer_nkey_seed: issuer_nkey_seed.into(), + issuer_public_key: None, + credentials: None, oidc_issuer_url: oidc_issuer_url.into(), oidc_audience: oidc_audience.into(), device_id_claim: "device_id".to_string(), device_id_prefix_strip: String::new(), roles_claim: DEFAULT_ROLES_CLAIM.to_string(), + oidc_application: None, admin_role: DEFAULT_ADMIN_ROLE.to_string(), device_role: DEFAULT_DEVICE_ROLE.to_string(), danger_accept_invalid_certs: false, + image_pull_secret: None, } } + /// Construct against a typed NATS account without copying its URL or name. + #[allow(clippy::too_many_arguments)] + pub fn from_account( + name: impl Into, + namespace: impl Into, + nats: &NatsAccountRef, + oidc_issuer_url: impl Into, + oidc_audience: impl Into, + nats_auth_user: impl Into, + nats_auth_pass: impl Into, + issuer_nkey_seed: impl Into, + ) -> Self { + Self::new( + name, + namespace, + nats.url(), + oidc_issuer_url, + oidc_audience, + nats_auth_user, + nats_auth_pass, + issuer_nkey_seed, + ) + .target_account(nats.account()) + } + + pub fn with_nats(mut self, nats: &NatsAccountRef) -> Self { + self.nats_url = nats.url().to_string(); + self.target_account = nats.account().to_string(); + self + } + + pub fn issuer_public_key(mut self, public_key: impl Into) -> Self { + self.issuer_public_key = Some(public_key.into()); + self + } + + pub fn credentials(mut self, credentials: &NatsAuthCalloutCredentialsRef) -> Self { + assert_eq!(credentials.namespace(), self.namespace); + self.credentials = Some(credentials.clone()); + self + } + + pub fn with_oidc( + mut self, + provider: &OidcProviderRef, + application: &OidcApplicationRef, + ) -> Self { + self.oidc_issuer_url = provider.issuer().to_string(); + self.oidc_application = Some(application.clone()); + self + } + + pub fn try_auth_callout_ref(&self) -> Option { + Some(NatsAuthCalloutRef::new( + self.nats_auth_user.clone(), + self.target_account.clone(), + self.credentials.clone()?, + )) + } + + /// Reference consumed by the NATS Score. This only reads desired state. + pub fn auth_callout_ref(&self) -> NatsAuthCalloutRef { + self.try_auth_callout_ref() + .expect("credentials must be set before exporting auth_callout_ref") + } + pub fn image(mut self, image: impl Into) -> Self { self.image = image.into(); self } + pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self { + self.image_pull_secret = secret.map(str::to_string); + self + } + pub fn target_account(mut self, account: impl Into) -> Self { self.target_account = account.into(); self @@ -166,16 +264,27 @@ impl NatsAuthCalloutScore { self } + pub fn device_id_claim(mut self, claim: impl Into) -> Self { + self.device_id_claim = claim.into(); + self + } + pub fn danger_accept_invalid_certs(mut self, accept: bool) -> Self { self.danger_accept_invalid_certs = accept; self } fn secret_name(&self) -> String { - format!("{}-secrets", self.name) + self.credentials + .as_ref() + .map(|credentials| credentials.secret_name().to_string()) + .unwrap_or_else(|| format!("{}-secrets", self.name)) } - fn build_secret(&self) -> Secret { + fn build_secret(&self) -> Option { + if self.credentials.is_some() { + return None; + } let mut data: BTreeMap = BTreeMap::new(); data.insert( "issuer-nkey-seed".to_string(), @@ -186,7 +295,7 @@ impl NatsAuthCalloutScore { ByteString(self.nats_auth_pass.as_bytes().to_vec()), ); - Secret { + Some(Secret { metadata: ObjectMeta { name: Some(self.secret_name()), namespace: Some(self.namespace.clone()), @@ -195,11 +304,43 @@ impl NatsAuthCalloutScore { data: Some(data), type_: Some("Opaque".to_string()), ..Default::default() - } + }) } fn build_deployment(&self) -> Deployment { let secret_name = self.secret_name(); + let seed_key = self + .credentials + .as_ref() + .map_or("issuer-nkey-seed", NatsAuthCalloutCredentialsRef::seed_key); + let password_key = self.credentials.as_ref().map_or( + "nats-auth-pass", + NatsAuthCalloutCredentialsRef::password_key, + ); + let (oidc_audience, roles_claim) = match &self.oidc_application { + Some(application) => ( + json!({ + "name": "OIDC_AUDIENCE", + "valueFrom": { "configMapKeyRef": { + "name": application.config_map_name(), + "key": application.project_id_key(), + "optional": false + }} + }), + json!({ + "name": "ROLES_CLAIM", + "valueFrom": { "configMapKeyRef": { + "name": application.config_map_name(), + "key": application.roles_claim_key(), + "optional": false + }} + }), + ), + None => ( + json!({ "name": "OIDC_AUDIENCE", "value": self.oidc_audience }), + json!({ "name": "ROLES_CLAIM", "value": self.roles_claim }), + ), + }; // Secrets are exposed via `valueFrom.secretKeyRef` rather than // a volume mount. Two reasons: @@ -213,10 +354,14 @@ impl NatsAuthCalloutScore { // either `_FILE` (mounted file) or `` (raw // env). We use `` here. // - // Rotation: a Secret value change does NOT auto-restart the - // pod when consumed via env. Operators rotate by `kubectl - // rollout restart`. Acceptable trade-off — the callout is a - // singleton and rotation is rare. + let mut config_hash = DefaultHasher::new(); + self.nats_auth_pass.hash(&mut config_hash); + self.issuer_nkey_seed.hash(&mut config_hash); + self.credentials.hash(&mut config_hash); + let image_pull_secrets = self + .image_pull_secret + .as_ref() + .map_or_else(|| json!(null), |name| json!([{ "name": name }])); let manifest = json!({ "metadata": { "name": self.name, @@ -227,8 +372,14 @@ impl NatsAuthCalloutScore { "replicas": 1, "selector": { "matchLabels": { "app": self.name } }, "template": { - "metadata": { "labels": { "app": self.name } }, + "metadata": { + "labels": { "app": self.name }, + "annotations": { + "harmony.nationtech.io/config-hash": format!("{:x}", config_hash.finish()) + } + }, "spec": { + "imagePullSecrets": image_pull_secrets, // No pod-level securityContext: leave // runAsUser / runAsGroup / fsGroup unset so // the SCC (or PSA) assigns them. @@ -244,7 +395,7 @@ impl NatsAuthCalloutScore { "valueFrom": { "secretKeyRef": { "name": secret_name, - "key": "nats-auth-pass", + "key": password_key, "optional": false } } @@ -253,16 +404,16 @@ impl NatsAuthCalloutScore { "valueFrom": { "secretKeyRef": { "name": secret_name, - "key": "issuer-nkey-seed", + "key": seed_key, "optional": false } } }, { "name": "OIDC_ISSUER_URL", "value": self.oidc_issuer_url }, - { "name": "OIDC_AUDIENCE", "value": self.oidc_audience }, + oidc_audience, { "name": "DEVICE_ID_CLAIM", "value": self.device_id_claim }, { "name": "DEVICE_ID_PREFIX_STRIP", "value": self.device_id_prefix_strip }, - { "name": "ROLES_CLAIM", "value": self.roles_claim }, + roles_claim, { "name": "ADMIN_ROLE", "value": self.admin_role }, { "name": "DEVICE_ROLE", "value": self.device_role }, { "name": "DANGER_ACCEPT_INVALID_CERTS", @@ -309,19 +460,68 @@ impl Interpret for NatsAuthCalloutInterpret { inventory: &Inventory, topology: &T, ) -> Result { + if let Some(application) = &self.score.oidc_application + && application.namespace() != self.score.namespace + { + return Err(InterpretError::new(format!( + "OIDC application ConfigMap is in namespace '{}', but callout is in '{}'", + application.namespace(), + self.score.namespace + ))); + } let secret = self.score.build_secret(); let deployment = self.score.build_deployment(); - K8sResourceScore::single(secret, Some(self.score.namespace.clone())) - .interpret(inventory, topology) - .await?; + if let Some(secret) = secret { + K8sResourceScore::single(secret, Some(self.score.namespace.clone())) + .interpret(inventory, topology) + .await?; + } K8sResourceScore::single(deployment, Some(self.score.namespace.clone())) .interpret(inventory, topology) .await?; + let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; + if let Some(credentials) = &self.score.credentials { + tokio::time::timeout(Duration::from_secs(120), async { + loop { + if k8s + .get_resource::( + credentials.secret_name(), + Some(credentials.namespace()), + ) + .await + .is_ok_and(|secret| secret.is_some()) + { + return; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| { + InterpretError::new( + "timed out waiting for NATS auth callout credentials".to_string(), + ) + })?; + } + + k8s.wait_until_deployment_ready( + &self.score.name, + Some(&self.score.namespace), + Some(Duration::from_secs(120)), + ) + .await + .map_err(|e| { + InterpretError::new(format!( + "callout deployment {}/{} not ready: {e}", + self.score.namespace, self.score.name + )) + })?; + Ok(Outcome::success(format!( - "callout deployment {}/{} applied", + "callout deployment {}/{} ready", self.score.namespace, self.score.name ))) } @@ -404,10 +604,86 @@ mod tests { assert!(s.danger_accept_invalid_certs); } + #[test] + fn typed_refs_wire_the_cycle_without_io() { + let account = NatsAccountRef::new( + "nats://fleet-nats.fleet-system.svc.cluster.local:4222".to_string(), + "ACME".to_string(), + ); + let credentials = crate::modules::nats::NatsAuthCalloutCredentialsScore::existing( + "fleet-system", + "callout", + ); + let score = sample_score() + .with_nats(&account) + .credentials(&credentials.credentials_ref()); + + assert_eq!(score.nats_url, account.url()); + let callout = score.auth_callout_ref(); + assert_eq!(callout.account(), account.account()); + assert_eq!(callout.credentials().secret_name(), "callout"); + assert_eq!(callout.auth_user(), "auth"); + } + + #[test] + fn missing_credentials_has_non_panicking_ref_accessor() { + assert!(sample_score().try_auth_callout_ref().is_none()); + } + + #[test] + fn typed_oidc_uses_provider_issuer_and_application_config_map() { + let provider = crate::modules::zitadel::ZitadelScore { + host: "identity.example.test".to_string(), + external_secure: false, + external_port: Some(8080), + ..Default::default() + } + .provider_ref(); + let application = crate::modules::zitadel::setup::ZitadelSetupScore { + namespace: "fleet-system".to_string(), + applications: vec![crate::modules::zitadel::ZitadelApplication { + project_name: "fleet".to_string(), + app_name: "fleet-callout".to_string(), + app_type: crate::modules::zitadel::ZitadelAppType::DeviceCode, + }], + ..Default::default() + } + .application_ref("fleet-callout"); + let deployment = sample_score() + .with_oidc(&provider, &application) + .build_deployment(); + let env = deployment.spec.unwrap().template.spec.unwrap().containers[0] + .env + .clone() + .unwrap(); + + let issuer = env + .iter() + .find(|var| var.name == "OIDC_ISSUER_URL") + .unwrap(); + assert_eq!( + issuer.value.as_deref(), + Some("http://identity.example.test:8080") + ); + for (name, key) in [ + ("OIDC_AUDIENCE", application.project_id_key()), + ("ROLES_CLAIM", application.roles_claim_key()), + ] { + let selector = env + .iter() + .find(|var| var.name == name) + .and_then(|var| var.value_from.as_ref()) + .and_then(|source| source.config_map_key_ref.as_ref()) + .unwrap(); + assert_eq!(selector.name, application.config_map_name()); + assert_eq!(selector.key, key); + } + } + #[test] fn secret_carries_seed_and_password_at_expected_keys() { let s = sample_score(); - let secret = s.build_secret(); + let secret = s.build_secret().unwrap(); assert_eq!( secret.metadata.name.as_deref(), Some("fleet-callout-secrets") diff --git a/harmony/src/modules/openbao/mod.rs b/harmony/src/modules/openbao/mod.rs index c6062bfc..001ad2fe 100644 --- a/harmony/src/modules/openbao/mod.rs +++ b/harmony/src/modules/openbao/mod.rs @@ -1,24 +1,28 @@ -pub mod identity; pub mod setup; -use std::str::FromStr; +use std::{str::FromStr, time::Duration}; +use async_trait::async_trait; use harmony_macros::hurl; +use harmony_types::id::Id; use non_blank_string_rs::NonBlankString; use serde::Serialize; use crate::{ - interpret::Interpret, + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, modules::helm::chart::{HelmChartScore, HelmRepository}, score::Score, topology::{HelmCommand, K8sclient, Topology}, }; -pub use identity::OpenBaoDeploymentSecretGrants; +pub use harmony_secret::OpenBaoDeploymentSecretGrants; pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser, cached_root_token}; const DEFAULT_NAMESPACE: &str = "openbao"; const DEFAULT_RELEASE: &str = "openbao"; +const JWT_AUTH_MOUNT: &str = "jwt"; /// Where one OpenBao instance lives — the single authority both the /// deploy ([`OpenbaoScore`]) and the setup ([`OpenbaoSetupScore`]) take, @@ -63,14 +67,44 @@ pub struct OpenbaoScore { /// issuer name because a bare bool can't address an issuer. #[serde(default)] pub tls_issuer: Option, + #[serde(default)] + pub node_port: Option, } impl OpenbaoScore { + pub fn new( + namespace: impl Into, + release: impl Into, + host: impl Into, + ) -> Self { + Self { + instance: OpenbaoInstance { + namespace: namespace.into(), + release: release.into(), + }, + host: host.into(), + openshift: false, + tls_issuer: None, + node_port: None, + } + } + + pub fn node_port(mut self, port: u16) -> Self { + self.node_port = Some(port); + self + } + + pub fn tls(mut self, issuer: impl Into) -> Self { + self.tls_issuer = Some(issuer.into()); + self + } + fn values(&self) -> String { let Self { host, openshift, tls_issuer, + node_port, instance: _, } = self; // Edge TLS: the listener stays plain HTTP behind the ingress, which @@ -81,6 +115,9 @@ impl OpenbaoScore { ), None => String::new(), }; + let service_exposure = node_port + .map(|port| format!("\n type: NodePort\n nodePort: {port}")) + .unwrap_or_default(); format!( r#"global: @@ -117,7 +154,7 @@ server: }} service: - enabled: true + enabled: true{service_exposure} ingress: enabled: true @@ -147,13 +184,31 @@ impl Score for OpenbaoScore { #[doc(hidden)] fn create_interpret(&self) -> Box> { - HelmChartScore { - namespace: Some(NonBlankString::from_str(&self.instance.namespace).unwrap()), - release_name: NonBlankString::from_str(&self.instance.release).unwrap(), + Box::new(OpenbaoInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug)] +struct OpenbaoInterpret { + score: OpenbaoScore, +} + +#[async_trait] +impl Interpret for OpenbaoInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let outcome = HelmChartScore { + namespace: Some(NonBlankString::from_str(&self.score.instance.namespace).unwrap()), + release_name: NonBlankString::from_str(&self.score.instance.release).unwrap(), chart_name: NonBlankString::from_str("openbao/openbao").unwrap(), chart_version: None, values_overrides: None, - values_yaml: Some(self.values()), + values_yaml: Some(self.score.values()), create_namespace: true, install_only: false, force_conflicts: false, @@ -163,7 +218,36 @@ impl Score for OpenbaoScore { true, )), } - .create_interpret() + .interpret(inventory, topology) + .await?; + topology + .k8s_client() + .await + .map_err(InterpretError::new)? + .wait_until_pod_running( + &self.score.instance.pod(), + Some(&self.score.instance.namespace), + Some(Duration::from_secs(180)), + ) + .await + .map_err(InterpretError::new)?; + Ok(outcome) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("Openbao") + } + + fn get_version(&self) -> Version { + Version::from("0.1.0").expect("static version") + } + + fn get_status(&self) -> InterpretStatus { + InterpretStatus::QUEUED + } + + fn get_children(&self) -> Vec { + Vec::new() } } @@ -178,10 +262,12 @@ mod tests { host: "bao.example".into(), openshift: false, tls_issuer: None, + node_port: Some(30425), } .values(); assert!(!v.contains("cert-manager.io/cluster-issuer")); assert!(!v.contains("secretName")); + assert!(v.contains("nodePort: 30425")); } #[test] @@ -191,6 +277,7 @@ mod tests { host: "bao.example".into(), openshift: false, tls_issuer: Some("letsencrypt".into()), + node_port: None, } .values(); assert!(v.contains("cert-manager.io/cluster-issuer: letsencrypt")); diff --git a/harmony/src/modules/openbao/setup.rs b/harmony/src/modules/openbao/setup.rs index 147af974..e9730654 100644 --- a/harmony/src/modules/openbao/setup.rs +++ b/harmony/src/modules/openbao/setup.rs @@ -1,19 +1,23 @@ use std::path::PathBuf; use async_trait::async_trait; +use harmony_config::{Config, ConfigError}; +use k8s_openapi::api::core::v1::ConfigMap; use log::{info, warn}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::{ data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, + modules::zitadel::{OidcApplicationRef, OidcProviderRef}, score::Score, topology::{K8sclient, Topology}, }; use harmony_types::id::Id; -use super::OpenbaoInstance; +use super::{JWT_AUTH_MOUNT, OpenbaoInstance}; const DEFAULT_KV_MOUNT: &str = "secret"; @@ -56,6 +60,23 @@ pub struct OpenbaoJwtAuth { pub max_ttl: String, } +impl OpenbaoJwtAuth { + pub fn oidc(role_name: impl Into) -> Self { + Self { + oidc_discovery_url: String::new(), + jwt_validation_pubkeys: String::new(), + bound_issuer: String::new(), + role_name: role_name.into(), + bound_audiences: String::new(), + user_claim: "sub".to_string(), + groups_claim: "groups".to_string(), + token_type: "batch".to_string(), + ttl: "1h".to_string(), + max_ttl: "1h".to_string(), + } + } +} + /// Score that initializes, unseals, and configures an already-deployed OpenBao /// instance. /// @@ -102,6 +123,8 @@ pub struct OpenbaoSetupScore { /// Optional JWT auth configuration (e.g., for Zitadel OIDC). #[serde(default)] pub jwt_auth: Option, + #[serde(default)] + pub oidc_application: Option, } fn default_kv_mount() -> String { @@ -116,10 +139,44 @@ impl Default for OpenbaoSetupScore { policies: Vec::new(), users: Vec::new(), jwt_auth: None, + oidc_application: None, } } } +impl OpenbaoSetupScore { + pub fn new(instance: OpenbaoInstance) -> Self { + Self { + instance, + ..Default::default() + } + } + + pub fn with_oidc_provider( + mut self, + provider: &OidcProviderRef, + mut jwt_auth: OpenbaoJwtAuth, + ) -> Self { + let issuer = provider.issuer().to_string(); + jwt_auth.oidc_discovery_url = issuer.clone(); + jwt_auth.bound_issuer = issuer; + jwt_auth.jwt_validation_pubkeys.clear(); + self.jwt_auth = Some(jwt_auth); + self + } + + pub fn with_oidc_application( + self, + provider: &OidcProviderRef, + application: &OidcApplicationRef, + jwt_auth: OpenbaoJwtAuth, + ) -> Self { + let mut score = self.with_oidc_provider(provider, jwt_auth); + score.oidc_application = Some(application.clone()); + score + } +} + impl Score for OpenbaoSetupScore { fn name(&self) -> String { "OpenbaoSetupScore".to_string() @@ -148,6 +205,15 @@ struct InitOutput { root_token: String, } +#[derive(Debug, Serialize, Deserialize, JsonSchema, Config)] +#[config(secret)] +struct OpenbaoRecovery { + namespace: String, + release: String, + keys: Vec, + root_token: String, +} + fn keys_dir() -> PathBuf { directories::BaseDirs::new() .map(|dirs| dirs.data_dir().join("harmony").join("openbao")) @@ -177,6 +243,75 @@ pub fn cached_root_token(instance: &OpenbaoInstance) -> Result { } impl OpenbaoSetupInterpret { + async fn save_recovery(&self, init: &InitOutput) -> Result<(), InterpretError> { + let recovery = OpenbaoRecovery { + namespace: self.score.instance.namespace.clone(), + release: self.score.instance.release.clone(), + keys: init.keys.clone(), + root_token: init.root_token.clone(), + }; + match harmony_config::set(&recovery).await { + Ok(()) => Ok(()), + Err(ConfigError::NoSources) => { + let path = keys_file(&self.score.instance); + std::fs::create_dir_all(keys_dir()) + .map_err(|e| InterpretError::new(format!("create recovery directory: {e}")))?; + std::fs::write( + &path, + serde_json::to_string_pretty(init) + .map_err(|e| InterpretError::new(format!("serialize recovery: {e}")))?, + ) + .map_err(|e| InterpretError::new(format!("write {path:?}: {e}")))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| InterpretError::new(format!("secure {path:?}: {e}")))?; + } + Ok(()) + } + Err(e) => Err(InterpretError::new(format!( + "persist OpenBao recovery material: {e}" + ))), + } + } + + async fn load_recovery(&self) -> Result { + match harmony_config::get::().await { + Ok(recovery) + if recovery.namespace == self.score.instance.namespace + && recovery.release == self.score.instance.release => + { + Ok(InitOutput { + keys: recovery.keys, + root_token: recovery.root_token, + }) + } + Ok(_) => Err(InterpretError::new( + "stored OpenBao recovery material belongs to another instance".to_string(), + )), + Err(ConfigError::NoSources) => { + let path = keys_file(&self.score.instance); + let content = std::fs::read_to_string(&path) + .map_err(|e| InterpretError::new(format!("read {path:?}: {e}")))?; + serde_json::from_str(&content) + .map_err(|e| InterpretError::new(format!("parse {path:?}: {e}"))) + } + Err(ConfigError::NotFound { .. }) if keys_file(&self.score.instance).exists() => { + let path = keys_file(&self.score.instance); + let content = std::fs::read_to_string(&path) + .map_err(|e| InterpretError::new(format!("read {path:?}: {e}")))?; + let init: InitOutput = serde_json::from_str(&content) + .map_err(|e| InterpretError::new(format!("parse {path:?}: {e}")))?; + self.save_recovery(&init).await?; + Ok(init) + } + Err(e) => Err(InterpretError::new(format!( + "load OpenBao recovery material: {e}" + ))), + } + } + async fn exec( &self, k8s: &harmony_k8s::K8sClient, @@ -211,28 +346,13 @@ impl OpenbaoSetupInterpret { // -- Step 1: Init --------------------------------------------------------- - async fn init(&self, k8s: &harmony_k8s::K8sClient) -> Result { - let dir = keys_dir(); - std::fs::create_dir_all(&dir).map_err(|e| { - InterpretError::new(format!("Failed to create keys directory {:?}: {}", dir, e)) - })?; - - let path = keys_file(&self.score.instance); - + async fn init(&self, k8s: &harmony_k8s::K8sClient) -> Result { // Source of truth for "is this vault initialized?" is OpenBao itself, // not a `bao status` pre-check parsed from stderr — that probe is // brittle because `exec_pod_capture_output` discards stdout on // non-zero exit, and `bao status` exits 2 on a fresh-and-uninitialised // vault. So we just attempt `operator init` and let OpenBao tell us // authoritatively via its error message. - if path.exists() { - warn!( - "[OpenbaoSetup] Existing unseal-keys file at {:?} will be \ - overwritten if OpenBao reports as freshly uninitialized", - path - ); - } - info!("[OpenbaoSetup] Probing init state via `bao operator init`..."); let output = self .exec(k8s, vec!["bao", "operator", "init", "-format=json"]) @@ -245,41 +365,13 @@ impl OpenbaoSetupInterpret { let init: InitOutput = serde_json::from_str(&stdout).map_err(|e| { InterpretError::new(format!("Failed to parse init output: {e}")) })?; - let json = serde_json::to_string_pretty(&init) - .map_err(|e| InterpretError::new(format!("Failed to serialize keys: {e}")))?; - std::fs::write(&path, json).map_err(|e| { - InterpretError::new(format!("Failed to write keys to {:?}: {e}", path)) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); - } - - info!("[OpenbaoSetup] Initialized, keys saved to {:?}", path); - Ok(init.root_token) + self.save_recovery(&init).await?; + info!("[OpenbaoSetup] Initialized"); + Ok(init) } Err(e) if e.contains("already initialized") => { - // Existing vault — keys file MUST be present and correspond - // to this install, otherwise we can never unseal it. - if !path.exists() { - return Err(InterpretError::new(format!( - "OpenBao reports already initialized but no local keys \ - file at {:?}. Either restore the keys file or destroy \ - the cluster (PV included) to re-init.", - path - ))); - } - info!( - "[OpenbaoSetup] Vault is already initialized; loading cached keys from {:?}", - path - ); - let content = std::fs::read_to_string(&path) - .map_err(|e| InterpretError::new(format!("Failed to read keys: {e}")))?; - let init: InitOutput = serde_json::from_str(&content) - .map_err(|e| InterpretError::new(format!("Failed to parse keys: {e}")))?; - Ok(init.root_token) + info!("[OpenbaoSetup] Vault is already initialized; loading recovery material"); + self.load_recovery().await } Err(e) => Err(InterpretError::new(format!( "OpenBao operator init failed: {e}" @@ -289,7 +381,11 @@ impl OpenbaoSetupInterpret { // -- Step 2: Unseal ------------------------------------------------------- - async fn unseal(&self, k8s: &harmony_k8s::K8sClient) -> Result<(), InterpretError> { + async fn unseal( + &self, + k8s: &harmony_k8s::K8sClient, + init: &InitOutput, + ) -> Result<(), InterpretError> { #[derive(Deserialize)] struct Status { sealed: bool, @@ -335,12 +431,6 @@ impl OpenbaoSetupInterpret { } info!("[OpenbaoSetup] Unsealing..."); - let path = keys_file(&self.score.instance); - let content = std::fs::read_to_string(&path) - .map_err(|e| InterpretError::new(format!("Failed to read keys: {e}")))?; - let init: InitOutput = serde_json::from_str(&content) - .map_err(|e| InterpretError::new(format!("Failed to parse keys: {e}")))?; - for key in &init.keys[0..3] { self.exec(k8s, vec!["bao", "operator", "unseal", key]) .await @@ -453,8 +543,34 @@ impl OpenbaoSetupInterpret { None => return Ok(()), }; + let bound_audiences = if let Some(application) = &self.score.oidc_application { + let namespace = application.namespace(); + let name = application.config_map_name(); + let key = application.project_id_key(); + tokio::time::timeout(std::time::Duration::from_secs(180), async { + loop { + if let Ok(Some(config_map)) = + k8s.get_resource::(name, Some(namespace)).await + && let Some(audience) = + config_map.data.and_then(|data| data.get(key).cloned()) + { + return audience; + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| { + InterpretError::new(format!( + "Timed out waiting for OIDC application ConfigMap {namespace}/{name}" + )) + })? + } else { + jwt.bound_audiences.clone() + }; + let _ = self - .bao(k8s, root_token, &["bao", "auth", "enable", "jwt"]) + .bao(k8s, root_token, &["bao", "auth", "enable", JWT_AUTH_MOUNT]) .await; // Configure JWT validation: static public keys when provided, @@ -478,7 +594,7 @@ impl OpenbaoSetupInterpret { &[ "bao", "write", - "auth/jwt/config", + &format!("auth/{JWT_AUTH_MOUNT}/config"), &validation_arg, &format!("bound_issuer={}", jwt.bound_issuer), ], @@ -493,20 +609,19 @@ impl OpenbaoSetupInterpret { ); } Err(e) => { - warn!( - "[OpenbaoSetup] JWT auth config failed (non-fatal): {}. \ - Ensure '{}' resolves from inside the cluster.", - e, jwt.oidc_discovery_url - ); + return Err(InterpretError::new(format!( + "Failed to configure JWT auth from '{}': {e}", + jwt.oidc_discovery_url + ))); } } let mut role_args = vec![ "bao".to_string(), "write".to_string(), - format!("auth/jwt/role/{}", jwt.role_name), + format!("auth/{JWT_AUTH_MOUNT}/role/{}", jwt.role_name), "role_type=jwt".to_string(), - format!("bound_audiences={}", jwt.bound_audiences), + format!("bound_audiences={bound_audiences}"), format!("user_claim={}", jwt.user_claim), format!("ttl={}", jwt.ttl), format!("max_ttl={}", jwt.max_ttl), @@ -553,8 +668,9 @@ impl Interpret for OpenbaoSetupInterpret { )) })?; - let root_token = self.init(&k8s).await?; - self.unseal(&k8s).await?; + let recovery = self.init(&k8s).await?; + self.unseal(&k8s, &recovery).await?; + let root_token = recovery.root_token; self.enable_kv(&k8s, &root_token).await?; if !self.score.users.is_empty() { @@ -565,10 +681,7 @@ impl Interpret for OpenbaoSetupInterpret { self.create_users(&k8s, &root_token).await?; self.configure_jwt(&k8s, &root_token).await?; - let mut details = vec![ - format!("root_token={}", root_token), - format!("kv_mount={}", self.score.kv_mount), - ]; + let mut details = vec![format!("kv_mount={}", self.score.kv_mount)]; for user in &self.score.users { details.push(format!("user={}", user.username)); } @@ -600,6 +713,22 @@ impl Interpret for OpenbaoSetupInterpret { #[cfg(test)] mod tests { use super::*; + use crate::modules::zitadel::{ZitadelScore, ZitadelSetupScore}; + + fn jwt_auth() -> OpenbaoJwtAuth { + OpenbaoJwtAuth { + oidc_discovery_url: String::new(), + jwt_validation_pubkeys: String::new(), + bound_issuer: String::new(), + role_name: "fleet".into(), + bound_audiences: "fleet-api".into(), + user_claim: "sub".into(), + groups_claim: "groups".into(), + token_type: "batch".into(), + ttl: "5m".into(), + max_ttl: "15m".into(), + } + } #[test] fn default_score_carries_expected_mounts() { @@ -608,4 +737,50 @@ mod tests { assert_eq!(s.instance.pod(), "openbao-0"); assert_eq!(s.kv_mount, "secret"); } + + #[test] + fn oidc_provider_supplies_discovery_and_issuer() { + let provider = ZitadelScore { + host: "auth.example".into(), + external_secure: true, + ..Default::default() + } + .provider_ref(); + let mut jwt = jwt_auth(); + jwt.jwt_validation_pubkeys = "stale static key".into(); + + let score = OpenbaoSetupScore::default().with_oidc_provider(&provider, jwt); + let jwt = score.jwt_auth.unwrap(); + + assert_eq!(jwt.oidc_discovery_url, "https://auth.example"); + assert_eq!(jwt.bound_issuer, "https://auth.example"); + assert!(jwt.jwt_validation_pubkeys.is_empty()); + } + + #[test] + fn oidc_application_supplies_cross_namespace_project_id_coordinate() { + let provider = ZitadelScore { + host: "auth.example".into(), + ..Default::default() + } + .provider_ref(); + let application = ZitadelSetupScore { + outputs_namespace: Some("fleet-system".into()), + ..Default::default() + } + .application( + "fleet", + "fleet", + crate::modules::zitadel::ZitadelAppType::DeviceCode, + ) + .application_ref("fleet"); + + let score = + OpenbaoSetupScore::default().with_oidc_application(&provider, &application, jwt_auth()); + let jwt = score.jwt_auth.as_ref().unwrap(); + + assert_eq!(jwt.oidc_discovery_url, "https://auth.example"); + assert_eq!(jwt.bound_issuer, "https://auth.example"); + assert_eq!(score.oidc_application, Some(application)); + } } diff --git a/harmony/src/modules/podman/mod.rs b/harmony/src/modules/podman/mod.rs index e4425e86..d2dbb560 100644 --- a/harmony/src/modules/podman/mod.rs +++ b/harmony/src/modules/podman/mod.rs @@ -1,10 +1,7 @@ mod interpret; mod score; -#[cfg(unix)] -mod topology; -pub use crate::topology::EnvVar; pub use interpret::PodmanV0Interpret; -pub use score::{PodmanService, PodmanV0Score, ReconcileScore, SecretEnvVar}; -#[cfg(unix)] -pub use topology::PodmanTopology; +pub use score::{ + EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount, +}; diff --git a/harmony/src/modules/podman/score.rs b/harmony/src/modules/podman/score.rs index 65d135d5..10fe9dd3 100644 --- a/harmony/src/modules/podman/score.rs +++ b/harmony/src/modules/podman/score.rs @@ -1,96 +1,14 @@ -//! Podman Score types and their `Score` / `Interpret` bindings. -//! -//! `ContainerRuntime` is a first-class harmony capability (see -//! [`crate::topology::ContainerRuntime`]); the types that describe a -//! set of containers a caller wants running live here next to the -//! trait impls that route them to interpretation. These types are -//! independent of any particular product (IoT fleet, OKD fleet, etc.) -//! — callers serialize them over whatever transport they like. - -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - use crate::{ interpret::Interpret, score::Score, - topology::{ContainerRuntime, EnvVar, RestartPolicy, Topology, VolumeMount}, + topology::{ContainerRuntime, Topology}, }; use super::interpret::PodmanV0Interpret; -/// An env var whose value the device fetches from its secret store at -/// apply time (ADR-025). `secret` names the entry under the -/// deployment's KV subtree (`///`); -/// the agent injects the fetched value under `name`. Values never -/// transit NATS — only this reference does. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -pub struct SecretEnvVar { - pub name: String, - pub secret: String, -} - -/// A single container managed by podman on the target host. -/// -/// Wire-compatible with prior releases: the new `env`, `volumes`, and -/// `restart_policy` fields all default to empty / `unless-stopped` so older -/// Deployment CRs without them deserialize unchanged. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -pub struct PodmanService { - pub name: String, - pub image: String, - pub ports: Vec, - /// Environment variables passed to the container. Order is preserved so - /// `PartialEq` is deterministic for drift detection. - #[serde(default)] - pub env: Vec, - /// Env vars resolved from the device's secret store at apply time. - /// `skip_serializing_if` keeps payloads byte-identical for specs - /// that don't use secrets (string-compare idempotency). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub secret_env: Vec, - /// Bind-mount volumes. Bind-only in v0; `host_path` is an absolute path - /// on the device's filesystem. - #[serde(default)] - pub volumes: Vec, - /// Restart policy on container exit. Defaults to `unless-stopped` — - /// matching docker-compose's typical behavior for long-running services. - #[serde(default)] - pub restart_policy: RestartPolicy, -} - -/// v0 Score for podman-based workloads. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] -pub struct PodmanV0Score { - pub services: Vec, -} - -impl PodmanV0Score { - /// Label value applied to every container this Score creates. The - /// caller uses the label to enumerate the set of containers that - /// belong to a given Score instance (e.g. to tear them down when - /// the corresponding desired-state entry is deleted). - /// - /// For v0 there is a single PodmanV0Score per desired-state key - /// and no multi-score scheduling on the target host, so a stable - /// join over the service names is an adequate identifier. When - /// v0.1 introduces multiple scores per host the caller will pass - /// an explicit deployment id. - pub fn deployment_label(&self) -> String { - let names: Vec<&str> = self.services.iter().map(|s| s.name.as_str()).collect(); - names.join(",") - } -} - -/// Wire envelope for a reconciler-style Score payload: externally -/// tagged so the JSON is `{"type": "PodmanV0", "data": { ... }}`. -/// Adding a new variant is additive — emitters stay opaque routers, -/// consumers learn the new variant, older consumers harmlessly -/// log-and-skip the unknown tag. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] -#[serde(tag = "type", content = "data")] -pub enum ReconcileScore { - PodmanV0(PodmanV0Score), -} +pub use harmony_reconciler_contracts::{ + EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount, +}; impl Score for PodmanV0Score { fn create_interpret(&self) -> Box> { @@ -115,159 +33,3 @@ impl Score for ReconcileScore { } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn svc(name: &str, image: &str) -> PodmanService { - PodmanService { - name: name.to_string(), - image: image.to_string(), - ports: vec![], - env: vec![], - secret_env: vec![], - volumes: vec![], - restart_policy: RestartPolicy::default(), - } - } - - #[test] - fn podman_v0_score_serializes_with_adjacent_tag() { - let score = ReconcileScore::PodmanV0(PodmanV0Score { - services: vec![PodmanService { - ports: vec!["8080:80".to_string()], - ..svc("web", "nginx:latest") - }], - }); - let json = serde_json::to_string(&score).unwrap(); - assert!(json.contains("\"type\":\"PodmanV0\"")); - assert!(json.contains("\"data\"")); - } - - #[test] - fn podman_v0_score_roundtrip() { - let score = ReconcileScore::PodmanV0(PodmanV0Score { - services: vec![ - PodmanService { - ports: vec!["8080:80".to_string()], - env: vec![EnvVar::new("LOG_LEVEL", "info")], - volumes: vec![VolumeMount { - host_path: "/var/lib/web/data".to_string(), - container_path: "/data".to_string(), - read_only: false, - }], - restart_policy: RestartPolicy::Always, - ..svc("web", "nginx:latest") - }, - PodmanService { - ports: vec!["3000:3000".to_string(), "9090:9090".to_string()], - ..svc("api", "myapp:1.0") - }, - ], - }); - let serialized = serde_json::to_string(&score).unwrap(); - let deserialized: ReconcileScore = serde_json::from_str(&serialized).unwrap(); - assert_eq!(score, deserialized); - } - - #[test] - fn legacy_payload_without_env_volumes_or_restart_deserializes() { - // Wire-compat: a Deployment CR built before these fields existed - // still round-trips into the new PodmanService. - let legacy = r#"{ - "type": "PodmanV0", - "data": { "services": [ - { "name": "web", "image": "nginx", "ports": ["8080:80"] } - ]} - }"#; - let parsed: ReconcileScore = serde_json::from_str(legacy).unwrap(); - let ReconcileScore::PodmanV0(score) = parsed; - assert_eq!(score.services.len(), 1); - assert!(score.services[0].env.is_empty()); - assert!(score.services[0].secret_env.is_empty()); - assert!(score.services[0].volumes.is_empty()); - assert_eq!( - score.services[0].restart_policy, - RestartPolicy::UnlessStopped - ); - } - - #[test] - fn secret_env_roundtrips_and_is_omitted_when_empty() { - // Omission when empty keeps payloads byte-identical to - // pre-secret specs — the agent's string-compare idempotency - // and old-agent compatibility both rely on it. - let plain = serde_json::to_string(&svc("web", "nginx")).unwrap(); - assert!(!plain.contains("secret_env"), "got: {plain}"); - - let with_secret = PodmanService { - secret_env: vec![SecretEnvVar { - name: "DB_PASSWORD".to_string(), - secret: "db_password".to_string(), - }], - ..svc("api", "myapp") - }; - let json = serde_json::to_string(&with_secret).unwrap(); - let back: PodmanService = serde_json::from_str(&json).unwrap(); - assert_eq!(back.secret_env, with_secret.secret_env); - } - - #[test] - fn restart_policy_serializes_kebab_case() { - // docker-compose users expect `unless-stopped`, `on-failure` — - // verify our serde rename produces that. - let s = serde_json::to_string(&RestartPolicy::UnlessStopped).unwrap(); - assert_eq!(s, "\"unless-stopped\""); - let s = serde_json::to_string(&RestartPolicy::OnFailure).unwrap(); - assert_eq!(s, "\"on-failure\""); - } - - #[test] - fn env_ordering_is_preserved_across_roundtrip() { - // Deterministic equality is what `matches_spec` drift detection - // relies on. If env reordered on roundtrip, agents would loop - // on recreate. - let svc = PodmanService { - env: vec![ - EnvVar::new("B", "2"), - EnvVar::new("A", "1"), - EnvVar::new("C", "3"), - ], - ..svc("api", "myapp") - }; - let json = serde_json::to_string(&svc).unwrap(); - let back: PodmanService = serde_json::from_str(&json).unwrap(); - assert_eq!(back.env, svc.env); - } - - #[test] - fn env_schema_is_object_not_tuple_for_crd_validation() { - // k8s apiextensions rejects tuple validation - // (`items: [, ]`) with `Forbidden: items must be a - // schema object and not an array`. schemars generates that for - // `Vec<(String, String)>`. The fix is the `EnvVar` struct, and - // this test pins it: if anyone reverts to a tuple, the operator's - // CRD install fails — catching it here saves a deploy round-trip. - let schema = schemars::schema_for!(PodmanService); - let json = serde_json::to_value(&schema).unwrap(); - let env_items = json - .pointer("/properties/env/items") - .expect("PodmanService schema must expose env.items"); - // Object form: `{"$ref": "#/definitions/EnvVar"}` or - // `{"type": "object", ...}`. Tuple form: `[...]`. We reject the - // latter. - assert!( - env_items.is_object(), - "env.items must be a single schema object, got: {env_items}" - ); - } - - #[test] - fn deployment_label_joins_service_names() { - let score = PodmanV0Score { - services: vec![svc("web", "nginx"), svc("api", "myapp")], - }; - assert_eq!(score.deployment_label(), "web,api"); - } -} diff --git a/harmony/src/modules/podman/topology.rs b/harmony/src/modules/podman/topology.rs deleted file mode 100644 index 3aeb53bd..00000000 --- a/harmony/src/modules/podman/topology.rs +++ /dev/null @@ -1,392 +0,0 @@ -use std::collections::HashMap; -use std::path::PathBuf; -use std::time::Duration; - -use async_trait::async_trait; -use futures_util::StreamExt; -use podman_api::Podman; -use podman_api::models::{ContainerMount, PortMapping}; -use podman_api::opts::{ - ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts, - ContainerRestartPolicy, ContainerStopOpts, PullOpts, -}; - -use crate::domain::topology::{ - ContainerRuntime, ContainerSpec, ContainerState, PreparationError, PreparationOutcome, - RestartPolicy, Topology, VolumeMount, -}; -use crate::executors::ExecutorError; - -/// Node-level Topology backed by a local Podman daemon reachable over its -/// REST API (the user systemd socket, typically at -/// `$XDG_RUNTIME_DIR/podman/podman.sock`). This is the concrete Topology used -/// by the IoT agent running on a Raspberry Pi or similar single-host device: -/// Scores reach in via the `ContainerRuntime` capability, everything else the -/// agent needs is delegated to `Inventory`. -pub struct PodmanTopology { - name: String, - podman: Podman, -} - -impl PodmanTopology { - /// Connect to the local user Podman socket at the path resolved from - /// `XDG_RUNTIME_DIR` (falls back to `/run/user//podman/podman.sock` - /// when that env var is unset). - pub fn from_default_socket() -> Result { - Self::from_unix_socket(default_user_socket()) - } - - /// Connect to an explicit Unix socket path. Useful for tests, or for - /// pointing at a system-level socket (`/run/podman/podman.sock`). - pub fn from_unix_socket(socket: PathBuf) -> Result { - let podman = Podman::unix(&socket); - Ok(Self { - name: format!("PodmanTopology({})", socket.display()), - podman, - }) - } - - fn containers(&self) -> podman_api::api::Containers { - self.podman.containers() - } - - async fn get_by_name( - &self, - name: &str, - ) -> Result, ExecutorError> { - let opts = ContainerListOpts::builder() - .all(true) - .filter([ContainerListFilter::Name(name.to_string())]) - .build(); - let list = self.containers().list(&opts).await.map_err(to_exec_error)?; - Ok(list.into_iter().next()) - } - - async fn ensure_image_present(&self, image: &str) -> Result<(), ExecutorError> { - // Fast path: image already in the local store → no network - // call, no rate-limit exposure. Matches the behaviour a - // Kubernetes `imagePullPolicy: IfNotPresent` would give, and - // it's the right default for a long-lived device agent — - // every podman `pull` against a public registry is rate- - // limited traffic we only want to spend when strictly - // necessary. Upgrades (different `image` string / tag) hit - // this function with a reference that's NOT locally - // present yet and still do the pull below. - let images = self.podman.images(); - if images.get(image).exists().await.map_err(to_exec_error)? { - return Ok(()); - } - - let opts = PullOpts::builder().reference(image).build(); - let mut stream = images.pull(&opts); - while let Some(event) = stream.next().await { - let event = event.map_err(to_exec_error)?; - if let Some(err) = event.error { - return Err(exec_err(format!("podman pull {image} failed: {err}"))); - } - } - Ok(()) - } - - async fn remove_container(&self, name_or_id: &str) -> Result<(), ExecutorError> { - let opts = ContainerDeleteOpts::builder().force(true).build(); - let container = self.containers().get(name_or_id); - match container.delete(&opts).await { - Ok(()) => Ok(()), - Err(e) if is_not_found(&e) => Ok(()), - Err(e) => Err(to_exec_error(e)), - } - } -} - -#[async_trait] -impl Topology for PodmanTopology { - fn name(&self) -> &str { - &self.name - } - - async fn ensure_ready(&self) -> Result { - // A quick ping — calling info() is the cheapest endpoint that - // exercises the socket end-to-end. - self.podman - .info() - .await - .map_err(|e| PreparationError::new(format!("podman socket unreachable: {e}")))?; - Ok(PreparationOutcome::Noop) - } -} - -#[async_trait] -impl ContainerRuntime for PodmanTopology { - async fn ensure_service_running(&self, spec: &ContainerSpec) -> Result<(), ExecutorError> { - let existing = self.get_by_name(&spec.name).await?; - - if let Some(existing) = existing.as_ref() { - if matches_spec(existing, spec) { - let running = existing.state.as_deref() == Some("running"); - if running { - return Ok(()); - } - // Same spec, just not running — start it rather than recreate. - let id = existing.id.clone().unwrap_or_else(|| spec.name.clone()); - self.containers() - .get(id) - .start(None) - .await - .map_err(to_exec_error)?; - return Ok(()); - } - // Drift — remove and recreate below. - self.remove_container(&spec.name).await?; - } - - self.ensure_image_present(&spec.image).await?; - - let mut labels = HashMap::new(); - labels.insert( - ContainerSpec::MANAGED_BY_LABEL.to_string(), - ContainerSpec::MANAGED_BY_VALUE.to_string(), - ); - for (k, v) in &spec.labels { - labels.insert(k.clone(), v.clone()); - } - - let mut port_mappings = Vec::with_capacity(spec.ports.len()); - for raw in &spec.ports { - port_mappings.push(parse_port_mapping(raw)?); - } - - let env_map: HashMap = spec - .env - .iter() - .map(|e| (e.name.clone(), e.value.clone())) - .collect(); - - let mounts: Vec = spec.volumes.iter().map(volume_to_mount).collect(); - - let mut builder = ContainerCreateOpts::builder() - .name(&spec.name) - .image(&spec.image) - .labels(labels) - .portmappings(port_mappings) - .env(env_map) - .restart_policy(map_restart_policy(spec.restart_policy)); - if !mounts.is_empty() { - builder = builder.mounts(mounts); - } - let opts = builder.build(); - - let created = self - .containers() - .create(&opts) - .await - .map_err(to_exec_error)?; - self.containers() - .get(created.id.clone()) - .start(None) - .await - .map_err(to_exec_error)?; - Ok(()) - } - - async fn remove_service(&self, name: &str) -> Result<(), ExecutorError> { - // First try a graceful stop; ignore failures so removal still happens. - let stop_opts = ContainerStopOpts::builder() - .timeout(STOP_TIMEOUT.as_secs() as usize) - .build(); - let container = self.containers().get(name); - if container.exists().await.unwrap_or(false) { - let _ = container.stop(&stop_opts).await; - } - self.remove_container(name).await - } - - async fn list_managed_services(&self) -> Result, ExecutorError> { - let opts = ContainerListOpts::builder() - .all(true) - .filter([ContainerListFilter::LabelKeyVal( - ContainerSpec::MANAGED_BY_LABEL.to_string(), - ContainerSpec::MANAGED_BY_VALUE.to_string(), - )]) - .build(); - let list = self.containers().list(&opts).await.map_err(to_exec_error)?; - Ok(list.into_iter().map(from_list_container).collect()) - } -} - -/// Per ROADMAP §5.6 — graceful shutdown = `podman stop` with a 5-minute -/// timeout, then SIGKILL (which Podman does on its own once the timeout -/// elapses). This matches the partner-visible behaviour we advertise. -const STOP_TIMEOUT: Duration = Duration::from_secs(300); - -fn default_user_socket() -> PathBuf { - if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") { - return PathBuf::from(format!("{dir}/podman/podman.sock")); - } - // unsafe fallback only for edge cases; the agent systemd unit runs - // under its own user with XDG_RUNTIME_DIR populated. - let uid = current_uid_from_proc(); - PathBuf::from(format!("/run/user/{uid}/podman/podman.sock")) -} - -// We'd rather not pull in the `libc` crate just to call getuid(2). The -// agent only runs on Linux, so reading /proc/self/status is the minimal -// portable route. -fn current_uid_from_proc() -> u32 { - match std::fs::read_to_string("/proc/self/status") { - Ok(s) => s - .lines() - .find_map(|l| l.strip_prefix("Uid:")) - .and_then(|rest| rest.split_whitespace().next()) - .and_then(|s| s.parse().ok()) - .unwrap_or(0), - Err(_) => 0, - } -} - -fn parse_port_mapping(raw: &str) -> Result { - let (host, container) = raw.split_once(':').ok_or_else(|| { - exec_err(format!( - "invalid port mapping '{raw}' — expected 'HOST:CONTAINER'" - )) - })?; - let host: u16 = host - .parse() - .map_err(|_| exec_err(format!("invalid host port '{host}' in mapping '{raw}'")))?; - let container: u16 = container.parse().map_err(|_| { - exec_err(format!( - "invalid container port '{container}' in mapping '{raw}'" - )) - })?; - Ok(PortMapping { - host_port: Some(host), - container_port: Some(container), - host_ip: None, - protocol: None, - range: None, - }) -} - -fn matches_spec(observed: &podman_api::models::ListContainer, spec: &ContainerSpec) -> bool { - let same_image = observed - .image - .as_deref() - .map(|i| i == spec.image) - .unwrap_or(false); - if !same_image { - return false; - } - let observed_ports = observed.ports.as_deref().unwrap_or(&[]); - if observed_ports.len() != spec.ports.len() { - return false; - } - for raw in &spec.ports { - let Ok(expected) = parse_port_mapping(raw) else { - return false; - }; - let present = observed_ports.iter().any(|p| { - p.host_port == expected.host_port && p.container_port == expected.container_port - }); - if !present { - return false; - } - } - // FIXME(redeploy-loop): this branch makes the agent's periodic - // reconcile non-idempotent for any non-trivial Deployment. - // Symptom: a service with env or volumes is destroyed and - // recreated every 30s tick (RECONCILE_INTERVAL), even when the - // observed container is already correct — operators see flapping - // container IDs, intermittent connectivity blips, log noise. - // - // Root cause: `podman list` (v5.x) doesn't surface env or mounts, - // so we can't compare them; the original author chose to declare - // "any spec with env/volumes is drifted" as a fail-safe. That's - // the wrong default for a polling reconciler — it weaponizes the - // poll into a re-creation loop. - // - // Right fix (out of scope for the demo, in scope for delivery): - // 1. Switch this code path to `containers.get(name).inspect()` - // which DOES return env + mounts. Compare structurally. - // 2. Treat absent fields on the inspect response as "unchanged", - // not "drifted". - // 3. Add an integration test that runs ensure_service_running - // twice on the same spec and asserts the container ID is - // unchanged. - // - // Layered next: the upcoming health-check addition to - // ContainerSpec gives the agent a separate signal to decide - // when to recreate (failed health checks → unhealthy → recreate) - // independent of the spec-drift check. - // - // Until fixed: avoid env / volumes in demo-time deployments to - // dodge the loop. The hello-web nginx demo doesn't have either, - // which is why it's stable. - if !spec.env.is_empty() || !spec.volumes.is_empty() { - return false; - } - // Restart policy: ListContainer doesn't surface it directly. We - // only force a recreate when the spec explicitly asks for something - // other than the default — so unchanged podman-default behaviour - // stays a NOOP, and explicit policy changes converge on next apply. - if spec.restart_policy != RestartPolicy::default() { - return false; - } - true -} - -fn volume_to_mount(v: &VolumeMount) -> ContainerMount { - // ContainerMount expresses options as a string Vec — Podman's - // post-create flag list. `ro`/`rw` go there. Bind-only in v0. - let mut options: Vec = Vec::new(); - options.push(if v.read_only { - "ro".to_string() - } else { - "rw".to_string() - }); - ContainerMount { - _type: Some("bind".to_string()), - source: Some(v.host_path.clone()), - destination: Some(v.container_path.clone()), - options: Some(options), - uid_mappings: None, - gid_mappings: None, - } -} - -fn map_restart_policy(p: RestartPolicy) -> ContainerRestartPolicy { - match p { - RestartPolicy::No => ContainerRestartPolicy::No, - RestartPolicy::UnlessStopped => ContainerRestartPolicy::UnlessStopped, - RestartPolicy::OnFailure => ContainerRestartPolicy::OnFailure, - RestartPolicy::Always => ContainerRestartPolicy::Always, - } -} - -fn from_list_container(c: podman_api::models::ListContainer) -> ContainerState { - ContainerState { - name: c - .names - .as_deref() - .and_then(|ns| ns.first().cloned()) - .or(c.id.clone()) - .unwrap_or_default(), - image: c.image.unwrap_or_default(), - running: c.state.as_deref() == Some("running"), - labels: c.labels.unwrap_or_default().into_iter().collect(), - } -} - -fn to_exec_error(e: podman_api::Error) -> ExecutorError { - ExecutorError::UnexpectedError(format!("podman: {e}")) -} - -fn exec_err(msg: impl Into) -> ExecutorError { - ExecutorError::UnexpectedError(msg.into()) -} - -fn is_not_found(e: &podman_api::Error) -> bool { - // The crate returns a rich Error enum; fall back to string matching - // because the exact variants for 404 vary by endpoint. Good enough for - // the "already-gone" idempotency path. - format!("{e}").to_lowercase().contains("no such container") -} diff --git a/harmony/src/modules/postgresql/capability.rs b/harmony/src/modules/postgresql/capability.rs index e66ab3e9..f55f5241 100644 --- a/harmony/src/modules/postgresql/capability.rs +++ b/harmony/src/modules/postgresql/capability.rs @@ -27,6 +27,37 @@ pub trait PostgreSQL: Send + Sync { // ) -> Result, String>; } +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct PostgreSQLRootAccountRef { + pub(crate) host: String, + pub(crate) port: u16, + pub(crate) namespace: String, + pub(crate) secret_name: String, + pub(crate) username_key: String, + pub(crate) password_key: String, +} + +impl PostgreSQLRootAccountRef { + pub fn host(&self) -> &str { + &self.host + } + pub fn port(&self) -> u16 { + self.port + } + pub fn namespace(&self) -> &str { + &self.namespace + } + pub fn secret_name(&self) -> &str { + &self.secret_name + } + pub fn username_key(&self) -> &str { + &self.username_key + } + pub fn password_key(&self) -> &str { + &self.password_key + } +} + #[derive(Clone, Debug, Serialize)] pub struct PostgreSQLConfig { pub cluster_name: String, diff --git a/harmony/src/modules/postgresql/mod.rs b/harmony/src/modules/postgresql/mod.rs index 4e6efd59..f4d7b418 100644 --- a/harmony/src/modules/postgresql/mod.rs +++ b/harmony/src/modules/postgresql/mod.rs @@ -1,4 +1,5 @@ pub mod capability; +pub use capability::*; mod score_connect; mod score_k8s; pub use score_connect::*; diff --git a/harmony/src/modules/postgresql/score_k8s.rs b/harmony/src/modules/postgresql/score_k8s.rs index 231b2d07..e607fb6e 100644 --- a/harmony/src/modules/postgresql/score_k8s.rs +++ b/harmony/src/modules/postgresql/score_k8s.rs @@ -2,7 +2,7 @@ use crate::data::Version; use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; use crate::inventory::Inventory; use crate::modules::k8s::resource::K8sResourceScore; -use crate::modules::postgresql::capability::PostgreSQLConfig; +use crate::modules::postgresql::capability::{PostgreSQLConfig, PostgreSQLRootAccountRef}; use crate::modules::postgresql::cnpg::{ Bootstrap, Cluster, ClusterSpec, ExternalCluster, Initdb, PgBaseBackup, ReplicaSpec, SecretKeySelector, Storage, @@ -16,9 +16,9 @@ use async_trait::async_trait; use harmony_k8s::KubernetesDistribution; use harmony_types::id::Id; use k8s_openapi::ByteString; -use k8s_openapi::api::core::v1::Service; use k8s_openapi::api::core::v1::{Pod, Secret}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::api::ListParams; use log::{debug, info, warn}; use serde::Serialize; @@ -60,6 +60,25 @@ impl K8sPostgreSQLScore { }, } } + + pub fn cluster_name(mut self, name: impl Into) -> Self { + self.config.cluster_name = name.into(); + self + } + + pub fn root_account_ref(&self) -> PostgreSQLRootAccountRef { + PostgreSQLRootAccountRef { + host: format!( + "{}-rw.{}.svc.cluster.local", + self.config.cluster_name, self.config.namespace + ), + port: 5432, + namespace: self.config.namespace.clone(), + secret_name: format!("{}-superuser", self.config.cluster_name), + username_key: "username".to_string(), + password_key: "password".to_string(), + } + } } impl Score for K8sPostgreSQLScore { @@ -195,9 +214,7 @@ impl K8sPostgreSQLInterpret { Ok(()) } - /// Waits for the cluster's `-rw` service to exist, indicating the primary - /// pod is running and the CNPG operator has created the service. - async fn wait_for_rw_service( + async fn wait_for_primary( &self, topology: &T, ) -> Result<(), InterpretError> { @@ -206,40 +223,52 @@ impl K8sPostgreSQLInterpret { .await .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {}", e)))?; - let service_name = format!("{}-rw", self.config.cluster_name); let namespace = &self.config.namespace; let timeout = std::time::Duration::from_secs(120); let start = std::time::Instant::now(); info!( - "Waiting for service '{}/{}' (up to {}s)...", + "Waiting for PostgreSQL primary in '{}' (up to {}s)...", namespace, - service_name, timeout.as_secs() ); loop { match k8s_client - .get_resource::(&service_name, Some(namespace)) + .list_resources::( + Some(namespace), + Some(ListParams::default().labels(&format!( + "cnpg.io/cluster={},role=primary", + self.config.cluster_name + ))), + ) .await { - Ok(Some(_)) => { - info!("Service '{}/{}' is ready", namespace, service_name); + Ok(pods) + if pods.items.iter().any(|pod| { + pod.status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition.type_ == "Ready" && condition.status == "True" + }) + }) + }) => + { + info!("PostgreSQL primary is ready in '{namespace}'"); return Ok(()); } - Ok(None) => { - debug!("Service '{service_name}' not yet created"); - } + Ok(_) => debug!("PostgreSQL primary is not ready"), Err(e) => { - debug!("Error checking service '{service_name}': {e}"); + debug!("Error checking PostgreSQL primary: {e}"); } } if start.elapsed() > timeout { return Err(InterpretError::new(format!( - "Timed out waiting for service '{}/{}' after {}s", + "Timed out waiting for PostgreSQL primary in '{}' after {}s", namespace, - service_name, timeout.as_secs() ))); } @@ -292,7 +321,7 @@ impl Interpret for K8sPostgr .await?; if self.config.wait_for_ready { - self.wait_for_rw_service(topology).await?; + self.wait_for_primary(topology).await?; } Ok(outcome) @@ -423,3 +452,26 @@ impl Interpret for K8sPostgr todo!() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn root_account_ref_has_deterministic_cnpg_coordinates() { + let mut score = K8sPostgreSQLScore::new("identity"); + score.config.cluster_name = "fleet-pg".to_string(); + + assert_eq!( + score.root_account_ref(), + PostgreSQLRootAccountRef { + host: "fleet-pg-rw.identity.svc.cluster.local".to_string(), + port: 5432, + namespace: "identity".to_string(), + secret_name: "fleet-pg-superuser".to_string(), + username_key: "username".to_string(), + password_key: "password".to_string(), + } + ); + } +} diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index a3522f76..7b00bac5 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -1,9 +1,7 @@ pub mod admin_auth; -pub mod device_groups; pub mod setup; pub use admin_auth::{ADMIN_API_SCOPES, DeviceCodeError, DeviceCodeFlowConfig, device_code_login}; -pub use device_groups::ZitadelDeviceGroups; pub use setup::{ MachineKeyType, MintedDeviceCredentials, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelClientIdExportScore, ZitadelCredentialsExportScore, @@ -12,7 +10,7 @@ pub use setup::{ }; use harmony_k8s::KubernetesDistribution; -use k8s_openapi::api::core::v1::Namespace; +use k8s_openapi::api::core::v1::{Namespace, Service, ServicePort, ServiceSpec}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::{ByteString, api::core::v1::Secret as K8sSecret}; use kube::{Error as KubeError, core::ErrorResponse}; @@ -25,7 +23,6 @@ use async_trait::async_trait; use harmony_config::Config; use harmony_macros::hurl; use harmony_types::id::Id; -use harmony_types::storage::StorageSize; use log::{debug, error, info, trace, warn}; use non_blank_string_rs::NonBlankString; use schemars::JsonSchema; @@ -36,7 +33,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, modules::helm::chart::{HelmChartScore, HelmRepository}, - modules::postgresql::capability::{PostgreSQL, PostgreSQLClusterRole, PostgreSQLConfig}, + modules::{k8s::resource::K8sResourceScore, postgresql::capability::PostgreSQLRootAccountRef}, score::Score, topology::{HelmCommand, K8sclient, Topology}, }; @@ -45,9 +42,89 @@ use crate::{ /// Public so callers can pre-create the namespace via `oc new-project` /// matching what the Score will use. pub const DEFAULT_NAMESPACE: &str = "zitadel"; -const PG_CLUSTER_NAME: &str = "zitadel-pg"; const MASTERKEY_SECRET_NAME: &str = "zitadel-masterkey"; +// Q: What's an oidc provider in harmony's context +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct OidcProviderRef { + host: String, + secure: bool, + port: Option, +} + +impl OidcProviderRef { + pub fn issuer(&self) -> String { + let scheme = if self.secure { "https" } else { "http" }; + let default_port = if self.secure { 443 } else { 80 }; + let port = self + .port + .filter(|port| *port != default_port) + .map(|port| format!(":{port}")) + .unwrap_or_default(); + format!("{scheme}://{}{port}", self.host) + } + + pub fn host(&self) -> &str { + &self.host + } + + pub fn secure(&self) -> bool { + self.secure + } + + pub fn port(&self) -> Option { + self.port + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct OidcApplicationRef { + namespace: String, + config_map: String, +} + +impl OidcApplicationRef { + pub fn namespace(&self) -> &str { + &self.namespace + } + + pub fn config_map_name(&self) -> &str { + &self.config_map + } + + pub fn project_id_key(&self) -> &'static str { + "project_id" + } + + pub fn client_id_key(&self) -> &'static str { + "client_id" + } + + pub fn roles_claim_key(&self) -> &'static str { + "roles_claim" + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct OidcMachineIdentityRef { + namespace: String, + secret: String, +} + +impl OidcMachineIdentityRef { + pub fn namespace(&self) -> &str { + &self.namespace + } + + pub fn secret_name(&self) -> &str { + &self.secret + } + + pub fn key_json_key(&self) -> &'static str { + "key.json" + } +} + /// Opinionated Zitadel deployment score. /// /// Deploys a PostgreSQL cluster (via the [`PostgreSQL`] trait) and the Zitadel @@ -121,6 +198,10 @@ pub struct ZitadelScore { /// a separate channel. #[serde(default)] pub password_change_required: bool, + #[serde(default)] + pub database: Option, + #[serde(default)] + pub node_port: Option, } #[allow(dead_code)] @@ -180,11 +261,47 @@ impl Default for ZitadelScore { namespace: DEFAULT_NAMESPACE.to_string(), cluster_issuer: "letsencrypt-prod".to_string(), password_change_required: false, + database: None, + node_port: None, } } } -impl Score for ZitadelScore { +impl ZitadelScore { + pub fn new(host: impl Into, namespace: impl Into) -> Self { + Self { + host: host.into(), + namespace: namespace.into(), + ..Default::default() + } + } + + pub fn http(mut self, port: Option) -> Self { + self.external_secure = false; + self.external_port = port; + self + } + + pub fn database(mut self, database: PostgreSQLRootAccountRef) -> Self { + self.database = Some(database); + self + } + + pub fn node_port(mut self, port: u16) -> Self { + self.node_port = Some(port); + self + } + + pub fn provider_ref(&self) -> OidcProviderRef { + OidcProviderRef { + host: self.host.clone(), + secure: self.external_secure, + port: self.external_port, + } + } +} + +impl Score for ZitadelScore { fn name(&self) -> String { "ZitadelScore".to_string() } @@ -199,6 +316,8 @@ impl Score for ZitadelSco namespace: self.namespace.clone(), cluster_issuer: self.cluster_issuer.clone(), password_change_required: self.password_change_required, + database: self.database.clone(), + node_port: self.node_port, }) } } @@ -214,10 +333,12 @@ struct ZitadelInterpret { namespace: String, cluster_issuer: String, password_change_required: bool, + database: Option, + node_port: Option, } #[async_trait] -impl Interpret for ZitadelInterpret { +impl Interpret for ZitadelInterpret { async fn execute( &self, inventory: &Inventory, @@ -243,72 +364,29 @@ impl Interpret for Zitade // --- Step 1: PostgreSQL ------------------------------------------- - let pg_config = PostgreSQLConfig { - cluster_name: PG_CLUSTER_NAME.to_string(), - // Single instance by default: storage durability comes from the - // backing layer (Ceph), so CNPG-level replication adds cost/ops - // without buying availability here. - instances: 1, - storage_size: StorageSize::gi(10), - role: PostgreSQLClusterRole::Primary, - namespace: self.namespace.to_string(), - wait_for_ready: true, - }; - - debug!( - "[Zitadel] Deploying PostgreSQL cluster '{}' — instances: {}, storage: 10Gi, namespace: '{}'", - pg_config.cluster_name, pg_config.instances, pg_config.namespace - ); - - topology.deploy(&pg_config).await.map_err(|e| { - let msg = format!( - "[Zitadel] PostgreSQL deployment failed for '{}': {e}", - pg_config.cluster_name - ); - error!("{msg}"); - InterpretError::new(msg) + let database = self.database.as_ref().ok_or_else(|| { + InterpretError::new("[Zitadel] PostgreSQLRootAccountRef is required".to_string()) })?; - - info!( - "[Zitadel] PostgreSQL cluster '{}' deployed", - pg_config.cluster_name - ); - - // --- Step 2: Resolve internal DB endpoint ------------------------- - - debug!( - "[Zitadel] Resolving internal endpoint for cluster '{}'", - pg_config.cluster_name - ); - - let endpoint = topology.get_endpoint(&pg_config).await.map_err(|e| { - let msg = format!( - "[Zitadel] Failed to resolve endpoint for cluster '{}': {e}", - pg_config.cluster_name - ); - error!("{msg}"); - InterpretError::new(msg) - })?; - - info!( - "[Zitadel] DB endpoint resolved — host: '{}', port: {}", - endpoint.host, endpoint.port - ); + if database.namespace() != self.namespace { + return Err(InterpretError::new(format!( + "[Zitadel] PostgreSQL credential Secret is in namespace '{}', not '{}'", + database.namespace(), + self.namespace + ))); + } // The CNPG-managed superuser secret contains 'password', 'username', // 'host', 'port', 'dbname', 'uri'. We reference 'password' directly // via env.valueFrom.secretKeyRef because CNPG's key names do not // match ZITADEL's required env var names. - let pg_user_secret = format!("{PG_CLUSTER_NAME}-app"); - let pg_superuser_secret = format!("{PG_CLUSTER_NAME}-superuser"); - let db_host = &endpoint.host; - let db_port = endpoint.port; + let pg_superuser_secret = database.secret_name(); + let pg_username_key = database.username_key(); + let pg_password_key = database.password_key(); + let db_host = database.host(); + let db_port = database.port(); let host = &self.host; - debug!("[Zitadel] DB credentials source — secret: '{pg_user_secret}', key: 'password'"); - debug!( - "[Zitadel] DB credentials source — superuser secret: '{pg_superuser_secret}', key: 'password'" - ); + debug!("[Zitadel] DB credentials source — superuser secret: '{pg_superuser_secret}'"); // Zitadel requires one symbol, one number and more. So let's force it. fn generate_secure_password(length: usize) -> String { @@ -599,22 +677,22 @@ env: valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: user + key: {pg_username_key} - name: ZITADEL_DATABASE_POSTGRES_USER_PASSWORD valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: password + key: {pg_password_key} - name: ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: user + key: {pg_username_key} - name: ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: password + key: {pg_password_key} podSecurityContext: runAsNonRoot: true runAsUser: {sc_run_as_user} @@ -722,6 +800,8 @@ login: db_port = db_port, admin_password = admin_password, pg_superuser_secret = pg_superuser_secret, + pg_username_key = pg_username_key, + pg_password_key = pg_password_key, cluster_issuer = self.cluster_issuer, sc_run_as_user = sc_run_as_user, sc_fs_group = sc_fs_group, @@ -783,22 +863,22 @@ env: valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: user + key: {pg_username_key} - name: ZITADEL_DATABASE_POSTGRES_USER_PASSWORD valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: password + key: {pg_password_key} - name: ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: user + key: {pg_username_key} - name: ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD valueFrom: secretKeyRef: name: "{pg_superuser_secret}" - key: password + key: {pg_password_key} podSecurityContext: runAsNonRoot: true runAsUser: 1000 @@ -885,6 +965,8 @@ login: db_port = db_port, admin_password = admin_password, pg_superuser_secret = pg_superuser_secret, + pg_username_key = pg_username_key, + pg_password_key = pg_password_key, password_change_required = self.password_change_required, ) } @@ -941,22 +1023,59 @@ login: "[Zitadel] Helm chart deployed successfully\n\n\ ===== ZITADEL DEPLOYMENT COMPLETE =====\n\ Login URL: {protocol}://{host}\n\ - Login name: {admin_login_name}\n\ - Password: {admin_password}\n\n\ - Credentials are persisted via harmony_secret and reused on\n\ - subsequent runs. Password must still be changed on first\n\ - login (Zitadel's PasswordChangeRequired). After rotation,\n\ - update the persisted secret or delete it to force regen.\n\ + Login name: {admin_login_name}\n\n\ + Credentials are persisted via harmony_config and reused on\n\ + subsequent runs.\n\ =========================================", protocol = protocol, host = self.host, admin_login_name = admin_login_name, - admin_password = admin_password ), Err(e) => error!("[Zitadel] Helm chart deployment failed: {e}"), } - result + result?; + + if let Some(port) = self.node_port { + let selector = BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "zitadel".to_string()), + ( + "app.kubernetes.io/instance".to_string(), + "zitadel".to_string(), + ), + ]); + return K8sResourceScore::single( + Service { + metadata: ObjectMeta { + name: Some("zitadel-nodeport".to_string()), + namespace: Some(self.namespace.clone()), + ..Default::default() + }, + spec: Some(ServiceSpec { + type_: Some("NodePort".to_string()), + selector: Some(selector), + ports: Some(vec![ServicePort { + name: Some("http".to_string()), + port: port.into(), + target_port: Some( + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int( + 8080, + ), + ), + node_port: Some(port.into()), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }, + Some(self.namespace.clone()), + ) + .interpret(inventory, topology) + .await; + } + + Ok(Outcome::success("Zitadel deployed".to_string())) } fn get_name(&self) -> InterpretName { @@ -994,3 +1113,43 @@ struct ZitadelMasterkey { #[config(secret)] masterkey: String, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn database_consumes_postgresql_root_account_ref() { + let database = PostgreSQLRootAccountRef { + host: "fleet-pg-rw.identity.svc.cluster.local".to_string(), + port: 5432, + namespace: "identity".to_string(), + secret_name: "fleet-pg-superuser".to_string(), + username_key: "username".to_string(), + password_key: "credential".to_string(), + }; + + let score = ZitadelScore { + namespace: "identity".to_string(), + ..Default::default() + } + .database(database.clone()); + + assert_eq!(score.database, Some(database)); + } + + #[test] + fn provider_ref_uses_the_declared_external_endpoint() { + let score = ZitadelScore { + host: "identity.localhost".to_string(), + external_secure: false, + external_port: Some(8080), + ..Default::default() + }; + + assert_eq!( + score.provider_ref().issuer(), + "http://identity.localhost:8080" + ); + } +} diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index f3e6ca73..4da19905 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use std::path::PathBuf; use async_trait::async_trait; +use harmony_config::{Config, ConfigError}; use log::{debug, info, warn}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::{ @@ -14,6 +16,8 @@ use crate::{ }; use harmony_types::id::Id; +use super::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; + const ADMIN_PAT_SECRET: &str = "iam-admin-pat"; const ZITADEL_NAMESPACE: &str = "zitadel"; @@ -239,6 +243,10 @@ pub struct ZitadelSetupScore { /// didn't override. #[serde(default = "default_zitadel_namespace")] pub namespace: String, + /// Namespace receiving generated ConfigMaps and Secrets. Defaults to + /// `namespace` when unset. + #[serde(default)] + pub outputs_namespace: Option, /// Override the Zitadel **org context** for management API calls /// — sent as the `x-zitadel-orgid` header on every request. /// @@ -307,6 +315,7 @@ impl Default for ZitadelSetupScore { skip_tls: false, endpoint: None, namespace: default_zitadel_namespace(), + outputs_namespace: None, admin_org_id: None, applications: Vec::new(), api_apps: Vec::new(), @@ -322,6 +331,122 @@ impl Default for ZitadelSetupScore { } } +impl ZitadelSetupScore { + pub fn for_provider( + provider: &OidcProviderRef, + namespace: impl Into, + outputs_namespace: impl Into, + ) -> Self { + Self { + host: provider.host().to_string(), + scheme: if provider.secure() { + ZitadelScheme::Https + } else { + ZitadelScheme::Http + }, + port: provider + .port() + .map(|port| port.try_into().expect("OIDC provider port exceeds u16")), + namespace: namespace.into(), + outputs_namespace: Some(outputs_namespace.into()), + ..Default::default() + } + } + + pub fn application( + mut self, + project: impl Into, + name: impl Into, + app_type: ZitadelAppType, + ) -> Self { + self.applications.push(ZitadelApplication { + project_name: project.into(), + app_name: name.into(), + app_type, + }); + self + } + + pub fn api_application(mut self, project: impl Into, name: impl Into) -> Self { + self.api_apps.push(ZitadelApiApp { + project_name: project.into(), + app_name: name.into(), + }); + self + } + + pub fn role( + mut self, + project: impl Into, + key: impl Into, + display_name: impl Into, + ) -> Self { + self.roles.push(ZitadelRole { + project_name: project.into(), + key: key.into(), + display_name: display_name.into(), + group: None, + }); + self + } + + pub fn machine_identity( + mut self, + project: impl Into, + username: impl Into, + name: impl Into, + roles: impl IntoIterator>, + ) -> Self { + self.machine_users.push(ZitadelMachineUser { + username: username.into(), + name: name.into(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(project.into()), + grant_roles: roles.into_iter().map(Into::into).collect(), + }); + self + } + + pub fn port_forward(mut self, service: impl Into) -> Self { + self.port_forward_service = Some(service.into()); + self + } + + pub fn groups_claim(mut self) -> Self { + self.groups_claim_action = true; + self + } + + fn resolved_outputs_namespace(&self) -> &str { + self.outputs_namespace.as_deref().unwrap_or(&self.namespace) + } + + pub fn application_ref(&self, name: &str) -> OidcApplicationRef { + assert!( + self.applications.iter().any(|app| app.app_name == name), + "OIDC application '{name}' is not declared by this ZitadelSetupScore" + ); + OidcApplicationRef { + namespace: self.resolved_outputs_namespace().to_string(), + config_map: format!("zitadel-{name}-oidc"), + } + } + + pub fn machine_identity_ref(&self, username: &str) -> OidcMachineIdentityRef { + assert!( + self.machine_users + .iter() + .any(|user| user.username == username), + "machine identity '{username}' is not declared by this ZitadelSetupScore" + ); + OidcMachineIdentityRef { + namespace: self.resolved_outputs_namespace().to_string(), + secret: format!("zitadel-{username}-machine"), + } + } +} + /// Function name doubles as the Action name — Zitadel requires the /// script's entry function to match. pub const GROUPS_CLAIM_ACTION_NAME: &str = "harmonyGroupsClaim"; @@ -338,8 +463,11 @@ pub const GROUPS_CLAIM_ACTION_SCRIPT: &str = r#"function harmonyGroupsClaim(ctx, }"#; /// Cached Zitadel provisioning results. -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Serialize, Deserialize, JsonSchema, Config)] +#[config(secret)] pub struct ZitadelClientConfig { + #[serde(default)] + pub host: Option, pub project_id: Option, /// `app_name` → `clientId` (for OIDC apps that have one). #[serde(default)] @@ -382,14 +510,53 @@ impl ZitadelClientConfig { } pub fn load() -> Option { - let path = Self::cache_path(); + Self::load_path(Self::cache_path()) + } + + async fn load_for_host(host: &str) -> Result { + match harmony_config::get::().await { + Ok(config) if config.host.as_deref() == Some(host) => Ok(config), + Ok(_) | Err(ConfigError::NotFound { .. }) => Ok(Self::default()), + Err(ConfigError::NoSources) => { + Ok(Self::load_path(Self::cache_path_for_host(host)).unwrap_or_default()) + } + Err(e) => Err(format!("load Zitadel provisioning state: {e}")), + } + } + + fn load_path(path: PathBuf) -> Option { std::fs::read_to_string(&path) .ok() .and_then(|s| serde_json::from_str(&s).ok()) } - fn save(&self) -> Result<(), String> { - let path = Self::cache_path(); + async fn save_for_host(&mut self, host: &str) -> Result<(), String> { + self.host = Some(host.to_string()); + match harmony_config::set(self).await { + Ok(()) => Ok(()), + Err(ConfigError::NoSources) => { + self.save_path(Self::cache_path_for_host(host))?; + self.save_path(Self::cache_path()) + } + Err(e) => Err(format!("persist Zitadel provisioning state: {e}")), + } + } + + fn cache_path_for_host(host: &str) -> PathBuf { + let host: String = host + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '-') { + c + } else { + '_' + } + }) + .collect(); + Self::cache_path().with_file_name(format!("client-config-{host}.json")) + } + + fn save_path(&self, path: PathBuf) -> Result<(), String> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) .map_err(|e| format!("Failed to create cache dir: {e}"))?; @@ -397,6 +564,12 @@ impl ZitadelClientConfig { let json = serde_json::to_string_pretty(self) .map_err(|e| format!("Failed to serialize config: {e}"))?; std::fs::write(&path, json).map_err(|e| format!("Failed to write cache: {e}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| format!("Failed to secure cache: {e}"))?; + } Ok(()) } @@ -1803,6 +1976,9 @@ impl ZitadelSetupInterpret { .await .map_err(InterpretError::new)?, }; + if config.machine_user_ids.get(&user.username) != Some(&user_id) { + config.machine_keys.remove(&user.username); + } config .machine_user_ids .insert(user.username.clone(), user_id.clone()); @@ -2238,7 +2414,7 @@ pub async fn mint_device_credentials( impl Interpret for ZitadelSetupInterpret { async fn execute( &self, - _inventory: &Inventory, + inventory: &Inventory, topology: &T, ) -> Result { let k8s = topology @@ -2327,7 +2503,9 @@ impl Interpret for ZitadelSetupInterpret { // dialing`. Both are now handled in `wait_until_ready`. me.wait_until_ready(&client, &pat).await?; - let mut config = ZitadelClientConfig::load().unwrap_or_default(); + let mut config = ZitadelClientConfig::load_for_host(&me.score.host) + .await + .map_err(InterpretError::new)?; let mut details = Vec::new(); @@ -2388,7 +2566,91 @@ impl Interpret for ZitadelSetupInterpret { details.push(format!("action:{GROUPS_CLAIM_ACTION_NAME}")); } - config.save().map_err(InterpretError::new)?; + config + .save_for_host(&me.score.host) + .await + .map_err(InterpretError::new)?; + + use crate::modules::k8s::resource::K8sResourceScore; + use k8s_openapi::api::core::v1::{ConfigMap, Secret}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use std::collections::BTreeMap; + + for app in &me.score.applications { + let app_ref = me.score.application_ref(&app.app_name); + let mut data = BTreeMap::new(); + let project_id = config + .project_id_by_name(&app.project_name) + .ok_or_else(|| { + InterpretError::new(format!( + "project ID for '{}' missing after provisioning", + app.project_name + )) + })?; + data.insert(app_ref.project_id_key().to_string(), project_id.clone()); + data.insert( + app_ref.roles_claim_key().to_string(), + format!("urn:zitadel:iam:org:project:{project_id}:roles"), + ); + data.insert( + app_ref.client_id_key().to_string(), + config + .client_id(&app.app_name) + .ok_or_else(|| { + InterpretError::new(format!( + "client ID for '{}' missing after provisioning", + app.app_name + )) + })? + .clone(), + ); + K8sResourceScore::single( + ConfigMap { + metadata: ObjectMeta { + name: Some(app_ref.config_map_name().to_string()), + namespace: Some(app_ref.namespace().to_string()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }, + Some(app_ref.namespace().to_string()), + ) + .interpret(inventory, topology) + .await?; + } + + for user in me + .score + .machine_users + .iter() + .filter(|user| user.machine_key.is_some()) + { + let identity_ref = me.score.machine_identity_ref(&user.username); + let key_json = config.machine_key(&user.username).ok_or_else(|| { + InterpretError::new(format!( + "machine key for '{}' missing after provisioning", + user.username + )) + })?; + K8sResourceScore::single( + Secret { + metadata: ObjectMeta { + name: Some(identity_ref.secret_name().to_string()), + namespace: Some(identity_ref.namespace().to_string()), + ..Default::default() + }, + string_data: Some(BTreeMap::from([( + identity_ref.key_json_key().to_string(), + key_json.clone(), + )])), + ..Default::default() + }, + Some(identity_ref.namespace().to_string()), + ) + .interpret(inventory, topology) + .await?; + } Ok(Outcome { status: InterpretStatus::SUCCESS, @@ -2765,6 +3027,44 @@ mod tests { assert!(cfg.machine_key("nope").is_none()); } + #[test] + fn refs_have_deterministic_kubernetes_coordinates() { + let setup = ZitadelSetupScore { + namespace: "identity".to_string(), + ..Default::default() + } + .application("fleet", "fleet-console", ZitadelAppType::DeviceCode) + .machine_identity("fleet", "fleet-callout", "Fleet callout", ["admin"]); + + let app = setup.application_ref("fleet-console"); + assert_eq!(app.namespace(), "identity"); + assert_eq!(app.config_map_name(), "zitadel-fleet-console-oidc"); + assert_eq!(app.project_id_key(), "project_id"); + assert_eq!(app.client_id_key(), "client_id"); + assert_eq!(app.roles_claim_key(), "roles_claim"); + + let machine = setup.machine_identity_ref("fleet-callout"); + assert_eq!(machine.namespace(), "identity"); + assert_eq!(machine.secret_name(), "zitadel-fleet-callout-machine"); + assert_eq!(machine.key_json_key(), "key.json"); + + let setup = ZitadelSetupScore { + namespace: "zitadel".to_string(), + outputs_namespace: Some("fleet-system".to_string()), + ..Default::default() + } + .application("fleet", "fleet-console", ZitadelAppType::DeviceCode) + .machine_identity("fleet", "fleet-callout", "Fleet callout", ["admin"]); + assert_eq!( + setup.application_ref("fleet-console").namespace(), + "fleet-system" + ); + assert_eq!( + setup.machine_identity_ref("fleet-callout").namespace(), + "fleet-system" + ); + } + fn score(host: &str) -> ZitadelSetupScore { ZitadelSetupScore { host: host.to_string(), diff --git a/harmony_app/Cargo.toml b/harmony_app/Cargo.toml index 3b333ea1..aed817e8 100644 --- a/harmony_app/Cargo.toml +++ b/harmony_app/Cargo.toml @@ -11,14 +11,15 @@ harmony-k8s = { path = "../harmony-k8s" } harmony_config = { path = "../harmony_config" } harmony_types = { path = "../harmony_types" } async-trait.workspace = true -anyhow.workspace = true tokio.workspace = true serde = { workspace = true } +serde_json = { workspace = true } serde_yaml = { workspace = true } schemars = "0.8" tempfile.workspace = true toml.workspace = true log.workspace = true +reqwest.workspace = true k8s-openapi.workspace = true thiserror.workspace = true docker-compose-types = "0.24" diff --git a/harmony_app/src/app.rs b/harmony_app/src/app.rs index cc86726b..252610ea 100644 --- a/harmony_app/src/app.rs +++ b/harmony_app/src/app.rs @@ -2,8 +2,8 @@ //! functions returning structured results — the UI-agnostic core every //! front-end (CLI, TUI, web) drives. -use anyhow::{Result, anyhow}; use async_trait::async_trait; +use harmony::interpret::InterpretStatus; use harmony::inventory::Inventory; use harmony::maestro::Maestro; use harmony::score::Score; @@ -12,6 +12,8 @@ use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::Pod; use crate::context::AppContext; +use crate::error::AppError; +use crate::publish::{ImageRefs, ImageSpec, build_images, publish_images}; /// Identity-only description of an app (ADR-026 §3): what it's called and /// where it runs. No behavioral knobs — those come from the context/profile. @@ -32,20 +34,34 @@ pub trait HarmonyApp: Send + Sync { /// The desired state for this context — the app composes its Scores, /// branching on `ctx.profile()`. Called by `deploy`/`ship`. - async fn scores(&self, ctx: &AppContext) -> Result>>>; + async fn scores( + &self, + ctx: &AppContext, + images: &ImageRefs, + ) -> Result>>, AppError>; async fn scores_with_options( &self, ctx: &AppContext, - _options: DeployOptions, - ) -> Result>>> { - self.scores(ctx).await + options: DeployOptions, + ) -> Result>>, AppError> { + self.scores(ctx, &options.images).await } - /// Build + distribute images for `ctx` (push to a registry, or import to - /// k3d). Default: nothing to build. Called by `ship`. - async fn publish(&self, _ctx: &AppContext) -> Result<()> { - Ok(()) + fn images(&self, _ctx: &AppContext) -> Result, AppError> { + Ok(Vec::new()) + } + + fn build(&self, ctx: &AppContext) -> Result { + Ok(build_images(&self.images(ctx)?, &ctx.publisher()?)?) + } + + fn publish(&self, ctx: &AppContext, images: &ImageRefs) -> Result { + Ok(publish_images( + &self.images(ctx)?, + images, + &ctx.publisher()?, + )?) } } @@ -62,9 +78,10 @@ pub struct DeployReport { pub steps: Vec, } -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub struct DeployOptions { pub force_conflicts: bool, + pub images: ImageRefs, } #[derive(Debug, Clone, serde::Serialize)] @@ -93,7 +110,7 @@ pub async fn deploy( app: &dyn HarmonyApp, topology: T, ctx: &AppContext, -) -> Result { +) -> Result { deploy_with_options(app, topology, ctx, DeployOptions::default()).await } @@ -102,7 +119,7 @@ pub async fn deploy_with_options( topology: T, ctx: &AppContext, options: DeployOptions, -) -> Result { +) -> Result { let scores = app.scores_with_options(ctx, options).await?; let to_run: Vec>> = scores.iter().map(|s| s.clone_box()).collect(); @@ -111,15 +128,24 @@ pub async fn deploy_with_options( maestro .prepare_topology() .await - .map_err(|e| anyhow!("topology preparation failed: {e}"))?; + .map_err(|e| AppError::Deploy(format!("topology preparation failed: {e}")))?; let mut steps = Vec::new(); - for s in to_run { - let name = s.name(); + for score in to_run { + let name = score.name(); let outcome = maestro - .interpret(s) + .interpret(score) .await - .map_err(|e| anyhow!("{name}: {e}"))?; + .map_err(|error| AppError::Deploy(format!("{name}: {error}")))?; + if !matches!( + outcome.status, + InterpretStatus::SUCCESS | InterpretStatus::NOOP + ) { + return Err(AppError::Deploy(format!( + "{name}: {}: {}", + outcome.status, outcome.message + ))); + } steps.push(StepOutcome { name, message: outcome.message, @@ -133,7 +159,7 @@ pub async fn ship( app: &dyn HarmonyApp, topology: T, ctx: &AppContext, -) -> Result { +) -> Result { ship_with_options(app, topology, ctx, DeployOptions::default()).await } @@ -142,22 +168,23 @@ pub async fn ship_with_options( topology: T, ctx: &AppContext, options: DeployOptions, -) -> Result { - app.publish(ctx).await?; - deploy_with_options(app, topology, ctx, options).await +) -> Result { + let images = app.build(ctx)?; + let images = app.publish(ctx, &images)?; + deploy_with_options(app, topology, ctx, DeployOptions { images, ..options }).await } /// Workload readiness in the app's namespace (operational, read-only). pub async fn status( app: &dyn HarmonyApp, ctx: &AppContext, -) -> Result { +) -> Result { let id = app.identity(); let client = ctx.k8s_client().await?; let deployments = client .list_resources::(Some(&id.namespace), None) .await - .map_err(|e| anyhow!("listing deployments: {e}"))?; + .map_err(|e| AppError::Deploy(format!("listing deployments: {e}")))?; let workloads = deployments .items .iter() @@ -182,13 +209,13 @@ pub async fn logs( app: &dyn HarmonyApp, ctx: &AppContext, tail: Option, -) -> Result> { +) -> Result, AppError> { let id = app.identity(); let client = ctx.k8s_client().await?; let pods = client .list_resources::(Some(&id.namespace), None) .await - .map_err(|e| anyhow!("listing pods: {e}"))?; + .map_err(|e| AppError::Deploy(format!("listing pods: {e}")))?; let mut out = Vec::new(); for p in pods.items { let pod = p.metadata.name.unwrap_or_default(); diff --git a/harmony_app/src/chart.rs b/harmony_app/src/chart.rs index 6b0300c4..0a1aceab 100644 --- a/harmony_app/src/chart.rs +++ b/harmony_app/src/chart.rs @@ -25,6 +25,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use crate::AppError; use crate::Profile; use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; @@ -60,6 +61,7 @@ pub struct DeployConfig { /// pull this app's images from a private registry, set on every pod's /// `imagePullSecrets`. By reference — the Secret is applied separately. pub image_pull_secrets: Vec, + pub images: BTreeMap, } /// Mount one key of a Secret as a file at `path` (its parent dir is the mount @@ -97,6 +99,7 @@ impl DeployConfig { extra_env: Vec::new(), secret_file_mounts: Vec::new(), image_pull_secrets: Vec::new(), + images: BTreeMap::new(), } } } @@ -115,6 +118,9 @@ const ACCESS_MODE_RWO: &str = "ReadWriteOnce"; /// used verbatim. pub fn service_image(cfg: &DeployConfig, svc: &ComposeService) -> String { if svc.build_context.is_some() { + if let Some(image) = cfg.images.get(&svc.name) { + return image.clone(); + } format!( "{}/{}/{}-{}:{}", cfg.registry, cfg.project, cfg.app_name, svc.name, cfg.version @@ -140,7 +146,7 @@ pub fn build_chart( app: &ComposeApp, cfg: &DeployConfig, out_dir: &Path, -) -> Result { +) -> Result { let mut chart = HelmChart::new(cfg.app_name.clone(), cfg.version.clone()); chart.version = cfg.chart_version.clone(); chart.description = format!("{} — imported from docker-compose by Harmony", cfg.app_name); @@ -151,12 +157,16 @@ pub fn build_chart( format!("deployment-{}.yaml", svc.name), &deployment(svc, cfg), ) - .map_err(|e| format!("serialize deployment {}: {e}", svc.name))?, + .map_err(|e| { + AppError::InvalidComposition(format!("serialize deployment {}: {e}", svc.name)) + })?, ); if let Some(service) = service(svc) { chart.add_resource( HelmResourceKind::from_serializable(format!("service-{}.yaml", svc.name), &service) - .map_err(|e| format!("serialize service {}: {e}", svc.name))?, + .map_err(|e| { + AppError::InvalidComposition(format!("serialize service {}: {e}", svc.name)) + })?, ); } } @@ -164,13 +174,13 @@ pub fn build_chart( for vol in app.mounted_volumes() { chart.add_resource( HelmResourceKind::from_serializable(format!("pvc-{vol}.yaml"), &pvc(&vol, cfg)) - .map_err(|e| format!("serialize pvc {vol}: {e}"))?, + .map_err(|e| AppError::InvalidComposition(format!("serialize pvc {vol}: {e}")))?, ); } chart .write_to(out_dir) - .map_err(|e| format!("writing chart: {e}")) + .map_err(|e| AppError::InvalidComposition(format!("writing chart: {e}"))) } fn labels(name: &str) -> BTreeMap { @@ -413,6 +423,7 @@ mod tests { extra_env: vec![], secret_file_mounts: vec![], image_pull_secrets: vec![], + images: BTreeMap::new(), }; let tmp = tempfile::tempdir().unwrap(); build_chart(&app, &cfg, tmp.path()).unwrap(); diff --git a/harmony_app/src/compose.rs b/harmony_app/src/compose.rs index 2e1e9e7a..eb4060b6 100644 --- a/harmony_app/src/compose.rs +++ b/harmony_app/src/compose.rs @@ -57,7 +57,7 @@ pub struct ComposeService { pub dockerfile: Option, /// `build.args` — baked into the image at `docker build` time (forwarded as /// `--build-arg`). Only the advanced build form carries them. - pub build_args: Vec<(String, String)>, + pub build_args: Vec<(String, Option)>, pub ports: Vec, pub env: Vec<(String, String)>, pub mounts: Vec, @@ -200,21 +200,23 @@ fn parse_short_port(service: &str, spec: &str) -> Result }) } -/// Flatten compose `build.args` into ordered key/value pairs, mirroring -/// `parse_env`. List items without `=` get an empty value (docker then resolves -/// them from the build environment). -fn parse_build_args(args: &Option) -> Vec<(String, String)> { +fn parse_build_args( + args: &Option, +) -> Vec<(String, Option)> { use docker_compose_types::BuildArgs; match args { None | Some(BuildArgs::Simple(_)) => Vec::new(), Some(BuildArgs::List(items)) => items .iter() .map(|item| match item.split_once('=') { - Some((k, v)) => (k.to_string(), v.to_string()), - None => (item.to_string(), String::new()), + Some((k, v)) => (k.to_string(), Some(v.to_string())), + None => (item.to_string(), None), }) .collect(), - Some(BuildArgs::KvPair(map)) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + Some(BuildArgs::KvPair(map)) => map + .iter() + .map(|(k, v)| (k.clone(), Some(v.clone()))) + .collect(), } } @@ -366,7 +368,17 @@ volumes: let app = ComposeApp::parse(yaml, Path::new(".")).unwrap(); assert_eq!( app.service("api").unwrap().build_args, - vec![("QUARKUS_BUILD_PROFILE".to_string(), "dev".to_string())] + vec![("QUARKUS_BUILD_PROFILE".to_string(), Some("dev".to_string()))] + ); + } + + #[test] + fn bare_build_arg_inherits_from_environment() { + let yaml = "services:\n api:\n build:\n context: .\n args: [TOKEN]\n"; + let app = ComposeApp::parse(yaml, Path::new(".")).unwrap(); + assert_eq!( + app.service("api").unwrap().build_args, + vec![("TOKEN".to_string(), None)] ); } diff --git a/harmony_app/src/context.rs b/harmony_app/src/context.rs index ed051de5..a5061f1e 100644 --- a/harmony_app/src/context.rs +++ b/harmony_app/src/context.rs @@ -12,7 +12,6 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::Arc; -use anyhow::{Context, Result, bail}; use harmony::topology::{K8sAnywhereConfig, K8sAnywhereTopology}; use harmony_config::{Config, ConfigClient, ConfigSource, LocalFileSource, PromptSource}; use harmony_k8s::K8sClient; @@ -22,6 +21,8 @@ use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use crate::profile::Profile; +use crate::publish::PublicationTopology; +use crate::{ContextError, error::io}; /// `.harmony/contexts.toml`: `[contexts.]` blocks. #[derive(Debug, Deserialize)] @@ -35,6 +36,10 @@ struct ContextsFile { #[derive(Debug, Deserialize)] struct ContextDef { profile: Profile, + registry: Option, + project: Option, + domain: Option, + image_pull_secret: Option, /// Let K8sAnywhereTopology autoprovision a local k3d cluster (offline; no /// pre-created cluster, no kubeconfig). The hands-off local/CI path. #[serde(default)] @@ -65,6 +70,10 @@ pub struct AppContext { name: String, profile: Profile, version: String, + registry: Option, + project: Option, + domain: Option, + image_pull_secret: Option, local_config_dir: Option, k3d_cluster: Option, /// True when K8sAnywhere autoprovisions the cluster: there is no kubeconfig @@ -79,42 +88,43 @@ pub struct AppContext { } impl AppContext { + /// Load context metadata without contacting the cluster credential source. + pub fn load_metadata( + name: &str, + version: impl Into, + local_config_dir: Option, + contexts_file: Option, + ) -> Result { + let def = read_context(name, contexts_file)?; + Ok(Self::from_definition( + name, + version.into(), + local_config_dir, + &def, + Arc::new(ConfigClient::new(Vec::new())), + None, + )) + } + /// Resolve a context by name from the in-repo `.harmony/contexts.toml`. pub async fn resolve( name: &str, version: impl Into, local_config_dir: Option, contexts_file: Option, - ) -> Result { - let path = match contexts_file { - Some(p) => { - if !p.is_file() { - bail!("contexts file not found: {}", p.display()); - } - p - } - None => find_contexts_file()?, - }; - let raw = std::fs::read_to_string(&path) - .with_context(|| format!("reading {}", path.display()))?; - let file: ContextsFile = - toml::from_str(&raw).with_context(|| format!("parsing {}", path.display()))?; - let def = file.contexts.get(name).with_context(|| { - format!( - "context '{name}' not in {} (have: {})", - path.display(), - file.contexts.keys().cloned().collect::>().join(", ") - ) - })?; + ) -> Result { + let def = read_context(name, contexts_file)?; info!( "Resolving deploy context '{name}' (profile {:?})", def.profile ); debug!("Context '{name}' definition: {def:?}"); - let config_sources = build_config_sources(def, local_config_dir.clone()) + let config_sources = build_config_sources(&def, local_config_dir.clone()) .await - .with_context(|| format!("building config sources for context '{name}'"))?; + .map_err(|e| { + ContextError::Config(format!("building config sources for context '{name}': {e}")) + })?; harmony_config::init(config_sources.clone()).await; let config_client = Arc::new(ConfigClient::new(config_sources)); let guard = match (def.autoprovision, &def.k3d, &def.openbao_namespace) { @@ -128,37 +138,53 @@ impl AppContext { } (false, None, Some(ns)) => { info!("Cluster access: kubeconfig from OpenBao (namespace '{ns}')"); - let access: ClusterAccess = config_client.get().await.with_context(|| { - format!( + let access: ClusterAccess = config_client.get().await.map_err(|e| { + ContextError::Config(format!( "loading cluster kubeconfig from OpenBao ({ns}/ClusterAccess) \ - — is HARMONY_ZITADEL_KEY_JSON set for this context?" - ) + — is HARMONY_ZITADEL_KEY_JSON set for this context?: {e}" + )) })?; Some(write_kubeconfig(access.kubeconfig.as_bytes())?) } - _ => bail!( - "context '{name}' must set exactly one of `autoprovision = true`, \ - `k3d`, or `openbao_namespace`" - ), + _ => unreachable!("context access mode was validated while loading"), }; - let k3d_cluster = if def.autoprovision { - Some(AUTOPROVISION_CLUSTER.to_string()) - } else { - def.k3d.clone() - }; + Ok(Self::from_definition( + name, + version.into(), + local_config_dir, + &def, + config_client, + guard, + )) + } - Ok(Self { + fn from_definition( + name: &str, + version: String, + local_config_dir: Option, + def: &ContextDef, + config_client: Arc, + guard: Option, + ) -> Self { + Self { name: name.to_string(), profile: def.profile, - version: version.into(), + version, + registry: def.registry.clone(), + project: def.project.clone(), + domain: def.domain.clone(), + image_pull_secret: def.image_pull_secret.clone(), local_config_dir, - k3d_cluster, + k3d_cluster: def + .autoprovision + .then(|| AUTOPROVISION_CLUSTER.to_string()) + .or_else(|| def.k3d.clone()), autoprovision: def.autoprovision, kubeconfig: guard.as_ref().map(|g| g.path().to_path_buf()), _kubeconfig_guard: guard, config_client, - }) + } } pub fn name(&self) -> &str { @@ -170,6 +196,44 @@ impl AppContext { pub fn version(&self) -> &str { &self.version } + pub fn registry(&self) -> Option<&str> { + self.registry.as_deref() + } + pub fn domain(&self) -> Result<&str, ContextError> { + self.domain + .as_deref() + .ok_or_else(|| ContextError::Missing("deploy context has no domain".to_string())) + } + pub fn service_host(&self, service: &str, namespace: &str) -> Result { + match self.profile { + Profile::Local => Ok(format!("{service}.{namespace}.svc.cluster.local")), + Profile::Prod => Ok(format!("{service}.{}", self.domain()?)), + } + } + pub fn image_pull_secret(&self) -> Result, ContextError> { + match self.profile { + Profile::Local => Ok(None), + Profile::Prod => self.image_pull_secret.as_deref().map(Some).ok_or_else(|| { + ContextError::Missing("prod context has no image_pull_secret".to_string()) + }), + } + } + pub fn image(&self, name: &str) -> Result { + match self.profile { + Profile::Local => Ok(format!("localhost/{name}:{}", self.version)), + Profile::Prod => Ok(format!( + "{}/{}/{}:{}", + self.registry.as_deref().ok_or_else(|| { + ContextError::Missing("prod context has no registry".to_string()) + })?, + self.project.as_deref().ok_or_else(|| { + ContextError::Missing("prod context has no project".to_string()) + })?, + name, + self.version + )), + } + } pub fn local_config_dir(&self) -> Option<&Path> { self.local_config_dir.as_deref() } @@ -180,6 +244,19 @@ impl AppContext { self.k3d_cluster.as_deref() } + pub fn publisher(&self) -> Result { + if let Some(cluster) = &self.k3d_cluster { + return Ok(PublicationTopology::K3d { + cluster: cluster.clone(), + }); + } + Ok(PublicationTopology::Registry { + registry: self.registry.clone().ok_or_else(|| { + ContextError::Missing("remote context has no registry".to_string()) + })?, + }) + } + /// The converge target. Under autoprovision K8sAnywhere installs + manages /// a local k3d cluster; otherwise it's pinned to the brokered kubeconfig /// (no env, so we never deploy to the wrong cluster). @@ -200,7 +277,7 @@ impl AppContext { /// A read client for operational verbs (status/logs) — no topology prep. /// Under autoprovision the cluster exists only after a deploy, so its /// kubeconfig is resolved lazily here from the managed k3d. - pub async fn k8s_client(&self) -> Result { + pub async fn k8s_client(&self) -> Result { let _guard; let path = match &self.kubeconfig { Some(p) => p.clone(), @@ -214,23 +291,66 @@ impl AppContext { }; K8sClient::from_kubeconfig(&path.to_string_lossy()) .await - .context("building k8s client from the context kubeconfig") + .ok_or_else(|| { + ContextError::Invalid("building k8s client from the context kubeconfig".to_string()) + }) } } +fn read_context(name: &str, contexts_file: Option) -> Result { + let path = match contexts_file { + Some(path) if path.is_file() => path, + Some(path) => { + return Err(ContextError::Missing(format!( + "contexts file not found: {}", + path.display() + ))); + } + None => find_contexts_file()?, + }; + let raw = + std::fs::read_to_string(&path).map_err(|e| io(format!("reading {}", path.display()), e))?; + let mut file: ContextsFile = toml::from_str(&raw).map_err(|source| ContextError::Parse { + action: format!("parsing {}", path.display()), + source, + })?; + let def = file.contexts.remove(name).ok_or_else(|| { + ContextError::Missing(format!( + "context '{name}' not in {} (have: {})", + path.display(), + file.contexts.keys().cloned().collect::>().join(", ") + )) + })?; + if !matches!( + ( + def.autoprovision, + def.k3d.is_some(), + def.openbao_namespace.is_some() + ), + (true, false, false) | (false, true, false) | (false, false, true) + ) { + return Err(ContextError::Invalid(format!( + "context '{name}' must set exactly one of `autoprovision = true`, \ + `k3d`, or `openbao_namespace`" + ))); + } + Ok(def) +} + /// Walk up from the cwd for `.harmony/contexts.toml` (git/cargo style). -fn find_contexts_file() -> Result { - let mut dir = std::env::current_dir().context("current dir")?; +fn find_contexts_file() -> Result { + let mut dir = std::env::current_dir().map_err(|e| io("current dir", e))?; loop { let candidate = dir.join(".harmony").join("contexts.toml"); if candidate.is_file() { return Ok(candidate); } if !dir.pop() { - bail!( + return Err(ContextError::Missing( "no .harmony/contexts.toml found (searched up from the cwd) — \ run from the project dir or pass --config " - ); + .to_string(), + )); } } } @@ -238,7 +358,7 @@ fn find_contexts_file() -> Result { async fn build_config_sources( def: &ContextDef, local_config_dir: Option, -) -> Result>> { +) -> Result>, ContextError> { let mut sources: Vec> = Vec::new(); if let Some(namespace) = &def.openbao_namespace { @@ -254,12 +374,16 @@ async fn build_config_sources( ), ) .await - .with_context(|| format!("reaching OpenBao for namespace '{namespace}'"))?; + .ok_or_else(|| { + ContextError::Config(format!("reaching OpenBao for namespace '{namespace}'")) + })?; sources.push(source); } else { let dir = local_config_dir .or_else(LocalFileSource::default_path) - .context("local contexts need a config directory")?; + .ok_or_else(|| { + ContextError::Missing("local contexts need a config directory".to_string()) + })?; sources.push(Arc::new(LocalFileSource::new(dir))); } @@ -267,17 +391,18 @@ async fn build_config_sources( Ok(sources) } -fn write_kubeconfig(contents: &[u8]) -> Result { - let mut file = - NamedTempFile::with_prefix("harmony-ctx-").context("create kubeconfig tempfile")?; - file.write_all(contents).context("write kubeconfig")?; - file.flush().context("flush kubeconfig")?; +fn write_kubeconfig(contents: &[u8]) -> Result { + let mut file = NamedTempFile::with_prefix("harmony-ctx-") + .map_err(|e| io("create kubeconfig tempfile", e))?; + file.write_all(contents) + .map_err(|e| io("write kubeconfig", e))?; + file.flush().map_err(|e| io("flush kubeconfig", e))?; Ok(file) } /// The k3d cluster's kubeconfig, written to a temp file. Prefers harmony's /// managed k3d binary, falling back to a `k3d` on PATH. -fn k3d_kubeconfig(cluster: &str) -> Result { +fn k3d_kubeconfig(cluster: &str) -> Result { let managed = harmony::config::HARMONY_DATA_DIR.join("k3d").join("k3d"); let k3d: OsString = if managed.exists() { managed.into_os_string() @@ -287,12 +412,12 @@ fn k3d_kubeconfig(cluster: &str) -> Result { let out = Command::new(&k3d) .args(["kubeconfig", "get", cluster]) .output() - .context("spawn k3d (installed and on PATH?)")?; + .map_err(|e| io("spawn k3d (installed and on PATH?)", e))?; if !out.status.success() { - bail!( + return Err(ContextError::Command(format!( "k3d kubeconfig get {cluster} failed: {}", String::from_utf8_lossy(&out.stderr) - ); + ))); } write_kubeconfig(&out.stdout) } diff --git a/harmony_app/src/deploy.rs b/harmony_app/src/deploy.rs index 5bc9c5a7..df8a29e3 100644 --- a/harmony_app/src/deploy.rs +++ b/harmony_app/src/deploy.rs @@ -7,15 +7,17 @@ use std::collections::BTreeMap; use std::path::Path; -use crate::{AppContext, AppIdentity, AppRef, Capability, DeployOptions, HarmonyApp, Profile}; -use anyhow::{Result, anyhow}; +use crate::{ + AppContext, AppError, AppIdentity, AppRef, Capability, DeployOptions, HarmonyApp, ImageRefs, + ImageSpec, Profile, +}; use async_trait::async_trait; use harmony::score::Score; use harmony::topology::{HelmCommand, K8sAnywhereTopology, K8sclient, Topology}; use crate::chart::{DeployConfig, cluster_issuer_for}; use crate::compose::ComposeApp; -use crate::score::{ComposeAppScore, PublicEndpoint, PublishedChart}; +use crate::score::{ComposeAppScore, PublicEndpoint}; struct Endpoint { service: String, @@ -40,8 +42,8 @@ pub struct ComposeDeploy { impl ComposeDeploy { /// Import the app from a compose dir. `namespace`/`project` default to the /// name; override fluently. - pub fn from_dir(name: impl Into, dir: impl AsRef) -> Result { - let app = ComposeApp::from_dir(dir.as_ref()).map_err(|e| e.to_string())?; + pub fn from_dir(name: impl Into, dir: impl AsRef) -> Result { + let app = ComposeApp::from_dir(dir.as_ref())?; Ok(Self::from_compose(name, app)) } @@ -109,7 +111,7 @@ impl ComposeDeploy { profile: Profile, version: &str, force_conflicts: bool, - ) -> Result { + ) -> Result { let public_endpoint = match &self.expose { Some(e) => Some(PublicEndpoint::from_compose( &self.app, @@ -128,12 +130,6 @@ impl ComposeDeploy { Ok(ComposeAppScore { namespace: self.namespace.clone(), release_name: self.name.clone(), - // Prod installs the published OCI chart; local renders from compose. - published_chart: (profile == Profile::Prod).then(|| PublishedChart { - registry: self.registry.clone(), - project: self.project.clone(), - version: version.to_string(), - }), public_endpoint, app: self.app.clone(), deploy, @@ -152,19 +148,39 @@ impl HarmonyApp for ComposeDeploy { } } - async fn scores(&self, ctx: &AppContext) -> Result>>> { - self.scores_with_options(ctx, DeployOptions::default()) - .await + async fn scores( + &self, + ctx: &AppContext, + images: &ImageRefs, + ) -> Result>>, AppError> { + self.scores_with_options( + ctx, + DeployOptions { + images: images.clone(), + ..Default::default() + }, + ) + .await } async fn scores_with_options( &self, ctx: &AppContext, options: DeployOptions, - ) -> Result>>> { - let app_score = self - .score(ctx.profile(), ctx.version(), options.force_conflicts) - .map_err(|e| anyhow!(e))?; + ) -> Result>>, AppError> { + let mut app_score = self.score(ctx.profile(), ctx.version(), options.force_conflicts)?; + app_score.deploy.images = self + .app + .services + .iter() + .filter(|service| service.build_context.is_some()) + .map(|service| { + Ok(( + service.name.clone(), + options.images.require(&service.name)?.to_string(), + )) + }) + .collect::>()?; let mut scores: Vec>> = vec![Box::new(app_score)]; let app_ref = self.app_ref(ctx.profile()); for capability in &self.capabilities { @@ -173,26 +189,22 @@ impl HarmonyApp for ComposeDeploy { Ok(scores) } - async fn publish(&self, ctx: &AppContext) -> Result<()> { - use crate::publish::{PublishTarget, publish}; - let target = match ctx.profile() { - Profile::Local => PublishTarget::K3dImport { - cluster: ctx.k3d_cluster().unwrap_or(&self.name).to_string(), - }, - // Push needs registry creds; for this example they come from env. - // (A prod app would load them from its secret store.) - Profile::Prod => PublishTarget::Registry { - user: std::env::var("REGISTRY_USER") - .map_err(|_| anyhow!("REGISTRY_USER required to push"))?, - token: std::env::var("REGISTRY_TOKEN") - .map_err(|_| anyhow!("REGISTRY_TOKEN required to push"))?, - }, - }; - publish( - &self.app, - &self.deploy_config(ctx.profile(), ctx.version()), - &target, - ) + fn images(&self, ctx: &AppContext) -> Result, AppError> { + let config = self.deploy_config(ctx.profile(), ctx.version()); + self.app + .services + .iter() + .filter_map(|service| { + let context = service.build_context.clone()?; + Some(Ok(ImageSpec { + name: service.name.clone(), + image: crate::service_image(&config, service), + dockerfile: context.join(service.dockerfile.as_deref().unwrap_or("Dockerfile")), + context, + build_args: service.build_args.clone(), + })) + }) + .collect() } } @@ -228,7 +240,6 @@ mod tests { .expose("frontend", "ts.local") .score(Profile::Local, "1.0.0", false) .unwrap(); - assert!(s.published_chart.is_none(), "local renders from compose"); assert_eq!(s.deploy.replicas, 1); assert!(!s.deploy.rolling); let ep = s.public_endpoint.unwrap(); @@ -239,17 +250,16 @@ mod tests { } #[test] - fn prod_profile_publishes_replicated_tls() { + fn prod_profile_renders_replicated_tls() { let s = CD::from_compose("ts", fixture()) .registry("hub.x") .project("p") .expose("frontend", "ts.x") .score(Profile::Prod, "2.0.0", false) .unwrap(); - let pc = s.published_chart.expect("prod installs published chart"); - assert_eq!(pc.registry, "hub.x"); - assert_eq!(pc.project, "p"); - assert_eq!(pc.version, "2.0.0"); + assert_eq!(s.deploy.registry, "hub.x"); + assert_eq!(s.deploy.project, "p"); + assert_eq!(s.deploy.version, "2.0.0"); assert_eq!(s.deploy.replicas, 1); assert!(s.deploy.rolling); assert_eq!( diff --git a/harmony_app/src/error.rs b/harmony_app/src/error.rs new file mode 100644 index 00000000..cb568169 --- /dev/null +++ b/harmony_app/src/error.rs @@ -0,0 +1,73 @@ +use thiserror::Error; + +use crate::compose::ComposeError; + +#[derive(Debug, Error)] +pub enum ContextError { + #[error("{action}: {source}")] + Io { + action: String, + source: std::io::Error, + }, + #[error("{action}: {source}")] + Parse { + action: String, + source: toml::de::Error, + }, + #[error("{0}")] + Config(String), + #[error("{0}")] + Command(String), + #[error("{0}")] + Missing(String), + #[error("{0}")] + Invalid(String), +} + +#[derive(Debug, Error)] +pub enum ImageError { + #[error("{action}: {source}")] + Io { + action: String, + source: std::io::Error, + }, + #[error("{action}: {source}")] + Parse { + action: &'static str, + source: serde_json::Error, + }, + #[error("{0}")] + Command(String), + #[error("{0}")] + Missing(String), + #[error("{0}")] + Invalid(String), +} + +#[derive(Debug, Error)] +pub enum AppError { + #[error(transparent)] + Context(#[from] ContextError), + #[error(transparent)] + Image(#[from] ImageError), + #[error(transparent)] + Composition(#[from] ComposeError), + #[error("{0}")] + InvalidComposition(String), + #[error("{0}")] + Deploy(String), +} + +pub(crate) fn io(action: impl Into, source: std::io::Error) -> ContextError { + ContextError::Io { + action: action.into(), + source, + } +} + +pub(crate) fn image_io(action: impl Into, source: std::io::Error) -> ImageError { + ImageError::Io { + action: action.into(), + source, + } +} diff --git a/harmony_app/src/lib.rs b/harmony_app/src/lib.rs index 87800026..9855a5e2 100644 --- a/harmony_app/src/lib.rs +++ b/harmony_app/src/lib.rs @@ -20,6 +20,7 @@ pub mod chart; pub mod compose; pub mod context; pub mod deploy; +pub mod error; pub mod profile; pub mod publish; pub mod score; @@ -33,5 +34,7 @@ pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image pub use compose::ComposeApp; pub use context::{AppContext, ClusterAccess}; pub use deploy::ComposeDeploy; +pub use error::{AppError, ContextError, ImageError}; pub use profile::Profile; -pub use score::{ComposeAppScore, PublicEndpoint, PublishedChart}; +pub use publish::{ImagePublisher, ImageRefs, ImageSpec, PublicationTopology}; +pub use score::{ComposeAppScore, PublicEndpoint}; diff --git a/harmony_app/src/publish.rs b/harmony_app/src/publish.rs index 34b533fc..803b9ebc 100644 --- a/harmony_app/src/publish.rs +++ b/harmony_app/src/publish.rs @@ -1,209 +1,451 @@ //! Build the per-service images, then distribute them to a target: push to //! an OCI registry (CD) or import into a local k3d cluster (dev). The build //! is identical either way — only distribution differs (ADR-026 §4). Plain -//! `anyhow` — binary glue, not library API. +use std::collections::BTreeMap; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{Command, Stdio}; -use anyhow::{Context, Result, bail}; +use crate::ImageError; +use crate::error::image_io; -use crate::chart::{DeployConfig, build_chart, service_image}; -use crate::compose::ComposeApp; - -/// Where built images (and, for a registry, the chart) are distributed. -pub enum PublishTarget { - /// Push images + chart to `cfg.registry` (CD path). - Registry { user: String, token: String }, - /// Import images into a k3d cluster; the chart is rendered at deploy - /// time (no OCI chart), so nothing is pushed (local dev). - K3dImport { cluster: String }, +#[derive(Debug, Clone)] +pub struct ImageSpec { + pub name: String, + pub image: String, + pub context: PathBuf, + pub dockerfile: PathBuf, + pub build_args: Vec<(String, Option)>, } -/// Build the images, then distribute them to `target`. -pub fn publish(app: &ComposeApp, cfg: &DeployConfig, target: &PublishTarget) -> Result<()> { - build_images(app, cfg)?; - match target { - PublishTarget::Registry { user, token } => push_to_registry(app, cfg, user, token), - PublishTarget::K3dImport { cluster } => import_to_k3d(app, cfg, cluster), +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct ImageRefs(BTreeMap); + +impl ImageRefs { + pub fn new(images: impl IntoIterator) -> Self { + Self(images.into_iter().collect()) + } + + pub fn require(&self, name: &str) -> Result<&str, ImageError> { + self.0.get(name).map(String::as_str).ok_or_else(|| { + ImageError::Missing(format!("image '{name}' was not published or supplied")) + }) + } + + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(name, image)| (name.as_str(), image.as_str())) } } -/// `docker build` each service that builds from source; prebuilt `image:` -/// services are left for the kubelet to pull. -pub fn build_images(app: &ComposeApp, cfg: &DeployConfig) -> Result<()> { - for svc in &app.services { - let Some(context) = &svc.build_context else { - log::info!("service '{}' uses prebuilt image, skipping build", svc.name); - continue; - }; - let image = service_image(cfg, svc); - log::info!("docker build {image}"); - let mut cmd = Command::new("docker"); - cmd.arg("build").arg("-t").arg(&image); - if let Some(dockerfile) = &svc.dockerfile { - cmd.arg("-f").arg(context.join(dockerfile)); +pub trait ImagePublisher { + fn archive_type(&self) -> &'static str; + fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result<(), ImageError>; +} + +pub enum PublicationTopology { + K3d { cluster: String }, + Registry { registry: String }, +} + +impl ImagePublisher for PublicationTopology { + fn archive_type(&self) -> &'static str { + match self { + Self::K3d { .. } => "oci", + Self::Registry { .. } => "docker", } - for (key, value) in &svc.build_args { - cmd.arg("--build-arg").arg(format!("{key}={value}")); + } + + fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result<(), ImageError> { + match self { + Self::K3d { cluster } => import_images_to_k3d(specs, images, cluster), + Self::Registry { registry } => push_images(specs, images, registry), + } + } +} + +pub fn build_images( + specs: &[ImageSpec], + publisher: &impl ImagePublisher, +) -> Result { + ensure_oci_builder()?; + std::fs::create_dir_all(image_artifact_dir()) + .map_err(|e| image_io("create image artifact directory", e))?; + let mut refs = BTreeMap::new(); + for spec in specs { + let metadata = + tempfile::NamedTempFile::new().map_err(|e| image_io("image build metadata file", e))?; + let archive = tempfile::Builder::new() + .suffix(".image") + .tempdir_in(image_artifact_dir()) + .map_err(|e| image_io("image build artifact", e))?; + let archive_path = archive.path().join("image.tar"); + run({ + let mut command = Command::new("docker"); + command + .args(["buildx", "build"]) + .arg("--metadata-file") + .arg(metadata.path()) + .arg("-f") + .arg(&spec.dockerfile) + .arg("-t") + .arg(&spec.image) + .args(["--builder", "harmony-oci", "--output"]) + .arg(format!( + "type={},dest={}", + publisher.archive_type(), + archive_path.display() + )); + for (name, value) in &spec.build_args { + command.arg("--build-arg").arg( + value + .as_ref() + .map_or_else(|| name.clone(), |value| format!("{name}={value}")), + ); + } + command.arg(&spec.context); + command + })?; + let metadata: serde_json::Value = serde_json::from_slice( + &std::fs::read(metadata.path()) + .map_err(|e| image_io("read docker build metadata", e))?, + ) + .map_err(|source| ImageError::Parse { + action: "parse docker build metadata", + source, + })?; + let digest = metadata["containerimage.digest"].as_str().ok_or_else(|| { + ImageError::Missing("docker build did not report containerimage.digest".to_string()) + })?; + let repository = spec + .image + .rsplit_once(':') + .map_or(spec.image.as_str(), |v| v.0); + let artifact = image_artifact(&spec.name, digest); + if artifact.exists() { + std::fs::remove_file(&artifact) + .map_err(|e| image_io("replace image build artifact", e))?; + } + std::fs::rename(archive_path, &artifact) + .map_err(|e| image_io("store image build artifact", e))?; + refs.insert(spec.name.clone(), format!("{repository}@{digest}")); + } + Ok(ImageRefs(refs)) +} + +pub fn publish_images( + specs: &[ImageSpec], + images: &ImageRefs, + publisher: &impl ImagePublisher, +) -> Result { + for spec in specs { + images.require(&spec.name)?; + } + publisher.publish(specs, images)?; + Ok(images.clone()) +} + +fn push_images(specs: &[ImageSpec], images: &ImageRefs, registry: &str) -> Result<(), ImageError> { + let user = std::env::var("REGISTRY_USER") + .map_err(|_| ImageError::Missing("REGISTRY_USER required to push".to_string()))?; + let token = std::env::var("REGISTRY_TOKEN") + .map_err(|_| ImageError::Missing("REGISTRY_TOKEN required to push".to_string()))?; + registry_login(registry, &user, &token)?; + for spec in specs { + let reference = images.require(&spec.name)?; + let (repository, expected_digest) = reference.split_once('@').ok_or_else(|| { + ImageError::Invalid("published image reference has no digest".to_string()) + })?; + let tagged_repository = spec + .image + .rsplit_once(':') + .map_or(spec.image.as_str(), |value| value.0); + if repository != tagged_repository { + return Err(ImageError::Invalid(format!( + "image '{}' belongs to repository '{repository}', expected '{tagged_repository}'", + spec.name + ))); + } + run({ + let mut command = Command::new("docker"); + command + .args(["load", "-i"]) + .arg(image_artifact(&spec.name, expected_digest)); + command + })?; + let output = Command::new("docker") + .arg("push") + .arg(&spec.image) + .output() + .map_err(|e| image_io("spawn docker push", e))?; + if !output.status.success() { + return Err(ImageError::Command(format!( + "docker push {} failed: {}", + spec.image, + String::from_utf8_lossy(&output.stderr) + ))); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let pushed_digest = parse_pushed_digest(&stdout) + .or_else(|| parse_pushed_digest(&stderr)) + .ok_or_else(|| { + ImageError::Missing("docker push did not report a digest".to_string()) + })?; + if pushed_digest != expected_digest { + return Err(ImageError::Invalid(format!( + "docker pushed {pushed_digest} for '{}', expected {expected_digest}", + spec.name + ))); } - cmd.arg(context); - run(cmd)?; } Ok(()) } -/// Log in, push each built image, then package + push the hydrated chart. -pub fn push_to_registry( - app: &ComposeApp, - cfg: &DeployConfig, - user: &str, - token: &str, -) -> Result<()> { - registry_login(&cfg.registry, user, token)?; - for svc in &app.services { - if svc.build_context.is_none() { - continue; +fn parse_pushed_digest(output: &str) -> Option<&str> { + let mut words = output.split_whitespace(); + while let Some(word) = words.next() { + if word == "digest:" { + return words.next(); } - let image = service_image(cfg, svc); - log::info!("docker push {image}"); - run({ - let mut c = Command::new("docker"); - c.arg("push").arg(&image); - c - })?; } - let tmp = tempfile::tempdir().context("chart tempdir")?; - let chart_dir = - build_chart(app, cfg, tmp.path()).map_err(|e| anyhow::anyhow!("hydrate chart: {e}"))?; - let tgz = helm_package(&chart_dir, tmp.path())?; - let oci_repo = format!("oci://{}/{}", cfg.registry, cfg.project); - log::info!("helm push {} {oci_repo}", tgz.display()); - run({ - let mut c = Command::new("helm"); - c.arg("push").arg(&tgz).arg(&oci_repo); - c - }) + None } -/// Load locally-built images into a k3d cluster's nodes — k3d has no -/// registry, so without this the `IfNotPresent` pulls find nothing. -pub fn import_to_k3d(app: &ComposeApp, cfg: &DeployConfig, cluster: &str) -> Result<()> { - let images: Vec = app - .services - .iter() - .filter(|s| s.build_context.is_some()) - .map(|s| service_image(cfg, s)) - .collect(); - if images.is_empty() { +fn ensure_oci_builder() -> Result<(), ImageError> { + let exists = Command::new("docker") + .args(["buildx", "inspect", "harmony-oci"]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false); + if !exists { + run({ + let mut command = Command::new("docker"); + command.args([ + "buildx", + "create", + "--name", + "harmony-oci", + "--driver", + "docker-container", + ]); + command + })?; + } + Ok(()) +} + +fn image_artifact_dir() -> PathBuf { + harmony::config::HARMONY_DATA_DIR.join("images") +} + +fn image_artifact(name: &str, digest: &str) -> PathBuf { + image_artifact_dir().join(format!( + "{}-{}.tar", + name.replace(|character: char| !character.is_ascii_alphanumeric(), "-"), + digest.replace(':', "-") + )) +} + +fn image_digest(reference: &str) -> Result<&str, ImageError> { + reference + .split_once('@') + .map(|value| value.1) + .ok_or_else(|| ImageError::Invalid("image reference has no digest".to_string())) +} + +fn import_images_to_k3d( + specs: &[ImageSpec], + images: &ImageRefs, + cluster: &str, +) -> Result<(), ImageError> { + if specs.is_empty() { return Ok(()); } - // Prefer harmony's managed k3d (the local-dev install location); fall - // back to a `k3d` on PATH for runners that ship their own. let managed = harmony::config::HARMONY_DATA_DIR.join("k3d").join("k3d"); let k3d: &std::ffi::OsStr = if managed.exists() { managed.as_os_str() } else { "k3d".as_ref() }; - // `ship` imports before it converges, but on the autoprovision path the - // cluster is created during converge — so ensure it exists first. K8sAnywhere - // then reuses this cluster (its installer starts an existing one). let exists = Command::new(k3d) .args(["cluster", "list", cluster, "--no-headers"]) .output() - .map(|o| o.status.success() && !o.stdout.is_empty()) + .map(|output| output.status.success() && !output.stdout.is_empty()) .unwrap_or(false); if !exists { - log::info!("k3d cluster create {cluster} (autoprovision)"); - let status = Command::new(k3d) - .args(["cluster", "create", cluster, "--wait"]) - .status() - .context("spawn k3d cluster create")?; - if !status.success() { - bail!("k3d cluster create {cluster} failed ({status})"); - } + run({ + let mut command = Command::new(k3d); + command.args(["cluster", "create", cluster, "--wait"]); + command + })?; } - log::info!("k3d image import {} -> {cluster}", images.join(" ")); - let status = Command::new(k3d) - .args(["image", "import", "-c", cluster]) - .args(&images) - .status() - .context("spawn k3d (installed and on PATH?)")?; - if !status.success() { - bail!("k3d image import failed ({status})"); + run({ + let mut command = Command::new(k3d); + command + .args(["image", "import", "-c", cluster, "--mode", "direct"]) + .args( + specs + .iter() + .map(|spec| { + images + .require(&spec.name) + .and_then(image_digest) + .map(|digest| image_artifact(&spec.name, digest)) + }) + .collect::, ImageError>>()?, + ); + command + })?; + + let output = Command::new(k3d) + .args(["node", "list", "-o", "json"]) + .output() + .map_err(|e| image_io("list k3d nodes", e))?; + if !output.status.success() { + return Err(ImageError::Command(format!( + "k3d node list failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + let nodes: serde_json::Value = + serde_json::from_slice(&output.stdout).map_err(|source| ImageError::Parse { + action: "parse k3d node list", + source, + })?; + let nodes = nodes + .as_array() + .ok_or_else(|| ImageError::Invalid("k3d node list did not return an array".to_string()))? + .iter() + .filter(|node| { + node["runtimeLabels"]["k3d.cluster"] == cluster + && matches!(node["role"].as_str(), Some("server" | "agent")) + }) + .collect::>(); + if nodes.is_empty() { + return Err(ImageError::Missing(format!( + "k3d cluster '{cluster}' has no server or agent nodes" + ))); + } + + for node in nodes { + let node = node["name"] + .as_str() + .ok_or_else(|| ImageError::Missing("k3d node has no name".to_string()))?; + for spec in specs { + let reference = images.require(&spec.name)?; + let expected = image_digest(reference)?; + let output = Command::new("docker") + .args([ + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "ls", + &format!("name=={}", spec.image), + ]) + .output() + .map_err(|e| image_io("inspect imported k3d image", e))?; + if !output.status.success() { + return Err(ImageError::Command(format!( + "inspect imported k3d image failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let actual = stdout + .lines() + .nth(1) + .and_then(|line| line.split_whitespace().nth(2)) + .ok_or_else(|| { + ImageError::Missing("imported k3d image was not found".to_string()) + })?; + if actual != expected { + return Err(ImageError::Invalid(format!( + "k3d imported {actual} for '{}', expected {expected}", + spec.name + ))); + } + let _ = Command::new("docker") + .args([ + "exec", node, "ctr", "-n", "k8s.io", "images", "rm", reference, + ]) + .output(); + run({ + let mut command = Command::new("docker"); + command.args([ + "exec", + node, + "ctr", + "-n", + "k8s.io", + "images", + "tag", + &spec.image, + reference, + ]); + command + })?; + } } Ok(()) } -fn registry_login(registry: &str, user: &str, token: &str) -> Result<()> { +fn registry_login(registry: &str, user: &str, token: &str) -> Result<(), ImageError> { login( "docker", &["login", registry, "-u", user, "--password-stdin"], token, - )?; - login( - "helm", - &[ - "registry", - "login", - registry, - "-u", - user, - "--password-stdin", - ], - token, ) } /// Feed the secret on stdin so it never lands in argv or the process table. -fn login(bin: &str, args: &[&str], password: &str) -> Result<()> { +fn login(bin: &str, args: &[&str], password: &str) -> Result<(), ImageError> { let mut child = Command::new(bin) .args(args) .stdin(Stdio::piped()) .spawn() - .with_context(|| format!("spawn {bin} (installed and on PATH?)"))?; + .map_err(|e| image_io(format!("spawn {bin} (installed and on PATH?)"), e))?; child .stdin .take() - .context("child stdin unavailable")? + .ok_or_else(|| ImageError::Missing("child stdin unavailable".to_string()))? .write_all(password.as_bytes()) - .with_context(|| format!("writing password to {bin}"))?; - let status = child.wait().with_context(|| format!("{bin} login"))?; + .map_err(|e| image_io(format!("writing password to {bin}"), e))?; + let status = child + .wait() + .map_err(|e| image_io(format!("{bin} login"), e))?; if !status.success() { - bail!("{bin} registry login failed ({status})"); + return Err(ImageError::Command(format!( + "{bin} registry login failed ({status})" + ))); } Ok(()) } -fn run(mut cmd: Command) -> Result<()> { +fn run(mut cmd: Command) -> Result<(), ImageError> { let status = cmd .status() - .with_context(|| format!("spawn {cmd:?} (is it installed and in PATH?)"))?; + .map_err(|e| image_io(format!("spawn {cmd:?} (is it installed and in PATH?)"), e))?; if !status.success() { - bail!("`{cmd:?}` failed ({status})"); + return Err(ImageError::Command(format!("`{cmd:?}` failed ({status})"))); } Ok(()) } -fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result { - let output = Command::new("helm") - .args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?]) - .output() - .context("spawn helm package")?; - if !output.status.success() { - bail!("helm package failed ({})", output.status); - } - // helm prints "…saved it to: "; the last token is the path. - String::from_utf8_lossy(&output.stdout) - .split_whitespace() - .last() - .map(PathBuf::from) - .context("helm package printed no path") -} +#[cfg(test)] +mod tests { + use super::*; -fn path_str(p: &Path) -> Result<&str> { - p.to_str() - .with_context(|| format!("path not utf-8: {}", p.display())) + #[test] + fn parses_docker_push_digest() { + assert_eq!( + parse_pushed_digest("latest: digest: sha256:abc123 size: 856"), + Some("sha256:abc123") + ); + } } diff --git a/harmony_app/src/score.rs b/harmony_app/src/score.rs index 24f6c4ac..01e45ea4 100644 --- a/harmony_app/src/score.rs +++ b/harmony_app/src/score.rs @@ -1,14 +1,13 @@ //! [`ComposeAppScore`] — converge a compose-imported app onto a cluster. //! //! Mirrors `FleetOperatorScore`: render a self-contained helm chart from -//! the imported compose at interpret time (or install an already-published -//! OCI chart on the CD path), `helm upgrade --install` via +//! the imported compose at interpret time, `helm upgrade --install` via //! [`HelmChartScore`], then layer the public Ingress on top. No //! `ApplicationScore`, no ArgoCD — the same build/publish/deploy split the //! fleet stack uses. use std::collections::BTreeMap; -use std::str::FromStr; +use std::{str::FromStr, time::Duration}; use async_trait::async_trait; use harmony_types::id::Id; @@ -26,18 +25,10 @@ use harmony::modules::k8s::resource::K8sResourceScore; use harmony::score::Score; use harmony::topology::{HelmCommand, K8sclient, Topology}; +use crate::AppError; use crate::chart::{DeployConfig, app_secret_name, build_chart}; use crate::compose::ComposeApp; -/// The already-published OCI chart to install (the CD path). When set, -/// the chart is pulled from the registry instead of rendered locally. -#[derive(Debug, Clone, Serialize)] -pub struct PublishedChart { - pub registry: String, - pub project: String, - pub version: String, -} - /// Expose one service through an Ingress (a deploy-only knob — the host /// is never read from compose). #[derive(Debug, Clone, Serialize)] @@ -59,13 +50,15 @@ impl PublicEndpoint { service: &str, host: impl Into, cluster_issuer: Option, - ) -> Result { + ) -> Result { let port = app .service(service) - .ok_or_else(|| format!("no compose service '{service}'"))? + .ok_or_else(|| AppError::InvalidComposition(format!("no compose service '{service}'")))? .ports .first() - .ok_or_else(|| format!("service '{service}' exposes no port"))? + .ok_or_else(|| { + AppError::InvalidComposition(format!("service '{service}' exposes no port")) + })? .container; Ok(Self { service: service.to_string(), @@ -82,9 +75,6 @@ pub struct ComposeAppScore { pub release_name: String, pub app: ComposeApp, pub deploy: DeployConfig, - /// `None` renders + installs the chart from the imported compose - /// (dev/e2e); `Some` installs the published OCI chart (CD). - pub published_chart: Option, pub public_endpoint: Option, /// App secrets (from OpenBao) applied as one Opaque `-secrets` /// Secret before the workload, loaded by every container via `envFrom`. @@ -112,6 +102,36 @@ struct ComposeAppInterpret { score: ComposeAppScore, } +async fn smoke_test_endpoint(endpoint: &PublicEndpoint) -> Result<(), InterpretError> { + let scheme = if endpoint.cluster_issuer.is_some() { + "https" + } else { + "http" + }; + let url = format!("{scheme}://{}", endpoint.host); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .map_err(|e| InterpretError::new(format!("build endpoint smoke-test client: {e}")))?; + let mut last_error = "endpoint did not respond".to_string(); + tokio::time::timeout(Duration::from_secs(180), async { + loop { + match client.get(&url).send().await { + Ok(response) if response.status().is_success() => return, + Ok(response) => last_error = format!("HTTP {}", response.status()), + Err(error) => last_error = error.to_string(), + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| { + InterpretError::new(format!( + "app endpoint {url} failed its smoke test after 180s: {last_error}" + )) + }) +} + #[async_trait] impl Interpret for ComposeAppInterpret { async fn execute( @@ -120,6 +140,13 @@ impl Interpret for ComposeAppInterpret topology: &T, ) -> Result { let s = &self.score; + let k8s = topology + .k8s_client() + .await + .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))?; + k8s.ensure_namespace(&s.namespace).await.map_err(|e| { + InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace)) + })?; // App secrets first, so the workload's `envFrom` can load them. The // values come from OpenBao at deploy time — never baked into the @@ -140,32 +167,35 @@ impl Interpret for ComposeAppInterpret .await?; } - // Install from the published OCI chart (CD) or a chart rendered - // from the imported compose (dev/e2e). Each branch keeps its - // tempdir alive across the install. - let helm_outcome = if let Some(p) = &s.published_chart { - let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, s.deploy.app_name); - info!( - "Installing '{}' from published chart {chart_ref}:{}", - s.release_name, p.version - ); - self.install(inventory, topology, &chart_ref, &p.version) - .await? - } else { - let tmp = tempfile::tempdir() - .map_err(|e| InterpretError::new(format!("chart tempdir: {e}")))?; - let chart_path = build_chart(&s.app, &s.deploy, tmp.path()) - .map_err(|e| InterpretError::new(format!("render chart: {e}")))?; - let chart_path = chart_path - .to_str() - .ok_or_else(|| InterpretError::new("chart path not utf-8".to_string()))?; - info!( - "Installing '{}' from rendered chart {chart_path}", - s.release_name - ); - self.install(inventory, topology, chart_path, &s.deploy.chart_version) - .await? - }; + let tmp = + tempfile::tempdir().map_err(|e| InterpretError::new(format!("chart tempdir: {e}")))?; + let chart_path = build_chart(&s.app, &s.deploy, tmp.path()) + .map_err(|e| InterpretError::new(format!("render chart: {e}")))?; + let chart_path = chart_path + .to_str() + .ok_or_else(|| InterpretError::new("chart path not utf-8".to_string()))?; + info!( + "Installing '{}' from rendered chart {chart_path}", + s.release_name + ); + let helm_outcome = self + .install(inventory, topology, chart_path, &s.deploy.chart_version) + .await?; + + for service in &s.app.services { + k8s.wait_until_deployment_ready( + &service.name, + Some(&s.namespace), + Some(Duration::from_secs(180)), + ) + .await + .map_err(|e| { + InterpretError::new(format!( + "app deployment {}/{} not ready: {e}", + s.namespace, service.name + )) + })?; + } // Public Ingress, applied after the chart so the backing Service // exists. Skipped when no endpoint is configured. @@ -196,6 +226,7 @@ impl Interpret for ComposeAppInterpret } .interpret(inventory, topology) .await?; + smoke_test_endpoint(ep).await?; } Ok(Outcome::success_with_details( @@ -222,7 +253,7 @@ impl Interpret for ComposeAppInterpret } impl ComposeAppInterpret { - async fn install( + async fn install( &self, inventory: &Inventory, topology: &T, @@ -230,18 +261,6 @@ impl ComposeAppInterpret { version: &str, ) -> Result { let s = &self.score; - // Ensure the namespace exists without patching it (a namespace-scoped - // deployer can't patch the cluster-scoped Namespace object), then install - // without helm `--create-namespace`. - topology - .k8s_client() - .await - .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))? - .ensure_namespace(&s.namespace) - .await - .map_err(|e| { - InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace)) - })?; HelmChartScore { namespace: Some(non_blank(&s.namespace, "namespace")?), release_name: non_blank(&s.release_name, "release_name")?, diff --git a/harmony_app/tests/helm_render.rs b/harmony_app/tests/helm_render.rs index 8ee2a07a..5f598001 100644 --- a/harmony_app/tests/helm_render.rs +++ b/harmony_app/tests/helm_render.rs @@ -38,6 +38,7 @@ fn generated_chart_passes_helm_lint_and_template() { extra_env: vec![], secret_file_mounts: vec![], image_pull_secrets: vec![], + images: Default::default(), }; let tmp = tempfile::tempdir().unwrap(); diff --git a/harmony_cli/Cargo.toml b/harmony_cli/Cargo.toml index 9bc6b3d4..4d539e56 100644 --- a/harmony_cli/Cargo.toml +++ b/harmony_cli/Cargo.toml @@ -10,6 +10,8 @@ default = ["tui"] tui = ["dep:harmony_tui"] [dependencies] +serde.workspace = true +serde_json.workspace = true assert_cmd = "2.0.17" anyhow = { workspace = true } clap = { version = "4.5.35", features = ["derive"] } diff --git a/harmony_cli/src/app.rs b/harmony_cli/src/app.rs index eb0eec9b..a4a34133 100644 --- a/harmony_cli/src/app.rs +++ b/harmony_cli/src/app.rs @@ -35,10 +35,22 @@ struct AppCli { /// `.harmony/contexts.toml` found by walking up from the current dir. #[arg(long, short = 'C', global = true, value_name = "FILE")] config: Option, + + /// Emit the versioned machine-readable output schema. + #[arg(long, global = true)] + json: bool, } #[derive(Subcommand, Debug)] enum Verb { + /// Build images and print digest-pinned references. + Build, + /// Publish already-built images to the selected context. + Publish { + /// Digest-pinned image as NAME=REFERENCE. Repeat for multi-image apps. + #[arg(long = "image", value_parser = parse_image, required = true)] + images: Vec<(String, String)>, + }, /// Build + publish + deploy. Ship { /// Pass `--force-conflicts` to the app chart's helm upgrade. @@ -50,6 +62,9 @@ enum Verb { /// Pass `--force-conflicts` to the app chart's helm upgrade. #[arg(long)] force_conflicts: bool, + /// Digest-pinned image as NAME=REFERENCE. Repeat for multi-image apps. + #[arg(long = "image", value_parser = parse_image)] + images: Vec<(String, String)>, }, /// Workload readiness in the app's namespace. Status, @@ -70,42 +85,101 @@ pub async fn app_main + 'static>(app: A) -> R let context = cli .context - .context("--context is required (there is no default context)")?; - let ctx = AppContext::resolve(&context, cli.tag, cli.local_config, cli.config).await?; + .or_else(|| std::env::var("HARMONY_CONTEXT").ok()) + .context("--context or HARMONY_CONTEXT is required")?; + let metadata_only = matches!(&cli.verb, Verb::Build | Verb::Publish { .. }); + let ctx = if metadata_only { + AppContext::load_metadata(&context, cli.tag, cli.local_config, cli.config)? + } else { + AppContext::resolve(&context, cli.tag, cli.local_config, cli.config).await? + }; match cli.verb { + Verb::Build => render_images(app.build(&ctx)?, cli.json), + Verb::Publish { images } => render_images( + app.publish(&ctx, &harmony_app::ImageRefs::new(images))?, + cli.json, + ), Verb::Ship { force_conflicts } => render_deploy( harmony_app::ship_with_options( &app, ctx.topology(), &ctx, - harmony_app::DeployOptions { force_conflicts }, + harmony_app::DeployOptions { + force_conflicts, + ..Default::default() + }, ) .await?, + cli.json, ), - Verb::Deploy { force_conflicts } => render_deploy( + Verb::Deploy { + force_conflicts, + images, + } => render_deploy( harmony_app::deploy_with_options( &app, ctx.topology(), &ctx, - harmony_app::DeployOptions { force_conflicts }, + harmony_app::DeployOptions { + force_conflicts, + images: harmony_app::ImageRefs::new(images), + }, ) .await?, + cli.json, ), - Verb::Status => render_status(harmony_app::status(&app, &ctx).await?), - Verb::Logs { tail } => render_logs(harmony_app::logs(&app, &ctx, Some(tail)).await?), + Verb::Status => render_status(harmony_app::status(&app, &ctx).await?, cli.json), + Verb::Logs { tail } => { + render_logs(harmony_app::logs(&app, &ctx, Some(tail)).await?, cli.json) + } } Ok(()) } -fn render_deploy(report: harmony_app::DeployReport) { +fn parse_image(value: &str) -> Result<(String, String), String> { + let (name, image) = value + .split_once('=') + .filter(|(name, _)| !name.is_empty()) + .ok_or_else(|| "expected NAME=IMAGE@sha256:DIGEST".to_string())?; + let (repository, digest) = image + .rsplit_once("@sha256:") + .ok_or_else(|| "expected NAME=IMAGE@sha256:DIGEST".to_string())?; + if repository.is_empty() + || digest.len() != 64 + || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err("expected NAME=IMAGE@sha256:DIGEST".to_string()); + } + Ok((name.to_string(), image.to_string())) +} + +fn render_deploy(report: harmony_app::DeployReport, json: bool) { + if json { + render_json(&report); + return; + } println!("\n🚀 Deployed:"); for step in report.steps { println!(" ✅ {} — {}", step.name, step.message); } } -fn render_status(report: harmony_app::StatusReport) { +fn render_images(images: harmony_app::ImageRefs, json: bool) { + if json { + render_json(&images); + return; + } + for (name, image) in images.iter() { + println!("{name}={image}"); + } +} + +fn render_status(report: harmony_app::StatusReport, json: bool) { + if json { + render_json(&report); + return; + } println!("\n📊 {} workloads:", report.namespace); if report.workloads.is_empty() { println!(" (none)"); @@ -116,8 +190,32 @@ fn render_status(report: harmony_app::StatusReport) { } } -fn render_logs(pods: Vec) { +fn render_logs(pods: Vec, json: bool) { + if json { + render_json(&pods); + return; + } for p in pods { println!("\n=== {} ===\n{}", p.pod, p.logs.trim_end()); } } + +fn render_json(value: &impl serde::Serialize) { + println!( + "{}", + serde_json::json!({ "schema_version": 1, "result": value }) + ); +} + +#[cfg(test)] +mod tests { + use super::parse_image; + + #[test] + fn image_reference_requires_a_full_sha256_digest() { + let digest = "a".repeat(64); + assert!(parse_image(&format!("operator=registry/operator@sha256:{digest}")).is_ok()); + assert!(parse_image("operator=registry/operator@sha256:abc").is_err()); + assert!(parse_image(&format!("operator=@sha256:{digest}")).is_err()); + } +} diff --git a/harmony_config/src/source/local_file.rs b/harmony_config/src/source/local_file.rs index c435cab1..c9b8fc81 100644 --- a/harmony_config/src/source/local_file.rs +++ b/harmony_config/src/source/local_file.rs @@ -5,10 +5,6 @@ use tokio::fs; use crate::{ConfigClass, ConfigError, ConfigSource}; /// Local file-backed config source (`/.json`). -/// -/// ⚠️ Cleartext on disk and ignores `ConfigClass`, like -/// [`SqliteSource`](crate::SqliteSource) — non-secret config only, via an -/// explicit [`ConfigClient::new`](crate::ConfigClient::new). pub struct LocalFileSource { base_path: PathBuf, } diff --git a/harmony_secret/Cargo.toml b/harmony_secret/Cargo.toml index 576dc819..c542b49a 100644 --- a/harmony_secret/Cargo.toml +++ b/harmony_secret/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true unexpected_cfgs = { level = "warn", check-cfg = ['cfg(secrete2etest)'] } [dependencies] +harmony-reconciler-contracts = { path = "../harmony-reconciler-contracts" } harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" } harmony_secret_derive = { version = "0.1.0", path = "../harmony_secret_derive" } serde = { version = "1.0.209", features = ["derive", "rc"] } diff --git a/harmony/src/modules/openbao/identity.rs b/harmony_secret/src/deployment_grants.rs similarity index 94% rename from harmony/src/modules/openbao/identity.rs rename to harmony_secret/src/deployment_grants.rs index 641b9276..047dec53 100644 --- a/harmony/src/modules/openbao/identity.rs +++ b/harmony_secret/src/deployment_grants.rs @@ -18,10 +18,9 @@ use reqwest::StatusCode; use serde_json::json; use tokio::sync::OnceCell; -use crate::modules::fleet::secret_access::{DeploymentSecretGrants, SecretAccessError}; -use harmony_reconciler_contracts::DeploymentName; +use harmony_reconciler_contracts::{DeploymentName, DeploymentSecretGrants, SecretAccessError}; -const DEFAULT_JWT_MOUNT: &str = "jwt"; +const JWT_AUTH_MOUNT: &str = "jwt"; pub struct OpenBaoDeploymentSecretGrants { client: reqwest::Client, @@ -44,7 +43,7 @@ impl OpenBaoDeploymentSecretGrants { token, kv_mount, secret_prefix, - DEFAULT_JWT_MOUNT.to_string(), + JWT_AUTH_MOUNT.to_string(), ) } @@ -213,7 +212,7 @@ path "{kv}/metadata/{prefix}/{dep}/*" {{ capabilities = ["read", "list"] }}"#, /// mount, and that it carries `policy`. async fn attach(&self, group: &str, policy: &str) -> Result<(), SecretAccessError> { match self.read_group_policies(group).await? { - Some(policies) if policies.iter().any(|p| p == policy) => return Ok(()), + Some(policies) if policies.iter().any(|p| p == policy) => {} Some(mut policies) => { policies.push(policy.to_string()); self.write_group_policies(group, &policies).await?; @@ -221,9 +220,9 @@ path "{kv}/metadata/{prefix}/{dep}/*" {{ capabilities = ["read", "list"] }}"#, None => { self.write_group_policies(group, &[policy.to_string()]) .await?; - self.ensure_group_alias(group).await?; } } + self.ensure_group_alias(group).await?; Ok(()) } @@ -272,12 +271,18 @@ path "{kv}/metadata/{prefix}/{dep}/*" {{ capabilities = ["read", "list"] }}"#, })), ) .await?; - // A concurrent sync may have just created the alias; OpenBao - // answers 400 "alias already exists" — treat as success. - if !resp.status().is_success() && resp.status() != StatusCode::BAD_REQUEST { + if resp.status().is_success() { + return Ok(()); + } + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let alias_exists = status == StatusCode::BAD_REQUEST + && body.to_ascii_lowercase().contains("alias") + && body.to_ascii_lowercase().contains("already"); + if !alias_exists { return Err(Self::err( format!("create group alias {group}"), - resp.status(), + format!("{status}: {body}"), )); } Ok(()) diff --git a/harmony_secret/src/lib.rs b/harmony_secret/src/lib.rs index a66ebf78..54577483 100644 --- a/harmony_secret/src/lib.rs +++ b/harmony_secret/src/lib.rs @@ -1,4 +1,5 @@ pub mod config; +mod deployment_grants; pub mod store; use crate::config::SECRET_NAMESPACE; @@ -30,6 +31,7 @@ pub use store::{OpenbaoSecretStore, OpenbaoStoreOptions, ZitadelJwtBearerConfig} use thiserror::Error; use tokio::sync::OnceCell; +pub use deployment_grants::OpenBaoDeploymentSecretGrants; pub use harmony_secret_derive::Secret; // The Secret trait remains the same. diff --git a/harmony_zitadel_auth/Cargo.toml b/harmony_zitadel_auth/Cargo.toml index afeeba43..7ff56ee2 100644 --- a/harmony_zitadel_auth/Cargo.toml +++ b/harmony_zitadel_auth/Cargo.toml @@ -10,9 +10,11 @@ default = [] axum = ["dep:axum", "dep:axum-extra"] [dependencies] +harmony-reconciler-contracts = { path = "../harmony-reconciler-contracts" } harmony_config = { path = "../harmony_config" } harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" } anyhow.workspace = true +async-trait.workspace = true base64.workspace = true chrono = { workspace = true, features = ["serde"] } rand.workspace = true diff --git a/harmony/src/modules/zitadel/device_groups.rs b/harmony_zitadel_auth/src/device_groups.rs similarity index 97% rename from harmony/src/modules/zitadel/device_groups.rs rename to harmony_zitadel_auth/src/device_groups.rs index 698afaae..fa677dd6 100644 --- a/harmony/src/modules/zitadel/device_groups.rs +++ b/harmony_zitadel_auth/src/device_groups.rs @@ -10,7 +10,7 @@ use std::collections::{HashMap, HashSet}; use async_trait::async_trait; use serde::Deserialize; -use crate::modules::fleet::groups::{DeviceGroupSource, GroupSourceError}; +use harmony_reconciler_contracts::{DeviceGroupSource, GroupSourceError}; pub struct ZitadelDeviceGroups { client: reqwest::Client, diff --git a/harmony_zitadel_auth/src/lib.rs b/harmony_zitadel_auth/src/lib.rs index 88a363ab..af932922 100644 --- a/harmony_zitadel_auth/src/lib.rs +++ b/harmony_zitadel_auth/src/lib.rs @@ -1,11 +1,13 @@ #[cfg(feature = "axum")] pub mod axum_login_flow; pub mod config; +mod device_groups; pub mod jwks; pub mod login; pub mod session; pub use config::{OperatorCookieKey, ZitadelAuthConfig}; +pub use device_groups::ZitadelDeviceGroups; pub use harmony_zitadel_jwt::{MachineKeyFile, ZitadelJwtBearer}; pub use jwks::JwksCache;