feat/typed-deploy-contexts #343

Open
johnride wants to merge 4 commits from feat/typed-deploy-contexts into docs/score-composition-via-refs
55 changed files with 2758 additions and 818 deletions

View File

@@ -1,10 +0,0 @@
[contexts.local]
profile = "local"
autoprovision = true
domain = "localhost"
# Customer contexts belong in their private repository and can be selected with
# `--config path/to/contexts.toml`. A production context also sets `registry`,
# `project`, `domain`, `image_pull_secret`, and brokered OpenBao cluster access.
# Its cluster contains the immutable `fleet-system/fleet-callout-credentials`
# Secret described in `fleet/README.md`.

16
Cargo.lock generated
View File

@@ -2833,6 +2833,8 @@ dependencies = [
"anyhow", "anyhow",
"harmony_app", "harmony_app",
"harmony_cli", "harmony_cli",
"harmony_macros",
"harmony_types",
"tokio", "tokio",
] ]
@@ -2893,6 +2895,19 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "example-fleet-typed-deploy"
version = "0.0.0"
dependencies = [
"anyhow",
"harmony",
"harmony-fleet-deploy",
"harmony_app",
"harmony_macros",
"harmony_types",
"tokio",
]
[[package]] [[package]]
name = "example-grafana" name = "example-grafana"
version = "0.1.0" version = "0.1.0"
@@ -4274,7 +4289,6 @@ dependencies = [
"tempfile", "tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"toml",
] ]
[[package]] [[package]]

View File

@@ -17,7 +17,7 @@ The private repository owns:
```text ```text
Cargo.toml Cargo.toml
src/main.rs # Fleet app + compiled context catalog src/main.rs # compiled Fleet deployment context
.github/workflows/e2e.yml # or the equivalent CI system .github/workflows/e2e.yml # or the equivalent CI system
README.md # operator setup and recovery README.md # operator setup and recovery
``` ```
@@ -31,32 +31,31 @@ are compiled into the binary; loading additional context definitions at runtime
is deferred until a real requirement justifies that interface. is deferred until a real requirement justifies that interface.
```rust ```rust
let contexts = ContextCatalog::new([ let context = Context {
NamedContext { name: context_name!("e2e"),
name: context_name!("e2e"), namespace: "fleet-e2e".parse()?,
target: ContextSpec::Remote(RemoteContext { spec: ContextSpec::Remote(RemoteContext {
profile: Profile::Prod, registry: oci_registry!("registry.example.com"),
registry: oci_registry!("registry.example.com"), repository: oci_repository!("harmony/e2e"),
registry_project: oci_repository!("harmony/e2e"), domain: domain!("e2e.example.com"),
domain: domain!("e2e.example.com"), image_pull_secret: Some("registry-pull".parse()?),
image_pull_secret: k8s_name!("registry-pull"), access: OpenBaoClusterAccess {
access: OpenBaoClusterAccess { namespace: openbao_namespace!("example"),
namespace: openbao_namespace!("example"), url: http_url!("https://secrets.nationtech.io"),
url: http_url!("https://secrets.nationtech.io"), role: OpenBaoRoleName::new("example-ci")?,
role: OpenBaoRoleName::new("example-ci")?, zitadel_url: http_url!("https://identity.example.com"),
zitadel_url: http_url!("https://identity.example.com"), zitadel_audience: OidcAudience::new("example")?,
zitadel_audience: OidcAudience::new("example")?, },
}, }),
}), };
}, deploy_fleet_with_context(context).await
])?;
``` ```
Concrete names and identity details stay private. Concrete names and identity details stay private.
### Context type design ### Context type design
`ContextSpec` should encode the access-mode choice instead of representing it `ContextSpec` encodes the access-mode choice instead of representing it
with unrelated optional fields: with unrelated optional fields:
```rust ```rust
@@ -66,16 +65,19 @@ pub enum ContextSpec {
} }
``` ```
`RemoteContext` carries required registry, domain, image-pull, and OpenBao `RemoteContext` carries required registry, domain, and OpenBao values directly;
values directly. `LocalContext` carries only its k3d settings. This makes its image-pull Secret is optional for public registries. `LocalContext` carries
only its k3d settings. This makes
impossible combinations unrepresentable and removes checks such as "exactly one impossible combinations unrepresentable and removes checks such as "exactly one
of autoprovision, k3d, or OpenBao must be set." of autoprovision, k3d, or OpenBao must be set."
Use existing domain types where they are already correct: Use existing domain types where they are already correct:
- `Profile` remains an enum. It does not need a string macro. - `Profile` remains an enum derived from the local or remote context variant.
- The context's `namespace` places every tenant Fleet component and custom
resource in one Kubernetes namespace.
- Kubernetes namespaces, Secret names, and other DNS-label resource names use - Kubernetes namespaces, Secret names, and other DNS-label resource names use
`K8sName` and a compile-time `k8s_name!` literal. `K8sName`.
- IP and MAC literals continue using the existing Harmony macros. - IP and MAC literals continue using the existing Harmony macros.
Add types only for distinct grammars or values that are easy to transpose: Add types only for distinct grammars or values that are easy to transpose:
@@ -104,8 +106,7 @@ compile-fail macro tests.
### Application integration ### Application integration
Change the lifecycle entry point from discovering `.harmony/contexts.toml` to The lifecycle entry point receives a compiled catalog:
receiving the catalog:
```rust ```rust
harmony_cli::app::app_main(FleetApp, contexts).await harmony_cli::app::app_main(FleetApp, contexts).await
@@ -152,19 +153,16 @@ The context should resolve once at the start of the combined build-and-publish
job. Build itself remains credential-free; publish consumes the registry job. Build itself remains credential-free; publish consumes the registry
credential fetched from OpenBao. credential fetched from OpenBao.
The current `harmony_app::context` TOML parser and `Option<String>` context
definition must also be replaced by the typed catalog above. This is a public
application-layer change; concrete context values remain private.
## Phase 1: cluster and identity ## Phase 1: cluster and identity
- [ ] Add the context domain types and their shared validators to - [x] Add the context domain types and their shared validators to
`harmony_types`. `harmony_types`.
- [ ] Add only the justified literal macros to `harmony_macros`, including - [x] Add only the justified literal macros to `harmony_macros`, including
compile-fail tests. compile-fail tests.
- [ ] Replace TOML discovery with a caller-supplied `ContextCatalog`. - [x] Replace TOML discovery with a caller-supplied `ContextCatalog`.
- [ ] Export the canonical `FleetApp` composition so the private deploy binary - [x] Export the canonical `FleetApp` composition so the private deploy binary
does not duplicate it. does not duplicate it.
- [x] Separate cluster-owned Fleet CRDs from tenant operator releases.
- [ ] Create a dedicated E2E namespace or namespace prefix on the public OKD - [ ] Create a dedicated E2E namespace or namespace prefix on the public OKD
cluster. cluster.
- [ ] Configure namespace-scoped RBAC for the OpenBao-brokered kubeconfig. - [ ] Configure namespace-scoped RBAC for the OpenBao-brokered kubeconfig.

View File

@@ -138,13 +138,10 @@ Eleven principles, grouped.
7. **Per-environment values are a Score computed as a function of the 7. **Per-environment values are a Score computed as a function of the
context.** Differences (prod = 3 replicas + managed Postgres; local = context.** Differences (prod = 3 replicas + managed Postgres; local =
1 replica + sqlite) are expressed in typed code that branches on a 1 replica + sqlite) are expressed in typed code that branches on a
**profile tag carried by the context** — the Pulumi-stack idea in pure profile derived from the typed context variant: local contexts use
Rust, validated by the compiler. The profile is a **structured field on `Profile::Local`, and remote contexts use `Profile::Prod`. A Score branches
the context, not encoded in its name**: a Score branches on on `ctx.profile()`, never on the context name. An independent profile input
`ctx.profile()`, never on the context name (a free human handle — is deferred until a third behavior has distinct semantics.
`myapp-prod` and `otherapp-prod` may share `profile = Prod`). (A dedicated
typed `Profile` input is deferred until divergence justifies it — Rule
of Three.)
8. **Reconciliation is invocation-driven (today).** Convergence happens 8. **Reconciliation is invocation-driven (today).** Convergence happens
when someone runs `harmony app deploy`, not continuously. Out-of-band when someone runs `harmony app deploy`, not continuously. Out-of-band

View File

@@ -1,8 +1,8 @@
# Harmony Application CLI — Use Cases & Commands # Harmony Application CLI — Use Cases & Commands
> **Status: design target (proposed).** Today `harmony_cli` runs Scores > **Status: partially implemented.** The app lifecycle provides build,
> via flags; none of the verbs below are implemented yet. This is the > publish, ship, deploy, status, and logs. Other verbs below remain the design
> living reference we are building toward. The *decisions and rationale* > target. The *decisions and rationale*
> live in [ADR-026](../adr/026-application-lifecycle-cli.md) — read that > live in [ADR-026](../adr/026-application-lifecycle-cli.md) — read that
> for the "why"; this doc is the "what" and "how". > for the "why"; this doc is the "what" and "how".
@@ -18,42 +18,45 @@ Four ideas carry the whole CLI:
- **Implicit app, explicit context.** The app is *this project's app* - **Implicit app, explicit context.** The app is *this project's app*
(inferred — you never name it). The **target is always explicit**: (inferred — you never name it). The **target is always explicit**:
`--context <name>` or `HARMONY_CONTEXT`. **There is no default context; `--context <name>` or `HARMONY_CONTEXT`. **There is no default context;
omitting it is a hard error.** Even local k3d is `--context local`. omitting it is a hard error.** Selection remains mandatory for local k3d and
binaries with one compiled target so shell history and CI identify the target.
- **Declarative vs operational.** Verbs that change desired state go - **Declarative vs operational.** Verbs that change desired state go
through the project's typed Scores and re-converge. Verbs that only through the project's typed Scores and re-converge. Verbs that only
read or are ephemeral talk to the cluster directly and never mutate read or are ephemeral talk to the cluster directly and never mutate
desired state. You never imperatively edit live state — you edit a desired state. You never imperatively edit live state — you edit a
Score and redeploy. Score and redeploy.
- **Config has three homes, none of them a config file.** Behavior → - **Config has three homes, none of them a config file.** Behavior →
typed Scores (in git). Target + credentials → the context. Secrets → typed Scores (in git). Targets → compiled contexts in the deploy binary.
OpenBao. Secrets → OpenBao.
### Contexts & profiles ### Contexts & profiles
A **context** is `{ cluster (endpoint + CA), tenant, credential source / A **context** is `{ cluster access, namespace, publication target, domain }`.
identity, profile }`. It carries **no role** — authorization is enforced It carries no authorization role; authorization comes from the deploy
server-side from the identity's token (Zitadel → OpenBao → RBAC), never a identity (Zitadel → OpenBao → RBAC). Contexts are typed Rust values compiled
client-side setting. Contexts are defined out-of-band (user/CI config), into the deploy binary. Private targets stay in private deploy repositories.
not in the project.
``` ```
harmony context list # what can I target? harmony context list # what can I target?
harmony context show myapp-prod # cluster, tenant, profile, identity (+ role, derived from token) harmony context show myapp-prod # cluster, tenant, profile, identity (+ role, derived from token)
``` ```
The **profile** (`local | staging | prod`) is a **structured field** the The context variant determines the profile: local contexts use `Profile::Local`
context carries — *not* its name. A Score branches on `ctx.profile()` to and remote contexts use production behavior through `Profile::Prod`. A staging
compute per-environment values (`prod` → 3 replicas + managed Postgres; cluster can have a staging context name, but it currently uses the same
`local` → 1 replica + sqlite), and **never parses the context name**. The replication, TLS, and credential behavior as production. Scores never infer
name (`myapp-prod`) is a free human handle for picking `--context`; the behavior from the context name. A separate staging profile should be added only
profile field is the authoritative value, so two differently-named contexts when it has distinct operational semantics. See ADR-026 §7.
can share `profile = Prod` (and `myapp-prod` vs `otherapp-prod` are two
different contexts at the same profile). See ADR-026 §7.
Credentials ride on the context and degrade: **local** uses the ambient Credentials ride on the context and degrade: **local** uses the ambient
k3d/kubeconfig; **remote** mints a short-lived, namespace-scoped token via k3d/kubeconfig; **remote** mints a short-lived, namespace-scoped token via
Zitadel→OpenBao. The CLI holds nothing standing (ADR-026 §10). Zitadel→OpenBao. The CLI holds nothing standing (ADR-026 §10).
For remote contexts, the kubeconfig stored in OpenBao selects the Kubernetes
cluster through its `current-context`. Context resolution logs that context,
the API server, and the deployment namespace. Deploy reports repeat the same
target so CI output records where convergence occurred.
--- ---
## `app` — the developer lifecycle ## `app` — the developer lifecycle
@@ -63,7 +66,7 @@ The verbs, by class:
| Verb | Class | Context? | What it does | | Verb | Class | Context? | What it does |
|---|---|---|---| |---|---|---|---|
| `check` | declarative | no | Compile + type-check the Scores. The only pre-deploy validation today (no live diff yet). | | `check` | declarative | no | Compile + type-check the Scores. The only pre-deploy validation today (no live diff yet). |
| `build` | declarative | no | Build a digest-pinned OCI image; prints the digest. Environment-agnostic. | | `build` | declarative | **yes** | Build a digest-pinned OCI image using target-specific naming. Resolves context metadata, but not credentials or cluster access. |
| `publish` | declarative | **yes** | Push the image to the registry (remote) **or** `k3d image import` (local). Topology-specific. | | `publish` | declarative | **yes** | Push the image to the registry (remote) **or** `k3d image import` (local). Topology-specific. |
| `deploy` | declarative | **yes** | Converge the Scores against the context, pinned to `--image <digest>`. **Does not build.** Returns only after smoke-test. | | `deploy` | declarative | **yes** | Converge the Scores against the context, pinned to `--image <digest>`. **Does not build.** Returns only after smoke-test. |
| `ship` | declarative | **yes** | `build` + `publish` + `deploy`, threading the digest. The everyday verb. | | `ship` | declarative | **yes** | `build` + `publish` + `deploy`, threading the digest. The everyday verb. |
@@ -81,7 +84,7 @@ The verbs, by class:
| I want to… | Command | | I want to… | Command |
|---|---| |---|---|
| Check my Scores compile/type-check | `harmony app check` | | Check my Scores compile/type-check | `harmony app check` |
| Build the image to test it builds | `harmony app build` | | Build the image to test it builds | `harmony app build --context local` |
| Deploy to local k3d and test | `harmony app ship --context local` | | Deploy to local k3d and test | `harmony app ship --context local` |
| See it running locally | `harmony app status --context local` · `harmony app logs --context local -f` | | See it running locally | `harmony app status --context local` · `harmony app logs --context local -f` |
@@ -92,10 +95,10 @@ The verbs, by class:
| Build + publish + deploy to prod | `harmony app ship --context myapp-prod` | | Build + publish + deploy to prod | `harmony app ship --context myapp-prod` |
| Deploy an already-built image | `harmony app deploy --context myapp-prod --image <digest>` | | Deploy an already-built image | `harmony app deploy --context myapp-prod --image <digest>` |
| Roll **forward** to a prior good build (recovery) | `harmony app deploy --context myapp-prod --image <prior-digest>` | | Roll **forward** to a prior good build (recovery) | `harmony app deploy --context myapp-prod --image <prior-digest>` |
| Build / publish as separate CI stages | `harmony app build``harmony app publish --context myapp-prod``harmony app deploy …` | | Build / publish as separate CI stages | `harmony app build --context myapp-prod``harmony app publish --context myapp-prod``harmony app deploy …` |
There is no rollback and no staging (the roll-forward-only model): you roll forward by There is no rollback command. Recovery deploys a known-good digest as a new
deploying a known-good digest. roll-forward operation.
### Operate (day-2) ### Operate (day-2)

View File

@@ -1,8 +1,8 @@
# Deploy Fleet to a remote cluster # Deploy Fleet to a remote cluster
Fleet uses the same lifecycle commands and Scores for local, staging, and Fleet uses the same lifecycle commands and Scores for every cluster. Remote
production clusters. The selected context changes the target and publication contexts currently use production-equivalent replication, TLS, exposure, and
behavior; there is no separate staging installer. credential behavior, including when the target cluster is used for staging.
## Prerequisites ## Prerequisites
@@ -11,49 +11,92 @@ behavior; there is no separate staging installer.
- The image registry contains credentials for `REGISTRY_USER` and - The image registry contains credentials for `REGISTRY_USER` and
`REGISTRY_TOKEN`. `REGISTRY_TOKEN`.
- The deploy identity can obtain the cluster kubeconfig through OpenBao. - The deploy identity can obtain the cluster kubeconfig through OpenBao.
- Public DNS points the context domain at the cluster ingress. - Public DNS points `zitadel`, `openbao`, and `nats` under the context domain at
the cluster ingress.
## Define the context ## Define the context
Remote contexts belong in the customer's private repository, not this public Remote contexts belong in the customer's private deploy repository, not this
repository. Create a context file like this: public repository. A deploy crate directly depends on `harmony-fleet-deploy`,
`harmony_app`, `harmony_macros`, `harmony_types`, `tokio`, and `anyhow`. Its
binary compiles the target into Rust:
```toml ```rust
[contexts.customer-prod] let context = Context {
profile = "prod" name: context_name!("customer-prod"),
registry = "hub.example.com" namespace: "customer-fleet".parse()?,
project = "customer" spec: ContextSpec::Remote(RemoteContext {
domain = "fleet.example.com" registry: oci_registry!("hub.example.com"),
image_pull_secret = "registry-pull" repository: oci_repository!("customer"),
openbao_namespace = "customer-prod" domain: domain!("fleet.example.com"),
openbao_url = "https://openbao.example.com" image_pull_secret: None,
zitadel_url = "https://sso.example.com" access: OpenBaoClusterAccess {
openbao_role = "customer-prod-cd" namespace: openbao_namespace!("customer-prod"),
zitadel_audience = "openbao" url: http_url!("https://openbao.example.com"),
role: OpenBaoRoleName::new("customer-prod-cd")?,
zitadel_url: http_url!("https://sso.example.com"),
zitadel_audience: OidcAudience::new("openbao")?,
},
}),
};
deploy_fleet_with_context(context).await
``` ```
[`examples/fleet_typed_deploy`](../../examples/fleet_typed_deploy) contains a
complete, compile-tested `Cargo.toml` and binary. Changing a compiled target
requires rebuilding the deploy binary; runtime context loading is deliberately
deferred.
Set `HARMONY_ZITADEL_KEY_JSON` to the deploy service user's Zitadel key JSON. Set `HARMONY_ZITADEL_KEY_JSON` to the deploy service user's Zitadel key JSON.
The context uses that identity to obtain its OpenBao-brokered kubeconfig. The context uses that identity to obtain its OpenBao-brokered kubeconfig.
Every Fleet component is deployed into the context's Kubernetes namespace.
The kubeconfig's `current-context` selects the cluster; resolution and deploy
output report the selected context, API server, and namespace.
## Provision callout credentials ## Install the Fleet CRDs
Before the first deploy, create the immutable Fleet CRDs have a cluster lifecycle independent of tenant Fleet deployments.
`fleet-system/fleet-callout-credentials` Secret. It must contain The cluster platform deploys `FleetCrdsScore` once before any tenant stack.
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass`. Tenant operator releases neither own nor remove the CRDs.
`Device` changed from cluster-scoped to namespaced with this tenancy model.
Kubernetes cannot change a CRD's scope in place. Existing development clusters
must delete the old `devices.fleet.nationtech.io` CRD before applying the new
CRD Score; Device objects are then rebuilt from the device-info KV stream.
Moving an existing stack from the historical `zitadel`, `openbao`, and
`fleet-system` namespaces into one tenant namespace is also an explicit data
migration, not an in-place Score update. Back up PostgreSQL and OpenBao, restore
them into the tenant namespace, verify identities and secrets, then retire the
old releases. Disposable local and E2E clusters should be recreated instead.
## Provision the tenant
Run the tenant provisioning binary with a platform-admin context. It applies
`TenantScore`, creates the Fleet deployer ServiceAccount and RBAC, issues its
kubeconfig, and stores `ClusterAccess` through Harmony Config. The credential
store role must be able to write the tenant's OpenBao namespace.
The deployer can create namespaced Roles and RoleBindings because Helm installs
the Fleet operator's runtime RBAC. Kubernetes prevents it from binding rights it
does not already hold, and its ClusterRole grants only `get` on its own
Namespace.
The generated kubeconfig currently contains a long-lived ServiceAccount token.
Deleting `<namespace>/fleet-deployer-token` revokes it; rerun tenant
provisioning to create and store a replacement. This remains the rotation path
until short-lived TokenRequest brokerage is implemented.
```bash ```bash
kubectl create namespace fleet-system --dry-run=client -o yaml | kubectl apply -f - cargo run --release --bin tenant-provision -- \
kubectl -n fleet-system create secret generic fleet-callout-credentials \ deploy --context platform-admin
--from-literal=issuer-nkey-seed="$ISSUER_NKEY_SEED" \
--from-literal=issuer-public-key="$ISSUER_PUBLIC_KEY" \
--from-literal=nats-auth-pass="$NATS_AUTH_PASS" \
--dry-run=client -o yaml | kubectl apply -f -
kubectl -n fleet-system patch secret fleet-callout-credentials \
--type=merge -p '{"immutable":true}'
``` ```
The registry pull Secret named by `image_pull_secret` must also exist in each Fleet creates the immutable NATS callout credential Secret on first deploy and
namespace that pulls private images. reuses it thereafter. Back up that Secret with the tenant's other state. Fleet
agents connect to `wss://nats.<context-domain>:443`; OKD terminates TLS at an
edge Route while in-cluster clients use the ClusterIP Service. Fleet images are
public, so the remote context uses `image_pull_secret: None`.
## Deploy ## Deploy
@@ -65,19 +108,23 @@ export REGISTRY_USER=...
export REGISTRY_TOKEN=... export REGISTRY_TOKEN=...
export HARMONY_ZITADEL_KEY_JSON=... export HARMONY_ZITADEL_KEY_JSON=...
cargo run -p harmony-fleet-deploy -- \ cargo run -- ship --context customer-prod
ship --context customer-prod --config path/to/contexts.toml
``` ```
These three environment variables are required today. The target CI model
keeps only `HARMONY_ZITADEL_KEY_JSON` in the runner and reads registry
credentials from OpenBao through `ConfigClient`; that publication work is not
implemented yet.
For CI, the stages can run independently. Pass the digest references printed by For CI, the stages can run independently. Pass the digest references printed by
`build` into `publish` and `deploy`: `build` into `publish` and `deploy`:
```bash ```bash
cargo run -p harmony-fleet-deploy -- build --context customer-prod --config path/to/contexts.toml cargo run -- build --context customer-prod
cargo run -p harmony-fleet-deploy -- publish --context customer-prod --config path/to/contexts.toml \ cargo run -- publish --context customer-prod \
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \ --image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:... --image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
cargo run -p harmony-fleet-deploy -- deploy --context customer-prod --config path/to/contexts.toml \ cargo run -- deploy --context customer-prod \
--image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \ --image operator=hub.example.com/customer/harmony-fleet-operator@sha256:... \
--image callout=hub.example.com/customer/harmony-nats-callout@sha256:... --image callout=hub.example.com/customer/harmony-nats-callout@sha256:...
``` ```
@@ -88,15 +135,17 @@ readiness check passes, including the operator's authenticated NATS startup.
## Verify ## Verify
```bash ```bash
cargo run -p harmony-fleet-deploy -- status --context customer-prod --config path/to/contexts.toml cargo run -- status --context customer-prod
cargo run -p harmony-fleet-deploy -- logs --context customer-prod --config path/to/contexts.toml cargo run -- logs --context customer-prod
kubectl -n fleet-system get deployments,pods kubectl -n customer-fleet get deployments,pods,routes
kubectl get deployments.fleet.nationtech.io,devices.fleet.nationtech.io kubectl -n customer-fleet get deployments.fleet.nationtech.io,devices.fleet.nationtech.io
``` ```
For local validation before a remote deploy: For local validation before a remote deploy:
```bash ```bash
cargo run -p harmony-fleet-deploy --bin harmony-fleet-crds-deploy -- \
deploy --context local
cargo run -p harmony-fleet-deploy -- ship --context local cargo run -p harmony-fleet-deploy -- ship --context local
HARMONY_FLEET_E2E=1 cargo test -p harmony-fleet-e2e --test ping -- --nocapture HARMONY_FLEET_E2E=1 cargo test -p harmony-fleet-e2e --test ping -- --nocapture
``` ```

View File

@@ -1,6 +0,0 @@
# Deploy contexts for this example (checked in — zero config).
# k3d cluster create compose-local && kubectl create ns timesheet
# compose-deploy ship --context local
[contexts.local]
profile = "local"
k3d = "compose-local"

View File

@@ -13,5 +13,7 @@ path = "src/main.rs"
[dependencies] [dependencies]
harmony_app = { path = "../../harmony_app" } harmony_app = { path = "../../harmony_app" }
harmony_cli = { path = "../../harmony_cli" } harmony_cli = { path = "../../harmony_cli" }
harmony_macros = { path = "../../harmony_macros" }
harmony_types = { path = "../../harmony_types" }
anyhow = { workspace = true } anyhow = { workspace = true }
tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] }

View File

@@ -11,14 +11,13 @@ app/ the "customer" project (their existing repo)
backend/ (Java + SQLite, Dockerfile) backend/ (Java + SQLite, Dockerfile)
frontend/ (React + nginx, Dockerfile) frontend/ (React + nginx, Dockerfile)
Harmony.toml identity only — the app name (ADR-026 §3) Harmony.toml identity only — the app name (ADR-026 §3)
.harmony/contexts.toml deploy targets (local / prod), checked in
src/ src/
compose.rs import docker-compose → typed model (loud on the unsupported) compose.rs import docker-compose → typed model (loud on the unsupported)
chart.rs model + profile knobs → a hydrated helm chart (typed k8s) chart.rs model + profile knobs → a hydrated helm chart (typed k8s)
publish.rs build images + (push to a registry | import to k3d) publish.rs build images + (push to a registry | import to k3d)
score.rs ComposeAppScore: helm upgrade --install + Ingress score.rs ComposeAppScore: helm upgrade --install + Ingress
deploy.rs ComposeDeploy — the declarative builder (impls HarmonyApp) deploy.rs ComposeDeploy — the declarative builder (impls HarmonyApp)
main.rs the whole app, declared main.rs the app and its compiled deploy contexts
``` ```
## The app, declared (`main.rs`) ## The app, declared (`main.rs`)
@@ -27,7 +26,12 @@ src/
let app = ComposeDeploy::from_dir("timesheet", "./app")? let app = ComposeDeploy::from_dir("timesheet", "./app")?
.expose("frontend", "timesheet.example.harmony.mcd") .expose("frontend", "timesheet.example.harmony.mcd")
.with(Postgres::managed()); // a managed CNPG database, wired in .with(Postgres::managed()); // a managed CNPG database, wired in
harmony_cli::app::app_main(app).await let contexts = ContextCatalog::new([Context {
name: context_name!("local"),
namespace: "timesheet".parse()?,
spec: ContextSpec::Local(LocalContext::ManagedK3d),
}])?;
harmony_cli::app::app_main(app, contexts).await
``` ```
That declaration gets you `ship` / `deploy` / `status` / `logs` over any That declaration gets you `ship` / `deploy` / `status` / `logs` over any
@@ -54,18 +58,14 @@ That declaration gets you `ship` / `deploy` / `status` / `logs` over any
## Run it (local k3d) ## Run it (local k3d)
```sh ```sh
k3d cluster create compose-local --servers 1 --wait cd examples/compose_java_react
kubectl --context k3d-compose-local create namespace timesheet
cd examples/compose_java_react # so the CLI finds .harmony/contexts.toml
cargo run --bin compose-deploy -- ship --context local # build → k3d import → deploy (+ Postgres) cargo run --bin compose-deploy -- ship --context local # build → k3d import → deploy (+ Postgres)
cargo run --bin compose-deploy -- status --context local cargo run --bin compose-deploy -- status --context local
cargo run --bin compose-deploy -- logs --context local --tail 50 cargo run --bin compose-deploy -- logs --context local --tail 50
``` ```
Running from elsewhere? Point at the contexts file with The binary accepts only contexts compiled into `main.rs`. A context is always
`--config <path>/.harmony/contexts.toml` (a context is always required — there required, so omitting `--context` and `HARMONY_CONTEXT` is an error.
is no default, so you never hit the wrong cluster).
**Production** uses the same verbs against a prod context, with cluster **Production** uses the same verbs against a prod context, with cluster
credentials brokered from OpenBao. credentials brokered from OpenBao.

View File

@@ -6,11 +6,11 @@
//! compose-deploy status --context local //! compose-deploy status --context local
//! compose-deploy logs --context local //! compose-deploy logs --context local
//! //!
//! Contexts live in `.harmony/contexts.toml`. `ship`/`deploy`/`status`/`logs` //! `ship`/`deploy`/`status`/`logs` and the context model all come from
//! and the context model all come from `harmony_app` — this file only //! `harmony_app` — this file only declares the app and its available target.
//! *declares* the app.
use harmony_app::{ComposeDeploy, Postgres}; use harmony_app::{ComposeDeploy, Context, ContextCatalog, ContextSpec, LocalContext, Postgres};
use harmony_macros::context_name;
#[tokio::main] #[tokio::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
@@ -19,5 +19,13 @@ async fn main() -> anyhow::Result<()> {
.expose("frontend", "timesheet.example.harmony.mcd") .expose("frontend", "timesheet.example.harmony.mcd")
.with(Postgres::managed()); // deploys a CNPG cluster + wires DATABASE_URL .with(Postgres::managed()); // deploys a CNPG cluster + wires DATABASE_URL
harmony_cli::app::app_main(app).await harmony_cli::app::app_main(
app,
ContextCatalog::new([Context {
name: context_name!("local"),
namespace: "timesheet".parse()?,
spec: ContextSpec::Local(LocalContext::ManagedK3d),
}])?,
)
.await
} }

View File

@@ -1,6 +1,6 @@
//! Install the harmony fleet server-side stack into the cluster //! Install the harmony fleet server-side stack into the cluster
//! `KUBECONFIG` points at: NATS + the harmony fleet operator (CRDs + //! `KUBECONFIG` points at: NATS + Fleet CRDs + the harmony fleet operator
//! RBAC + Deployment), and optionally a central Zitadel OIDC //! (RBAC + Deployment), and optionally a central Zitadel OIDC
//! identity provider, via [`FleetServerScore`]. //! identity provider, via [`FleetServerScore`].
//! //!
//! This is the framework-side replacement for the //! This is the framework-side replacement for the
@@ -51,7 +51,7 @@ use harmony::modules::nats::NatsScore;
use harmony::modules::zitadel::ZitadelScore; use harmony::modules::zitadel::ZitadelScore;
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_fleet_deploy::FleetOperatorScore; use harmony_fleet_deploy::{FleetCrdsScore, FleetOperatorScore};
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command( #[command(
@@ -151,7 +151,7 @@ async fn main() -> Result<()> {
// inner Scores directly so it can keep using the basic NATS // inner Scores directly so it can keep using the basic NATS
// helm chart for k3d-style local installs. // helm chart for k3d-style local installs.
let mut scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = let mut scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> =
vec![Box::new(nats), Box::new(operator)]; vec![Box::new(nats), Box::new(FleetCrdsScore), Box::new(operator)];
if let Some(host) = cli.zitadel_host { if let Some(host) = cli.zitadel_host {
// Default external_secure logic: HTTPS unless the host is a // Default external_secure logic: HTTPS unless the host is a

View File

@@ -0,0 +1,14 @@
[package]
name = "example-fleet-typed-deploy"
edition = "2024"
version = "0.0.0"
description = "Compile-tested consumer example for a private Fleet deployment context"
[dependencies]
anyhow = "1"
tokio = { version = "1.52", features = ["full"] }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
harmony = { path = "../../harmony" }
harmony_app = { path = "../../harmony_app" }
harmony_macros = { path = "../../harmony_macros" }
harmony_types = { path = "../../harmony_types" }

View File

@@ -0,0 +1,8 @@
use example_fleet_typed_deploy::{credential_store, platform_context, tenant_config};
use harmony_fleet_deploy::provision_fleet_tenant_with_context;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
provision_fleet_tenant_with_context(platform_context()?, tenant_config(), credential_store()?)
.await
}

View File

@@ -0,0 +1,71 @@
use harmony::topology::tenant::{ResourceLimits, TenantConfig};
use harmony_app::{Context, ContextSpec, OpenBaoClusterAccess, RemoteContext};
use harmony_macros::{
context_name, domain, http_url, oci_registry, oci_repository, openbao_namespace,
};
pub fn fleet_context() -> anyhow::Result<Context> {
Ok(Context {
name: context_name!("customer-prod"),
namespace: tenant_namespace(),
spec: ContextSpec::Remote(RemoteContext {
registry: oci_registry!("registry.example.com"),
repository: oci_repository!("customer/fleet"),
domain: domain!("fleet.example.com"),
image_pull_secret: None,
access: tenant_access("fleet-deployer")?,
}),
})
}
pub fn platform_context() -> anyhow::Result<Context> {
Ok(Context {
name: context_name!("platform-admin"),
namespace: "platform-system".parse()?,
spec: ContextSpec::Remote(RemoteContext {
registry: oci_registry!("registry.example.com"),
repository: oci_repository!("customer/fleet"),
domain: domain!("fleet.example.com"),
image_pull_secret: None,
access: OpenBaoClusterAccess {
namespace: openbao_namespace!("platform/admin"),
url: http_url!("https://secrets.example.com"),
role: "platform-admin".parse()?,
zitadel_url: http_url!("https://identity.example.com"),
zitadel_audience: "openbao".parse()?,
},
}),
})
}
pub fn tenant_config() -> TenantConfig {
TenantConfig {
id: "customer-fleet".into(),
name: tenant_namespace().to_string(),
resource_limits: ResourceLimits {
storage_total_gb: 30.0,
..Default::default()
},
..Default::default()
}
}
pub fn credential_store() -> anyhow::Result<OpenBaoClusterAccess> {
tenant_access("tenant-provisioner")
}
fn tenant_namespace() -> harmony_types::k8s_name::K8sName {
"customer-fleet"
.parse()
.expect("static Kubernetes namespace is valid")
}
fn tenant_access(role: &str) -> anyhow::Result<OpenBaoClusterAccess> {
Ok(OpenBaoClusterAccess {
namespace: openbao_namespace!("customer/fleet"),
url: http_url!("https://secrets.example.com"),
role: role.parse()?,
zitadel_url: http_url!("https://identity.example.com"),
zitadel_audience: "openbao".parse()?,
})
}

View File

@@ -0,0 +1,7 @@
use example_fleet_typed_deploy::fleet_context;
use harmony_fleet_deploy::deploy_fleet_with_context;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
deploy_fleet_with_context(fleet_context()?).await
}

View File

@@ -23,9 +23,17 @@ the Fleet images, and deploys PostgreSQL, Zitadel, NATS with auth callout,
OpenBao, and the Fleet operator: OpenBao, and the Fleet operator:
```bash ```bash
cargo run -p harmony-fleet-deploy --bin harmony-fleet-crds-deploy -- \
deploy --context local
cargo run -p harmony-fleet-deploy -- ship --context local cargo run -p harmony-fleet-deploy -- ship --context local
``` ```
The first command manages the cluster-wide Fleet CRD definitions. The tenant
deployment has a separate lifecycle and puts PostgreSQL, Zitadel, NATS,
OpenBao, the callout, and the operator in the namespace compiled into its
context. Both `Deployment` and `Device` resources are namespaced, and each
operator watches only its tenant namespace.
The command returns only after the operator has authenticated through Zitadel The command returns only after the operator has authenticated through Zitadel
and initialized its NATS JetStream state. The credentials Score creates an and initialized its NATS JetStream state. The credentials Score creates an
immutable local Kubernetes Secret on the first run and reuses it thereafter. immutable local Kubernetes Secret on the first run and reuses it thereafter.
@@ -117,30 +125,34 @@ k3d cluster delete fleet-e2e
## Production deploys ## Production deploys
Production contexts live in the customer's private repository. They provide a Production contexts live in the customer's private repository. They provide a
registry, project, domain, image-pull Secret, and OpenBao-brokered kubeconfig. tenant namespace, registry, project, domain, optional image-pull Secret, and
OpenBao-brokered kubeconfig. The kubeconfig's current context selects the
cluster.
The cluster must also contain the immutable The cluster must also contain the immutable
`fleet-system/fleet-callout-credentials` Secret with `<context namespace>/fleet-callout-credentials` Secret with
`issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass` keys. The deploy `issuer-nkey-seed`, `issuer-public-key`, and `nats-auth-pass` keys. The deploy
validates this Secret before configuring NATS. validates this Secret before configuring NATS.
```bash ```bash
cargo run -p harmony-fleet-deploy -- \ cargo run -- ship --context customer-prod
ship --context customer-prod \
--config path/to/contexts.toml
``` ```
Run this from the private deploy repository whose binary calls
`deploy_fleet_with_context` with its compiled context.
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap. See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
### Connecting to the operator ### Connecting to the operator
The operator runs as a single-replica Deployment in `--namespace` (default `fleet-system`). The operator runs as a single-replica Deployment in the context namespace.
```bash ```bash
export FLEET_NAMESPACE=customer-fleet
# Tail logs # Tail logs
kubectl -n fleet-system logs deploy/harmony-fleet-operator -f kubectl -n "$FLEET_NAMESPACE" logs deploy/harmony-fleet-operator -f
# Port-forward the embedded web dashboard (web-frontend feature) # Port-forward the embedded web dashboard (web-frontend feature)
kubectl -n fleet-system port-forward deploy/harmony-fleet-operator 18080:18080 kubectl -n "$FLEET_NAMESPACE" port-forward deploy/harmony-fleet-operator 18080:18080
# Or run the dashboard standalone with seeded fake data — no NATS, no cluster # Or run the dashboard standalone with seeded fake data — no NATS, no cluster
cargo run -p harmony-fleet-operator --features web-frontend -- serve-web --mock cargo run -p harmony-fleet-operator --features web-frontend -- serve-web --mock

View File

@@ -16,6 +16,10 @@ path = "src/lib.rs"
name = "harmony-fleet-deploy" name = "harmony-fleet-deploy"
path = "src/main.rs" path = "src/main.rs"
[[bin]]
name = "harmony-fleet-crds-deploy"
path = "src/bin/harmony-fleet-crds-deploy.rs"
[dependencies] [dependencies]
harmony = { path = "../../harmony", features = ["podman"] } harmony = { path = "../../harmony", features = ["podman"] }
harmony_cli = { path = "../../harmony_cli" } harmony_cli = { path = "../../harmony_cli" }

View File

@@ -0,0 +1,389 @@
use crate::{FleetCrdsScore, FleetOperatorScore};
use anyhow::Result;
use async_trait::async_trait;
use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService};
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore};
use harmony::modules::postgresql::K8sPostgreSQLScore;
use harmony::modules::tenant::{TenantCredentialScore, TenantScore};
use harmony::modules::zitadel::{ZitadelAppType, ZitadelScore, ZitadelSetupScore};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, tenant::TenantConfig};
use harmony_app::{
AppContext, AppError, AppIdentity, HarmonyApp, ImageRefs, ImageSpec, OpenBaoClusterAccess,
Profile,
};
use harmony_config::ConfigClient;
use harmony_types::k8s_name::K8sName;
use k8s_openapi::api::rbac::v1::PolicyRule;
use std::sync::Arc;
const PROJECT: &str = "fleet";
const ADMIN_ROLE: &str = "fleet-admin";
const DEVICE_ROLE: &str = "device";
const OPERATOR_APP: &str = "harmony-fleet-operator";
const OPERATOR_USER: &str = "fleet-operator";
const NATS_ACCOUNT: &str = "FLEET";
pub struct FleetApp;
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
fn identity(&self, ctx: &AppContext) -> AppIdentity {
AppIdentity {
name: "fleet".to_string(),
namespace: ctx.namespace().to_string(),
}
}
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
Ok(vec![
ImageSpec {
name: "operator".to_string(),
image: ctx.image("harmony-fleet-operator"),
context: ".".into(),
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
build_args: Vec::new(),
},
ImageSpec {
name: "callout".to_string(),
image: ctx.image("harmony-nats-callout"),
context: ".".into(),
dockerfile: "nats/callout/Dockerfile".into(),
build_args: Vec::new(),
},
])
}
async fn scores(
&self,
ctx: &AppContext,
images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let namespace = ctx.namespace();
let image_pull_secret = ctx.image_pull_secret();
let postgres = K8sPostgreSQLScore::new(namespace).cluster_name("zitadel-pg");
let database = postgres.root_account_ref();
let mut zitadel =
ZitadelScore::new(ctx.service_host("zitadel"), namespace).database(database);
if ctx.profile() == Profile::Local {
// Q : should that be handled by harmony internally?
zitadel = zitadel.http(Some(8080));
}
let provider = zitadel.provider_ref();
let identity = ZitadelSetupScore::for_provider(&provider, namespace, namespace)
.application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
.api_application(PROJECT, "nats")
.role(PROJECT, ADMIN_ROLE, "Fleet Admin")
.role(PROJECT, DEVICE_ROLE, "Device")
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
.port_forward("zitadel")
.groups_claim();
let application = identity.application_ref(OPERATOR_APP);
let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
let credentials =
NatsAuthCalloutCredentialsScore::generated(namespace, "fleet-callout-credentials");
let service = match ctx.profile() {
// FIXME this should definitely not surface up to the end user deployment crate. This is
// topology dependent and is the topology's job (local k3d vs regular cluster) to decide
// how to expose a port and then the score ref should be appropriate
Profile::Local => NatsService::NodePort(30422),
Profile::Prod => NatsService::ClusterIp,
};
// I feel like this should not be a standalone nats score but rather be configuration passed
// to the main nats score that is installing the nats cluster
let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
.with_jetstream_size("2Gi");
let nats = if ctx.profile() == Profile::Prod {
nats.websocket(ctx.service_host("nats"), "letsencrypt-prod")
} else {
nats
};
let account = nats.account_ref();
let callout =
NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
.credentials(&credentials.credentials_ref())
.with_oidc(&provider, &application)
.image(images.require("callout")?)
.image_pull_secret(image_pull_secret.clone())
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE)
.device_id_claim("client_id");
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao"));
if ctx.profile() == Profile::Prod {
openbao = openbao.tls("letsencrypt-prod");
}
let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
&provider,
&application,
OpenbaoJwtAuth::oidc("fleet-device"),
);
let operator = FleetOperatorScore::new(images.require("operator")?)
.namespace(namespace)
.image_pull_secret(image_pull_secret)
.messaging(&nats.client_ref())
.identity(&provider, &application, &operator_identity);
Ok(vec![
Box::new(postgres),
Box::new(zitadel),
Box::new(credentials),
Box::new(nats),
Box::new(identity),
Box::new(callout),
Box::new(openbao),
Box::new(openbao_setup),
Box::new(operator),
])
}
}
pub struct FleetCrdsApp;
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for FleetCrdsApp {
fn identity(&self, ctx: &AppContext) -> AppIdentity {
AppIdentity {
name: "fleet-crds".to_string(),
namespace: ctx.namespace().to_string(),
}
}
async fn scores(
&self,
_ctx: &AppContext,
_images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
Ok(vec![Box::new(FleetCrdsScore)])
}
}
pub struct FleetTenantProvisionApp {
tenant: TenantConfig,
credential_store: Arc<ConfigClient>,
}
impl FleetTenantProvisionApp {
pub fn new(tenant: TenantConfig, credential_store: Arc<ConfigClient>) -> Self {
Self {
tenant,
credential_store,
}
}
pub async fn from_openbao(
tenant: TenantConfig,
credential_store: &OpenBaoClusterAccess,
) -> Result<Self> {
let source = harmony_config::openbao_source(
credential_store.namespace.as_ref(),
Some(credential_store.url.to_string()),
Some(credential_store.zitadel_url.to_string()),
Some(credential_store.zitadel_audience.to_string()),
Some(credential_store.role.to_string()),
)
.await
.ok_or_else(|| anyhow::anyhow!("tenant credential store is unavailable"))?;
Ok(Self::new(tenant, Arc::new(ConfigClient::new(vec![source]))))
}
}
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for FleetTenantProvisionApp {
fn identity(&self, _ctx: &AppContext) -> AppIdentity {
AppIdentity {
name: "fleet-tenant".to_string(),
namespace: self.tenant.name.clone(),
}
}
async fn scores(
&self,
_ctx: &AppContext,
_images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let namespace = self
.tenant
.name
.parse::<K8sName>()
.map_err(|error| AppError::InvalidComposition(error.to_string()))?;
Ok(vec![
Box::new(TenantScore {
config: self.tenant.clone(),
}),
Box::new(TenantCredentialScore::new(
namespace,
"fleet-deployer"
.parse()
.expect("static Kubernetes name is valid"),
fleet_deployer_rules(),
self.credential_store.clone(),
)),
])
}
}
fn fleet_deployer_rules() -> Vec<PolicyRule> {
// Helm creates the operator's Role and RoleBinding, so the deployer can
// delegate its existing tenant permissions but cannot cross namespaces.
let verbs = || {
[
"get", "list", "watch", "create", "update", "patch", "delete",
]
.map(String::from)
.to_vec()
};
vec![
PolicyRule {
api_groups: Some(vec![String::new()]),
resources: Some(
[
"configmaps",
"persistentvolumeclaims",
"pods",
"secrets",
"serviceaccounts",
"services",
]
.map(String::from)
.to_vec(),
),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec![String::new()]),
resources: Some(vec!["pods/log".to_string()]),
verbs: vec!["get".to_string()],
..Default::default()
},
PolicyRule {
api_groups: Some(vec![String::new()]),
resources: Some(vec![
"pods/exec".to_string(),
"pods/portforward".to_string(),
]),
verbs: vec!["create".to_string()],
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["apps".to_string()]),
resources: Some(
["deployments", "replicasets", "statefulsets"]
.map(String::from)
.to_vec(),
),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["batch".to_string()]),
resources: Some(vec!["jobs".to_string()]),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["networking.k8s.io".to_string()]),
resources: Some(["ingresses", "networkpolicies"].map(String::from).to_vec()),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["route.openshift.io".to_string()]),
resources: Some(vec!["routes".to_string()]),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["policy".to_string()]),
resources: Some(vec!["poddisruptionbudgets".to_string()]),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["rbac.authorization.k8s.io".to_string()]),
resources: Some(vec!["roles".to_string(), "rolebindings".to_string()]),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["postgresql.cnpg.io".to_string()]),
resources: Some(vec!["clusters".to_string()]),
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["fleet.nationtech.io".to_string()]),
resources: Some(
[
"deployments",
"deployments/finalizers",
"deployments/status",
"devices",
"devices/status",
]
.map(String::from)
.to_vec(),
),
verbs: verbs(),
..Default::default()
},
]
}
#[cfg(test)]
mod tenant_tests {
use super::*;
use harmony::topology::tenant::TenantNetworkPolicy;
use harmony_types::id::Id;
#[test]
fn deployer_permissions_exclude_cluster_resources() {
let rules = fleet_deployer_rules();
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&["postgresql.cnpg.io".to_string()])
&& rule.resources.as_deref() == Some(&["clusters".to_string()])
}));
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()])
&& rule.resources.as_deref() == Some(&["routes".to_string()])
}));
assert!(rules.iter().all(|rule| {
!rule.resources.as_ref().is_some_and(|resources| {
resources
.iter()
.any(|resource| resource == "customresourcedefinitions")
})
}));
}
#[tokio::test]
async fn tenant_provisioning_runs_tenant_before_credentials() {
let app = FleetTenantProvisionApp::new(
TenantConfig {
id: Id::from("customer-fleet"),
name: "customer-fleet".to_string(),
resource_limits: Default::default(),
network_policy: TenantNetworkPolicy::default(),
},
Arc::new(ConfigClient::new(Vec::new())),
);
let context = harmony_app::Context {
name: "platform-admin".parse().unwrap(),
namespace: "platform-system".parse().unwrap(),
spec: harmony_app::ContextSpec::Local(harmony_app::LocalContext::ManagedK3d),
};
let context = AppContext::load_metadata(&context, "test", None);
let scores = app.scores(&context, &ImageRefs::default()).await.unwrap();
assert_eq!(scores[0].name(), "customer-fleet [TenantScore]");
assert_eq!(scores[1].name(), "customer-fleet [TenantCredentialScore]");
}
}

View File

@@ -0,0 +1,13 @@
use harmony_app::{Context, ContextSpec, LocalContext};
use harmony_fleet_deploy::deploy_fleet_crds_with_context;
use harmony_macros::context_name;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
deploy_fleet_crds_with_context(Context {
name: context_name!("local"),
namespace: "fleet-system".parse()?,
spec: ContextSpec::Local(LocalContext::ManagedK3d),
})
.await
}

View File

@@ -11,12 +11,34 @@
//! it does not own provider operations or a Fleet-wide aggregate Score. //! it does not own provider operations or a Fleet-wide aggregate Score.
pub mod agent; pub mod agent;
mod app;
mod device_setup; mod device_setup;
pub mod operator; pub mod operator;
pub use agent::{FleetAgentScore, PodTarget}; pub use agent::{FleetAgentScore, PodTarget};
pub use app::{FleetApp, FleetCrdsApp, FleetTenantProvisionApp};
pub use device_setup::{ pub use device_setup::{
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
HostsEntry, merge_hosts_file, HostsEntry, merge_hosts_file,
}; };
pub use operator::{FleetOperatorScore, OperatorCredentials}; pub use operator::{FleetCrdsScore, FleetOperatorScore, OperatorCredentials};
pub async fn deploy_fleet_with_context(context: harmony_app::Context) -> anyhow::Result<()> {
let contexts = harmony_app::ContextCatalog::new([context])?;
harmony_cli::app::app_main(FleetApp, contexts).await
}
pub async fn deploy_fleet_crds_with_context(context: harmony_app::Context) -> anyhow::Result<()> {
let contexts = harmony_app::ContextCatalog::new([context])?;
harmony_cli::app::app_main(FleetCrdsApp, contexts).await
}
pub async fn provision_fleet_tenant_with_context(
context: harmony_app::Context,
tenant: harmony::topology::tenant::TenantConfig,
credential_store: harmony_app::OpenBaoClusterAccess,
) -> anyhow::Result<()> {
let app = FleetTenantProvisionApp::from_openbao(tenant, &credential_store).await?;
let contexts = harmony_app::ContextCatalog::new([context])?;
harmony_cli::app::app_main(app, contexts).await
}

View File

@@ -1,153 +1,13 @@
use anyhow::Result; use harmony_app::{Context, ContextSpec, LocalContext};
use async_trait::async_trait; use harmony_fleet_deploy::deploy_fleet_with_context;
use harmony::modules::nats::{NatsAuthCalloutCredentialsScore, NatsScore, NatsService}; use harmony_macros::context_name;
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore};
use harmony::modules::postgresql::K8sPostgreSQLScore;
use harmony::modules::zitadel::{ZitadelAppType, ZitadelScore, ZitadelSetupScore};
use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology;
use harmony_app::{AppContext, AppError, AppIdentity, HarmonyApp, ImageRefs, ImageSpec, Profile};
use harmony_fleet_deploy::FleetOperatorScore;
const PROJECT: &str = "fleet";
const ADMIN_ROLE: &str = "fleet-admin";
const DEVICE_ROLE: &str = "device";
const OPERATOR_APP: &str = "harmony-fleet-operator";
const OPERATOR_USER: &str = "fleet-operator";
const NATS_ACCOUNT: &str = "FLEET";
struct FleetApp;
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
fn identity(&self) -> AppIdentity {
AppIdentity {
name: "fleet".to_string(),
namespace: "fleet-system".to_string(),
}
}
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
Ok(vec![
ImageSpec {
name: "operator".to_string(),
image: ctx.image("harmony-fleet-operator")?,
context: ".".into(),
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
build_args: Vec::new(),
},
ImageSpec {
name: "callout".to_string(),
image: ctx.image("harmony-nats-callout")?,
context: ".".into(),
dockerfile: "nats/callout/Dockerfile".into(),
build_args: Vec::new(),
},
])
}
async fn scores(
&self,
ctx: &AppContext,
images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let namespace = "fleet-system";
let zitadel_namespace = "zitadel";
let openbao_namespace = "openbao";
let image_pull_secret = ctx.image_pull_secret()?;
let postgres = K8sPostgreSQLScore::new(zitadel_namespace).cluster_name("zitadel-pg");
let database = postgres.root_account_ref();
let mut zitadel = ZitadelScore::new(
ctx.service_host("zitadel", zitadel_namespace)?,
zitadel_namespace,
)
.database(database);
if ctx.profile() == Profile::Local {
// Q : should that be handled by harmony internally?
zitadel = zitadel.http(Some(8080));
}
let provider = zitadel.provider_ref();
let identity = ZitadelSetupScore::for_provider(&provider, zitadel_namespace, namespace)
.application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
.api_application(PROJECT, "nats")
.role(PROJECT, ADMIN_ROLE, "Fleet Admin")
.role(PROJECT, DEVICE_ROLE, "Device")
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
.port_forward("zitadel")
.groups_claim();
let application = identity.application_ref(OPERATOR_APP);
let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
let credentials = match ctx.profile() {
// Q: this looks weird. Why do we need to generate creds locally but not in prod? I
// would think that in both cases creds should be "ensured". Created if not existing.
// Maybe in prod it should use harmony config with a remote openbao and store them there
// (as we plan on using external secrets asap)
Profile::Local => NatsAuthCalloutCredentialsScore::generated,
Profile::Prod => NatsAuthCalloutCredentialsScore::existing,
}(namespace, "fleet-callout-credentials");
let service = match ctx.profile() {
// FIXME this should definitely not surface up to the end user deployment crate. This is
// topology dependent and is the topology's job (local k3d vs regular cluster) to decide
// how to expose a port and then the score ref should be appropriate
Profile::Local => NatsService::NodePort(30422),
Profile::Prod => NatsService::LoadBalancer,
};
// I feel like this should not be a standalone nats score but rather be configuration passed
// to the main nats score that is installing the nats cluster
let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
.with_jetstream_size("2Gi");
let account = nats.account_ref();
let callout =
NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
.credentials(&credentials.credentials_ref())
.with_oidc(&provider, &application)
.image(images.require("callout")?)
.image_pull_secret(image_pull_secret)
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE)
.device_id_claim("client_id");
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
let mut openbao = OpenbaoScore::new(
openbao_namespace,
"openbao",
ctx.service_host("openbao", openbao_namespace)?,
);
if ctx.profile() == Profile::Prod {
openbao = openbao.tls("letsencrypt-prod");
}
let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
&provider,
&application,
OpenbaoJwtAuth::oidc("fleet-device"),
);
let operator = FleetOperatorScore::new(images.require("operator")?)
.namespace(namespace)
.image_pull_secret(image_pull_secret)
.messaging(&nats.client_ref())
.identity(&provider, &application, &operator_identity);
Ok(vec![
Box::new(postgres),
Box::new(zitadel),
Box::new(credentials),
Box::new(nats),
Box::new(identity),
Box::new(callout),
Box::new(openbao),
Box::new(openbao_setup),
Box::new(operator),
])
}
}
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> anyhow::Result<()> {
harmony_cli::app::app_main(FleetApp).await deploy_fleet_with_context(Context {
name: context_name!("local"),
namespace: "fleet-system".parse()?,
spec: ContextSpec::Local(LocalContext::ManagedK3d),
})
.await
} }

View File

@@ -10,37 +10,30 @@
//! chart with user-facing values, layer a templating pass on top of //! chart with user-facing values, layer a templating pass on top of
//! this output. //! this output.
//! //!
//! Parity with `install` subcommand: both install the same two CRDs
//! (`Deployment`, `Device`). `install` applies the CRDs only, for
//! the host-side-operator path; `chart` packages CRDs + RBAC + the
//! operator Deployment into a helm chart the cluster runs itself.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::hash::{DefaultHasher, Hash, Hasher}; use std::hash::{DefaultHasher, Hash, Hasher};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use harmony_types::k8s_name::K8sName;
use k8s_openapi::ByteString; use k8s_openapi::ByteString;
use k8s_openapi::api::apps::v1::{ use k8s_openapi::api::apps::v1::{
Deployment as K8sDeployment, DeploymentSpec as K8sDeploymentSpec, Deployment as K8sDeployment, DeploymentSpec as K8sDeploymentSpec,
}; };
use k8s_openapi::api::core::v1::{ use k8s_openapi::api::core::v1::{
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile, Capabilities, Container, EnvVar, EnvVarSource, ObjectFieldSelector, PodSpec, PodTemplateSpec,
Secret, SecretKeySelector, SecretVolumeSource, SecurityContext, Service, ServiceAccount, SeccompProfile, Secret, SecretKeySelector, SecretVolumeSource, SecurityContext, Service,
ServicePort, ServiceSpec, Volume, VolumeMount, ServiceAccount, ServicePort, ServiceSpec, Volume, VolumeMount,
}; };
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject}; use k8s_openapi::api::rbac::v1::{PolicyRule, Role, RoleBinding, RoleRef, Subject};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
use kube::CustomResourceExt;
use kube::api::ObjectMeta; use kube::api::ObjectMeta;
use serde::Serialize; use serde::Serialize;
use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
use harmony_fleet_operator::{Deployment, Device};
/// Inputs for chart generation. Default values are aimed at a /// Inputs for chart generation. Default values are aimed at a
/// local-dev k3d install; override via the `chart` subcommand flags. /// local-dev k3d install; override via the `chart` subcommand flags.
@@ -82,7 +75,7 @@ pub struct ChartOptions {
pub web_cookie_key_json: Option<String>, pub web_cookie_key_json: Option<String>,
pub identity: Option<OperatorIdentityRefs>, pub identity: Option<OperatorIdentityRefs>,
pub identity_version: Option<String>, pub identity_version: Option<String>,
pub image_pull_secret: Option<String>, pub image_pull_secret: Option<K8sName>,
} }
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
@@ -124,8 +117,8 @@ impl Default for ChartOptions {
pub const CHART_NAME: &str = "harmony-fleet-operator-chart"; pub const CHART_NAME: &str = "harmony-fleet-operator-chart";
pub const RELEASE_NAME: &str = "harmony-fleet-operator"; pub const RELEASE_NAME: &str = "harmony-fleet-operator";
pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator"; pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator";
pub const CLUSTER_ROLE: &str = "harmony-fleet-operator"; pub const ROLE: &str = "harmony-fleet-operator";
pub const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator"; pub const ROLE_BINDING: &str = "harmony-fleet-operator";
pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets"; pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets";
/// Port the operator UI listens on and the Service exposes. Mirrors the /// Port the operator UI listens on and the Service exposes. Mirrors the
/// operator binary's bind default (`harmony-fleet-operator`'s /// operator binary's bind default (`harmony-fleet-operator`'s
@@ -165,26 +158,15 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
chart.version = chart_version; chart.version = chart_version;
chart.description = "IoT operator — Deployment CRD → NATS KV".to_string(); chart.description = "IoT operator — Deployment CRD → NATS KV".to_string();
chart.add_resource(HelmResourceKind::Crd(crd_with_keep_annotation(
Deployment::crd(),
)));
chart.add_resource(HelmResourceKind::Crd(crd_with_keep_annotation(
Device::crd(),
)));
chart.add_resource(HelmResourceKind::ServiceAccount(service_account())); chart.add_resource(HelmResourceKind::ServiceAccount(service_account()));
chart.add_resource(HelmResourceKind::ClusterRole(cluster_role())); chart.add_resource(
// The CRB's subject must reference the ServiceAccount's namespace. HelmResourceKind::from_serializable("role.yaml", &role())
// Since the chart itself is namespace-neutral (helm assigns the .context("serializing operator Role")?,
// release namespace to the SA + Deployment at install time), we );
// emit a literal helm template token so helm substitutes the chart.add_resource(
// release namespace at the same moment. This is the one chart HelmResourceKind::from_serializable("rolebinding.yaml", &role_binding())
// resource that can't be made namespace-neutral by simply omitting .context("serializing operator RoleBinding")?,
// the field — `subjects[].namespace` is part of the resource );
// identity and must point somewhere concrete after rendering.
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
"{{ .Release.Namespace }}",
)));
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts))); chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
chart.add_resource( chart.add_resource(
HelmResourceKind::from_serializable("service.yaml", &operator_service()) HelmResourceKind::from_serializable("service.yaml", &operator_service())
@@ -237,20 +219,6 @@ pub fn operator_secret(opts: &ChartOptions) -> Option<Secret> {
}) })
} }
/// Annotate a CRD with `helm.sh/resource-policy: keep` so
/// `helm uninstall` **does not** cascade-delete the CRD and its
/// CRs. Without this, uninstall wipes every `Deployment` + `Device`
/// CR in the cluster via the GC → agents notice the desired-state
/// KV deletes → the whole fleet tears down its containers. One
/// typo on uninstall would be catastrophic. `keep` makes uninstall
/// idempotent and data-preserving; the user explicitly `kubectl
/// delete crd …` if they actually want to wipe.
fn crd_with_keep_annotation(mut crd: CustomResourceDefinition) -> CustomResourceDefinition {
let annotations = crd.metadata.annotations.get_or_insert_with(BTreeMap::new);
annotations.insert("helm.sh/resource-policy".to_string(), "keep".to_string());
crd
}
// Namespace-neutral: helm fills in the release namespace at install // Namespace-neutral: helm fills in the release namespace at install
// time, and the direct-apply path (`K8sResourceScore::single(sa, // time, and the direct-apply path (`K8sResourceScore::single(sa,
// Some(ns))`) injects the namespace through its second argument. // Some(ns))`) injects the namespace through its second argument.
@@ -266,11 +234,11 @@ fn service_account() -> ServiceAccount {
/// Verbs the operator actually uses — nothing aspirational. Tightening /// Verbs the operator actually uses — nothing aspirational. Tightening
/// later is a matter of deleting a line. /// later is a matter of deleting a line.
fn cluster_role() -> ClusterRole { fn role() -> Role {
let group = "fleet.nationtech.io".to_string(); let group = "fleet.nationtech.io".to_string();
ClusterRole { Role {
metadata: ObjectMeta { metadata: ObjectMeta {
name: Some(CLUSTER_ROLE.to_string()), name: Some(ROLE.to_string()),
..Default::default() ..Default::default()
}, },
rules: Some(vec![ rules: Some(vec![
@@ -323,25 +291,24 @@ fn cluster_role() -> ClusterRole {
..Default::default() ..Default::default()
}, },
]), ]),
..Default::default()
} }
} }
fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding { fn role_binding() -> RoleBinding {
ClusterRoleBinding { RoleBinding {
metadata: ObjectMeta { metadata: ObjectMeta {
name: Some(CLUSTER_ROLE_BINDING.to_string()), name: Some(ROLE_BINDING.to_string()),
..Default::default() ..Default::default()
}, },
role_ref: RoleRef { role_ref: RoleRef {
api_group: "rbac.authorization.k8s.io".to_string(), api_group: "rbac.authorization.k8s.io".to_string(),
kind: "ClusterRole".to_string(), kind: "Role".to_string(),
name: CLUSTER_ROLE.to_string(), name: ROLE.to_string(),
}, },
subjects: Some(vec![Subject { subjects: Some(vec![Subject {
kind: "ServiceAccount".to_string(), kind: "ServiceAccount".to_string(),
name: SERVICE_ACCOUNT.to_string(), name: SERVICE_ACCOUNT.to_string(),
namespace: Some(namespace.to_string()), namespace: Some("{{ .Release.Namespace }}".to_string()),
..Default::default() ..Default::default()
}]), }]),
} }
@@ -403,6 +370,17 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
)]); )]);
let mut env = vec![ let mut env = vec![
EnvVar {
name: "FLEET_NAMESPACE".to_string(),
value_from: Some(EnvVarSource {
field_ref: Some(ObjectFieldSelector {
field_path: "metadata.namespace".to_string(),
..Default::default()
}),
..Default::default()
}),
..Default::default()
},
EnvVar { EnvVar {
name: "NATS_URL".to_string(), name: "NATS_URL".to_string(),
value: Some(opts.nats_url.clone()), value: Some(opts.nats_url.clone()),
@@ -517,7 +495,7 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
service_account_name: Some(SERVICE_ACCOUNT.to_string()), service_account_name: Some(SERVICE_ACCOUNT.to_string()),
image_pull_secrets: opts.image_pull_secret.as_ref().map(|name| { image_pull_secrets: opts.image_pull_secret.as_ref().map(|name| {
vec![k8s_openapi::api::core::v1::LocalObjectReference { vec![k8s_openapi::api::core::v1::LocalObjectReference {
name: name.clone(), name: name.0.clone(),
}] }]
}), }),
containers: vec![Container { containers: vec![Container {
@@ -599,21 +577,6 @@ fn container_security_context() -> SecurityContext {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn crds_carry_keep_annotation() {
let crd = crd_with_keep_annotation(Deployment::crd());
assert_eq!(
crd.metadata
.annotations
.as_ref()
.and_then(|a| a.get("helm.sh/resource-policy"))
.map(String::as_str),
Some("keep"),
"CRDs must carry the keep annotation so helm uninstall doesn't \
cascade-delete CRs and wipe the fleet"
);
}
#[test] #[test]
fn security_context_is_locked_down() { fn security_context_is_locked_down() {
let sc = container_security_context(); let sc = container_security_context();
@@ -666,8 +629,8 @@ mod tests {
// resources — a patch on `*/status` is forbidden without an explicit // resources — a patch on `*/status` is forbidden without an explicit
// grant. Lock both so adding a status subresource can't silently 403. // grant. Lock both so adding a status subresource can't silently 403.
#[test] #[test]
fn cluster_role_grants_status_subresources() { fn role_grants_status_subresources() {
let role = cluster_role(); let role = role();
let grants_patch = |resource: &str| { let grants_patch = |resource: &str| {
role.rules.as_ref().unwrap().iter().any(|r| { role.rules.as_ref().unwrap().iter().any(|r| {
r.resources r.resources
@@ -680,6 +643,62 @@ mod tests {
assert!(grants_patch("devices/status")); assert!(grants_patch("devices/status"));
} }
#[test]
fn chart_is_namespaced_and_excludes_crds() {
let tmp = tempfile::tempdir().unwrap();
let chart_path = build_chart(&ChartOptions {
output_dir: tmp.path().to_path_buf(),
..Default::default()
})
.unwrap();
let templates = chart_path.join("templates");
assert!(templates.join("role.yaml").exists());
assert!(templates.join("rolebinding.yaml").exists());
assert!(!templates.join(format!("clusterrole-{ROLE}.yaml")).exists());
assert!(
!templates
.join(format!("clusterrolebinding-{ROLE_BINDING}.yaml"))
.exists()
);
assert!(std::fs::read_dir(templates).unwrap().all(|entry| {
!entry
.unwrap()
.file_name()
.to_string_lossy()
.starts_with("crd-")
}));
}
#[test]
fn deployment_reads_fleet_namespace_from_pod_metadata() {
let deployment = operator_deployment(&ChartOptions::default());
let env = deployment
.spec
.unwrap()
.template
.spec
.unwrap()
.containers
.into_iter()
.next()
.unwrap()
.env
.unwrap();
let namespace = env
.iter()
.find(|env| env.name == "FLEET_NAMESPACE")
.unwrap();
assert_eq!(
namespace
.value_from
.as_ref()
.and_then(|source| source.field_ref.as_ref())
.map(|field| field.field_path.as_str()),
Some("metadata.namespace")
);
}
// The chart bakes these env names at publish time; the operator's // The chart bakes these env names at publish time; the operator's
// ConfigClient derives them from the struct names at runtime. Lock // ConfigClient derives them from the struct names at runtime. Lock
// them together so a rename can't silently desync the two. // them together so a rename can't silently desync the two.

View File

@@ -10,5 +10,5 @@
pub(crate) mod chart; pub(crate) mod chart;
pub mod score; pub mod score;
pub use chart::OperatorCredentials; pub use chart::{OperatorCredentials, SERVICE_ACCOUNT};
pub use score::FleetOperatorScore; pub use score::{FleetCrdsScore, FleetOperatorScore};

View File

@@ -1,8 +1,8 @@
//! [`FleetOperatorScore`] install the harmony fleet operator and //! [`FleetOperatorScore`] installs the harmony fleet operator into a
//! its CRDs into a Kubernetes cluster. //! Kubernetes namespace; [`FleetCrdsScore`] manages its CRDs separately.
//! //!
//! Renders a self-contained helm chart (CRDs + ServiceAccount + //! Renders a self-contained helm chart (ServiceAccount + Role +
//! ClusterRole + ClusterRoleBinding + Deployment) into a tempdir at //! RoleBinding + Deployment) into a tempdir at
//! interpret-time and delegates to //! interpret-time and delegates to
//! [`HelmChartScore`](crate::modules::helm::chart::HelmChartScore) //! [`HelmChartScore`](crate::modules::helm::chart::HelmChartScore)
//! for the actual install. The tempdir is dropped after the helm //! for the actual install. The tempdir is dropped after the helm
@@ -27,6 +27,7 @@ use std::{str::FromStr, time::Duration};
use async_trait::async_trait; use async_trait::async_trait;
use harmony_types::id::Id; use harmony_types::id::Id;
use harmony_types::k8s_name::K8sName;
use log::info; use log::info;
use serde::Serialize; use serde::Serialize;
@@ -40,7 +41,10 @@ use harmony::modules::nats::NatsClientRef;
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
use harmony::score::Score; use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, Topology}; use harmony::topology::{HelmCommand, K8sclient, Topology};
use harmony_fleet_operator::{Deployment, Device};
use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret}; use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
use kube::api::ListParams; use kube::api::ListParams;
use crate::operator::chart; use crate::operator::chart;
@@ -48,6 +52,95 @@ use crate::operator::chart::{
ChartOptions, OperatorCredentials, OperatorIdentityRefs, build_chart, operator_secret, ChartOptions, OperatorCredentials, OperatorIdentityRefs, build_chart, operator_secret,
}; };
#[derive(Debug, Clone, Copy, Default, Serialize)]
pub struct FleetCrdsScore;
fn fleet_crds() -> Vec<CustomResourceDefinition> {
vec![Deployment::crd(), Device::crd()]
}
impl<T: Topology + K8sclient> Score<T> for FleetCrdsScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(FleetCrdsInterpret)
}
fn name(&self) -> String {
"FleetCrdsScore".to_string()
}
}
#[derive(Debug)]
struct FleetCrdsInterpret;
#[async_trait]
impl<T: Topology + K8sclient> Interpret<T> for FleetCrdsInterpret {
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let client = topology
.k8s_client()
.await
.map_err(|error| InterpretError::new(format!("get Kubernetes client: {error}")))?;
let crds = fleet_crds();
let device_name = crds
.iter()
.find(|crd| crd.spec.names.kind == "Device")
.and_then(|crd| crd.metadata.name.as_deref())
.expect("generated Device CRD has a name");
if let Some(scope) = client
.crd_scope(device_name)
.await
.map_err(|error| InterpretError::new(format!("read Device CRD: {error}")))?
&& scope != "Namespaced"
{
return Err(InterpretError::new(format!(
"Device CRD scope is '{scope}', but this release requires 'Namespaced'; back up or recreate Device resources, delete {device_name}, then rerun the CRD deployment"
)));
}
let names = crds
.iter()
.filter_map(|crd| crd.metadata.name.clone())
.collect::<Vec<_>>();
K8sResourceScore {
resource: crds,
namespace: None,
}
.create_interpret()
.execute(inventory, topology)
.await?;
for name in names {
client
.wait_for_crd(&name, Some(Duration::from_secs(60)))
.await
.map_err(|error| {
InterpretError::new(format!("Fleet CRD '{name}' not registered: {error}"))
})?;
}
client.invalidate_discovery().await;
Ok(Outcome::success("Fleet CRDs are registered".to_string()))
}
fn get_name(&self) -> InterpretName {
InterpretName::K8sResource
}
fn get_version(&self) -> Version {
Version::from(env!("CARGO_PKG_VERSION")).expect("package version")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::SUCCESS
}
fn get_children(&self) -> Vec<Id> {
Vec::new()
}
}
/// Declarative install of the harmony fleet operator. Construct via /// Declarative install of the harmony fleet operator. Construct via
/// [`new`](Self::new), tune with the builder-style methods, hand to /// [`new`](Self::new), tune with the builder-style methods, hand to
/// a topology that implements [`HelmCommand`]. /// a topology that implements [`HelmCommand`].
@@ -75,7 +168,7 @@ pub struct FleetOperatorScore {
/// unauthenticated (dev/e2e). /// unauthenticated (dev/e2e).
pub web_auth: Option<WebAuth>, pub web_auth: Option<WebAuth>,
pub identity: Option<OperatorIdentityRefs>, pub identity: Option<OperatorIdentityRefs>,
pub image_pull_secret: Option<String>, pub image_pull_secret: Option<K8sName>,
} }
/// The dashboard's auth inputs the operator reads via `ConfigClient`. /// The dashboard's auth inputs the operator reads via `ConfigClient`.
@@ -162,8 +255,8 @@ impl FleetOperatorScore {
self self
} }
pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self { pub fn image_pull_secret(mut self, secret: Option<K8sName>) -> Self {
self.image_pull_secret = secret.map(str::to_string); self.image_pull_secret = secret;
self self
} }
@@ -481,6 +574,20 @@ fn non_blank(value: &str, field: &str) -> Result<NonBlankString, InterpretError>
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn fleet_crds_are_namespaced_resources() {
let crds = fleet_crds();
assert_eq!(crds.len(), 2);
for crd in crds {
assert_eq!(
crd.spec.scope,
"Namespaced",
"{}",
crd.metadata.name.unwrap()
);
}
}
#[test] #[test]
fn defaults_target_fleet_system() { fn defaults_target_fleet_system() {
let s = FleetOperatorScore::new("operator:dev"); let s = FleetOperatorScore::new("operator:dev");

View File

@@ -0,0 +1,23 @@
use std::process::Command;
fn run(args: &[&str]) -> String {
let output = Command::new(env!("CARGO_BIN_EXE_harmony-fleet-deploy"))
.args(args)
.env_remove("HARMONY_CONTEXT")
.output()
.unwrap();
assert!(!output.status.success());
String::from_utf8(output.stderr).unwrap()
}
#[test]
fn missing_context_lists_compiled_contexts() {
let stderr = run(&["build"]);
assert!(stderr.contains("--context or HARMONY_CONTEXT is required (available: local)"));
}
#[test]
fn unknown_context_lists_compiled_contexts() {
let stderr = run(&["build", "--context", "unknown"]);
assert!(stderr.contains("context 'unknown' not found (have: local)"));
}

View File

@@ -29,11 +29,10 @@ use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_fleet_deploy::agent::PodTarget; use harmony_fleet_deploy::agent::PodTarget;
use harmony_fleet_deploy::operator::OperatorCredentials; use harmony_fleet_deploy::operator::OperatorCredentials;
use harmony_fleet_deploy::{FleetAgentScore, FleetOperatorScore}; use harmony_fleet_deploy::{FleetAgentScore, FleetCrdsScore, FleetOperatorScore};
use harmony_fleet_operator::Deployment as FleetDeployment;
use k3d_rs::{K3d, PortMapping}; use k3d_rs::{K3d, PortMapping};
use kube::Client; use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams}; use kube::api::{Api, DeleteParams};
use thiserror::Error; use thiserror::Error;
use tokio::sync::OnceCell; use tokio::sync::OnceCell;
@@ -306,6 +305,10 @@ impl Stack {
} }
}; };
if opts.deploy_operator { if opts.deploy_operator {
FleetCrdsScore
.interpret(&Inventory::autoload(), &topology)
.await
.map_err(BringUpError::OperatorDeploy)?;
let mut operator = FleetOperatorScore::new(opts.operator_image.clone()) let mut operator = FleetOperatorScore::new(opts.operator_image.clone())
.namespace(namespace.clone()) .namespace(namespace.clone())
.messaging(&nats_ref) .messaging(&nats_ref)
@@ -607,15 +610,22 @@ async fn connect_nats_admin(url: &str) -> Result<async_nats::Client, BringUpErro
} }
async fn prune_stale_operator_cluster_resources(client: &Client) { async fn prune_stale_operator_cluster_resources(client: &Client) {
prune_stale_fleet_deployments(client).await;
let crds: Api<CustomResourceDefinition> = Api::all(client.clone()); let crds: Api<CustomResourceDefinition> = Api::all(client.clone());
for name in [ if let Ok(Some(crd)) = crds.get_opt("devices.fleet.nationtech.io").await
"deployments.fleet.nationtech.io", && crd.spec.scope == "Cluster"
"devices.fleet.nationtech.io", {
] { tracing::warn!("recreating legacy cluster-scoped Device CRD for disposable E2E cluster");
delete_cluster_resource(&crds, name, "CRD").await; let _ = crds
.delete("devices.fleet.nationtech.io", &DeleteParams::default())
.await;
for _ in 0..30 {
if matches!(crds.get_opt("devices.fleet.nationtech.io").await, Ok(None)) {
break;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
} }
let cluster_roles: Api<ClusterRole> = Api::all(client.clone()); let cluster_roles: Api<ClusterRole> = Api::all(client.clone());
delete_cluster_resource(&cluster_roles, "harmony-fleet-operator", "ClusterRole").await; delete_cluster_resource(&cluster_roles, "harmony-fleet-operator", "ClusterRole").await;
let cluster_role_bindings: Api<ClusterRoleBinding> = Api::all(client.clone()); let cluster_role_bindings: Api<ClusterRoleBinding> = Api::all(client.clone());
@@ -627,35 +637,6 @@ async fn prune_stale_operator_cluster_resources(client: &Client) {
.await; .await;
} }
async fn prune_stale_fleet_deployments(client: &Client) {
let deployments: Api<FleetDeployment> = Api::all(client.clone());
let Ok(list) = deployments.list(&Default::default()).await else {
return;
};
for deployment in list {
let Some(name) = deployment.metadata.name else {
continue;
};
let Some(namespace) = deployment.metadata.namespace else {
continue;
};
let namespaced: Api<FleetDeployment> = Api::namespaced(client.clone(), &namespace);
let patch = serde_json::json!({
"metadata": {
"finalizers": null
}
});
let _ = namespaced
.patch(&name, &PatchParams::default(), &Patch::Merge(&patch))
.await;
let _ = namespaced.delete(&name, &DeleteParams::default()).await;
}
}
async fn delete_cluster_resource<K>(api: &Api<K>, name: &str, kind: &str) async fn delete_cluster_resource<K>(api: &Api<K>, name: &str, kind: &str)
where where
K: kube::Resource<DynamicType = ()> K: kube::Resource<DynamicType = ()>

View File

@@ -1,9 +1,14 @@
use harmony_fleet_deploy::operator::SERVICE_ACCOUNT;
use harmony_fleet_e2e::{StackOptions, shared_stack}; use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_fleet_operator::crd::{ use harmony_fleet_operator::crd::{
Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore, Deployment, DeploymentSpec, Device, DeviceSpec, PodmanService, PodmanV0Score, ReconcileScore,
Rollout, RolloutStrategy, Rollout, RolloutStrategy,
}; };
use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key}; use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, DeploymentName, desired_state_key};
use k8s_openapi::api::authorization::v1::{
ResourceAttributes, SubjectAccessReview, SubjectAccessReviewSpec,
};
use k8s_openapi::api::core::v1::Namespace;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client; use kube::Client;
use kube::api::{Api, DeleteParams, ObjectMeta, PostParams}; use kube::api::{Api, DeleteParams, ObjectMeta, PostParams};
@@ -42,8 +47,7 @@ async fn operator_writes_desired_state_for_matching_device() -> anyhow::Result<(
let stack = operator_stack().await?; let stack = operator_stack().await?;
let client = Client::try_default().await?; let client = Client::try_default().await?;
// Device CRs are cluster-scoped; Deployment CRs are namespaced. let devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
let devices: Api<Device> = Api::all(client.clone());
let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace); let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
create_device(&devices, "desired-state-device").await?; create_device(&devices, "desired-state-device").await?;
@@ -64,7 +68,7 @@ async fn operator_deletes_desired_state_when_deployment_is_deleted() -> anyhow::
let stack = operator_stack().await?; let stack = operator_stack().await?;
let client = Client::try_default().await?; let client = Client::try_default().await?;
let devices: Api<Device> = Api::all(client.clone()); let devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace); let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
create_device(&devices, "cleanup-device").await?; create_device(&devices, "cleanup-device").await?;
@@ -81,6 +85,106 @@ async fn operator_deletes_desired_state_when_deployment_is_deleted() -> anyhow::
Ok(()) Ok(())
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
if !e2e_enabled() {
skip_e2e();
return Ok(());
}
let stack = operator_stack().await?;
let client = Client::try_default().await?;
let namespace = format!(
"e2e-isolation-{}",
&uuid::Uuid::new_v4().simple().to_string()[..8]
);
let namespaces: Api<Namespace> = Api::all(client.clone());
namespaces
.create(
&PostParams::default(),
&Namespace {
metadata: ObjectMeta {
name: Some(namespace.clone()),
labels: Some(std::collections::BTreeMap::from([(
"harmony.io/managed-by".to_string(),
"fleet-e2e".to_string(),
)])),
..Default::default()
},
..Default::default()
},
)
.await?;
let devices: Api<Device> = Api::namespaced(client.clone(), &namespace);
let deployments: Api<Deployment> = Api::namespaced(client.clone(), &namespace);
create_device(&devices, "other-tenant-device").await?;
create_fleet_deployment(&deployments, "other-tenant-deployment").await?;
let sentinel = format!(
"isolation-sentinel-{}",
&uuid::Uuid::new_v4().simple().to_string()[..8]
);
let tenant_devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
let tenant_deployments: Api<Deployment> = Api::namespaced(client.clone(), &stack.namespace);
create_device(&tenant_devices, &sentinel).await?;
create_fleet_deployment(&tenant_deployments, &sentinel).await?;
wait_for_desired_state_entry(&stack, &sentinel, &sentinel, true).await?;
let deployment = deployments.get("other-tenant-deployment").await?;
assert!(
deployment
.metadata
.finalizers
.unwrap_or_default()
.is_empty()
);
wait_for_desired_state_entry(
&stack,
"other-tenant-device",
"other-tenant-deployment",
false,
)
.await?;
let reviews: Api<SubjectAccessReview> = Api::all(client);
let review = reviews
.create(
&PostParams::default(),
&SubjectAccessReview {
metadata: ObjectMeta::default(),
spec: SubjectAccessReviewSpec {
resource_attributes: Some(ResourceAttributes {
group: Some("fleet.nationtech.io".to_string()),
namespace: Some(namespace.clone()),
resource: Some("devices".to_string()),
verb: Some("list".to_string()),
..Default::default()
}),
user: Some(format!(
"system:serviceaccount:{}:{SERVICE_ACCOUNT}",
stack.namespace
)),
..Default::default()
},
status: None,
},
)
.await?;
assert!(!review.status.is_some_and(|status| status.allowed));
tenant_deployments
.delete(&sentinel, &DeleteParams::default())
.await?;
tenant_devices
.delete(&sentinel, &DeleteParams::default())
.await?;
namespaces
.delete(&namespace, &DeleteParams::default())
.await?;
Ok(())
}
async fn operator_stack() -> anyhow::Result<Arc<harmony_fleet_e2e::Stack>> { async fn operator_stack() -> anyhow::Result<Arc<harmony_fleet_e2e::Stack>> {
let _ = tracing_subscriber::fmt() let _ = tracing_subscriber::fmt()
.with_env_filter( .with_env_filter(

View File

@@ -54,8 +54,8 @@ pub struct Context {
pub kv: Store, pub kv: Store,
} }
pub async fn run(client: Client, kv: Store) -> anyhow::Result<()> { pub async fn run(client: Client, namespace: &str, kv: Store) -> anyhow::Result<()> {
let api: Api<Deployment> = Api::all(client.clone()); let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
let ctx = Arc::new(Context { client, kv }); let ctx = Arc::new(Context { client, kv });
tracing::info!("starting Deployment controller"); tracing::info!("starting Deployment controller");

View File

@@ -82,9 +82,7 @@ pub struct AggregateLastError {
pub at: String, pub at: String,
} }
/// A physical/virtual device registered with the fleet. Cluster-scoped /// A physical/virtual device registered with a tenant's fleet.
/// because devices aren't tenant-isolated by namespace — they're
/// infrastructure, the same way K8s Nodes are cluster-scoped.
/// ///
/// Created by the operator from `DeviceInfo` entries in the NATS /// Created by the operator from `DeviceInfo` entries in the NATS
/// `device-info` bucket. Agents never touch the kube apiserver /// `device-info` bucket. Agents never touch the kube apiserver
@@ -104,6 +102,7 @@ pub struct AggregateLastError {
kind = "Device", kind = "Device",
plural = "devices", plural = "devices",
shortname = "fleetdev", shortname = "fleetdev",
namespaced,
status = "DeviceStatus" status = "DeviceStatus"
)] )]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@@ -141,3 +140,15 @@ pub enum Reachability {
/// Last heartbeat is older than the freshness window. /// Last heartbeat is older than the freshness window.
Stale, Stale,
} }
#[cfg(test)]
mod tests {
use kube::CustomResourceExt;
use super::Device;
#[test]
fn device_is_namespaced() {
assert_eq!(Device::crd().spec.scope, "Namespaced");
}
}

View File

@@ -2,7 +2,7 @@
//! //!
//! Agents publish a `DeviceInfo` payload to NATS on startup + on //! Agents publish a `DeviceInfo` payload to NATS on startup + on
//! label/inventory change. This reconciler watches that bucket and //! label/inventory change. This reconciler watches that bucket and
//! materializes each entry as a cluster-scoped `Device` custom //! materializes each entry as a namespaced `Device` custom
//! resource, so label selectors and `kubectl get devices -l …` //! resource, so label selectors and `kubectl get devices -l …`
//! work the way they do for K8s Nodes. //! work the way they do for K8s Nodes.
//! //!
@@ -22,7 +22,11 @@ use crate::crd::{Device, DeviceSpec};
const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler"; const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler";
pub async fn run(client: Client, js: async_nats::jetstream::Context) -> Result<()> { pub async fn run(
client: Client,
namespace: &str,
js: async_nats::jetstream::Context,
) -> Result<()> {
let bucket = js let bucket = js
.create_key_value(async_nats::jetstream::kv::Config { .create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DEVICE_INFO.to_string(), bucket: BUCKET_DEVICE_INFO.to_string(),
@@ -30,11 +34,11 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> Result<(
}) })
.await?; .await?;
run_loop(client, bucket).await run_loop(client, namespace, bucket).await
} }
async fn run_loop(client: Client, bucket: Store) -> Result<()> { async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()> {
let devices: Api<Device> = Api::all(client); let devices: Api<Device> = Api::namespaced(client, namespace);
// `watch_with_history` replays every current entry then streams // `watch_with_history` replays every current entry then streams
// live updates. Matches the aggregator's pattern and means we // live updates. Matches the aggregator's pattern and means we
// don't need a separate cold-start KV scan here. // don't need a separate cold-start KV scan here.
@@ -58,7 +62,7 @@ async fn run_loop(client: Client, bucket: Store) -> Result<()> {
continue; continue;
} }
}; };
if let Err(e) = upsert_device(&devices, &info).await { if let Err(e) = upsert_device(&devices, namespace, &info).await {
tracing::warn!( tracing::warn!(
device = %info.device_id, device = %info.device_id,
error = %e, error = %e,
@@ -79,7 +83,7 @@ async fn run_loop(client: Client, bucket: Store) -> Result<()> {
Ok(()) Ok(())
} }
async fn upsert_device(api: &Api<Device>, info: &DeviceInfo) -> Result<()> { async fn upsert_device(api: &Api<Device>, namespace: &str, info: &DeviceInfo) -> Result<()> {
let name = info.device_id.to_string(); let name = info.device_id.to_string();
let mut device = Device::new( let mut device = Device::new(
&name, &name,
@@ -87,6 +91,7 @@ async fn upsert_device(api: &Api<Device>, info: &DeviceInfo) -> Result<()> {
inventory: info.inventory.clone(), inventory: info.inventory.clone(),
}, },
); );
device.metadata.namespace = Some(namespace.to_string());
device.metadata.labels = Some(clean_labels(&info.labels)); device.metadata.labels = Some(clean_labels(&info.labels));
api.patch( api.patch(

View File

@@ -32,7 +32,11 @@ const STALE_AFTER: Duration = Duration::from_secs(90);
/// How often to re-evaluate freshness and patch changed devices. /// How often to re-evaluate freshness and patch changed devices.
const TICK: Duration = Duration::from_secs(30); const TICK: Duration = Duration::from_secs(30);
pub async fn run(client: Client, js: async_nats::jetstream::Context) -> Result<()> { pub async fn run(
client: Client,
namespace: &str,
js: async_nats::jetstream::Context,
) -> Result<()> {
let bucket = js let bucket = js
.create_key_value(async_nats::jetstream::kv::Config { .create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DEVICE_HEARTBEAT.to_string(), bucket: BUCKET_DEVICE_HEARTBEAT.to_string(),
@@ -41,7 +45,7 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> Result<(
.await?; .await?;
let heartbeats: Mutex<HashMap<String, DateTime<Utc>>> = Mutex::new(HashMap::new()); let heartbeats: Mutex<HashMap<String, DateTime<Utc>>> = Mutex::new(HashMap::new());
let devices: Api<Device> = Api::all(client); let devices: Api<Device> = Api::namespaced(client, namespace);
tokio::try_join!( tokio::try_join!(
watch_heartbeats(&bucket, &heartbeats), watch_heartbeats(&bucket, &heartbeats),

View File

@@ -189,6 +189,7 @@ fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet
pub async fn run( pub async fn run(
client: Client, client: Client,
namespace: &str,
js: async_nats::jetstream::Context, js: async_nats::jetstream::Context,
secret_grants: Option<Arc<dyn DeploymentSecretGrants>>, secret_grants: Option<Arc<dyn DeploymentSecretGrants>>,
group_source: Option<Arc<dyn DeviceGroupSource>>, group_source: Option<Arc<dyn DeviceGroupSource>>,
@@ -212,9 +213,9 @@ pub async fn run(
let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default())); let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default()));
seed_owned_targets(&desired_bucket, &state).await?; seed_owned_targets(&desired_bucket, &state).await?;
let deployments_api: Api<Deployment> = Api::all(client.clone()); let deployments_api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
let devices_api: Api<Device> = Api::all(client.clone()); let devices_api: Api<Device> = Api::namespaced(client.clone(), namespace);
let patch_api: Api<Deployment> = Api::all(client); let patch_api: Api<Deployment> = Api::namespaced(client, namespace);
tracing::info!( tracing::info!(
owned = state owned = state
@@ -803,9 +804,8 @@ async fn patch_tick(api: &Api<Deployment>, state: &SharedFleetState) -> anyhow::
}; };
for (key, aggregate) in dirty { for (key, aggregate) in dirty {
let ns_api: Api<Deployment> = Api::namespaced(api.clone().into_client(), &key.namespace);
let status = json!({ "status": { "aggregate": aggregate } }); let status = json!({ "status": { "aggregate": aggregate } });
if let Err(e) = ns_api if let Err(e) = api
.patch_status(&key.name, &PatchParams::default(), &Patch::Merge(&status)) .patch_status(&key.name, &PatchParams::default(), &Patch::Merge(&status))
.await .await
{ {

View File

@@ -49,6 +49,15 @@ struct Cli {
)] )]
kv_bucket: String, kv_bucket: String,
/// Kubernetes namespace containing this tenant's Fleet resources.
#[arg(
long,
env = "FLEET_NAMESPACE",
default_value = "fleet-system",
global = true
)]
tenant_namespace: String,
/// `[credentials]` TOML payload (same shape the agent reads from /// `[credentials]` TOML payload (same shape the agent reads from
/// `/etc/fleet-agent/config.toml`). Mounted into the Pod from the /// `/etc/fleet-agent/config.toml`). Mounted into the Pod from the
/// operator's Secret. Empty string means "no auth — bare connect" /// operator's Secret. Empty string means "no auth — bare connect"
@@ -137,6 +146,7 @@ async fn main() -> Result<()> {
run( run(
&cli.nats_url, &cli.nats_url,
&cli.kv_bucket, &cli.kv_bucket,
&cli.tenant_namespace,
&credentials_toml, &credentials_toml,
&cli.openbao_url, &cli.openbao_url,
&cli.openbao_token, &cli.openbao_token,
@@ -149,7 +159,7 @@ async fn main() -> Result<()> {
addr, addr,
css_from, css_from,
live_reload, live_reload,
} => serve_web(mock, addr, css_from, live_reload).await, } => serve_web(mock, addr, css_from, live_reload, &cli.tenant_namespace).await,
} }
} }
@@ -162,6 +172,7 @@ async fn serve_web(
addr: std::net::SocketAddr, addr: std::net::SocketAddr,
css_from: Option<PathBuf>, css_from: Option<PathBuf>,
live_reload: bool, live_reload: bool,
tenant_namespace: &str,
) -> Result<()> { ) -> Result<()> {
use std::sync::Arc; use std::sync::Arc;
@@ -170,7 +181,10 @@ async fn serve_web(
let fleet: Arc<dyn FleetService> = if mock { let fleet: Arc<dyn FleetService> = if mock {
Arc::new(MockFleetService::default()) Arc::new(MockFleetService::default())
} else { } else {
Arc::new(RealFleetService::new(Client::try_default().await?)) Arc::new(RealFleetService::new(
Client::try_default().await?,
tenant_namespace,
))
}; };
serve_dashboard(fleet, addr, css_from, live_reload).await serve_dashboard(fleet, addr, css_from, live_reload).await
} }
@@ -230,15 +244,16 @@ async fn serve_dashboard(
/// (e.g. Zitadel not yet reachable for JWKS) is logged but never tears /// (e.g. Zitadel not yet reachable for JWKS) is logged but never tears
/// down reconcile — the read UI is best-effort, the controller is not. /// down reconcile — the read UI is best-effort, the controller is not.
#[cfg(feature = "web-frontend")] #[cfg(feature = "web-frontend")]
fn spawn_dashboard(client: Client) { fn spawn_dashboard(client: Client, tenant_namespace: &str) {
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use service::real::RealFleetService; use service::real::RealFleetService;
let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT)); let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT));
let tenant_namespace = tenant_namespace.to_string();
tokio::spawn(async move { tokio::spawn(async move {
let fleet = Arc::new(RealFleetService::new(client)); let fleet = Arc::new(RealFleetService::new(client, tenant_namespace));
if let Err(e) = serve_dashboard(fleet, addr, None, false).await { if let Err(e) = serve_dashboard(fleet, addr, None, false).await {
tracing::error!(error = %e, "dashboard server exited; reconcile continues"); tracing::error!(error = %e, "dashboard server exited; reconcile continues");
} }
@@ -248,6 +263,7 @@ fn spawn_dashboard(client: Client) {
async fn run( async fn run(
nats_url: &str, nats_url: &str,
bucket: &str, bucket: &str,
tenant_namespace: &str,
credentials_toml: &str, credentials_toml: &str,
openbao_url: &Option<String>, openbao_url: &Option<String>,
openbao_token: &Option<String>, openbao_token: &Option<String>,
@@ -316,7 +332,7 @@ async fn run(
// it reads CRs only, no NATS). Built only with the web-frontend // it reads CRs only, no NATS). Built only with the web-frontend
// feature; absent from the lean reconcile-only image. // feature; absent from the lean reconcile-only image.
#[cfg(feature = "web-frontend")] #[cfg(feature = "web-frontend")]
spawn_dashboard(client.clone()); spawn_dashboard(client.clone(), tenant_namespace);
// Concurrent tasks: // Concurrent tasks:
// controller — CR validation + finalizer-cleanup // controller — CR validation + finalizer-cleanup
@@ -329,9 +345,9 @@ async fn run(
let dr_client = client.clone(); let dr_client = client.clone();
let dr_js = js.clone(); let dr_js = js.clone();
tokio::select! { tokio::select! {
r = controller::run(ctl_client, desired_state_kv) => r, r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r,
r = device_reconciler::run(dr_client, dr_js) => r, r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r,
r = fleet_aggregator::run(client, js, secret_grants, group_source) => r, r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r,
} }
} }

View File

@@ -35,26 +35,28 @@ const REGION_LABEL: &str = "region";
pub struct RealFleetService { pub struct RealFleetService {
kube: Client, kube: Client,
namespace: String,
/// In-memory ack set. Alerts are derived from live CR state and /// In-memory ack set. Alerts are derived from live CR state and
/// have no store of their own, so acks don't survive a restart. /// have no store of their own, so acks don't survive a restart.
acked_alerts: Mutex<HashSet<String>>, acked_alerts: Mutex<HashSet<String>>,
} }
impl RealFleetService { impl RealFleetService {
pub fn new(kube: Client) -> Self { pub fn new(kube: Client, namespace: impl Into<String>) -> Self {
Self { Self {
kube, kube,
namespace: namespace.into(),
acked_alerts: Mutex::new(HashSet::new()), acked_alerts: Mutex::new(HashSet::new()),
} }
} }
async fn device_crs(&self) -> anyhow::Result<Vec<DeviceCr>> { async fn device_crs(&self) -> anyhow::Result<Vec<DeviceCr>> {
let api: Api<DeviceCr> = Api::all(self.kube.clone()); let api: Api<DeviceCr> = Api::namespaced(self.kube.clone(), &self.namespace);
Ok(api.list(&ListParams::default()).await?.items) Ok(api.list(&ListParams::default()).await?.items)
} }
async fn deployment_crs(&self) -> anyhow::Result<Vec<DeploymentCr>> { async fn deployment_crs(&self) -> anyhow::Result<Vec<DeploymentCr>> {
let api: Api<DeploymentCr> = Api::all(self.kube.clone()); let api: Api<DeploymentCr> = Api::namespaced(self.kube.clone(), &self.namespace);
Ok(api.list(&ListParams::default()).await?.items) Ok(api.list(&ListParams::default()).await?.items)
} }
@@ -328,7 +330,7 @@ impl FleetService for RealFleetService {
} }
async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail> { async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail> {
let api: Api<DeviceCr> = Api::all(self.kube.clone()); let api: Api<DeviceCr> = Api::namespaced(self.kube.clone(), &self.namespace);
let patch = json!({ "metadata": { "labels": { BLACKLIST_LABEL: "true" } } }); let patch = json!({ "metadata": { "labels": { BLACKLIST_LABEL: "true" } } });
api.patch(id, &PatchParams::default(), &Patch::Merge(&patch)) api.patch(id, &PatchParams::default(), &Patch::Merge(&patch))
.await .await

View File

@@ -8,6 +8,79 @@ use tokio::sync::{OnceCell, RwLock};
use crate::types::KubernetesDistribution; use crate::types::KubernetesDistribution;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ClusterConnection {
pub name: String,
pub server: String,
pub tls_server_name: Option<String>,
pub proxy_url: Option<String>,
}
impl ClusterConnection {
fn from_kubeconfig(
kubeconfig: &Kubeconfig,
options: &KubeConfigOptions,
) -> Result<Self, String> {
let context_name = options
.context
.as_ref()
.or(kubeconfig.current_context.as_ref())
.ok_or("kubeconfig has no selected context")?;
let context = kubeconfig
.contexts
.iter()
.find(|context| &context.name == context_name)
.and_then(|context| context.context.as_ref())
.ok_or_else(|| format!("kubeconfig context '{context_name}' not found"))?;
let cluster_name = options.cluster.as_ref().unwrap_or(&context.cluster);
let cluster = kubeconfig
.clusters
.iter()
.find(|cluster| &cluster.name == cluster_name)
.and_then(|cluster| cluster.cluster.as_ref())
.ok_or_else(|| format!("kubeconfig cluster '{cluster_name}' not found"))?;
if cluster.insecure_skip_tls_verify == Some(true) {
return Err("cannot issue tenant credentials for an insecure cluster".to_string());
}
let server = safe_endpoint(
cluster
.server
.as_deref()
.ok_or_else(|| format!("kubeconfig cluster '{cluster_name}' has no server"))?,
"cluster server",
true,
)?;
let proxy_url = cluster
.proxy_url
.as_deref()
.map(|url| safe_endpoint(url, "cluster proxy", false))
.transpose()?;
Ok(Self {
name: cluster_name.clone(),
server,
tls_server_name: cluster.tls_server_name.clone(),
proxy_url,
})
}
}
fn safe_endpoint(endpoint: &str, name: &str, require_https: bool) -> Result<String, String> {
let url = url::Url::parse(endpoint).map_err(|error| format!("invalid {name}: {error}"))?;
if require_https && url.scheme() != "https" {
return Err(format!("{name} must use HTTPS"));
}
if !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(format!(
"{name} must not contain credentials, a query, or a fragment"
));
}
Ok(endpoint.to_string())
}
// TODO not cool, should use a proper configuration mechanism // TODO not cool, should use a proper configuration mechanism
// cli arg, env var, config file // cli arg, env var, config file
fn read_dry_run_from_env() -> bool { fn read_dry_run_from_env() -> bool {
@@ -26,6 +99,7 @@ pub struct K8sClient {
/// API discovery cache. Wrapped in `RwLock` so it can be invalidated /// API discovery cache. Wrapped in `RwLock` so it can be invalidated
/// after installing CRDs or operators that register new API groups. /// after installing CRDs or operators that register new API groups.
pub(crate) discovery: Arc<RwLock<Option<Arc<Discovery>>>>, pub(crate) discovery: Arc<RwLock<Option<Arc<Discovery>>>>,
cluster_connection: Option<ClusterConnection>,
} }
impl Serialize for K8sClient { impl Serialize for K8sClient {
@@ -55,6 +129,7 @@ impl K8sClient {
client, client,
k8s_distribution: Arc::new(OnceCell::new()), k8s_distribution: Arc::new(OnceCell::new()),
discovery: Arc::new(RwLock::new(None)), discovery: Arc::new(RwLock::new(None)),
cluster_connection: None,
} }
} }
@@ -72,6 +147,10 @@ impl K8sClient {
self.dry_run self.dry_run
} }
pub fn cluster_connection(&self) -> Option<&ClusterConnection> {
self.cluster_connection.as_ref()
}
pub async fn try_default() -> Result<Self, Error> { pub async fn try_default() -> Result<Self, Error> {
Ok(Self::new(Client::try_default().await?)) Ok(Self::new(Client::try_default().await?))
} }
@@ -96,8 +175,104 @@ impl K8sClient {
return None; return None;
} }
}; };
Some(Self::new( let connection = ClusterConnection::from_kubeconfig(&k, opts)
Client::try_from(Config::from_custom_kubeconfig(k, opts).await.unwrap()).unwrap(), .map_err(|error| error!("Tenant credentials unavailable for {path}: {error}"))
)) .ok();
let config = match Config::from_custom_kubeconfig(k, opts).await {
Ok(config) => config,
Err(error) => {
error!("Failed to load kubeconfig from {path}: {error}");
return None;
}
};
let client = match Client::try_from(config) {
Ok(client) => client,
Err(error) => {
error!("Failed to create Kubernetes client from {path}: {error}");
return None;
}
};
let mut client = Self::new(client);
client.cluster_connection = connection;
Some(client)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn connection_uses_selected_context_without_authentication() {
let kubeconfig: Kubeconfig = serde_yaml::from_str(
r#"
current-context: admin
contexts:
- name: admin
context:
cluster: shared
user: cluster-admin
clusters:
- name: shared
cluster:
server: https://api.example.com:6443
certificate-authority-data: Y2E=
users:
- name: cluster-admin
user:
token: super-secret
"#,
)
.unwrap();
assert_eq!(
ClusterConnection::from_kubeconfig(&kubeconfig, &KubeConfigOptions::default()).unwrap(),
ClusterConnection {
name: "shared".to_string(),
server: "https://api.example.com:6443".to_string(),
tls_server_name: None,
proxy_url: None,
}
);
}
#[test]
fn connection_rejects_credentials_in_server_url() {
let kubeconfig: Kubeconfig = serde_yaml::from_str(
r#"
current-context: admin
contexts:
- name: admin
context: { cluster: shared }
clusters:
- name: shared
cluster: { server: "https://admin:secret@api.example.com" }
"#,
)
.unwrap();
assert!(
ClusterConnection::from_kubeconfig(&kubeconfig, &KubeConfigOptions::default()).is_err()
);
}
#[test]
fn connection_rejects_plain_http_server() {
let kubeconfig: Kubeconfig = serde_yaml::from_str(
r#"
current-context: admin
contexts:
- name: admin
context: { cluster: shared }
clusters:
- name: shared
cluster: { server: "http://api.example.com" }
"#,
)
.unwrap();
assert!(
ClusterConnection::from_kubeconfig(&kubeconfig, &KubeConfigOptions::default()).is_err()
);
} }
} }

View File

@@ -12,6 +12,6 @@ pub mod port_forward;
pub mod resources; pub mod resources;
pub mod types; pub mod types;
pub use client::K8sClient; pub use client::{ClusterConnection, K8sClient};
pub use port_forward::PortForwardHandle; pub use port_forward::PortForwardHandle;
pub use types::{DrainOptions, KubernetesDistribution, NodeFile, ScopeResolver, WriteMode}; pub use types::{DrainOptions, KubernetesDistribution, NodeFile, ScopeResolver, WriteMode};

View File

@@ -149,14 +149,28 @@ impl K8sClient {
Ok(!crds.items.is_empty()) Ok(!crds.items.is_empty())
} }
/// Polls until a CRD is registered in the API server. pub async fn crd_scope(&self, name: &str) -> Result<Option<String>, Error> {
let api: Api<CustomResourceDefinition> = Api::all(self.client.clone());
Ok(api.get_opt(name).await?.map(|crd| crd.spec.scope))
}
/// Polls until a CRD is established in the API server.
pub async fn wait_for_crd(&self, name: &str, timeout: Option<Duration>) -> Result<(), Error> { pub async fn wait_for_crd(&self, name: &str, timeout: Option<Duration>) -> Result<(), Error> {
let timeout = timeout.unwrap_or(Duration::from_secs(60)); let timeout = timeout.unwrap_or(Duration::from_secs(60));
let start = std::time::Instant::now(); let start = std::time::Instant::now();
let poll = Duration::from_secs(2); let poll = Duration::from_secs(2);
loop { loop {
if self.has_crd(name).await? { let api: Api<CustomResourceDefinition> = Api::all(self.client.clone());
if let Some(crd) = api.get_opt(name).await?
&& crd.status.as_ref().is_some_and(|status| {
status.conditions.as_ref().is_some_and(|conditions| {
conditions.iter().any(|condition| {
condition.type_ == "Established" && condition.status == "True"
})
})
})
{
return Ok(()); return Ok(());
} }
if start.elapsed() > timeout { if start.elapsed() > timeout {

View File

@@ -9,12 +9,14 @@ use serde::Serialize;
use crate::data::Version; use crate::data::Version;
use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use crate::inventory::Inventory; use crate::inventory::Inventory;
use crate::modules::nats::score_nats_k8s::{WebSocketRouteCfg, websocket_route_score};
use crate::modules::nats::{ use crate::modules::nats::{
NatsAccountRef, NatsAuthCalloutCredentialsRef, NatsAuthCalloutRef, NatsClientRef, NatsAccountRef, NatsAuthCalloutCredentialsRef, NatsAuthCalloutRef, NatsClientRef,
NatsHelmChartScore, NatsHelmChartScore,
}; };
use crate::score::Score; use crate::score::Score;
use crate::topology::{HelmCommand, K8sclient, Topology}; use crate::topology::{HelmCommand, K8sclient, Topology};
use harmony_k8s::KubernetesDistribution;
/// Authentication mode the deployed NATS server enforces. /// Authentication mode the deployed NATS server enforces.
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
@@ -78,6 +80,7 @@ pub struct NatsScore {
/// per-device KV buckets a small fleet uses. /// per-device KV buckets a small fleet uses.
pub jetstream_size: String, pub jetstream_size: String,
pub image: Option<String>, pub image: Option<String>,
pub websocket: Option<WebSocketRouteCfg>,
} }
#[derive(Debug, Clone, Copy, Serialize)] #[derive(Debug, Clone, Copy, Serialize)]
@@ -96,6 +99,7 @@ impl NatsScore {
service: NatsService::ClusterIp, service: NatsService::ClusterIp,
jetstream_size: "10Gi".to_string(), jetstream_size: "10Gi".to_string(),
image: None, image: None,
websocket: None,
} }
} }
@@ -123,6 +127,7 @@ impl NatsScore {
service, service,
jetstream_size: "10Gi".to_string(), jetstream_size: "10Gi".to_string(),
image: None, image: None,
websocket: None,
} }
} }
@@ -142,6 +147,7 @@ impl NatsScore {
service, service,
jetstream_size: "10Gi".to_string(), jetstream_size: "10Gi".to_string(),
image: None, image: None,
websocket: None,
} }
} }
@@ -165,6 +171,14 @@ impl NatsScore {
self self
} }
pub fn websocket(mut self, host: impl Into<String>, cluster_issuer: impl Into<String>) -> Self {
self.websocket = Some(WebSocketRouteCfg {
host: host.into(),
cluster_issuer: cluster_issuer.into(),
});
self
}
pub fn with_auth_callout(mut self, callout: &NatsAuthCalloutRef) -> Self { pub fn with_auth_callout(mut self, callout: &NatsAuthCalloutRef) -> Self {
assert_eq!(callout.credentials().namespace(), self.namespace); assert_eq!(callout.credentials().namespace(), self.namespace);
self.auth = NatsAuth::Callout { self.auth = NatsAuth::Callout {
@@ -210,6 +224,7 @@ impl NatsScore {
&self.jetstream_size, &self.jetstream_size,
&self.auth, &self.auth,
self.image.as_deref(), self.image.as_deref(),
self.websocket.is_some(),
)?; )?;
serde_yaml::to_string(&values).map_err(NatsScoreError::RenderValues) serde_yaml::to_string(&values).map_err(NatsScoreError::RenderValues)
} }
@@ -273,9 +288,17 @@ struct NatsValuesSecretKeyRef {
struct NatsValuesConfig { struct NatsValuesConfig {
cluster: NatsValuesCluster, cluster: NatsValuesCluster,
jetstream: NatsValuesJetstream, jetstream: NatsValuesJetstream,
#[serde(skip_serializing_if = "Option::is_none")]
websocket: Option<NatsValuesWebSocket>,
merge: NatsValuesConfigMerge, merge: NatsValuesConfigMerge,
} }
#[derive(Debug, Serialize)]
struct NatsValuesWebSocket {
enabled: bool,
no_tls: bool,
}
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
struct NatsValuesCluster { struct NatsValuesCluster {
enabled: bool, enabled: bool,
@@ -377,6 +400,7 @@ fn build_values(
jetstream_size: &str, jetstream_size: &str,
auth: &NatsAuth, auth: &NatsAuth,
image: Option<&str>, image: Option<&str>,
websocket: bool,
) -> Result<NatsValues, NatsScoreError> { ) -> Result<NatsValues, NatsScoreError> {
let (authorization, accounts, container) = match auth { let (authorization, accounts, container) = match auth {
NatsAuth::None => (None, BTreeMap::new(), None), NatsAuth::None => (None, BTreeMap::new(), None),
@@ -472,6 +496,10 @@ fn build_values(
}, },
}, },
}, },
websocket: websocket.then_some(NatsValuesWebSocket {
enabled: true,
no_tls: true,
}),
merge: NatsValuesConfigMerge { merge: NatsValuesConfigMerge {
authorization, authorization,
accounts, accounts,
@@ -589,6 +617,27 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for NatsInterpret {
) )
.await .await
.map_err(InterpretError::new)?; .map_err(InterpretError::new)?;
if let Some(websocket) = &self.score.websocket {
let client = topology.k8s_client().await.map_err(InterpretError::new)?;
if client
.get_k8s_distribution()
.await
.map_err(|error| InterpretError::new(error.to_string()))?
!= KubernetesDistribution::OpenshiftFamily
{
return Err(InterpretError::new(
"NATS WebSocket exposure currently requires OpenShift".to_string(),
));
}
websocket_route_score(
&self.score.release_name,
&self.score.namespace,
&websocket.host,
&websocket.cluster_issuer,
)
.interpret(inventory, topology)
.await?;
}
Ok(outcome) Ok(outcome)
} }
@@ -670,6 +719,20 @@ mod tests {
); );
} }
#[test]
fn websocket_enables_plain_chart_listener() {
let score = user_pass(test_creds()).websocket("nats.example.com", "letsencrypt-prod");
let values = parse(&score.values_yaml().expect("renders"));
assert_eq!(
values["config"]["websocket"]["enabled"],
serde_yaml::Value::Bool(true)
);
assert_eq!(
values["config"]["websocket"]["no_tls"],
serde_yaml::Value::Bool(true)
);
}
#[test] #[test]
fn values_escape_special_yaml_characters() { fn values_escape_special_yaml_characters() {
// Passwords with YAML-significant characters used to round-trip // Passwords with YAML-significant characters used to round-trip

View File

@@ -45,6 +45,34 @@ pub struct WebSocketRouteCfg {
pub cluster_issuer: String, pub cluster_issuer: String,
} }
pub(crate) fn websocket_route_score(
name: &str,
namespace: &str,
host: &str,
cluster_issuer: &str,
) -> OKDRouteScore {
OKDRouteScore::new(
&format!("{name}-ws"),
namespace,
RouteSpec {
to: RouteTargetReference {
kind: "Service".to_string(),
name: name.to_string(),
weight: Some(100),
},
host: Some(host.to_string()),
port: Some(RoutePort { target_port: 8080 }),
tls: Some(TLSConfig {
termination: "edge".to_string(),
insecure_edge_termination_policy: Some("Redirect".to_string()),
..Default::default()
}),
..Default::default()
},
)
.with_annotation("cert-manager.io/cluster-issuer", cluster_issuer)
}
/// Auth-callout configuration for the NATS server. When `Some`, the /// Auth-callout configuration for the NATS server. When `Some`, the
/// rendered Helm values include an `authorization.auth_callout` block /// rendered Helm values include an `authorization.auth_callout` block
/// referencing this issuer pubkey, plus an `accounts.<account>` block /// referencing this issuer pubkey, plus an `accounts.<account>` block
@@ -271,31 +299,12 @@ impl NatsK8sInterpret {
) -> Result<Outcome, InterpretError> { ) -> Result<Outcome, InterpretError> {
match distribution { match distribution {
KubernetesDistribution::OpenshiftFamily => { KubernetesDistribution::OpenshiftFamily => {
OKDRouteScore::new( websocket_route_score(
&format!("{}-ws", nats_cluster.name), &nats_cluster.name,
&nats_cluster.namespace, &nats_cluster.namespace,
RouteSpec { &ws.host,
to: RouteTargetReference { &ws.cluster_issuer,
kind: "Service".to_string(),
name: nats_cluster.name.clone(),
weight: Some(100),
},
host: Some(ws.host.clone()),
// Chart names this Service port `websocket` and
// exposes it on 8080. We target by number to keep
// RoutePort u16-typed and avoid string-vs-int
// dance in the OpenAPI shape.
port: Some(RoutePort { target_port: 8080 }),
tls: Some(TLSConfig {
termination: "edge".to_string(),
insecure_edge_termination_policy: Some("Redirect".to_string()),
..Default::default()
}),
wildcard_policy: None,
..Default::default()
},
) )
.with_annotation("cert-manager.io/cluster-issuer", ws.cluster_issuer.clone())
.interpret(inventory, topology) .interpret(inventory, topology)
.await .await
} }

View File

@@ -37,6 +37,7 @@ use std::collections::BTreeMap;
use std::hash::{DefaultHasher, Hash, Hasher}; use std::hash::{DefaultHasher, Hash, Hasher};
use async_trait::async_trait; use async_trait::async_trait;
use harmony_types::k8s_name::K8sName;
use k8s_openapi::ByteString; use k8s_openapi::ByteString;
use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::apps::v1::Deployment;
use k8s_openapi::api::core::v1::Secret; use k8s_openapi::api::core::v1::Secret;
@@ -122,7 +123,7 @@ pub struct NatsAuthCalloutScore {
/// for local dev — Zitadel-on-k3d typically uses HTTP, but in /// for local dev — Zitadel-on-k3d typically uses HTTP, but in
/// development with a self-signed Zitadel cert this is the escape hatch). /// development with a self-signed Zitadel cert this is the escape hatch).
pub danger_accept_invalid_certs: bool, pub danger_accept_invalid_certs: bool,
pub image_pull_secret: Option<String>, pub image_pull_secret: Option<K8sName>,
} }
impl NatsAuthCalloutScore { impl NatsAuthCalloutScore {
@@ -244,8 +245,8 @@ impl NatsAuthCalloutScore {
self self
} }
pub fn image_pull_secret(mut self, secret: Option<&str>) -> Self { pub fn image_pull_secret(mut self, secret: Option<K8sName>) -> Self {
self.image_pull_secret = secret.map(str::to_string); self.image_pull_secret = secret;
self self
} }

View File

@@ -1,50 +1,380 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait; use async_trait::async_trait;
use chrono::{DateTime, Utc}; use harmony_config::{Config, ConfigClient};
use serde::Serialize; use harmony_k8s::ClusterConnection;
use harmony_types::id::Id;
use harmony_types::k8s_name::K8sName;
use k8s_openapi::api::core::v1::{Secret, ServiceAccount};
use k8s_openapi::api::rbac::v1::{
ClusterRole, ClusterRoleBinding, PolicyRule, Role, RoleBinding, RoleRef, Subject,
};
use kube::api::ObjectMeta;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::{interpret::InterpretError, score::Score, topology::Topology}; use crate::data::Version;
use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use crate::inventory::Inventory;
use crate::score::Score;
use crate::topology::{K8sclient, Topology};
/// Create and manage Tenant Credentials. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
/// pub struct ClusterAccess {
/// This is meant to be used by cluster administrators who need to provide their tenant users and #[config(secret)]
/// services with credentials to access their resources. pub kubeconfig: String,
#[derive(Debug, Clone, Serialize)] }
pub struct TenantCredentialScore;
impl<T: Topology + TenantCredentialManager> Score<T> for TenantCredentialScore { #[derive(Clone, Serialize)]
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> { pub struct TenantCredentialScore {
todo!() namespace: K8sName,
name: K8sName,
rules: Vec<PolicyRule>,
#[serde(skip)]
store: Arc<ConfigClient>,
}
impl std::fmt::Debug for TenantCredentialScore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TenantCredentialScore")
.field("namespace", &self.namespace)
.field("name", &self.name)
.field("rules", &self.rules)
.finish_non_exhaustive()
}
}
impl TenantCredentialScore {
pub fn new(
namespace: K8sName,
name: K8sName,
rules: Vec<PolicyRule>,
store: Arc<ConfigClient>,
) -> Self {
Self {
namespace,
name,
rules,
store,
}
}
fn service_account(&self) -> ServiceAccount {
ServiceAccount {
metadata: ObjectMeta {
name: Some(self.name.to_string()),
namespace: Some(self.namespace.to_string()),
..Default::default()
},
automount_service_account_token: Some(false),
..Default::default()
}
}
fn role(&self) -> Role {
Role {
metadata: ObjectMeta {
name: Some(self.name.to_string()),
namespace: Some(self.namespace.to_string()),
..Default::default()
},
rules: Some(self.rules.clone()),
}
}
fn role_binding(&self) -> RoleBinding {
RoleBinding {
metadata: ObjectMeta {
name: Some(self.name.to_string()),
namespace: Some(self.namespace.to_string()),
..Default::default()
},
role_ref: RoleRef {
api_group: "rbac.authorization.k8s.io".to_string(),
kind: "Role".to_string(),
name: self.name.to_string(),
},
subjects: Some(vec![self.subject()]),
}
}
fn cluster_role_name(&self) -> String {
format!("{}-{}-namespace-reader", self.namespace, self.name)
}
fn cluster_role(&self) -> ClusterRole {
ClusterRole {
metadata: ObjectMeta {
name: Some(self.cluster_role_name()),
..Default::default()
},
rules: Some(vec![PolicyRule {
api_groups: Some(vec![String::new()]),
resource_names: Some(vec![self.namespace.to_string()]),
resources: Some(vec!["namespaces".to_string()]),
verbs: vec!["get".to_string()],
..Default::default()
}]),
aggregation_rule: None,
}
}
fn cluster_role_binding(&self) -> ClusterRoleBinding {
ClusterRoleBinding {
metadata: ObjectMeta {
name: Some(self.cluster_role_name()),
..Default::default()
},
role_ref: RoleRef {
api_group: "rbac.authorization.k8s.io".to_string(),
kind: "ClusterRole".to_string(),
name: self.cluster_role_name(),
},
subjects: Some(vec![self.subject()]),
}
}
fn subject(&self) -> Subject {
Subject {
kind: "ServiceAccount".to_string(),
name: self.name.to_string(),
namespace: Some(self.namespace.to_string()),
..Default::default()
}
}
fn token_secret(&self) -> Secret {
Secret {
metadata: ObjectMeta {
name: Some(format!("{}-token", self.name)),
namespace: Some(self.namespace.to_string()),
annotations: Some(BTreeMap::from([(
"kubernetes.io/service-account.name".to_string(),
self.name.to_string(),
)])),
..Default::default()
},
type_: Some("kubernetes.io/service-account-token".to_string()),
..Default::default()
}
}
}
impl<T: Topology + K8sclient> Score<T> for TenantCredentialScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(TenantCredentialInterpret {
score: self.clone(),
})
} }
fn name(&self) -> String { fn name(&self) -> String {
"TenantCredentialScore".into() format!("{} [TenantCredentialScore]", self.namespace)
} }
} }
#[derive(Debug)]
struct TenantCredentialInterpret {
score: TenantCredentialScore,
}
#[async_trait] #[async_trait]
pub trait TenantCredentialManager { impl<T: Topology + K8sclient> Interpret<T> for TenantCredentialInterpret {
async fn create_user(&self) -> Result<TenantCredentialBundle, InterpretError>; async fn execute(
&self,
_inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let client = topology
.k8s_client()
.await
.map_err(|error| InterpretError::new(format!("get Kubernetes client: {error}")))?;
if client.is_dry_run() {
return Err(InterpretError::new(
"tenant credentials cannot be issued in dry-run mode".to_string(),
));
}
let connection = client.cluster_connection().cloned().ok_or_else(|| {
InterpretError::new(
"tenant credentials require a secure topology loaded from a kubeconfig".to_string(),
)
})?;
let namespace = self.score.namespace.as_ref();
client
.apply(&self.score.service_account(), Some(namespace))
.await
.map_err(|error| {
InterpretError::new(format!("apply deployer ServiceAccount: {error}"))
})?;
client
.apply(&self.score.role(), Some(namespace))
.await
.map_err(|error| InterpretError::new(format!("apply deployer Role: {error}")))?;
client
.apply(&self.score.role_binding(), Some(namespace))
.await
.map_err(|error| InterpretError::new(format!("apply deployer RoleBinding: {error}")))?;
client
.apply(&self.score.cluster_role(), None)
.await
.map_err(|error| {
InterpretError::new(format!("apply namespace reader ClusterRole: {error}"))
})?;
client
.apply(&self.score.cluster_role_binding(), None)
.await
.map_err(|error| {
InterpretError::new(format!(
"apply namespace reader ClusterRoleBinding: {error}"
))
})?;
let token_secret = self.score.token_secret();
client
.apply(&token_secret, Some(namespace))
.await
.map_err(|error| {
InterpretError::new(format!("apply deployer token Secret: {error}"))
})?;
let secret_name = token_secret.metadata.name.as_deref().unwrap();
let started = std::time::Instant::now();
let (token, certificate_authority_data) = loop {
if let Some(secret) = client
.get_resource::<Secret>(secret_name, Some(namespace))
.await
.map_err(|error| {
InterpretError::new(format!("read deployer token Secret: {error}"))
})?
&& let Some(data) = secret.data
&& let (Some(token), Some(ca)) = (data.get("token"), data.get("ca.crt"))
{
break (
String::from_utf8(token.0.clone()).map_err(|error| {
InterpretError::new(format!("deployer token is not UTF-8: {error}"))
})?,
base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &ca.0),
);
}
if started.elapsed() >= Duration::from_secs(60) {
return Err(InterpretError::new(
"Kubernetes did not populate the deployer token Secret within 60 seconds"
.to_string(),
));
}
tokio::time::sleep(Duration::from_secs(1)).await;
};
let kubeconfig = tenant_kubeconfig(
&connection,
namespace,
self.score.name.as_ref(),
&token,
&certificate_authority_data,
)?;
self.score
.store
.set(&ClusterAccess { kubeconfig })
.await
.map_err(|error| InterpretError::new(format!("store tenant ClusterAccess: {error}")))?;
Ok(Outcome::success(format!(
"tenant deployer access stored for namespace '{}'",
self.score.namespace
)))
}
fn get_name(&self) -> InterpretName {
InterpretName::Custom("TenantCredentialInterpret")
}
fn get_version(&self) -> Version {
Version::from(env!("CARGO_PKG_VERSION")).expect("package version")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::SUCCESS
}
fn get_children(&self) -> Vec<Id> {
Vec::new()
}
} }
#[derive(Debug, Clone)] fn tenant_kubeconfig(
pub struct CredentialMetadata { connection: &ClusterConnection,
pub tenant_id: String, namespace: &str,
pub credential_id: String, user: &str,
pub description: String, token: &str,
pub created_at: DateTime<Utc>, certificate_authority_data: &str,
pub expires_at: Option<DateTime<Utc>>, ) -> Result<String, InterpretError> {
let context = format!("{user}@{}", connection.name);
let mut cluster = serde_json::Map::from_iter([
("server".to_string(), connection.server.clone().into()),
(
"certificate-authority-data".to_string(),
certificate_authority_data.into(),
),
]);
if let Some(name) = &connection.tls_server_name {
cluster.insert("tls-server-name".to_string(), name.clone().into());
}
if let Some(url) = &connection.proxy_url {
cluster.insert("proxy-url".to_string(), url.clone().into());
}
serde_yaml::to_string(&serde_json::json!({
"apiVersion": "v1",
"kind": "Config",
"clusters": [{ "name": connection.name, "cluster": cluster }],
"users": [{ "name": user, "user": { "token": token } }],
"contexts": [{
"name": context,
"context": {
"cluster": connection.name,
"user": user,
"namespace": namespace,
}
}],
"current-context": context,
}))
.map_err(|error| InterpretError::new(format!("serialize tenant kubeconfig: {error}")))
} }
#[derive(Debug, Clone)] #[cfg(test)]
pub enum CredentialData { mod tests {
/// Used to store login instructions destined to a human. Akin to AWS login instructions email use super::*;
/// upon new console user creation.
PlainText(String),
}
pub struct TenantCredentialBundle { #[test]
_metadata: CredentialMetadata, fn kubeconfig_contains_only_tenant_identity() {
_content: CredentialData, let kubeconfig = tenant_kubeconfig(
} &ClusterConnection {
name: "shared".to_string(),
server: "https://api.example.com:6443".to_string(),
tls_server_name: None,
proxy_url: None,
},
"customer-fleet",
"fleet-deployer",
"tenant-token",
"Y2E=",
)
.unwrap();
impl TenantCredentialBundle {} assert!(kubeconfig.contains("tenant-token"));
assert!(kubeconfig.contains("namespace: customer-fleet"));
assert!(!kubeconfig.contains("cluster-admin"));
}
#[test]
fn score_serialization_excludes_config_destination() {
let score = TenantCredentialScore::new(
"customer-fleet".parse().unwrap(),
"fleet-deployer".parse().unwrap(),
Vec::new(),
Arc::new(ConfigClient::new(Vec::new())),
);
let serialized = serde_json::to_string(&score).unwrap();
assert!(!serialized.contains("store"));
}
}

View File

@@ -17,7 +17,6 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true } serde_yaml = { workspace = true }
schemars = "0.8" schemars = "0.8"
tempfile.workspace = true tempfile.workspace = true
toml.workspace = true
log.workspace = true log.workspace = true
reqwest.workspace = true reqwest.workspace = true
k8s-openapi.workspace = true k8s-openapi.workspace = true

View File

@@ -30,7 +30,7 @@ pub struct AppIdentity {
/// compile error, not a runtime surprise. /// compile error, not a runtime surprise.
#[async_trait] #[async_trait]
pub trait HarmonyApp<T: Topology>: Send + Sync { pub trait HarmonyApp<T: Topology>: Send + Sync {
fn identity(&self) -> AppIdentity; fn identity(&self, ctx: &AppContext) -> AppIdentity;
/// The desired state for this context — the app composes its Scores, /// The desired state for this context — the app composes its Scores,
/// branching on `ctx.profile()`. Called by `deploy`/`ship`. /// branching on `ctx.profile()`. Called by `deploy`/`ship`.
@@ -53,14 +53,14 @@ pub trait HarmonyApp<T: Topology>: Send + Sync {
} }
fn build(&self, ctx: &AppContext) -> Result<ImageRefs, AppError> { fn build(&self, ctx: &AppContext) -> Result<ImageRefs, AppError> {
Ok(build_images(&self.images(ctx)?, &ctx.publisher()?)?) Ok(build_images(&self.images(ctx)?, &ctx.publisher())?)
} }
fn publish(&self, ctx: &AppContext, images: &ImageRefs) -> Result<ImageRefs, AppError> { fn publish(&self, ctx: &AppContext, images: &ImageRefs) -> Result<ImageRefs, AppError> {
Ok(publish_images( Ok(publish_images(
&self.images(ctx)?, &self.images(ctx)?,
images, images,
&ctx.publisher()?, &ctx.publisher(),
)?) )?)
} }
} }
@@ -75,6 +75,9 @@ pub struct StepOutcome {
#[derive(Debug, Clone, serde::Serialize)] #[derive(Debug, Clone, serde::Serialize)]
pub struct DeployReport { pub struct DeployReport {
pub context: String,
pub namespace: String,
pub cluster: Option<String>,
pub steps: Vec<StepOutcome>, pub steps: Vec<StepOutcome>,
} }
@@ -151,7 +154,12 @@ pub async fn deploy_with_options<T: Topology + Send + Sync + 'static>(
message: outcome.message, message: outcome.message,
}); });
} }
Ok(DeployReport { steps }) Ok(DeployReport {
context: ctx.name().to_string(),
namespace: ctx.namespace().to_string(),
cluster: ctx.cluster_target().map(str::to_string),
steps,
})
} }
/// Build + publish, then deploy (ADR-026 §4). /// Build + publish, then deploy (ADR-026 §4).
@@ -179,7 +187,7 @@ pub async fn status<T: Topology>(
app: &dyn HarmonyApp<T>, app: &dyn HarmonyApp<T>,
ctx: &AppContext, ctx: &AppContext,
) -> Result<StatusReport, AppError> { ) -> Result<StatusReport, AppError> {
let id = app.identity(); let id = app.identity(ctx);
let client = ctx.k8s_client().await?; let client = ctx.k8s_client().await?;
let deployments = client let deployments = client
.list_resources::<Deployment>(Some(&id.namespace), None) .list_resources::<Deployment>(Some(&id.namespace), None)
@@ -210,7 +218,7 @@ pub async fn logs<T: Topology>(
ctx: &AppContext, ctx: &AppContext,
tail: Option<i64>, tail: Option<i64>,
) -> Result<Vec<PodLogs>, AppError> { ) -> Result<Vec<PodLogs>, AppError> {
let id = app.identity(); let id = app.identity(ctx);
let client = ctx.k8s_client().await?; let client = ctx.k8s_client().await?;
let pods = client let pods = client
.list_resources::<Pod>(Some(&id.namespace), None) .list_resources::<Pod>(Some(&id.namespace), None)

View File

@@ -1,10 +1,4 @@
//! [`AppContext`] — the selected deploy target (ADR-026 §1/§10). Contexts are //! Compiled deploy contexts and their resolved runtime form.
//! defined **in-repo** at `.harmony/contexts.toml` (found by walking up from
//! the cwd), so collaborators and CI need zero local setup. A context owns
//! *cluster access* — either a local k3d cluster, or a kubeconfig brokered
//! from OpenBao via `harmony_config` (Zitadel-authenticated, the CI path).
//! App *secrets* are separate (the app loads those); the context only says
//! how to reach the cluster.
use std::ffi::OsString; use std::ffi::OsString;
use std::io::Write; use std::io::Write;
@@ -12,226 +6,281 @@ use std::path::{Path, PathBuf};
use std::process::Command; use std::process::Command;
use std::sync::Arc; use std::sync::Arc;
use harmony::modules::tenant::ClusterAccess;
use harmony::topology::{K8sAnywhereConfig, K8sAnywhereTopology}; use harmony::topology::{K8sAnywhereConfig, K8sAnywhereTopology};
use harmony_config::{Config, ConfigClient, ConfigSource, LocalFileSource, PromptSource}; use harmony_config::{ConfigClient, ConfigSource, LocalFileSource, PromptSource};
use harmony_k8s::K8sClient; use harmony_k8s::K8sClient;
use harmony_types::context::{
ContextName, DomainName, HttpUrl, OciRegistry, OciRepository, OidcAudience, OpenBaoNamespace,
OpenBaoRoleName,
};
use harmony_types::k8s_name::K8sName;
use log::{debug, info}; use log::{debug, info};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
use crate::profile::Profile; use crate::profile::Profile;
use crate::publish::PublicationTopology; use crate::publish::PublicationTopology;
use crate::{ContextError, error::io}; use crate::{ContextError, error::io};
/// `.harmony/contexts.toml`: `[contexts.<name>]` blocks. #[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Debug, Deserialize)] pub struct Context {
struct ContextsFile { pub name: ContextName,
contexts: std::collections::HashMap<String, ContextDef>, pub namespace: K8sName,
pub spec: ContextSpec,
} }
/// One named context. Cluster access is exactly one of: `autoprovision` #[derive(Clone, Debug, PartialEq, Eq)]
/// (K8sAnywhere installs a local k3d), `k3d` (an existing local cluster), or pub enum ContextSpec {
/// `openbao_namespace` (kubeconfig brokered from OpenBao, the CI/prod path). Local(LocalContext),
#[derive(Debug, Deserialize)] Remote(RemoteContext),
struct ContextDef { }
profile: Profile,
registry: Option<String>, #[derive(Clone, Debug, PartialEq, Eq)]
project: Option<String>, pub enum LocalContext {
domain: Option<String>, ManagedK3d,
image_pull_secret: Option<String>, ExistingK3d { cluster: K8sName },
/// Let K8sAnywhereTopology autoprovision a local k3d cluster (offline; no }
/// pre-created cluster, no kubeconfig). The hands-off local/CI path.
#[serde(default)] #[derive(Clone, Debug, PartialEq, Eq)]
autoprovision: bool, pub struct RemoteContext {
k3d: Option<String>, pub registry: OciRegistry,
openbao_namespace: Option<String>, pub repository: OciRepository,
openbao_url: Option<String>, pub domain: DomainName,
zitadel_url: Option<String>, pub image_pull_secret: Option<K8sName>,
openbao_role: Option<String>, pub access: OpenBaoClusterAccess,
zitadel_audience: Option<String>, }
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OpenBaoClusterAccess {
pub namespace: OpenBaoNamespace,
pub url: HttpUrl,
pub role: OpenBaoRoleName,
pub zitadel_url: HttpUrl,
pub zitadel_audience: OidcAudience,
}
#[derive(Clone, Debug)]
pub struct ContextCatalog {
contexts: Vec<Context>,
}
impl ContextCatalog {
pub fn new(contexts: impl IntoIterator<Item = Context>) -> Result<Self, ContextError> {
let contexts = contexts.into_iter().collect::<Vec<_>>();
if contexts.is_empty() {
return Err(ContextError::Invalid(
"context catalog must not be empty".to_string(),
));
}
for (index, context) in contexts.iter().enumerate() {
if contexts[..index]
.iter()
.any(|candidate| candidate.name == context.name)
{
return Err(ContextError::Invalid(format!(
"duplicate context '{}'",
context.name
)));
}
}
Ok(Self { contexts })
}
pub fn require(&self, name: impl AsRef<str>) -> Result<&Context, ContextError> {
let name = name.as_ref();
self.contexts
.iter()
.find(|context| context.name.as_ref() == name)
.ok_or_else(|| {
ContextError::Missing(format!(
"context '{name}' not found (have: {})",
self.available_names()
))
})
}
pub fn names(&self) -> impl Iterator<Item = &ContextName> {
self.contexts.iter().map(|context| &context.name)
}
pub fn available_names(&self) -> String {
self.names()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", ")
}
}
impl From<&ContextSpec> for Profile {
fn from(spec: &ContextSpec) -> Self {
match spec {
ContextSpec::Local(_) => Self::Local,
ContextSpec::Remote(_) => Self::Prod,
}
}
} }
/// The cluster K8sAnywhereTopology autoprovisions (its K3DInstallationScore
/// default). Named so operational verbs and `publish` (k3d image import) find
/// the same cluster the deploy created.
const AUTOPROVISION_CLUSTER: &str = "harmony"; const AUTOPROVISION_CLUSTER: &str = "harmony";
/// Cluster credential brokered from OpenBao — its own secret
/// (`secret/<ns>/ClusterAccess`), separate from the app's secrets, so
/// "how to reach the cluster" and "the app's config" stay distinct.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Config)]
pub struct ClusterAccess {
#[config(secret)]
pub kubeconfig: String,
}
pub struct AppContext { pub struct AppContext {
name: String, context: Context,
profile: Profile,
version: String, version: String,
registry: Option<String>,
project: Option<String>,
domain: Option<String>,
image_pull_secret: Option<String>,
local_config_dir: Option<PathBuf>, local_config_dir: Option<PathBuf>,
k3d_cluster: Option<String>,
/// True when K8sAnywhere autoprovisions the cluster: there is no kubeconfig
/// at resolve time (the cluster is created during deploy), so the topology
/// is built env-free and operational verbs resolve the k3d kubeconfig lazily.
autoprovision: bool,
/// The brokered kubeconfig (k3d/OpenBao paths). `None` under autoprovision.
kubeconfig: Option<PathBuf>, kubeconfig: Option<PathBuf>,
/// The kubeconfig file must outlive every client/topology built from it.
_kubeconfig_guard: Option<NamedTempFile>, _kubeconfig_guard: Option<NamedTempFile>,
config_client: Arc<ConfigClient>, config_client: Arc<ConfigClient>,
cluster_target: Option<String>,
} }
impl AppContext { impl AppContext {
/// Load context metadata without contacting the cluster credential source. /// Load context metadata without contacting the cluster credential source.
pub fn load_metadata( pub fn load_metadata(
name: &str, context: &Context,
version: impl Into<String>, version: impl Into<String>,
local_config_dir: Option<PathBuf>, local_config_dir: Option<PathBuf>,
contexts_file: Option<PathBuf>, ) -> Self {
) -> Result<Self, ContextError> { Self::new(
let def = read_context(name, contexts_file)?; context,
Ok(Self::from_definition(
name,
version.into(), version.into(),
local_config_dir, local_config_dir,
&def,
Arc::new(ConfigClient::new(Vec::new())), Arc::new(ConfigClient::new(Vec::new())),
None, None,
)) context_cluster_hint(&context.spec),
)
} }
/// Resolve a context by name from the in-repo `.harmony/contexts.toml`.
pub async fn resolve( pub async fn resolve(
name: &str, context: &Context,
version: impl Into<String>, version: impl Into<String>,
local_config_dir: Option<PathBuf>, local_config_dir: Option<PathBuf>,
contexts_file: Option<PathBuf>,
) -> Result<Self, ContextError> { ) -> Result<Self, ContextError> {
let def = read_context(name, contexts_file)?; let name = &context.name;
info!( info!(
"Resolving deploy context '{name}' (profile {:?})", "Resolving deploy context '{name}' (profile {:?})",
def.profile Profile::from(&context.spec)
); );
debug!("Context '{name}' definition: {def:?}"); debug!("Context '{name}' definition: {:?}", context.spec);
let config_sources = build_config_sources(&def, local_config_dir.clone()) let config_sources = build_config_sources(&context.spec, local_config_dir.clone())
.await .await
.map_err(|e| { .map_err(|e| {
ContextError::Config(format!("building config sources for context '{name}': {e}")) ContextError::Config(format!("building config sources for context '{name}': {e}"))
})?; })?;
harmony_config::init(config_sources.clone()).await; harmony_config::init(config_sources.clone()).await;
let config_client = Arc::new(ConfigClient::new(config_sources)); let config_client = Arc::new(ConfigClient::new(config_sources));
let guard = match (def.autoprovision, &def.k3d, &def.openbao_namespace) { let (guard, cluster_target) = match &context.spec {
(true, None, None) => { ContextSpec::Local(LocalContext::ManagedK3d) => {
info!("Cluster access: autoprovision local k3d ('{AUTOPROVISION_CLUSTER}')"); info!("Cluster access: autoprovision local k3d ('{AUTOPROVISION_CLUSTER}')");
None (None, Some(AUTOPROVISION_CLUSTER.to_string()))
} }
(false, Some(cluster), None) => { ContextSpec::Local(LocalContext::ExistingK3d { cluster }) => {
info!("Cluster access: existing local k3d cluster '{cluster}'"); info!("Cluster access: existing local k3d cluster '{cluster}'");
Some(k3d_kubeconfig(cluster)?) (Some(k3d_kubeconfig(&cluster.0)?), Some(cluster.to_string()))
} }
(false, None, Some(ns)) => { ContextSpec::Remote(remote) => {
info!("Cluster access: kubeconfig from OpenBao (namespace '{ns}')"); info!(
"Cluster access: kubeconfig from OpenBao (namespace '{}')",
remote.access.namespace
);
let access: ClusterAccess = config_client.get().await.map_err(|e| { let access: ClusterAccess = config_client.get().await.map_err(|e| {
ContextError::Config(format!( ContextError::Config(format!(
"loading cluster kubeconfig from OpenBao ({ns}/ClusterAccess) \ "loading cluster kubeconfig from OpenBao ({}/ClusterAccess) \
is HARMONY_ZITADEL_KEY_JSON set for this context?: {e}" - is HARMONY_ZITADEL_KEY_JSON set for this context?: {e}",
remote.access.namespace
)) ))
})?; })?;
Some(write_kubeconfig(access.kubeconfig.as_bytes())?) let target = kubeconfig_target(&access.kubeconfig)?;
(
Some(write_kubeconfig(access.kubeconfig.as_bytes())?),
Some(target),
)
} }
_ => unreachable!("context access mode was validated while loading"),
}; };
Ok(Self::from_definition( info!(
name, "Deployment target: context='{name}', cluster='{}', namespace='{}'",
cluster_target
.as_deref()
.unwrap_or("kubeconfig current-context"),
context.namespace
);
Ok(Self::new(
context,
version.into(), version.into(),
local_config_dir, local_config_dir,
&def,
config_client, config_client,
guard, guard,
cluster_target,
)) ))
} }
fn from_definition( fn new(
name: &str, context: &Context,
version: String, version: String,
local_config_dir: Option<PathBuf>, local_config_dir: Option<PathBuf>,
def: &ContextDef,
config_client: Arc<ConfigClient>, config_client: Arc<ConfigClient>,
guard: Option<NamedTempFile>, guard: Option<NamedTempFile>,
cluster_target: Option<String>,
) -> Self { ) -> Self {
Self { Self {
name: name.to_string(), context: context.clone(),
profile: def.profile,
version, version,
registry: def.registry.clone(),
project: def.project.clone(),
domain: def.domain.clone(),
image_pull_secret: def.image_pull_secret.clone(),
local_config_dir, local_config_dir,
k3d_cluster: def kubeconfig: guard.as_ref().map(|guard| guard.path().to_path_buf()),
.autoprovision
.then(|| AUTOPROVISION_CLUSTER.to_string())
.or_else(|| def.k3d.clone()),
autoprovision: def.autoprovision,
kubeconfig: guard.as_ref().map(|g| g.path().to_path_buf()),
_kubeconfig_guard: guard, _kubeconfig_guard: guard,
config_client, config_client,
cluster_target,
} }
} }
pub fn name(&self) -> &str { pub fn name(&self) -> &str {
&self.name self.context.name.as_ref()
}
pub fn namespace(&self) -> &str {
self.context.namespace.as_ref()
}
pub fn cluster_target(&self) -> Option<&str> {
self.cluster_target.as_deref()
} }
pub fn profile(&self) -> Profile { pub fn profile(&self) -> Profile {
self.profile Profile::from(&self.context.spec)
} }
pub fn version(&self) -> &str { pub fn version(&self) -> &str {
&self.version &self.version
} }
pub fn registry(&self) -> Option<&str> { pub fn registry(&self) -> Option<&str> {
self.registry.as_deref() match &self.context.spec {
} ContextSpec::Local(_) => None,
pub fn domain(&self) -> Result<&str, ContextError> { ContextSpec::Remote(remote) => Some(remote.registry.as_ref()),
self.domain
.as_deref()
.ok_or_else(|| ContextError::Missing("deploy context has no domain".to_string()))
}
pub fn service_host(&self, service: &str, namespace: &str) -> Result<String, ContextError> {
match self.profile {
Profile::Local => Ok(format!("{service}.{namespace}.svc.cluster.local")),
Profile::Prod => Ok(format!("{service}.{}", self.domain()?)),
} }
} }
pub fn image_pull_secret(&self) -> Result<Option<&str>, ContextError> { pub fn domain(&self) -> Option<&str> {
match self.profile { match &self.context.spec {
Profile::Local => Ok(None), ContextSpec::Local(_) => None,
Profile::Prod => self.image_pull_secret.as_deref().map(Some).ok_or_else(|| { ContextSpec::Remote(remote) => Some(remote.domain.as_ref()),
ContextError::Missing("prod context has no image_pull_secret".to_string())
}),
} }
} }
pub fn image(&self, name: &str) -> Result<String, ContextError> { pub fn service_host(&self, service: &str) -> String {
match self.profile { match &self.context.spec {
Profile::Local => Ok(format!("localhost/{name}:{}", self.version)), ContextSpec::Local(_) => {
Profile::Prod => Ok(format!( format!("{service}.{}.svc.cluster.local", self.context.namespace)
}
ContextSpec::Remote(remote) => format!("{service}.{}", remote.domain),
}
}
pub fn image_pull_secret(&self) -> Option<K8sName> {
match &self.context.spec {
ContextSpec::Local(_) => None,
ContextSpec::Remote(remote) => remote.image_pull_secret.clone(),
}
}
pub fn image(&self, name: &str) -> String {
match &self.context.spec {
ContextSpec::Local(_) => format!("localhost/{name}:{}", self.version),
ContextSpec::Remote(remote) => format!(
"{}/{}/{}:{}", "{}/{}/{}:{}",
self.registry.as_deref().ok_or_else(|| { remote.registry, remote.repository, name, self.version
ContextError::Missing("prod context has no registry".to_string()) ),
})?,
self.project.as_deref().ok_or_else(|| {
ContextError::Missing("prod context has no project".to_string())
})?,
name,
self.version
)),
} }
} }
pub fn local_config_dir(&self) -> Option<&Path> { pub fn local_config_dir(&self) -> Option<&Path> {
@@ -241,49 +290,50 @@ impl AppContext {
&self.config_client &self.config_client
} }
pub fn k3d_cluster(&self) -> Option<&str> { pub fn k3d_cluster(&self) -> Option<&str> {
self.k3d_cluster.as_deref() match &self.context.spec {
} ContextSpec::Local(LocalContext::ManagedK3d) => Some(AUTOPROVISION_CLUSTER),
ContextSpec::Local(LocalContext::ExistingK3d { cluster }) => Some(cluster.0.as_str()),
pub fn publisher(&self) -> Result<PublicationTopology, ContextError> { ContextSpec::Remote(_) => None,
if let Some(cluster) = &self.k3d_cluster { }
return Ok(PublicationTopology::K3d { }
cluster: cluster.clone(),
}); pub fn publisher(&self) -> PublicationTopology {
match &self.context.spec {
ContextSpec::Local(local) => PublicationTopology::K3d {
cluster: match local {
LocalContext::ManagedK3d => AUTOPROVISION_CLUSTER.to_string(),
LocalContext::ExistingK3d { cluster } => cluster.to_string(),
},
},
ContextSpec::Remote(remote) => PublicationTopology::Registry {
registry: remote.registry.to_string(),
},
} }
Ok(PublicationTopology::Registry {
registry: self.registry.clone().ok_or_else(|| {
ContextError::Missing("remote context has no registry".to_string())
})?,
})
} }
/// The converge target. Under autoprovision K8sAnywhere installs + manages
/// a local k3d cluster; otherwise it's pinned to the brokered kubeconfig
/// (no env, so we never deploy to the wrong cluster).
pub fn topology(&self) -> K8sAnywhereTopology { pub fn topology(&self) -> K8sAnywhereTopology {
if self.autoprovision { if matches!(
self.context.spec,
ContextSpec::Local(LocalContext::ManagedK3d)
) {
return K8sAnywhereTopology::with_config(K8sAnywhereConfig::local_k3d()); return K8sAnywhereTopology::with_config(K8sAnywhereConfig::local_k3d());
} }
let kubeconfig = self let kubeconfig = self
.kubeconfig .kubeconfig
.as_ref() .as_ref()
.expect("non-autoprovision context resolves a kubeconfig"); .expect("resolved non-managed context has a kubeconfig");
K8sAnywhereTopology::with_config(K8sAnywhereConfig::kubeconfig( K8sAnywhereTopology::with_config(K8sAnywhereConfig::kubeconfig(
kubeconfig.to_string_lossy().to_string(), kubeconfig.to_string_lossy().to_string(),
None, None,
)) ))
} }
/// A read client for operational verbs (status/logs) — no topology prep.
/// Under autoprovision the cluster exists only after a deploy, so its
/// kubeconfig is resolved lazily here from the managed k3d.
pub async fn k8s_client(&self) -> Result<K8sClient, ContextError> { pub async fn k8s_client(&self) -> Result<K8sClient, ContextError> {
let _guard; let _guard;
let path = match &self.kubeconfig { let path = match &self.kubeconfig {
Some(p) => p.clone(), Some(path) => path.clone(),
None => { None => {
let cluster = self.k3d_cluster.as_deref().unwrap_or(AUTOPROVISION_CLUSTER); let guard = k3d_kubeconfig(self.k3d_cluster().unwrap_or(AUTOPROVISION_CLUSTER))?;
let guard = k3d_kubeconfig(cluster)?;
let path = guard.path().to_path_buf(); let path = guard.path().to_path_buf();
_guard = guard; _guard = guard;
path path
@@ -297,94 +347,87 @@ impl AppContext {
} }
} }
fn read_context(name: &str, contexts_file: Option<PathBuf>) -> Result<ContextDef, ContextError> { fn context_cluster_hint(spec: &ContextSpec) -> Option<String> {
let path = match contexts_file { match spec {
Some(path) if path.is_file() => path, ContextSpec::Local(LocalContext::ManagedK3d) => Some(AUTOPROVISION_CLUSTER.to_string()),
Some(path) => { ContextSpec::Local(LocalContext::ExistingK3d { cluster }) => Some(cluster.to_string()),
return Err(ContextError::Missing(format!( ContextSpec::Remote(_) => None,
"contexts file not found: {}",
path.display()
)));
}
None => find_contexts_file()?,
};
let raw =
std::fs::read_to_string(&path).map_err(|e| io(format!("reading {}", path.display()), e))?;
let mut file: ContextsFile = toml::from_str(&raw).map_err(|source| ContextError::Parse {
action: format!("parsing {}", path.display()),
source,
})?;
let def = file.contexts.remove(name).ok_or_else(|| {
ContextError::Missing(format!(
"context '{name}' not in {} (have: {})",
path.display(),
file.contexts.keys().cloned().collect::<Vec<_>>().join(", ")
))
})?;
if !matches!(
(
def.autoprovision,
def.k3d.is_some(),
def.openbao_namespace.is_some()
),
(true, false, false) | (false, true, false) | (false, false, true)
) {
return Err(ContextError::Invalid(format!(
"context '{name}' must set exactly one of `autoprovision = true`, \
`k3d`, or `openbao_namespace`"
)));
} }
Ok(def)
} }
/// Walk up from the cwd for `.harmony/contexts.toml` (git/cargo style). fn kubeconfig_target(contents: &str) -> Result<String, ContextError> {
fn find_contexts_file() -> Result<PathBuf, ContextError> { let config: serde_yaml::Value = serde_yaml::from_str(contents)
let mut dir = std::env::current_dir().map_err(|e| io("current dir", e))?; .map_err(|error| ContextError::Invalid(format!("parsing brokered kubeconfig: {error}")))?;
loop { let current = config
let candidate = dir.join(".harmony").join("contexts.toml"); .get("current-context")
if candidate.is_file() { .and_then(|value| value.as_str())
return Ok(candidate); .ok_or_else(|| {
} ContextError::Invalid("brokered kubeconfig has no current-context".into())
if !dir.pop() { })?;
return Err(ContextError::Missing( let cluster = config
"no .harmony/contexts.toml found (searched up from the cwd) — \ .get("contexts")
run from the project dir or pass --config <path>" .and_then(|value| value.as_sequence())
.to_string(), .ok_or_else(|| ContextError::Invalid("brokered kubeconfig has no contexts".into()))?
)); .iter()
} .find(|entry| entry.get("name").and_then(|name| name.as_str()) == Some(current))
} .and_then(|entry| entry.get("context"))
.and_then(|context| context.get("cluster"))
.and_then(|cluster| cluster.as_str())
.ok_or_else(|| {
ContextError::Invalid(format!(
"brokered kubeconfig current-context '{current}' has no cluster"
))
})?;
let server = config
.get("clusters")
.and_then(|value| value.as_sequence())
.ok_or_else(|| ContextError::Invalid("brokered kubeconfig has no clusters".into()))?
.iter()
.find(|entry| entry.get("name").and_then(|name| name.as_str()) == Some(cluster))
.and_then(|entry| entry.get("cluster"))
.and_then(|cluster| cluster.get("server"))
.and_then(|server| server.as_str())
.ok_or_else(|| {
ContextError::Invalid(format!(
"brokered kubeconfig cluster '{cluster}' has no server"
))
})?;
Ok(format!("{cluster} via context {current} ({server})"))
} }
async fn build_config_sources( async fn build_config_sources(
def: &ContextDef, spec: &ContextSpec,
local_config_dir: Option<PathBuf>, local_config_dir: Option<PathBuf>,
) -> Result<Vec<Arc<dyn ConfigSource>>, ContextError> { ) -> Result<Vec<Arc<dyn ConfigSource>>, ContextError> {
let mut sources: Vec<Arc<dyn ConfigSource>> = Vec::new(); let mut sources: Vec<Arc<dyn ConfigSource>> = Vec::new();
if let Some(namespace) = &def.openbao_namespace { match spec {
let source = harmony_config::openbao_source( ContextSpec::Remote(remote) => {
namespace, let access = &remote.access;
def.openbao_url.clone(), let source = harmony_config::openbao_source(
def.zitadel_url.clone(), access.namespace.as_ref(),
def.zitadel_audience.clone(), Some(access.url.to_string()),
Some( Some(access.zitadel_url.to_string()),
def.openbao_role Some(access.zitadel_audience.to_string()),
.clone() Some(access.role.to_string()),
.unwrap_or_else(|| format!("{namespace}-cd")), )
), .await
)
.await
.ok_or_else(|| {
ContextError::Config(format!("reaching OpenBao for namespace '{namespace}'"))
})?;
sources.push(source);
} else {
let dir = local_config_dir
.or_else(LocalFileSource::default_path)
.ok_or_else(|| { .ok_or_else(|| {
ContextError::Missing("local contexts need a config directory".to_string()) ContextError::Config(format!(
"reaching OpenBao for namespace '{}'",
access.namespace
))
})?; })?;
sources.push(Arc::new(LocalFileSource::new(dir))); sources.push(source);
}
ContextSpec::Local(_) => {
let dir = local_config_dir
.or_else(LocalFileSource::default_path)
.ok_or_else(|| {
ContextError::Missing("local contexts need a config directory".to_string())
})?;
sources.push(Arc::new(LocalFileSource::new(dir)));
}
} }
sources.push(Arc::new(PromptSource::new())); sources.push(Arc::new(PromptSource::new()));
@@ -400,8 +443,6 @@ fn write_kubeconfig(contents: &[u8]) -> Result<NamedTempFile, ContextError> {
Ok(file) Ok(file)
} }
/// The k3d cluster's kubeconfig, written to a temp file. Prefers harmony's
/// managed k3d binary, falling back to a `k3d` on PATH.
fn k3d_kubeconfig(cluster: &str) -> Result<NamedTempFile, ContextError> { fn k3d_kubeconfig(cluster: &str) -> Result<NamedTempFile, ContextError> {
let managed = harmony::config::HARMONY_DATA_DIR.join("k3d").join("k3d"); let managed = harmony::config::HARMONY_DATA_DIR.join("k3d").join("k3d");
let k3d: OsString = if managed.exists() { let k3d: OsString = if managed.exists() {
@@ -421,3 +462,91 @@ fn k3d_kubeconfig(cluster: &str) -> Result<NamedTempFile, ContextError> {
} }
write_kubeconfig(&out.stdout) write_kubeconfig(&out.stdout)
} }
#[cfg(test)]
mod tests {
use super::*;
fn local(name: &str, spec: LocalContext) -> Context {
Context {
name: name.parse().unwrap(),
namespace: "test-app".parse().unwrap(),
spec: ContextSpec::Local(spec),
}
}
fn remote() -> Context {
Context {
name: "prod".parse().unwrap(),
namespace: "test-app".parse().unwrap(),
spec: ContextSpec::Remote(RemoteContext {
registry: "registry.example.com".parse().unwrap(),
repository: "team/apps".parse().unwrap(),
domain: "example.com".parse().unwrap(),
image_pull_secret: Some("registry-auth".parse().unwrap()),
access: OpenBaoClusterAccess {
namespace: "team/prod".parse().unwrap(),
url: "https://bao.example.com".parse().unwrap(),
role: "deployer".parse().unwrap(),
zitadel_url: "https://auth.example.com".parse().unwrap(),
zitadel_audience: "harmony".parse().unwrap(),
},
}),
}
}
#[test]
fn catalog_rejects_empty_and_duplicate_contexts() {
assert!(ContextCatalog::new(Vec::new()).is_err());
let first = local("dev", LocalContext::ManagedK3d);
assert!(ContextCatalog::new(vec![first.clone(), first]).is_err());
}
#[test]
fn local_metadata_uses_local_images_and_k3d_publication() {
let context = local("dev", LocalContext::ManagedK3d);
let resolved = AppContext::load_metadata(&context, "1.2.3", None);
assert_eq!(resolved.profile(), Profile::Local);
assert_eq!(resolved.namespace(), "test-app");
assert_eq!(resolved.image("api"), "localhost/api:1.2.3");
assert!(matches!(
resolved.publisher(),
PublicationTopology::K3d { cluster } if cluster == "harmony"
));
}
#[test]
fn remote_metadata_uses_compiled_registry_domain_and_pull_secret() {
let context = remote();
let resolved = AppContext::load_metadata(&context, "2.0.0", None);
assert_eq!(resolved.profile(), Profile::Prod);
assert_eq!(
resolved.image("api"),
"registry.example.com/team/apps/api:2.0.0"
);
assert_eq!(resolved.domain(), Some("example.com"));
assert_eq!(resolved.image_pull_secret().unwrap().0, "registry-auth");
}
#[test]
fn kubeconfig_target_reports_context_and_server() {
let kubeconfig = r#"
current-context: tenant-admin
contexts:
- name: tenant-admin
context:
cluster: shared-okd
clusters:
- name: shared-okd
cluster:
server: https://api.example.com:6443
"#;
assert_eq!(
kubeconfig_target(kubeconfig).unwrap(),
"shared-okd via context tenant-admin (https://api.example.com:6443)"
);
}
}

View File

@@ -30,7 +30,6 @@ struct Endpoint {
/// topology can host. /// topology can host.
pub struct ComposeDeploy<T: Topology = K8sAnywhereTopology> { pub struct ComposeDeploy<T: Topology = K8sAnywhereTopology> {
name: String, name: String,
namespace: String,
project: String, project: String,
registry: String, registry: String,
app: ComposeApp, app: ComposeApp,
@@ -40,8 +39,7 @@ pub struct ComposeDeploy<T: Topology = K8sAnywhereTopology> {
} }
impl<T: Topology> ComposeDeploy<T> { impl<T: Topology> ComposeDeploy<T> {
/// Import the app from a compose dir. `namespace`/`project` default to the /// Import the app from a compose dir. The project defaults to the app name.
/// name; override fluently.
pub fn from_dir(name: impl Into<String>, dir: impl AsRef<Path>) -> Result<Self, AppError> { pub fn from_dir(name: impl Into<String>, dir: impl AsRef<Path>) -> Result<Self, AppError> {
let app = ComposeApp::from_dir(dir.as_ref())?; let app = ComposeApp::from_dir(dir.as_ref())?;
Ok(Self::from_compose(name, app)) Ok(Self::from_compose(name, app))
@@ -50,7 +48,6 @@ impl<T: Topology> ComposeDeploy<T> {
pub fn from_compose(name: impl Into<String>, app: ComposeApp) -> Self { pub fn from_compose(name: impl Into<String>, app: ComposeApp) -> Self {
let name = name.into(); let name = name.into();
Self { Self {
namespace: name.clone(),
project: name.clone(), project: name.clone(),
registry: "localhost".to_string(), registry: "localhost".to_string(),
name, name,
@@ -61,10 +58,6 @@ impl<T: Topology> ComposeDeploy<T> {
} }
} }
pub fn namespace(mut self, ns: impl Into<String>) -> Self {
self.namespace = ns.into();
self
}
pub fn project(mut self, p: impl Into<String>) -> Self { pub fn project(mut self, p: impl Into<String>) -> Self {
self.project = p.into(); self.project = p.into();
self self
@@ -93,10 +86,10 @@ impl<T: Topology> ComposeDeploy<T> {
self self
} }
fn app_ref(&self, profile: Profile) -> AppRef<'_> { fn app_ref<'a>(&'a self, namespace: &'a str, profile: Profile) -> AppRef<'a> {
AppRef { AppRef {
name: &self.name, name: &self.name,
namespace: &self.namespace, namespace,
profile, profile,
} }
} }
@@ -108,6 +101,7 @@ impl<T: Topology> ComposeDeploy<T> {
/// Derive the deploy Score for a profile (pure — the testable core). /// Derive the deploy Score for a profile (pure — the testable core).
pub fn score( pub fn score(
&self, &self,
namespace: &str,
profile: Profile, profile: Profile,
version: &str, version: &str,
force_conflicts: bool, force_conflicts: bool,
@@ -125,10 +119,10 @@ impl<T: Topology> ComposeDeploy<T> {
deploy.extra_env = self deploy.extra_env = self
.capabilities .capabilities
.iter() .iter()
.flat_map(|c| c.env(&self.app_ref(profile))) .flat_map(|c| c.env(&self.app_ref(namespace, profile)))
.collect(); .collect();
Ok(ComposeAppScore { Ok(ComposeAppScore {
namespace: self.namespace.clone(), namespace: namespace.to_string(),
release_name: self.name.clone(), release_name: self.name.clone(),
public_endpoint, public_endpoint,
app: self.app.clone(), app: self.app.clone(),
@@ -141,10 +135,10 @@ impl<T: Topology> ComposeDeploy<T> {
#[async_trait] #[async_trait]
impl<T: Topology + HelmCommand + K8sclient> HarmonyApp<T> for ComposeDeploy<T> { impl<T: Topology + HelmCommand + K8sclient> HarmonyApp<T> for ComposeDeploy<T> {
fn identity(&self) -> AppIdentity { fn identity(&self, ctx: &AppContext) -> AppIdentity {
AppIdentity { AppIdentity {
name: self.name.clone(), name: self.name.clone(),
namespace: self.namespace.clone(), namespace: ctx.namespace().to_string(),
} }
} }
@@ -168,7 +162,12 @@ impl<T: Topology + HelmCommand + K8sclient> HarmonyApp<T> for ComposeDeploy<T> {
ctx: &AppContext, ctx: &AppContext,
options: DeployOptions, options: DeployOptions,
) -> Result<Vec<Box<dyn Score<T>>>, AppError> { ) -> Result<Vec<Box<dyn Score<T>>>, AppError> {
let mut app_score = self.score(ctx.profile(), ctx.version(), options.force_conflicts)?; let mut app_score = self.score(
ctx.namespace(),
ctx.profile(),
ctx.version(),
options.force_conflicts,
)?;
app_score.deploy.images = self app_score.deploy.images = self
.app .app
.services .services
@@ -182,7 +181,7 @@ impl<T: Topology + HelmCommand + K8sclient> HarmonyApp<T> for ComposeDeploy<T> {
}) })
.collect::<Result<_, AppError>>()?; .collect::<Result<_, AppError>>()?;
let mut scores: Vec<Box<dyn Score<T>>> = vec![Box::new(app_score)]; let mut scores: Vec<Box<dyn Score<T>>> = vec![Box::new(app_score)];
let app_ref = self.app_ref(ctx.profile()); let app_ref = self.app_ref(ctx.namespace(), ctx.profile());
for capability in &self.capabilities { for capability in &self.capabilities {
scores.extend(capability.scores(&app_ref)); scores.extend(capability.scores(&app_ref));
} }
@@ -228,7 +227,7 @@ mod tests {
#[test] #[test]
fn namespace_and_project_default_to_name() { fn namespace_and_project_default_to_name() {
let s = CD::from_compose("timesheet", fixture()) let s = CD::from_compose("timesheet", fixture())
.score(Profile::Local, "0.1.0", false) .score("timesheet", Profile::Local, "0.1.0", false)
.unwrap(); .unwrap();
assert_eq!(s.namespace, "timesheet"); assert_eq!(s.namespace, "timesheet");
assert_eq!(s.release_name, "timesheet"); assert_eq!(s.release_name, "timesheet");
@@ -238,7 +237,7 @@ mod tests {
fn local_profile_renders_locally_rwo_http() { fn local_profile_renders_locally_rwo_http() {
let s = CD::from_compose("ts", fixture()) let s = CD::from_compose("ts", fixture())
.expose("frontend", "ts.local") .expose("frontend", "ts.local")
.score(Profile::Local, "1.0.0", false) .score("timesheet", Profile::Local, "1.0.0", false)
.unwrap(); .unwrap();
assert_eq!(s.deploy.replicas, 1); assert_eq!(s.deploy.replicas, 1);
assert!(!s.deploy.rolling); assert!(!s.deploy.rolling);
@@ -255,7 +254,7 @@ mod tests {
.registry("hub.x") .registry("hub.x")
.project("p") .project("p")
.expose("frontend", "ts.x") .expose("frontend", "ts.x")
.score(Profile::Prod, "2.0.0", false) .score("timesheet", Profile::Prod, "2.0.0", false)
.unwrap(); .unwrap();
assert_eq!(s.deploy.registry, "hub.x"); assert_eq!(s.deploy.registry, "hub.x");
assert_eq!(s.deploy.project, "p"); assert_eq!(s.deploy.project, "p");
@@ -272,7 +271,7 @@ mod tests {
fn secrets_are_carried_to_the_score() { fn secrets_are_carried_to_the_score() {
let s = CD::from_compose("ts", fixture()) let s = CD::from_compose("ts", fixture())
.secret("DB_PASSWORD", "dev") .secret("DB_PASSWORD", "dev")
.score(Profile::Local, "0.1.0", false) .score("timesheet", Profile::Local, "0.1.0", false)
.unwrap(); .unwrap();
assert_eq!(s.app_secrets.get("DB_PASSWORD").unwrap(), "dev"); assert_eq!(s.app_secrets.get("DB_PASSWORD").unwrap(), "dev");
} }
@@ -281,7 +280,7 @@ mod tests {
fn postgres_capability_wires_database_url_by_reference() { fn postgres_capability_wires_database_url_by_reference() {
let s = CD::from_compose("ts", fixture()) let s = CD::from_compose("ts", fixture())
.with(Postgres::managed()) .with(Postgres::managed())
.score(Profile::Local, "0.1.0", false) .score("timesheet", Profile::Local, "0.1.0", false)
.unwrap(); .unwrap();
let db = s let db = s
.deploy .deploy

View File

@@ -9,11 +9,6 @@ pub enum ContextError {
action: String, action: String,
source: std::io::Error, source: std::io::Error,
}, },
#[error("{action}: {source}")]
Parse {
action: String,
source: toml::de::Error,
},
#[error("{0}")] #[error("{0}")]
Config(String), Config(String),
#[error("{0}")] #[error("{0}")]

View File

@@ -10,9 +10,9 @@
//! a future TUI, and a web UI are all just front-ends that call these verbs //! a future TUI, and a web UI are all just front-ends that call these verbs
//! and render the results — none of them owns the logic. //! and render the results — none of them owns the logic.
//! //!
//! A [`Context`](context::AppContext) is the selected target + profile //! A [`Context`] defines a compiled deployment target. [`AppContext`] resolves
//! (ADR-026 §1/§10): the verbs converge the *same* Scores wherever the //! its credentials and runtime state. The verbs converge the same Scores for
//! context points; only the context changes between local and prod. //! local and production targets (ADR-026 §1/§10).
pub mod app; pub mod app;
pub mod capabilities; pub mod capabilities;
@@ -32,9 +32,13 @@ pub use app::{
pub use capabilities::{AppRef, Capability, Monitoring, Postgres, ZitadelAuth}; pub use capabilities::{AppRef, Capability, Monitoring, Postgres, ZitadelAuth};
pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image}; pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image};
pub use compose::ComposeApp; pub use compose::ComposeApp;
pub use context::{AppContext, ClusterAccess}; pub use context::{
AppContext, Context, ContextCatalog, ContextSpec, LocalContext, OpenBaoClusterAccess,
RemoteContext,
};
pub use deploy::ComposeDeploy; pub use deploy::ComposeDeploy;
pub use error::{AppError, ContextError, ImageError}; pub use error::{AppError, ContextError, ImageError};
pub use harmony::modules::tenant::ClusterAccess;
pub use profile::Profile; pub use profile::Profile;
pub use publish::{ImagePublisher, ImageRefs, ImageSpec, PublicationTopology}; pub use publish::{ImagePublisher, ImageRefs, ImageSpec, PublicationTopology};
pub use score::{ComposeAppScore, PublicEndpoint}; pub use score::{ComposeAppScore, PublicEndpoint};

View File

@@ -1,7 +1,6 @@
//! The deploy profile — the typed environment tag a [`Context`] carries //! Deployment behavior derived from the local or remote [`Context`] variant
//! (ADR-026 §7). The app's `scores()` branches on it; the *meaning* of each //! (ADR-026 §7). Apps branch on it; storage, replicas, and TLS policy remain
//! profile (storage class, replicas, TLS, …) lives in the app/deploy code, //! in app deployment code.
//! not here, so this stays a small generic tag.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]

View File

@@ -9,7 +9,7 @@ use std::path::PathBuf;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use harmony::topology::K8sAnywhereTopology; use harmony::topology::K8sAnywhereTopology;
use harmony_app::{AppContext, HarmonyApp}; use harmony_app::{AppContext, ContextCatalog, HarmonyApp};
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command(version)] #[command(version)]
@@ -17,8 +17,7 @@ struct AppCli {
#[command(subcommand)] #[command(subcommand)]
verb: Verb, verb: Verb,
/// Target cluster context (required; there is no default, so you never /// Target cluster context compiled into this deploy binary.
/// deploy to the wrong cluster). Defined in `.harmony/contexts.toml`.
#[arg(long, global = true)] #[arg(long, global = true)]
context: Option<String>, context: Option<String>,
@@ -31,11 +30,6 @@ struct AppCli {
#[arg(long, global = true)] #[arg(long, global = true)]
local_config: Option<PathBuf>, local_config: Option<PathBuf>,
/// Path to the contexts file. Default: the nearest
/// `.harmony/contexts.toml` found by walking up from the current dir.
#[arg(long, short = 'C', global = true, value_name = "FILE")]
config: Option<PathBuf>,
/// Emit the versioned machine-readable output schema. /// Emit the versioned machine-readable output schema.
#[arg(long, global = true)] #[arg(long, global = true)]
json: bool, json: bool,
@@ -75,23 +69,30 @@ enum Verb {
}, },
} }
/// Entry point for a per-app deploy binary: `fn main() { app_main(MyApp) }`. /// Entry point for a per-app deploy binary.
/// The CLI front-end drives `K8sAnywhereTopology`; the app layer itself is /// The CLI front-end drives `K8sAnywhereTopology`; the app layer itself is
/// topology-generic (a different front-end can drive another topology). /// topology-generic (a different front-end can drive another topology).
pub async fn app_main<A: HarmonyApp<K8sAnywhereTopology> + 'static>(app: A) -> Result<()> { pub async fn app_main<A: HarmonyApp<K8sAnywhereTopology> + 'static>(
app: A,
contexts: ContextCatalog,
) -> Result<()> {
crate::cli_logger::init(); crate::cli_logger::init();
crate::cli_reporter::init(); crate::cli_reporter::init();
let cli = AppCli::parse(); let cli = AppCli::parse();
let context = cli let available = contexts.available_names();
let context_name = cli
.context .context
.or_else(|| std::env::var("HARMONY_CONTEXT").ok()) .or_else(|| std::env::var("HARMONY_CONTEXT").ok())
.context("--context or HARMONY_CONTEXT is required")?; .with_context(|| {
format!("--context or HARMONY_CONTEXT is required (available: {available})")
})?;
let context = contexts.require(&context_name)?;
let metadata_only = matches!(&cli.verb, Verb::Build | Verb::Publish { .. }); let metadata_only = matches!(&cli.verb, Verb::Build | Verb::Publish { .. });
let ctx = if metadata_only { let ctx = if metadata_only {
AppContext::load_metadata(&context, cli.tag, cli.local_config, cli.config)? AppContext::load_metadata(context, cli.tag, cli.local_config)
} else { } else {
AppContext::resolve(&context, cli.tag, cli.local_config, cli.config).await? AppContext::resolve(context, cli.tag, cli.local_config).await?
}; };
match cli.verb { match cli.verb {
@@ -159,6 +160,15 @@ fn render_deploy(report: harmony_app::DeployReport, json: bool) {
render_json(&report); render_json(&report);
return; return;
} }
println!(
"\nTarget: context={} cluster={} namespace={}",
report.context,
report
.cluster
.as_deref()
.unwrap_or("kubeconfig current-context"),
report.namespace
);
println!("\n🚀 Deployed:"); println!("\n🚀 Deployed:");
for step in report.steps { for step in report.steps {
println!("{}{}", step.name, step.message); println!("{}{}", step.name, step.message);

View File

@@ -214,6 +214,9 @@ impl ConfigClient {
} }
pub async fn set<T: Config>(&self, config: &T) -> Result<(), ConfigError> { pub async fn set<T: Config>(&self, config: &T) -> Result<(), ConfigError> {
if self.sources.is_empty() {
return Err(ConfigError::NoSources);
}
let value = serde_json::to_value(config).map_err(|e| ConfigError::Serialization { let value = serde_json::to_value(config).map_err(|e| ConfigError::Serialization {
key: T::KEY.to_string(), key: T::KEY.to_string(),
source: e, source: e,
@@ -613,6 +616,20 @@ mod tests {
assert_eq!(result1, config); assert_eq!(result1, config);
} }
#[tokio::test]
async fn test_set_fails_with_no_sources() {
let manager = ConfigClient::new(vec![]);
let config = TestConfig {
name: "not-written".to_string(),
count: 0,
};
assert!(matches!(
manager.set(&config).await,
Err(ConfigError::NoSources)
));
}
#[tokio::test] #[tokio::test]
async fn test_derive_macro_emits_standard_class_with_no_secret_fields() { async fn test_derive_macro_emits_standard_class_with_no_secret_fields() {
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Config)] #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Config)]

View File

@@ -6,6 +6,89 @@ use serde_yaml::Value;
use syn::LitStr; use syn::LitStr;
use syn::parse_macro_input; use syn::parse_macro_input;
fn context_literal<T>(input: TokenStream, kind: &str) -> Result<LitStr, TokenStream>
where
T: std::str::FromStr,
T::Err: std::fmt::Display,
{
let literal =
syn::parse::<LitStr>(input).map_err(|error| TokenStream::from(error.to_compile_error()))?;
literal.value().parse::<T>().map_err(|error| {
TokenStream::from(
syn::Error::new(literal.span(), format!("invalid {kind} literal: {error}"))
.to_compile_error(),
)
})?;
Ok(literal)
}
macro_rules! context_macro {
($(#[$meta:meta])* $macro:ident, $type:ident, $kind:literal) => {
$(#[$meta])*
#[proc_macro]
pub fn $macro(input: TokenStream) -> TokenStream {
match context_literal::<harmony_types::context::$type>(input, $kind) {
Ok(literal) => quote! {
#literal
.parse::<::harmony_types::context::$type>()
.expect(concat!("validated ", $kind, " literal"))
}
.into(),
Err(error) => error,
}
}
};
}
context_macro!(
/// Rejects invalid context-name literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::context_name!("Prod");
/// ```
context_name, ContextName, "context name"
);
context_macro!(
/// Rejects invalid domain literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::domain!("https://example.com");
/// ```
domain, DomainName, "domain name"
);
context_macro!(
/// Rejects invalid HTTP URL literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::http_url!("ftp://example.com");
/// ```
http_url, HttpUrl, "HTTP URL"
);
context_macro!(
/// Rejects invalid OCI registry literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::oci_registry!("https://registry.example.com");
/// ```
oci_registry, OciRegistry, "OCI registry"
);
context_macro!(
/// Rejects invalid OCI repository literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::oci_repository!("Team/app");
/// ```
oci_repository, OciRepository, "OCI repository"
);
context_macro!(
/// Rejects invalid OpenBao namespace literals during macro expansion.
///
/// ```compile_fail
/// harmony_macros::openbao_namespace!("Team/app");
/// ```
openbao_namespace, OpenBaoNamespace, "OpenBao namespace"
);
#[proc_macro] #[proc_macro]
pub fn ip(input: TokenStream) -> TokenStream { pub fn ip(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as LitStr); let input = parse_macro_input!(input as LitStr);

View File

@@ -0,0 +1,23 @@
use harmony_macros::{
context_name, domain, http_url, oci_registry, oci_repository, openbao_namespace,
};
use harmony_types::context::{
ContextName, DomainName, HttpUrl, OciRegistry, OciRepository, OpenBaoNamespace,
};
#[test]
fn context_literals_produce_validated_types() {
let context: ContextName = context_name!("prod.eu");
let domain: DomainName = domain!("api.example.com");
let url: HttpUrl = http_url!("https://api.example.com/v1");
let registry: OciRegistry = oci_registry!("registry.example.com:5000");
let repository: OciRepository = oci_repository!("team/my_app");
let namespace: OpenBaoNamespace = openbao_namespace!("team/app");
assert_eq!(context.as_ref(), "prod.eu");
assert_eq!(domain.as_ref(), "api.example.com");
assert_eq!(url.as_ref(), "https://api.example.com/v1");
assert_eq!(registry.as_ref(), "registry.example.com:5000");
assert_eq!(repository.as_ref(), "team/my_app");
assert_eq!(namespace.as_ref(), "team/app");
}

View File

@@ -0,0 +1,333 @@
use std::{fmt, str::FromStr};
use url::{Host, Url};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ContextValueError {
kind: &'static str,
value: String,
requirement: &'static str,
}
impl ContextValueError {
fn new(kind: &'static str, value: &str, requirement: &'static str) -> Self {
Self {
kind,
value: value.to_owned(),
requirement,
}
}
}
impl fmt::Display for ContextValueError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"invalid {} '{}': {}",
self.kind, self.value, self.requirement
)
}
}
impl std::error::Error for ContextValueError {}
macro_rules! string_value {
($name:ident, $kind:literal, $requirement:literal, $validate:expr) => {
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct $name(String);
impl FromStr for $name {
type Err = ContextValueError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if !($validate)(value) {
return Err(ContextValueError::new($kind, value, $requirement));
}
Ok(Self(value.to_owned()))
}
}
impl fmt::Display for $name {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl AsRef<str> for $name {
fn as_ref(&self) -> &str {
&self.0
}
}
};
}
fn is_context_name(value: &str) -> bool {
let bytes = value.as_bytes();
(1..=63).contains(&bytes.len())
&& bytes.first().is_some_and(u8::is_ascii_alphanumeric)
&& bytes.last().is_some_and(u8::is_ascii_alphanumeric)
&& bytes.iter().all(|byte| {
byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-')
})
}
fn is_domain_name(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 253
&& value.split('.').all(|label| {
let bytes = label.as_bytes();
!bytes.is_empty()
&& bytes.len() <= 63
&& bytes.first().is_some_and(u8::is_ascii_alphanumeric)
&& bytes.last().is_some_and(u8::is_ascii_alphanumeric)
&& bytes
.iter()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-')
})
}
fn is_http_url(value: &str) -> bool {
let Some((scheme, authority)) = value.split_once("://") else {
return false;
};
if !(scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https"))
|| authority.starts_with('/')
{
return false;
}
Url::parse(value).is_ok_and(|url| {
matches!(url.scheme(), "http" | "https")
&& url.host().is_some()
&& url.username().is_empty()
&& url.password().is_none()
&& url.fragment().is_none()
})
}
fn is_oci_registry(value: &str) -> bool {
if value.is_empty()
|| !value.is_ascii()
|| value.contains(['/', '@', '?', '#'])
|| value.ends_with(':')
|| value.chars().any(char::is_whitespace)
{
return false;
}
Url::parse(&format!("http://{value}")).is_ok_and(|url| match url.host() {
Some(Host::Domain(host)) => is_domain_name(host),
Some(Host::Ipv4(_) | Host::Ipv6(_)) => true,
None => false,
})
}
fn is_oci_component(component: &str) -> bool {
let bytes = component.as_bytes();
if bytes.is_empty() || !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
return false;
}
let mut index = 1;
while index < bytes.len() {
if bytes[index].is_ascii_lowercase() || bytes[index].is_ascii_digit() {
index += 1;
continue;
}
match bytes[index] {
b'.' => index += 1,
b'_' => {
index += 1;
if bytes.get(index) == Some(&b'_') {
index += 1;
}
}
b'-' => {
while bytes.get(index) == Some(&b'-') {
index += 1;
}
}
_ => return false,
}
if !bytes
.get(index)
.is_some_and(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
{
return false;
}
}
true
}
fn is_oci_repository(value: &str) -> bool {
!value.is_empty() && value.split('/').all(is_oci_component)
}
fn is_openbao_namespace(value: &str) -> bool {
!value.is_empty()
&& value.split('/').all(|component| {
!component.is_empty()
&& component.bytes().all(|byte| {
byte.is_ascii_lowercase()
|| byte.is_ascii_digit()
|| matches!(byte, b'_' | b'-')
})
})
}
fn is_unmodified_text(value: &str) -> bool {
!value.is_empty() && value.trim() == value && !value.chars().any(char::is_control)
}
string_value!(
ContextName,
"context name",
"expected 1-63 lowercase ASCII letters, digits, '.', '_', or '-', beginning and ending with a letter or digit",
is_context_name
);
string_value!(
DomainName,
"domain name",
"expected a lowercase DNS subdomain of at most 253 characters with labels of at most 63 characters",
is_domain_name
);
string_value!(
HttpUrl,
"HTTP URL",
"expected an http(s) URL with a host and without userinfo or a fragment",
is_http_url
);
string_value!(
OciRegistry,
"OCI registry",
"expected a localhost, IP, or DNS authority with an optional port",
is_oci_registry
);
string_value!(
OciRepository,
"OCI repository",
"expected lowercase slash-separated OCI repository components",
is_oci_repository
);
string_value!(
OpenBaoNamespace,
"OpenBao namespace",
"expected slash-separated lowercase ASCII letters, digits, '_', or '-'",
is_openbao_namespace
);
string_value!(
OpenBaoRoleName,
"OpenBao role name",
"expected nonempty text without surrounding whitespace or control characters",
is_unmodified_text
);
string_value!(
OidcAudience,
"OIDC audience",
"expected nonempty text without surrounding whitespace or control characters",
is_unmodified_text
);
impl OpenBaoRoleName {
pub fn new(value: impl Into<String>) -> Result<Self, ContextValueError> {
value.into().parse()
}
}
impl OidcAudience {
pub fn new(value: impl Into<String>) -> Result<Self, ContextValueError> {
value.into().parse()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn accepts<T: FromStr>(values: &[&str]) {
for value in values {
assert!(value.parse::<T>().is_ok(), "should accept {value:?}");
}
}
fn rejects<T: FromStr>(values: &[&str]) {
for value in values {
assert!(value.parse::<T>().is_err(), "should reject {value:?}");
}
}
#[test]
fn context_names() {
accepts::<ContextName>(&["a", "prod.eu_1", "a-b"]);
rejects::<ContextName>(&["", "-prod", "prod-", "Prod", "a/b", &"a".repeat(64)]);
}
#[test]
fn domain_names() {
accepts::<DomainName>(&["localhost", "api.example.com", "1.example"]);
rejects::<DomainName>(&[
"",
"Example.com",
"*.example.com",
"example.com.",
"https://example.com",
"example.com:443",
"a..com",
&format!("{}.com", "a".repeat(64)),
]);
}
#[test]
fn http_urls() {
accepts::<HttpUrl>(&["http://localhost", "https://example.com/a?q=1"]);
rejects::<HttpUrl>(&[
"ftp://example.com",
"https:///path",
"https://user@example.com",
"https://example.com/#part",
]);
}
#[test]
fn oci_registries() {
accepts::<OciRegistry>(&[
"localhost",
"localhost:5000",
"registry.example.com",
"127.0.0.1:5000",
"[::1]:5000",
]);
rejects::<OciRegistry>(&[
"https://registry.example.com",
"registry.example.com/path",
"user@registry.example.com",
"registry.example.com?x=1",
"registry.example.com:",
"bad_name.example.com",
]);
}
#[test]
fn oci_repositories() {
accepts::<OciRepository>(&["app", "team/my_app", "team/a--b", "a/b.c"]);
rejects::<OciRepository>(&[
"", "/app", "app/", "app//api", ".", "..", "Team/app", "a..b", "a_-b",
]);
}
#[test]
fn openbao_namespaces() {
accepts::<OpenBaoNamespace>(&["team", "team/app_1", "a-b/c"]);
rejects::<OpenBaoNamespace>(&["", "/team", "team/", "team//app", ".", "..", "Team"]);
}
#[test]
fn role_names_and_audiences() {
accepts::<OpenBaoRoleName>(&["role", "Team Role"]);
accepts::<OidcAudience>(&["api://harmony", "harmony audience"]);
rejects::<OpenBaoRoleName>(&["", " role", "role ", "role\n"]);
rejects::<OidcAudience>(&["", " audience", "audience\0"]);
assert_eq!(OpenBaoRoleName::new("role").unwrap().as_ref(), "role");
assert_eq!(OidcAudience::new("audience").unwrap().as_ref(), "audience");
}
}

View File

@@ -47,6 +47,16 @@ pub enum K8sNameError {
InvalidFormat(String), InvalidFormat(String),
} }
impl std::fmt::Display for K8sNameError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::InvalidFormat(message) => f.write_str(message),
}
}
}
impl std::error::Error for K8sNameError {}
impl From<&K8sName> for String { impl From<&K8sName> for String {
fn from(value: &K8sName) -> Self { fn from(value: &K8sName) -> Self {
value.0.clone() value.0.clone()
@@ -59,6 +69,12 @@ impl std::fmt::Display for K8sName {
} }
} }
impl AsRef<str> for K8sName {
fn as_ref(&self) -> &str {
&self.0
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View File

@@ -1,3 +1,4 @@
pub mod context;
pub mod firewall; pub mod firewall;
pub mod id; pub mod id;
pub mod k8s_name; pub mod k8s_name;
@@ -6,3 +7,8 @@ pub mod rfc1123;
pub mod ssh; pub mod ssh;
pub mod storage; pub mod storage;
pub mod switch; pub mod switch;
pub use context::{
ContextName, ContextValueError, DomainName, HttpUrl, OciRegistry, OciRepository, OidcAudience,
OpenBaoNamespace, OpenBaoRoleName,
};