NationTech/harmony
3
0
Fork 2
Code Issues 38 Pull Requests 8 Actions Packages Projects Releases 1 Wiki Activity

feat/harmony-auth-ui #344

Merged
johnride merged 47 commits from feat/harmony-auth-ui into master 2026-07-28 14:21:44 +00:00
Conversation 0 Commits 47 Files Changed 158 +18694 -3880
158 changed files with 18694 additions and 3880 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
.opencode/skills/code-derived-doc-review/SKILL.md Normal file
View File

@@ -0,0 +1,70 @@
---
name: code-derived-doc-review
description: Use when writing or reviewing architecture and design documentation to derive claims from code, map claims to tests, run an external review, and improve clarity and information density with the humanizer skill.
license: AGPL-3.0-only
compatibility: opencode
---
# Code-derived documentation review
Keep design documentation factual, concise, and difficult to drift away from
the codebase.
## Workflow
1. Read the complete document set before editing.
2. Identify the authoritative document for decisions, operational behavior,
wire contracts, and implementation details. Remove duplicate specifications.
3. Extract every architecturally significant claim: ownership, trust boundary,
ordering, state transition, failure policy, durability guarantee, limit, and
security constraint.
4. For each claim, locate:
- the implementation owner with file and symbol;
- the test that observes the behavior;
- the coverage level: direct, partial, or missing.
5. Treat an unsupported claim as a finding. Correct the prose, add a behavioral
test when the guarantee is required, or label the gap explicitly.
6. Ask an independent read-only reviewer to compare the finished documentation
with code and tests. Address findings, then request confirmation.
7. Load the `humanizer` skill. Remove repetition, inflated certainty, generic
conclusions, and AI-style transitions without removing facts or caveats.
8. Run a density pass: each fact has one authoritative home; diagrams summarize
and do not introduce unique guarantees; planning files point to maintained
documentation instead of restating it.
9. Validate links, documentation builds, diagram syntax, formatting, and stale
terminology.
## Claim table
Use this shape during review:
| Architectural claim | Documentation | Implementation owner | Validating test | Coverage |
|---|---|---|---|---|
Coverage means:
- **Direct:** the test observes the documented behavior at the owning boundary.
- **Partial:** the test covers a component or serialization detail but not the
full guarantee.
- **Missing:** no automated test proves the claim.
Do not count a compile check, helper-only test, or repeated production logic as
behavioral proof. Do not imply integration coverage when only a unit test exists.
## External review prompt
Ask the reviewer to report findings first with file and line references. Require
checks for factual drift, hidden failure cases, security overclaims, duplicate
authority, unsupported certainty, unclear ownership, and missing tests. The
reviewer must not edit the files.
## Completion standard
Documentation is complete when:
- each significant claim is traceable to code;
- each required guarantee has a direct test or an explicit coverage gap;
- one source owns each fact;
- external review findings are resolved;
- the prose passes the humanizer and density checks;
- documentation and diagram validation pass.

206
Cargo.lock generated
View File

@@ -1750,6 +1750,27 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
[[package]]
name = "const_format"
version = "0.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e"
dependencies = [
"const_format_proc_macros",
"konst",
]
[[package]]
name = "const_format_proc_macros"
version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "constant_time_eq"
version = "0.4.2"
@@ -3498,12 +3519,14 @@ name = "example_harmony_apply_deployment"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"async-trait",
"harmony",
"harmony-fleet-deploy",
"harmony-fleet-operator",
"harmony-reconciler-contracts",
"harmony_app",
"harmony_cli",
"k8s-openapi",
"kube",
"serde_json",
"tokio",
]
@@ -3642,6 +3665,16 @@ dependencies = [
"serde",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -4012,6 +4045,7 @@ dependencies = [
"tokio-util",
"url",
"uuid",
"vaultrs",
"virt",
"walkdir",
"webbrowser",
@@ -4019,18 +4053,22 @@ dependencies = [
[[package]]
name = "harmony-fleet-agent"
version = "0.1.0"
version = "0.1.2"
dependencies = [
"anyhow",
"async-nats",
"async-trait",
"chrono",
"clap",
"fs2",
"futures-util",
"harmony-fleet-auth",
"harmony-reconciler-contracts",
"harmony_secret",
"oci-client",
"podman-api",
"reqwest 0.12.28",
"sd-notify",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -4039,6 +4077,7 @@ dependencies = [
"toml",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
@@ -4071,6 +4110,7 @@ dependencies = [
"harmony",
"harmony-fleet-auth",
"harmony-fleet-operator",
"harmony-reconciler-contracts",
"harmony_app",
"harmony_cli",
"harmony_config",
@@ -4082,6 +4122,7 @@ dependencies = [
"kube",
"log",
"non-blank-string-rs",
"oci-client",
"schemars 0.8.22",
"serde",
"serde_json",
@@ -4137,7 +4178,6 @@ dependencies = [
"async-trait",
"axum",
"axum-extra",
"base64 0.22.1",
"chrono",
"clap",
"dotenvy",
@@ -4161,6 +4201,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"url",
"uuid",
]
[[package]]
@@ -4229,6 +4270,7 @@ dependencies = [
"schemars 0.8.22",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
]
@@ -4317,6 +4359,50 @@ dependencies = [
"url",
]
[[package]]
name = "harmony_auth"
version = "0.1.0"
dependencies = [
"async-trait",
"chrono",
"reqwest 0.12.28",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
"uuid",
]
[[package]]
name = "harmony_auth_cli"
version = "0.1.0"
dependencies = [
"clap",
"harmony_auth",
"serde_json",
"tokio",
"tracing-subscriber",
]
[[package]]
name = "harmony_auth_ui"
version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"axum-extra",
"clap",
"harmony_auth",
"maud",
"reqwest 0.12.28",
"serde",
"tokio",
"tower-http",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
name = "harmony_cli"
version = "0.1.0"
@@ -4569,6 +4655,7 @@ name = "harmony_zitadel_jwt"
version = "0.1.0"
dependencies = [
"anyhow",
"base64 0.22.1",
"chrono",
"jsonwebtoken",
"reqwest 0.12.28",
@@ -4748,6 +4835,15 @@ dependencies = [
"itoa",
]
[[package]]
name = "http-auth"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822"
dependencies = [
"memchr",
]
[[package]]
name = "http-body"
version = "0.4.6"
@@ -5522,6 +5618,21 @@ dependencies = [
"simple_asn1",
]
[[package]]
name = "jwt"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f"
dependencies = [
"base64 0.13.1",
"crypto-common 0.1.7",
"digest 0.10.7",
"hmac 0.12.1",
"serde",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "k3d-rs"
version = "0.1.0"
@@ -5554,6 +5665,21 @@ dependencies = [
"serde_json",
]
[[package]]
name = "konst"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb"
dependencies = [
"konst_macro_rules",
]
[[package]]
name = "konst_macro_rules"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
[[package]]
name = "kube"
version = "1.1.0"
@@ -6216,6 +6342,49 @@ dependencies = [
"memchr",
]
[[package]]
name = "oci-client"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b74df13319e08bc386d333d3dc289c774c88cc543cae31f5347db07b5ec2172"
dependencies = [
"bytes 1.11.1",
"chrono",
"futures-util",
"http 1.4.0",
"http-auth",
"jwt",
"lazy_static",
"oci-spec",
"olpc-cjson",
"regex",
"reqwest 0.12.28",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.18",
"tokio",
"tracing",
"unicase",
]
[[package]]
name = "oci-spec"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc3da52b83ce3258fbf29f66ac784b279453c2ac3c22c5805371b921ede0d308"
dependencies = [
"const_format",
"derive_builder 0.20.2",
"getset",
"regex",
"serde",
"serde_json",
"strum 0.27.2",
"strum_macros 0.27.2",
"thiserror 2.0.18",
]
[[package]]
name = "octocrab"
version = "0.44.1"
@@ -6272,6 +6441,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "olpc-cjson"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083"
dependencies = [
"serde",
"serde_json",
"unicode-normalization",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -7905,6 +8085,15 @@ dependencies = [
"untrusted",
]
[[package]]
name = "sd-notify"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4"
dependencies = [
"libc",
]
[[package]]
name = "sec1"
version = "0.3.0"
@@ -9428,6 +9617,12 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
version = "0.3.18"
@@ -9564,6 +9759,7 @@ dependencies = [
"getrandom 0.4.2",
"js-sys",
"rand 0.10.1",
"serde_core",
"wasm-bindgen",
]

5
Cargo.toml
View File

@@ -36,6 +36,9 @@ members = [
"fleet/harmony-fleet-deploy",
"fleet/harmony-fleet-e2e",
"harmony-reconciler-contracts",
"harmony_auth",
"harmony_auth_cli",
"harmony_auth_ui",
"examples/fleet_server_install",
"nats/jwt",
"nats/callout",
@@ -85,6 +88,7 @@ convert_case = "0.8"
chrono = "0.4"
similar = "2"
uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] }
fs2 = "0.4"
pretty_assertions = "1.4.1"
tempfile = "3.20.0"
bollard = "0.19.1"
@@ -106,6 +110,7 @@ reqwest = { version = "0.12", features = [
"http2",
"json",
], default-features = false }
oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] }
assertor = "0.0.4"
tokio-test = "0.4"
anyhow = "1.0"

2
book.toml
View File

@@ -2,7 +2,7 @@
title = "Harmony"
description = "Infrastructure orchestration that treats your platform like first-class code"
src = "docs"
build-dir = "book"
# build-dir = "book"
authors = ["NationTech"]
[output.html]

16
docs/SUMMARY.md
View File

@@ -6,6 +6,17 @@
- [Typed Score References](./concepts/score-references.md)
- [Getting Started Guide](./guides/getting-started.md)
## Fleet
- [Overview and Architecture](./guides/fleet.md)
- [Enrollment and Device Secrets](./guides/fleet-device-secrets.md)
- [Podman Deployments and Canary Rollout](./guides/fleet-podman-deployments.md)
- [Application Continuous Delivery](./guides/fleet-application-cd.md)
- [Tasks and Upgrades](./guides/fleet-tasks-upgrades.md)
- [Control-plane Operations](./guides/fleet-staging-install.md)
- [Zitadel FAQ](./guides/fleet-zitadel-faq.md)
- [Manual Token Mint](./guides/fleet-manual-token-mint.md)
## Use Cases
- [PostgreSQL on Local K3D](./use-cases/postgresql-on-local-k3d.md)
@@ -22,13 +33,13 @@
- [Developer Guide](./guides/developer-guide.md)
- [Application CLI — Use Cases & Commands](./guides/application-cli.md)
- [Harmony Auth CLI](./guides/harmony-auth-cli.md)
- [Application Capabilities — .with(...)](./guides/application-capabilities.md)
- [Writing a Score](./guides/writing-a-score.md)
- [Writing a Topology](./guides/writing-a-topology.md)
- [Adding Capabilities](./guides/adding-capabilities.md)
- [Web Authentication and CSRF Security](./guides/web-auth-security.md)
- [Operator Dashboard SSO (Zitadel) — setup](./guides/operator-dashboard-sso.md)
- [Fleet Device Secrets](./guides/fleet-device-secrets.md)
## Configuration
@@ -37,6 +48,9 @@
## Reference Designs
- [Fleet Score References](./reference/fleet-score-references.md)
- [Fleet Agent Upgrades](./design/fleet-agent-upgrades.md)
- [Fleet Tasks](./design/fleet-tasks.md)
- [System-upgrade Executor](./design/fleet-system-upgrades.md)
## Architecture Decision Records

380
docs/adr/022-fleet-agent-upgrade.md
View File

@@ -4,353 +4,79 @@ Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-05-06
Last Updated Date: 2026-05-06
Last Updated Date: 2026-07-22
## Status
Accepted (design); implementation deferred — see roadmap
`ROADMAP/fleet_platform/v0_2_plan.md`.
Accepted. This revision replaces the dual-active design and the later signed
switch-authorization design.
## Context
The v0.1 fleet agent ships as a single static aarch64-musl binary
sitting at `/usr/local/bin/fleet-agent`, started by a systemd
unit dropped at install time by `FleetDeviceSetupScore`. Every
managed device runs one. Today the only "upgrade procedure" is
`scp` + `systemctl restart` — fine for the bring-up phase, not
fine once paying customers run real workloads on the fleet.
Harmony Fleet currently manages IoT devices whose agents reconcile Podman
workloads from NATS desired state. Agent replacement must preserve one workload
owner across process or power loss.
Without a defined upgrade story we cannot ship a v0.1 agent into
the field. The contract a customer needs is:
NationTech fleet administrators, the operator, and the NATS control plane share
one administrative trust domain. Device identity comes from a Zitadel JWT whose
signature is validated through Zitadel JWKS and whose registered claims are
checked before its `device_id` is interpolated into device-scoped NATS subjects.
Separate artifact and cutover signatures are omitted because they would remain
inside this trust domain while adding another key lifecycle.
1. New agent versions can be rolled out without operator-side
manual intervention per device.
2. Workloads currently reconciled on the device do not flap
(start/stop/start) during the upgrade.
3. A failed new version automatically reverts to the last
known-good version, on its own, without page.
4. The operator (the central one in the cluster, not the human)
sees what version each device is running, can drive a target
version per device, and observes upgrade progress.
The agent itself is the only process on-device with full context
on what's reconciling and what's healthy. Anything centralized
(Ansible-pushed, OS-package-managed) doesn't have that signal.
The agent must be the one driving its own swap, with the
operator coordinating but not executing.
Future MDC fleets and their OKD control plane are outside this decision.
## Decision
We adopt a **K8s rolling-update–shape upgrade**, single-host,
agent-driven, operator-coordinated. Old version stays alive until
new is verified healthy from the operator's vantage point; only
then does the operator signal old to exit. **No version is ever
erased from disk.** Symlinks select the active binary.
The operator CAS-writes one per-device attempt. Once accepted, its UUID is bound
to its complete content. The live agent forwards it to a root updater and keeps
reconciling while the updater downloads, verifies, installs, and probes the
candidate.
### On-disk layout
After probe success, the updater owns one local transaction:
```
/usr/bin/fleet-agent-v0.1.1 ← versioned binary, immutable
/usr/bin/fleet-agent-v0.1.2 ← versioned binary, immutable
/usr/bin/fleet-agent-v0.1.3 ← versioned binary, immutable
/usr/local/bin/fleet-agent → symlink to current versioned binary
```text
stop old service
select candidate atomically
start candidate
commit on strict readiness, otherwise select and start the previous binary
```
- Versioned binaries are the source of truth. They live forever
(history-preserving, no GC). Disk use is bounded by humans
cleaning up explicitly, not by the upgrade procedure.
- The systemd unit installed by `FleetDeviceSetupScore` references
`/usr/local/bin/fleet-agent`. Symlink swap is the cutover
primitive — atomic on POSIX (`renameat2`).
- Naming convention: exact crate version string, `v<MAJOR>.<MINOR>.<PATCH>`,
no build metadata in the path. Build metadata lives in the agent's
reported version string but not in the file path (otherwise you
can't predict the path from a version pin).
The old agent attempts to drain its current mutation only after systemd sends
SIGTERM. Shutdown is bounded by systemd, so an operation that exceeds the stop
limit may be killed rather than drained.
### State machine on the agent side
Artifacts use HTTPS and a SHA-256 digest carried by NATS intent. The updater
trusts its `fleet-agent` socket group, derives all managed paths, accepts no
command, and executes candidates without root privileges. It has no NATS,
OpenBao, or operator connection.
```
Running ──[operator publishes desired_version != current]──▶ Draining
▲ │
│ │
│ ▼
│ Staging
│ │
│ ▼
│ Verifying
│ │
│ ▼
│ ┌──────────────────────────────[smoke fails]────────┤
│ │ │
│ [revert: symlink → previous, ▼
│ stay at current] Cutover-Ready
│ │
│ [Cutover-Ready persists ≥ T_OPERATOR_OBSERVE │
│ until operator publishes stop_signal] │
│ ▼
└────────────────────────────────────────────────────── Stopping
│
▼
(exit)
```
There is no artifact signature, second cutover authorization, device trust key,
or extended probation.
States in detail:
The active service reports ready only after local dependencies, NATS, the
complete desired-state snapshot, activation target-version check, transaction
recovery, and required startup publications succeed. Strict readiness may reject
a valid candidate during a coincident network outage. Retry and failure
classification will be refined separately.
- **Running** — normal reconcile loop.
- **Draining** — refuses to start new podman services for new
desired-state writes. In-flight reconciles complete and report
their final state to the operator. Existing services stay
running. Heartbeat continues. State is published as part of the
agent's heartbeat (`agent_state: "draining"`).
- **Staging** — fetch new versioned binary URL (signed,
hash-pinned), verify, place at `/usr/bin/fleet-agent-v<new>`.
Set chmod, ownership. No other state mutation.
- **Verifying** — invoke the staged binary with `--self-test`. New
binary parses its config, opens NATS connection, validates JWT,
prints version + "ok", exits 0. **No state mutation.** Catches
obvious breakage (missing dependency, wrong arch, corrupt
download, broken config-schema migration) before swap.
- **Cutover-Ready** — staged binary is healthy. Old agent updates
the symlink atomically:
```
ln -sfn /usr/bin/fleet-agent-v0.1.2 /usr/local/bin/fleet-agent.new
mv -T /usr/local/bin/fleet-agent.new /usr/local/bin/fleet-agent
```
Old agent then `systemctl start fleet-agent-v0.1.2.service` (a
parallel transient service, not `systemctl restart` of itself).
Both old and new are now running. New publishes its first
heartbeat with `version=v0.1.2`. Operator sees two heartbeats
per device for a brief window.
- **Stopping** — operator publishes a stop signal to the old
agent's NATS subject. Old agent receives, gracefully exits.
systemd's `Restart=on-failure` does *not* trigger because the
exit is `success` (rc=0, code-path-driven). New agent is now
the only one running. systemd unit is reconfigured to point at
the *current* symlink target on its next restart, but that's
cosmetic — the symlink already does the job.
### Operator-side coordination
The operator is the only source of truth for "what version should
this device run". One new field per device, two new subjects.
**New on `Device` CR / KV `device-info`:**
- `current_version` — what the agent is running right now.
Reported in heartbeat; reflected to the CR.
- `desired_version` — what the operator wants the agent to run.
Set by operator-side logic (default: latest published; eventually
canary / %-based).
**New NATS subjects (per-device, scoped by callout permissions):**
- `device-cmd.<device_id>.upgrade-stop` — operator → old agent.
Payload: `{"reason": "...", "deadline_ms": ...}`. Sent only after
operator has observed a heartbeat from the new version with
`current_version == desired_version` AND `agent_state == "running"`.
- `device-state.<device_id>.upgrade` — agent → operator. Status
events: `staging`, `verifying`, `cutover-ready`, `failed`, `done`.
Drives `Device.status.upgrade.{phase, last_error, ...}`.
The operator only emits `upgrade-stop` after it has independently
verified the new agent is up. **Old agent does not stop itself
based on its own observations.** This is the load-bearing
property: the same operator that disagreed with the upgrade
("haven't seen new version's heartbeat") would never have sent
the stop signal. Single-source-of-truth handoff.
### Failure modes and rollback
- **Staging fails (download / hash mismatch):** Agent stays in
`Running`. Reports `phase: "failed"`, `last_error`. Operator
sees the failure; can fix the artifact + retry by re-publishing
the same desired_version (any change to desired_version
re-triggers the state machine).
- **Verifying fails (smoke test rc != 0):** Agent stays in
`Running`. Reports failure. Staged binary stays on disk for
inspection. Operator can collect, debug, ship a fixed version.
- **Cutover-ready, but new agent never publishes a heartbeat
with the new version within T_HEARTBEAT_TIMEOUT (suggested
60s):** Old agent reverts the symlink, stops the parallel
systemd transient service, transitions back to Running with
the old version. Reports `failed`. Same recovery path.
- **Operator never sends stop signal (e.g., operator-side
outage):** Old agent stays in Cutover-Ready indefinitely. Both
agents are running; only the new one is publishing as the
active one (the old one's writes are gated on its state). This
is expensive (2× resource use) but safe — the operator is the
authoritative coordinator and any other behavior would risk
losing both agents at once.
- **Both agents alive but new agent crashes:** systemd's
`Restart=on-failure` on the new agent's transient unit retries.
If it can't come back, the operator never sends the stop signal,
the old agent stays Cutover-Ready, and a human investigates.
The fleet keeps working on the old version — the rollback is
implicit.
- **Operator publishes an older `desired_version`:** Reverse
rollout. Same mechanism, just with old/new swapped. The "new"
binary is older, but the procedure is identical. The fact that
no version is ever GC'd is what makes this work.
### What this isn't
- **Not fleet-wide.** Per-device. Fleet-wide canary / %-based
rollouts are operator-side orchestration **on top of** this
primitive. The operator would publish `desired_version` to a
rolling subset of devices and watch heartbeats. Out of scope
for v0.2 — single-device upgrade is sufficient for a 100-Pi
fleet which is more than the 12-month customer roadmap.
- **Not blue/green of the entire OS.** We swap one userspace
binary. The OS, podman, the systemd unit text, the kernel — all
unchanged. Out of scope.
- **Not a package manager.** Versioned binaries land at fixed
paths because we control them. apt / dpkg / OSTree are
orthogonal and not in the loop.
## Rationale
- **No version ever erased.** Trivializes rollback (the previous
binary is a `ln -sfn` away). Simplifies the failure tree:
every "what if" branch resolves to "old still on disk". Disk
cost on aarch64-musl is ~5–10 MB per version — at 12 versions
/ year, that's 100 MB after a decade of upgrades. Negligible
compared to Pi storage.
- **Symlink swap as cutover.** POSIX-atomic. No daemon state.
Cheap to revert. Compatible with systemd unit references that
point at a stable path.
- **Old verifies new, then reports up.** This is the load-bearing
property: it places the verification at the agent (which has
the only complete view of its own runtime state) but the
*commitment* at the operator (which is the only thing safe to
trust as the cluster-wide source of truth). Either side alone
can fail safe; only consensus advances the upgrade.
- **Operator-driven stop, not agent self-stop.** A self-stopping
agent could decide to exit before the operator agrees, leaving
the cluster blind. Forcing the stop through the operator means
any disagreement keeps the old agent alive — which is the
desired bias.
- **Drains in-flight work first.** Mirrors K8s pod-shutdown
semantics. A workload reconciling at the moment of swap
finishes its current step, reports state, then queues. New
agent picks up the queue once it's the active version. No
observable flap on the workload.
- **Heartbeat-driven version reporting.** The agent already
publishes heartbeats; adding the version field is one line.
No new transport.
Every pre-commit activation failure biases executable selection toward the
previous binary. `rollback-failed` blocks new attempts until direct repair.
Successful readiness commits immediately. Failures after commit use systemd's
normal restart policy and do not trigger protocol rollback.
## Consequences
**Pros:**
- One process owns workload mutation; there is a bounded interval with no agent
during cutover.
- Preparation does not delay workload reconciliation.
- Rollback changes only the executable. Persistent changes made before commit
must remain readable by the previous release.
- The first updater-capable release requires `FleetDeviceSetupScore`. The root
updater remains outside automatic upgrades.
- NationTech administration, the `fleet-agent` socket, the Zitadel identity
chain, the auth callout, and NATS are trusted control-plane components.
- Bounded blast radius per upgrade (one device).
- Rollback is the same code path as upgrade — no special-case
bug class.
- Operator's view is monotonic: heartbeats with versions are
immutable history; there's no "did the upgrade really happen"
state.
- Old agent never decides to exit on its own. The most dangerous
failure mode in self-upgrading software (premature exit) is
designed out.
- Compatible with eventual fleet-wide rollouts (canary, %-based)
which become operator-side orchestration on top of this
primitive.
**Cons:**
- Briefly runs two agents per device (Cutover-Ready window).
Memory and connection-count both ~2× during that window.
Acceptable for the upgrade duration (typically <60s).
- Requires reliable connectivity between agent and operator to
complete the handoff. A device whose NATS link fails mid-
upgrade stays in Cutover-Ready until link recovers.
- Disk grows monotonically with version count. Bounded by human
cleanup. We do not GC.
- New NATS subjects, new heartbeat fields, new `Device.status`
fields. Schema bump that operators-in-the-field need to handle
(the operator must understand "old agent reporting no version
field" as `version: unknown`, not crash).
## Alternatives considered
1. **OS-package upgrade (apt / dpkg / OSTree).** *Pros:* zero
custom code, standard toolchain, GPG-signed.
*Cons:* Loses the "agent verifies the new agent before swap"
property. apt's restart hook flips the symlink and `systemctl
restart`s; if the new binary is broken, the device is bricked
until human intervention. Doesn't drain in-flight work. Doesn't
know about NATS-managed pause states. Couples the upgrade
schedule to the distro's repo, not to the cluster operator's
intent. Rejected.
2. **Pull-from-OCI-registry on each agent restart.** *Pros:* same
primitive as podman / kube node-image-rotation.
*Cons:* Coupling to a registry the device must reach — many
customer fleets are on private subnets without registry
access. Would mean shipping a registry mirror per fleet. Adds
a dependency for a problem we can solve with a signed binary
on a CDN.
3. **Two systemd units, blue/green at the unit level.**
`fleet-agent-v0.1.1.service` and `fleet-agent-v0.1.2.service`,
ratchet via systemctl enable/disable. *Pros:* no symlink dance.
*Cons:* duplicates a lot of unit-file content; harder to
reason about what the "active" unit is (you have to ask
systemd, not `readlink`); doesn't compose well with the
`ExecStart=/usr/local/bin/fleet-agent` line we already ship.
Symlink swap is the lighter primitive.
4. **Self-stopping agent (no operator stop signal).** New agent
tells old agent "I'm up, you can go" via NATS. *Pros:* one
fewer subject.
*Cons:* The new agent is also the agent we're least sure of
— putting it in charge of the old one's lifecycle inverts the
trust model. If the new agent has a bug that causes it to
announce ready prematurely, the cluster goes blind. The
operator path is the conservative choice.
5. **Operator-pushed binary (instead of agent-pulled).** The
operator sshes / executes a one-off command per device.
*Pros:* operator controls timing precisely.
*Cons:* Reintroduces SSH as a control plane (we just spent a
month getting rid of it for the enrollment flow). Doesn't
scale to fleets where most devices are NATted away from the
operator.
## Implementation milestones
(For a future implementer; not committed to a date here. Lives
in the v0.2+ backlog.)
1. **M1** — Versioned binary layout: builds produce
`fleet-agent-v<version>` artifacts; install Score writes them
to `/usr/bin/fleet-agent-v<version>` + creates
`/usr/local/bin/fleet-agent` symlink. Existing tests cover the
rest.
2. **M2** — Version field in heartbeat + `Device.status.current_version`
reflection on the operator side. No upgrade behavior yet.
3. **M3** — `desired_version` field on the device-info KV +
operator setter. No agent-side action yet.
4. **M4** — Agent state machine, end to end, gated by a feature
flag. Operator publishes desired_version → agent does the
dance → operator sends stop signal → done. Includes failure-
mode tests (download fail, smoke fail, heartbeat-timeout
revert).
5. **M5** — Remove the feature flag. Default-on.
6. **M6** — Operator-side rollout strategies (canary, %-based) —
only after M5 has been in production for 30 days against a
real fleet.
## Additional Notes
- Binary signing + signature verification is in scope for the
`Staging` step but the *which* signing scheme (cosign / Rekor
/ minisign) is deferred until the M1 implementation. Whatever
we pick must work on aarch64-musl Pi devices without
additional system dependencies.
- The N-versions-on-disk policy is "all of them, forever" per
the constraint above. If disk pressure becomes real on some
customer fleet, a manual GC tool can prune `/usr/bin/fleet-agent-v*`
by date — never automatic, never as part of the upgrade
itself.
- See JG's *Pour l'amour des compilateurs* talk (Botpress
Meetup, 2026-04-30) for the framing applied here:
cardinality-matched types and operator-as-coordinator are the
same idea, applied to one function and to one platform.
The [fleet agent upgrade guide](../design/fleet-agent-upgrades.md) owns the
current sequence, recovery behavior, limits, status, and repair constraints.
Wire types remain authoritative in `harmony-reconciler-contracts`.

18
docs/coding-guide.md
View File

@@ -215,6 +215,24 @@ pub use network::NetworkConfig;
---
## Testing
Test infrastructure behavior against the real component whenever practical:
use k3d for Kubernetes, Podman for containers, libvirt for VMs, and disposable
devices for privileged or destructive paths. A mock that reimplements an
external system can confirm the mock rather than the integration.
Fakes remain appropriate at domain and use-case boundaries when testing core
algorithms, failure transitions, or authorization decisions. Keep them behind
the same capability boundary as the real adapter, and retain at least one test
through the real adapter for behavior owned by that external system.
E2E harnesses compose the same Scores as production. They may change the
Topology, credentials, namespace, or target device, but must not maintain a
parallel manifest or deployment path.
---
## Commit Style
Follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/):

228
docs/design/fleet-agent-upgrades.md Normal file
View File

@@ -0,0 +1,228 @@
# Fleet agent upgrades
## Scope
This design updates the Harmony agent on IoT devices that reconcile Podman
workloads from NATS desired state. It covers one agent binary on one Linux
device. Operating-system, Podman, workload, and future MDC/OKD upgrades are
outside its scope.
![Fleet agent upgrade architecture](../diagrams/fleet-agent-upgrade-architecture.svg)
## Ownership
| Component | Responsibility |
|---|---|
| Fleet operator | Select the desired release and CAS-write the current per-device attempt |
| Active agent | Validate and forward intent, reconcile during preparation, drain on SIGTERM, publish status |
| Root updater | Prepare and probe the candidate, stop, switch, start, commit or roll back |
| systemd | Serialize service ownership and enforce startup and shutdown limits |
The active process holds an exclusive lock, so only one agent can mutate
workloads. Preparation does not pause Podman reconciliation. Running containers
continue under their restart policies during agent cutover.
The updater has no NATS, OpenBao, or operator client. Its Unix socket accepts
`upgrade` and read-only `status` requests. It derives every managed path and
accepts no shell command or caller-selected destination.
## Trust boundary
The operator CAS-writes `AgentUpgradeAttempt` to the device's
`agent-upgrade-intent` key. CAS orders competing writers; it does not make the KV
key append-only. After local acceptance, the agent and updater bind an attempt
UUID to the digest of its complete content.
An attempt contains the device, target version, architecture, artifact URL, byte
limit, SHA-256, and UUID. It has no source version or expiry: a device returning
after days or months can upgrade directly to the current target. Any required
data migration belongs in the binary and must handle the versions it supports.
Artifacts are either direct HTTPS URLs or anonymous `oci://` references with an
explicit tag or manifest digest. OCI artifacts must declare the Harmony agent
artifact type and contain exactly one Harmony agent binary layer. HTTPS
authenticates transport; SHA-256 binds the downloaded bytes to the NATS attempt.
There is no artifact signature, second cutover authorization, device upgrade
key, or OpenBao dependency. NationTech administrators, the operator, and NATS
share one administrative trust domain, so a separate release-signing domain is
not required today.
The updater trusts callers admitted by the `fleet-agent` socket group to forward
NATS intent faithfully. Compromise of that account includes control of agent
upgrades. Installed binaries still run as `fleet-agent`, not root.
The auth callout validates the Zitadel JWT signature through Zitadel JWKS and
checks its registered claims. It then extracts `device_id`, rejects characters
unsafe for NATS subjects, and interpolates the value into device-scoped data
permissions. No unverified client-supplied device ID is used.
## Upgrade flow
![Fleet agent upgrade sequence](../diagrams/fleet-agent-upgrade-sequence.svg)
### Intent
When desired and reported versions differ, the operator writes an attempt with
a new UUID. The agent rejects an attempt for another device, the wrong
architecture, or an invalid UUID. Reuse of a UUID with different content is
rejected. Intent has no age or previous-version gate.
The per-device intent key can later be replaced by CAS. An unchanged release
whose attempt has failed is reused rather than retried automatically. A new
attempt currently requires changed release metadata or direct control-plane
repair.
### Preparation and probe
The agent publishes `preparing` and sends one `upgrade` request. The updater
writes `preparing` to its journal before it:
1. downloads over HTTPS with redirects disabled;
2. enforces the attempt limit and 100 MiB compiled ceiling;
3. verifies SHA-256;
4. installs or re-verifies `/usr/lib/harmony-fleet/fleet-agent-v<version>`;
5. runs `<candidate> --self-test` as `fleet-agent` and requires its compiled
version to match the target.
The live agent continues reconciling during these steps.
Probe mode parses the real configuration, reaches Podman when enabled,
authenticates to NATS, opens required buckets, reads per-device keys, creates a
filtered desired-state consumer, checks the updater socket, and consumes a
complete snapshot. It starts no loops, publishes no normal status or heartbeat,
and mutates no workload. Creating the ephemeral consumer is a JetStream
management operation. The probe does not prove publish permissions; active
startup exercises its startup writes before readiness.
NATS connection attempts are limited to 15 seconds over a 3-minute retry
window. The complete probe is limited to 4 minutes.
### Stop, switch, start
After probe success, the updater writes `activating` and runs
`systemctl stop fleet-agent.service`. On SIGTERM, the old agent stops admitting
new runtime mutations and waits for the current mutation. It then attempts to
publish acknowledged `stopping` status with `reason=updater`, records drain
duration, flushes NATS, notifies systemd, and exits.
This drain is bounded by `TimeoutStopSec=60s`. systemd may kill an agent whose
current Podman mutation does not finish in time. The `stopping` status is
diagnostic; the updater does not wait for it. Cutover proceeds when systemd
reports the service inactive.
The updater atomically replaces `/usr/local/bin/fleet-agent`, syncs its parent
directory, and starts the service. It never deliberately runs old and new
reconcilers together.
### Readiness and commit
The new process sends `READY=1` only after it:
- acquires the process lock and parses configuration;
- reaches Podman when enabled;
- reconnects to NATS and opens required buckets;
- publishes device information;
- reads and validates the recovered updater transaction;
- verifies its compiled version against the activation target;
- consumes a complete desired-state snapshot;
- publishes transaction status when an upgrade is active.
The local transaction need not match the current NATS intent during recovery.
Its stored digest, target, and exact previous binary path are authoritative after
cutover starts. Rollback restores that path and does not depend on a source
version declared by the operator.
Successful systemd readiness commits immediately. A network failure after a
successful probe can therefore cause rollback. After `committed`, later crashes
use the normal systemd restart policy; this protocol does not roll back a
candidate that was already ready.
## Recovery
![Fleet agent durable transaction and recovery](../diagrams/fleet-agent-upgrade-recovery.svg)
The updater journal stores attempt identity, previous and target paths, bounded
errors, and up to 16 timed transitions.
| Durable phase | Recovery |
|---|---|
| `preparing` | Restore the previous link, record `failed`, start the previous service; restore or start failure enters `rollback-failed` |
| `activating` | Record `rolling-back`, restore the previous link, stop, then start the previous service |
| `committed` | Keep the target |
| `rolling-back` | Restore the previous link and repeat stop/start |
| `failed` | Keep the previous target; no automatic retry |
| `rollback-failed` | Refuse every new attempt until direct root repair |
Before the updater reports systemd readiness, interrupted pre-commit recovery
has selected the previous symlink and persisted its recovery phase. Service
stop/start completion continues asynchronously after updater readiness so the
agent can query local status while starting. Corrupt journal data prevents the
updater socket from starting; state is not inferred from filesystem contents.
Journal and binary writes use temporary files, file sync, atomic rename, and
parent-directory sync. Symlink selection uses a temporary symlink, atomic
rename, and parent-directory sync.
`rollback-failed` has no remote clear operation. Repair requires root access to
inspect the journal, active symlink, managed binaries, and both systemd units.
The updater must not be unquarantined until the previous binary is selected and
starts successfully.
## Status
Public phases are `preparing`, `stopping`, `starting`, `complete`,
`rolling-back`, `failed`, and `rollback-failed`. The wire contract in
[`harmony-reconciler-contracts/src/upgrade.rs`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs)
is authoritative for fields and serialization.
Status includes attempt and target identity, current phase, bounded detail and
error, attempt and transition timing, completed drain duration, typed reason,
boot and systemd invocation IDs when available, one current journald unit
reference, and up to 16 transitions. Log content remains in journald.
The updater never publishes NATS status. The active agent polls the local
transaction and reflects changes, including `complete`, into NATS and the Device
CR.
## Limits
| Limit | Value |
|---|---:|
| HTTPS connect / complete request | 15 seconds / 5 minutes |
| Artifact size | 100 MiB |
| NATS connection attempt / retry window | 15 seconds / 3 minutes |
| Candidate probe | 4 minutes |
| systemd stop / updater stop bound | 60 / 65 seconds |
| systemd start / updater start bound | 4 minutes / 4 minutes 15 seconds |
| Socket request-read / status response / upgrade response | 10 seconds / 15 seconds / 20 minutes |
| Socket request or response | 1 MiB |
| Concurrent socket handlers | 32 |
| Retained transitions | 16 |
| Detail or error | 1,024 characters |
Only one mutating socket request runs at a time; `status` remains available
during preparation and recovery. Versioned binaries are not garbage-collected.
Devices currently have many GB available, so collection is deferred until disk
pressure makes retention an operational concern.
Rollback changes only the executable. Configuration, local databases, Podman
labels, and any persistent format touched before commit must remain readable by
the previous release.
## Bootstrap
`FleetDeviceSetupScore` installs the bootstrap binary, updater and agent units,
socket ownership, state directories, and active symlink. The root updater is not
self-updated. Repairing or replacing it requires another device setup operation.
Running device setup during an active transaction is unsupported.
Devices installed before OCI support require one device setup operation before
their first `oci://` upgrade. This replaces the bootstrap updater; an older
updater cannot fetch an OCI artifact even when the active agent can accept the
intent.
## Related decisions
- [ADR-016: Harmony agent and global mesh](../adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md)
- [ADR-022: Fleet agent upgrade procedure](../adr/022-fleet-agent-upgrade.md)
- [ADR-023: Deploy architecture](../adr/023-deploy-architecture.md)

128
docs/design/fleet-system-upgrades.md Normal file
View File

@@ -0,0 +1,128 @@
# Fleet system-upgrade executor
## Summary
System upgrade is a privileged built-in Fleet task. `TaskRun` owns placement,
the frozen target plan, deadlines, and aggregate status as described in
[Fleet tasks](./fleet-tasks.md). This document covers the device executor.
The unprivileged agent requests one fixed operation from the existing root
updater over its Unix socket. The updater accepts no shell command, package
name, repository, path, unit, or reboot argument.
System and agent upgrades share the updater process and mutation lock. System
upgrades have a separate protocol operation, journal, and state machine because
package upgrades are not atomically reversible.
The first executor runs an apt full upgrade using the device's configured,
signed repositories and then requires a reboot. It supports Debian and
Raspberry Pi OS. It does not configure repositories, select packages, roll back
packages, cancel an active apt/dpkg process, or retry a terminal failure.
## Trust boundary
The updater socket admits only the `fleet-agent` Unix group. The agent can ask
for `AptFullUpgradeV1`, but cannot alter what that operation does. A compromised
agent can invoke the upgrade and is treated as a compromised device; adding a
second key held by the same agent would not improve that boundary.
A system-upgrade request contains only:
```text
attemptId
runUid
deviceId
expiresAt
```
The updater validates the identifiers and expiry before starting. Duplicate
requests with the same content return the durable result. Reusing an attempt ID
with different content fails. Expiry prevents an intent delayed in NATS from
starting later; it does not interrupt apt/dpkg after mutation begins.
The updater publishes protocol and `AptFullUpgradeV1` capability through device
observation. Planning fails if a selected device lacks that capability. The
privileged updater is installed out of band by `FleetDeviceSetupScore` and does
not self-update.
The updater has no NATS credentials. The agent relays updater status. A
compromised agent can therefore falsify status, which is consistent with the
device-compromise trust model.
## Privileged state machine
Agent and system upgrades share one durable mutation lock. System upgrades use
one journal per attempt under
`/var/lib/harmony-fleet-updater/system-upgrades/`.
Device-reported phases are:
```text
blocked
preflight
applying
rebooting
verifying
complete
failed
repair-required
```
`blocked` is non-terminal and covers an active agent upgrade or a bounded wait
for apt/dpkg locks. `repair-required` forbids automatic retry.
`AptFullUpgradeV1` has one compiled transaction:
1. Confirm package state is healthy and no configured apt source disables
repository authentication.
2. Run `apt-get update`.
3. Run noninteractive `apt-get full-upgrade`, preserving locally modified
configuration files.
4. Audit dpkg state.
5. Record `rebooting` and the current boot ID, then request reboot.
6. After startup, require a changed boot ID and healthy dpkg state.
Durable recovery is:
| Journal state | Recovery |
|---|---|
| `preflight` | Repeat non-mutating checks. |
| `applying` | Audit dpkg and enter `repair-required`; never rerun the interrupted full upgrade. |
| `rebooting`, old boot ID | Retry the fixed reboot request up to a bound, then report `failed`. |
| `rebooting`, changed boot ID | Persist `verifying`. |
| `verifying` | Audit package state, then complete or require repair. |
| `complete`, `failed`, `repair-required` | Return the same terminal result for duplicate intent. |
The updater persists `applying` before spawning apt and `rebooting` before
requesting reboot. Startup fails if more than one journal appears active or a
journal is corrupt. Journal writes use a temporary
file, fsync, and atomic rename. `repair-required` remains until an operator
repairs the device.
## Task result
Disconnect after `rebooting` is expected; request timeout is not a failure
signal. A device succeeds only after:
- the updater observes a boot ID different from the pre-upgrade boot ID;
- post-boot dpkg state is healthy;
- the agent reconnects and relays `complete`;
- the agent records a post-completion heartbeat before the run deadline.
The operator owns the run deadline. A device that never returns fails the run.
Already upgraded devices are not rolled back.
## Invariants
- A device runs at most one privileged mutation at a time.
- An expired attempt cannot start.
- Restart does not repeat a completed attempt.
- Corrupt or incomplete package state fails closed.
- All apt, dpkg, and reboot arguments are compiled into the updater.
## Remaining work
1. Prove the path on a disposable Debian VM, then Raspberry Pi OS.
2. Test updater and agent restart, package failure, reboot failure, expiry, and
duplicate delivery against real systemd, apt, and dpkg.
3. Add UTC recurring schedules after the direct run passes those tests.

142
docs/design/fleet-tasks.md Normal file
View File

@@ -0,0 +1,142 @@
# Fleet tasks
## Summary
Fleet tasks are finite jobs selected with the same group and label rules as
Fleet deployments. A task freezes its targets, runs once on each device, and
retains a terminal result.
The first release supports directly created, multi-device `TaskRun`s and the
built-in system upgrade. Recurring schedules come after the direct path is
proven on Debian and Raspberry Pi OS.
## `TaskRun`
```yaml
apiVersion: fleet.nationtech.io/v1alpha1
kind: TaskRun
metadata:
name: upgrade-device-1
spec:
allowedGroups: [production]
targetSelector:
matchLabels:
device-id: device-1
deadlineSeconds: 21600
systemUpgradeV1: {}
```
`systemUpgradeV1` is empty because apt, dpkg, repository, and reboot policy is
compiled into the updater. The API cannot pass package names, repository paths,
commands, or reboot arguments.
Admission rejects changes to the spec. The run starts immediately and has one
phase:
```text
Planning | Running | Complete | Failed
```
During `Planning`, the operator freezes every matching device that supports
`AptFullUpgradeV1`. No match fails with `NoTargets`.
Status records target, success, and failure counts; start and completion times;
a reason; and the latest bounded error. The CR never contains an unbounded
per-device map.
## Placement
Task placement uses the deployment aggregator's rule:
```text
allowed group membership AND targetSelector
```
`allowedGroups` requires at least one group. `matchLabels` is a conjunction, and
an empty selector matches every authorized device.
`matchExpressions` fails closed until implemented.
The operator checks group authorization again before creating the intent. A
device revoked before release fails the run with `TargetRevoked`; another device
is never substituted into the frozen plan. A group-source error retries
planning instead of becoming `NoTargets`.
The target plan is created atomically in JetStream before the run enters
`Running`. Concurrent operator instances therefore load the same frozen plan.
The run UID is also the attempt ID, so intent creation and updater execution are
idempotent across restarts.
## Execution
The operator writes the fixed system-upgrade attempt to:
```text
system-upgrade-intent: <device-id>.<run-uid>
```
The agent publishes updater-owned status to:
```text
system-upgrade-status: <device-id>.<run-uid>
```
The agent watches current and new intents, so reboot does not lose the active
run. The operator accepts terminal status only when its attempt and run IDs
match. For success, the agent records the first heartbeat after updater
completion in the durable run status; that heartbeat must precede the deadline.
A late result cannot reopen a failed run.
Intent and status buckets are file-backed, keep one value per key, and have byte
and age limits. The operator removes a terminal intent; terminal status remains
for bounded diagnostics.
The deadline is copied into the attempt as `expiresAt`. Expiry prevents a
delayed intent from starting. It does not interrupt apt or dpkg after the
updater has accepted the attempt.
## Canary release
If any matched device has the `canary=true` label, the operator releases only
those devices first. Every canary must complete successfully before the
operator releases the remaining devices. A failed, missing, or timed-out canary
stops the rollout. Completed devices are not rolled back.
When no matched device has `canary=true`, the operator releases the full
frozen target set immediately. There is no percentage, batch-size, or
concurrency setting.
The frozen target set must remain bounded outside the CR, while the CR retains
aggregate counts and the latest error.
## Recurring schedules
A later namespaced `TaskSchedule` creates immutable `TaskRun` resources. Daily,
weekly, yearly, and five-field cron forms use UTC. Time-zone configuration is
not supported.
One schedule has at most one active run. Ticks during an active run are skipped,
and restart creates at most one catch-up run for the latest eligible tick.
Suspension prevents new runs without stopping an active run.
## Other workloads
Container and agent-upgrade tasks remain deferred. Add a workload discriminator
only when a second workload is implemented. Agent upgrades continue to use
their existing activation, health-check, and rollback state machine.
## Invariants
- A run UID identifies one immutable target plan and operation.
- Canary and non-canary membership is frozen with the target plan.
- Group authorization is current when an intent is released.
- A device executes one run attempt at most once.
- A terminal run never advances.
- Restart does not repeat a completed device attempt.
- No Kubernetes status map grows with fleet size.
## Remaining work
1. Prove package upgrade, reboot, restart recovery, expiry, and duplicate intent
handling on a disposable Debian VM and Raspberry Pi OS device.
2. Add UTC schedules.
3. Add dashboard run history and alerts.

63
docs/diagrams/fleet-agent-upgrade-architecture.svg Normal file
View File

@@ -0,0 +1,63 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="540" viewBox="0 0 1200 540" role="img" aria-labelledby="title desc">
<title id="title">IoT fleet agent upgrade ownership and trust architecture</title>
<desc id="desc">The fleet operator writes one authenticated NATS upgrade intent. The active agent forwards it while continuing Podman reconciliation. A root updater downloads by HTTPS, verifies SHA-256, probes as fleet-agent, and performs the systemd stop, symlink switch, and start transaction.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
<marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
<style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
</defs>
<rect width="1200" height="540" fill="#ffffff"/>
<g class="sans">
<text x="42" y="50" fill="#0f172a" font-size="29" font-weight="750">One intent, one active workload owner</text>
<text x="42" y="78" fill="#64748b" font-size="16">Authenticated NATS carries upgrade intent. HTTPS and SHA-256 bind the candidate bytes.</text>
<rect x="35" y="110" width="300" height="360" rx="18" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
<text x="60" y="142" fill="#475569" font-size="12" font-weight="750" letter-spacing="1.4">CONTROL PLANE</text>
<rect x="65" y="175" width="240" height="90" rx="12" fill="#172554"/>
<text x="88" y="205" fill="#bfdbfe" font-size="12" font-weight="700">DESIRED RELEASE</text>
<text x="88" y="235" fill="#fff" font-size="18" font-weight="700" class="mono">Device.spec</text>
<text x="88" y="256" fill="#bfdbfe" font-size="14">version and artifact metadata</text>
<rect x="65" y="315" width="240" height="110" rx="12" fill="#0f766e"/>
<text x="88" y="345" fill="#ccfbf1" font-size="12" font-weight="700">FLEET OPERATOR</text>
<text x="88" y="376" fill="#fff" font-size="17" font-weight="700">current attempt</text>
<text x="88" y="400" fill="#ccfbf1" font-size="14">CAS and status reflection</text>
<path d="M185 265V315" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<rect x="390" y="110" width="250" height="360" rx="18" fill="#f0fdfa" stroke="#5eead4" stroke-width="1.5"/>
<text x="415" y="142" fill="#115e59" font-size="12" font-weight="750" letter-spacing="1.4">AUTHENTICATED MESH</text>
<rect x="425" y="178" width="180" height="245" rx="14" fill="#fff" stroke="#99f6e4"/>
<text x="515" y="216" text-anchor="middle" fill="#0f172a" font-size="21" font-weight="750">NATS KV</text>
<text x="445" y="265" fill="#115e59" font-size="12" class="mono">agent-upgrade-intent</text>
<text x="445" y="309" fill="#115e59" font-size="12" class="mono">agent-upgrade-status</text>
<text x="445" y="365" fill="#64748b" font-size="12">device-scoped data subjects</text>
<text x="445" y="390" fill="#64748b" font-size="12">identity from verified JWT</text>
<path d="M305 370H425" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<path d="M425 400H305" stroke="#0f766e" stroke-width="2" marker-end="url(#arrow)"/>
<rect x="695" y="110" width="470" height="360" rx="18" fill="#fff7ed" stroke="#fdba74" stroke-width="1.5"/>
<text x="720" y="142" fill="#9a3412" font-size="12" font-weight="750" letter-spacing="1.4">IOT PODMAN DEVICE</text>
<rect x="730" y="175" width="180" height="110" rx="12" fill="#172554"/>
<text x="750" y="203" fill="#bfdbfe" font-size="12" font-weight="700">UNPRIVILEGED</text>
<text x="750" y="234" fill="#fff" font-size="19" font-weight="750">Fleet agent</text>
<text x="750" y="259" fill="#bfdbfe" font-size="13">forward · reconcile · report</text>
<rect x="960" y="175" width="170" height="110" rx="12" fill="#7c2d12"/>
<text x="980" y="203" fill="#fed7aa" font-size="12" font-weight="700">ROOT</text>
<text x="980" y="234" fill="#fff" font-size="19" font-weight="750">Updater</text>
<text x="980" y="259" fill="#fed7aa" font-size="13">prepare · switch · recover</text>
<path d="M605 230H730" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<path d="M910 230H960" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/>
<text x="935" y="216" text-anchor="middle" fill="#9a3412" font-size="11">upgrade</text>
<rect x="730" y="340" width="180" height="80" rx="12" fill="#eef2ff" stroke="#a5b4fc"/>
<text x="750" y="372" fill="#312e81" font-size="17" font-weight="700">Podman workloads</text>
<text x="750" y="397" fill="#64748b" font-size="13">reconcile until SIGTERM</text>
<rect x="960" y="330" width="170" height="100" rx="12" fill="#fffbeb" stroke="#fbbf24"/>
<text x="980" y="360" fill="#78350f" font-size="17" font-weight="700">systemd + symlink</text>
<text x="980" y="385" fill="#92400e" font-size="13">stop · inactive · switch</text>
<text x="980" y="408" fill="#92400e" font-size="13">start · strict readiness</text>
<path d="M820 285V340" stroke="#64748b" stroke-width="2"/>
<path d="M1045 285V330" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/>
<text x="42" y="510" fill="#64748b" font-size="14">NationTech administration is one trust domain; device_id comes from a JWKS-verified Zitadel JWT.</text>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.4 KiB

45
docs/diagrams/fleet-agent-upgrade-recovery.svg Normal file
View File

@@ -0,0 +1,45 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="650" viewBox="0 0 1200 650" role="img" aria-labelledby="title desc">
<title id="title">Fleet agent durable upgrade states and recovery</title>
<desc id="desc">Preparing keeps the previous agent active; interruption selects the previous binary and fails, while recovery failure quarantines the updater. Activating covers stop, switch, strict start, and readiness; interruption enters rollback. Readiness commits immediately.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
<marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
<marker id="arrow-bad" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#be123c"/></marker>
<style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
</defs>
<rect width="1200" height="650" fill="#ffffff"/>
<g class="sans">
<text x="40" y="48" fill="#0f172a" font-size="29" font-weight="750">The durable journal controls recovery</text>
<text x="40" y="76" fill="#64748b" font-size="16">Every interruption before commit attempts to restore the previous binary. Rollback failure blocks new attempts.</text>
<rect x="35" y="110" width="520" height="230" rx="18" fill="#f0fdfa" stroke="#5eead4"/>
<text x="60" y="142" fill="#115e59" font-size="12" font-weight="750" letter-spacing="1.3">PREPARATION: PREVIOUS AGENT RECONCILES</text>
<rect x="85" y="195" width="170" height="76" rx="12" fill="#fff" stroke="#99f6e4"/><text x="170" y="226" text-anchor="middle" fill="#0f172a" font-size="18" font-weight="700" class="mono">no journal</text><text x="170" y="251" text-anchor="middle" fill="#64748b" font-size="13">existing target stays active</text>
<rect x="335" y="195" width="170" height="76" rx="12" fill="#0f766e"/><text x="420" y="226" text-anchor="middle" fill="#fff" font-size="18" font-weight="700" class="mono">preparing</text><text x="420" y="251" text-anchor="middle" fill="#ccfbf1" font-size="13">download, verify, probe</text>
<path d="M255 233H335" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
<text x="295" y="220" text-anchor="middle" fill="#115e59" font-size="12">upgrade request</text>
<rect x="610" y="110" width="555" height="230" rx="18" fill="#fff7ed" stroke="#fdba74"/>
<text x="635" y="142" fill="#9a3412" font-size="12" font-weight="750" letter-spacing="1.3">ACTIVATION: STOP, SWITCH, STRICT START</text>
<rect x="650" y="195" width="180" height="76" rx="12" fill="#b45309"/><text x="740" y="226" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">activating</text><text x="740" y="251" text-anchor="middle" fill="#ffedd5" font-size="12">not committed</text>
<rect x="930" y="195" width="180" height="76" rx="12" fill="#172554"/><text x="1020" y="226" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">committed</text><text x="1020" y="251" text-anchor="middle" fill="#bfdbfe" font-size="12">target retained</text>
<path d="M555 233H650" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/><text x="602" y="220" text-anchor="middle" fill="#9a3412" font-size="12">probe passes</text>
<path d="M830 233H930" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/><text x="880" y="220" text-anchor="middle" fill="#115e59" font-size="12">READY=1</text>
<text x="880" y="292" text-anchor="middle" fill="#64748b" font-size="12">immediate commit, no wait period</text>
<rect x="280" y="400" width="190" height="80" rx="12" fill="#be123c"/><text x="375" y="432" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">rolling-back</text><text x="375" y="458" text-anchor="middle" fill="#ffe4e6" font-size="13">restore previous target</text>
<path d="M740 271V350H470V420" fill="none" stroke="#be123c" stroke-width="2.5" marker-end="url(#arrow-bad)"/>
<text x="535" y="350" fill="#9f1239" font-size="13">activation interrupted before commit</text>
<rect x="85" y="535" width="180" height="72" rx="12" fill="#f1f5f9" stroke="#94a3b8"/><text x="175" y="565" text-anchor="middle" fill="#0f172a" font-size="18" font-weight="700" class="mono">failed</text><text x="175" y="588" text-anchor="middle" fill="#64748b" font-size="12">previous service selected</text>
<rect x="390" y="535" width="220" height="72" rx="12" fill="#fff1f2" stroke="#fb7185"/><text x="500" y="565" text-anchor="middle" fill="#9f1239" font-size="17" font-weight="700" class="mono">rollback-failed</text><text x="500" y="588" text-anchor="middle" fill="#be123c" font-size="12">hard quarantine</text>
<path d="M330 480V510H175V535" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/><text x="235" y="505" fill="#115e59" font-size="12">restore and start succeed</text>
<path d="M420 271V365H175V535" fill="none" stroke="#64748b" stroke-width="2.2" marker-end="url(#arrow)"/><text x="210" y="385" fill="#475569" font-size="12">preparation interrupted: select previous, fail</text>
<path d="M455 271V380H500V535" fill="none" stroke="#be123c" stroke-width="2" marker-end="url(#arrow-bad)"/><text x="515" y="400" fill="#9f1239" font-size="12">restore or start fails</text>
<path d="M420 480V510H500V535" fill="none" stroke="#be123c" stroke-width="2.5" marker-end="url(#arrow-bad)"/><text x="500" y="505" fill="#9f1239" font-size="12">restore or start fails</text>
<text x="700" y="558" fill="#475569" font-size="14" font-weight="700">Durability sequence</text>
<text x="700" y="581" fill="#64748b" font-size="13">temporary write, file sync, atomic rename, directory sync</text>
<text x="700" y="610" fill="#64748b" font-size="13">No automatic rollback after committed.</text>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 6.0 KiB

38
docs/diagrams/fleet-agent-upgrade-sequence.svg Normal file
View File

@@ -0,0 +1,38 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="830" viewBox="0 0 1200 830" role="img" aria-labelledby="title desc">
<title id="title">IoT fleet agent stop, switch, start upgrade sequence</title>
<desc id="desc">The operator CAS-writes the current NATS attempt. The active agent forwards one upgrade request and keeps reconciling while the updater downloads, verifies, installs, and probes. The updater stops the old service, waits for inactivity, switches the symlink, starts the candidate, and commits after strict readiness. The restarted agent publishes completion.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
<marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
<style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
</defs>
<rect width="1200" height="830" fill="#ffffff"/>
<g class="sans">
<text x="40" y="44" fill="#0f172a" font-size="28" font-weight="750">Prepare live, then stop, switch, start</text>
<text x="40" y="70" fill="#64748b" font-size="15">One authenticated intent controls the complete transaction. Strict readiness commits immediately.</text>
<g font-size="14" font-weight="700" text-anchor="middle">
<rect x="35" y="96" width="180" height="46" rx="10" fill="#0f766e"/><text x="125" y="125" fill="#fff">Fleet operator</text>
<rect x="275" y="96" width="150" height="46" rx="10" fill="#f0fdfa" stroke="#5eead4"/><text x="350" y="125" fill="#115e59">NATS KV</text>
<rect x="485" y="96" width="180" height="46" rx="10" fill="#172554"/><text x="575" y="125" fill="#fff">Active agent</text>
<rect x="725" y="96" width="180" height="46" rx="10" fill="#7c2d12"/><text x="815" y="125" fill="#fff">Root updater</text>
<rect x="965" y="96" width="200" height="46" rx="10" fill="#fffbeb" stroke="#fbbf24"/><text x="1065" y="125" fill="#78350f">systemd / candidate</text>
</g>
<g stroke="#cbd5e1" stroke-width="1.5" stroke-dasharray="5 5"><path d="M125 142V805"/><path d="M350 142V805"/><path d="M575 142V805"/><path d="M815 142V805"/><path d="M1065 142V805"/></g>
<g font-size="13">
<text x="45" y="178" fill="#64748b" font-weight="700">1</text><path d="M125 185H350" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="237" y="178" text-anchor="middle" fill="#115e59">CAS current attempt</text>
<text x="45" y="228" fill="#64748b" font-weight="700">2</text><path d="M350 235H575" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="462" y="228" text-anchor="middle" fill="#115e59">authenticated intent</text>
<path d="M575 275H815" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="695" y="268" text-anchor="middle" fill="#9a3412">one upgrade request</text>
<rect x="500" y="302" width="150" height="62" rx="8" fill="#eef2ff"/><text x="575" y="326" text-anchor="middle" fill="#312e81" font-weight="700">keep reconciling</text><text x="575" y="347" text-anchor="middle" fill="#64748b">no early drain</text>
<text x="45" y="395" fill="#64748b" font-weight="700">3</text><rect x="730" y="375" width="170" height="94" rx="8" fill="#fff7ed" stroke="#fdba74"/><text x="815" y="400" text-anchor="middle" fill="#9a3412" font-weight="700">preparing</text><text x="815" y="422" text-anchor="middle" fill="#9a3412">HTTPS + SHA-256</text><text x="815" y="443" text-anchor="middle" fill="#9a3412">install + fsync</text><text x="815" y="462" text-anchor="middle" fill="#9a3412">probe as fleet-agent</text>
<text x="45" y="505" fill="#64748b" font-weight="700">4</text><path d="M815 512H1065" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="940" y="505" text-anchor="middle" fill="#9a3412">activating: systemctl stop</text>
<path d="M1065 548H575" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="820" y="541" text-anchor="middle" fill="#9a3412">SIGTERM</text>
<rect x="500" y="565" width="150" height="78" rx="8" fill="#eef2ff"/><text x="575" y="588" text-anchor="middle" fill="#312e81" font-weight="700">attempt current drain</text><text x="575" y="608" text-anchor="middle" fill="#64748b">status + flush if done</text><text x="575" y="628" text-anchor="middle" fill="#64748b">60 s systemd bound</text>
<path d="M1065 660H815" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="940" y="653" text-anchor="middle" fill="#115e59">inactive observed</text>
<text x="45" y="693" fill="#64748b" font-weight="700">5</text><rect x="980" y="680" width="170" height="76" rx="8" fill="#fffbeb" stroke="#fbbf24"/><text x="1065" y="704" text-anchor="middle" fill="#78350f" font-weight="700">atomic symlink switch</text><text x="1065" y="726" text-anchor="middle" fill="#92400e">start + strict READY=1</text><text x="1065" y="746" text-anchor="middle" fill="#92400e">3 min NATS bound</text>
<text x="45" y="785" fill="#64748b" font-weight="700">6</text><path d="M1065 792H815" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="940" y="785" text-anchor="middle" fill="#115e59">READY=1: commit locally</text>
<path d="M575 812H350" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="462" y="805" text-anchor="middle" fill="#115e59">agent reflects complete status</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.5 KiB

147
docs/guides/fleet-application-cd.md Normal file
View File

@@ -0,0 +1,147 @@
# Fleet application continuous delivery
An application repository owns a Rust deployment crate whose binary is named
`harmony`. It declares image builds, runtime services, Fleet placement, and
compiled deployment contexts. Harmony supplies the release tag at runtime and
passes registry-returned digests into one Fleet `Deployment`.
The reference two-service manifest is
[`examples/harmony_apply_deployment`](../../examples/harmony_apply_deployment/src/main.rs).
It builds frontend and backend images and advances them together as one rollout
revision.
## Production command
After checks pass, the production pipeline runs:
```bash
harmony ship \
--context production \
--tag "$CI_COMMIT_SHA" \
--wait
```
`ship` performs these steps in order:
1. Build every image declared by the Rust manifest.
2. Push each image to the context's registry repository.
3. Record the digest returned by the registry.
4. Apply one namespaced Fleet `Deployment` containing those digests.
5. Wait for the operator to report the exact applied rollout revision.
6. Exit successfully when all frozen targets converge, or fail with rollout
counts and the latest device error.
The command prints the supplied tag and every deployed digest. The pipeline is
roll-forward only. A failed rollout blocks that run. Shipping a repaired image
or deployment spec changes the Fleet `Deployment` and creates the next
generation; rerunning an unchanged commit does not. Harmony does not rewrite a
Git manifest or roll the application back to an old tag.
## One CI secret
The runner receives exactly one secret variable,
`HARMONY_ZITADEL_KEY_JSON`.
This is the tenant-scoped Zitadel machine key. Harmony exchanges it for the
short-lived identity used to read the rest of the deployment configuration from
the tenant's OpenBao instance.
The compiled context contains non-secret coordinates such as the registry
hostname, repository, OpenBao URL, Zitadel URL, and OpenBao role. OpenBao holds:
- `RegistryCredentials` in the context namespace, using a push-scoped registry
robot account;
- Kubernetes cluster access for the tenant namespace;
- application secrets;
- pull-only registry credentials under
`<device-prefix>/registry/device-pull/<reference>`.
Each device pull secret is JSON scoped to one registry authority. The authority
includes the port when the registry uses one:
```json
{
"registry": "hub.nationtech.io:5000",
"username": "tenant-device-pull",
"password": "..."
}
```
The credential schema requires all three string fields: `registry`, `username`,
and `password`.
The agent normalizes the host name and requires the image registry authority,
including its port, to match before forwarding credentials to Podman.
Do not inject registry passwords, kubeconfigs, OpenBao tokens, or device pull
credentials into CI variables. `publish` loads `RegistryCredentials` through
the resolved context and uses a temporary Docker configuration directory.
## Separate push and pull identities
CI and devices never share registry credentials.
| Identity | Registry access | OpenBao reader |
|---|---|---|
| CI publisher | Push to the application's repositories | Tenant CI machine identity |
| Device puller | Pull only from the application's repositories | Device groups authorized for the Fleet Deployment |
The Rust manifest places only an `image_pull_secret` reference in each private
service. The operator adds exact referenced pull-secret paths to that
Deployment's OpenBao policy. The agent reads the credential only when Podman
needs to pull a missing image and sends it through Podman's registry-auth
header. It does not run `podman login`, write an auth file, or place credentials
in desired state, container environment, labels, or logs.
The registry should provide separate repository-scoped robot accounts for push
and pull. For the first hosted deployments, those repositories can live under
`hub.nationtech.io/<tenant>`.
## Rust manifest
The application crate compiles to `harmony`:
```toml
[[bin]]
name = "harmony"
path = "src/main.rs"
```
Its `HarmonyApp` implementation declares stable build and deployment policy:
- frontend and backend build contexts and Dockerfiles;
- one production context and tenant namespace;
- one Fleet Deployment name;
- allowed device groups and placement labels;
- ports, environment, application-secret references, and pull-secret
references.
Those values change only when application architecture or policy changes. A
normal release supplies a new commit tag to the same compiled manifest.
## Deploy an existing image
`deploy` skips build and publication:
```bash
harmony deploy \
--context production \
--tag "$CI_COMMIT_SHA" \
--image frontend=hub.nationtech.io/example/frontend@sha256:... \
--image backend=hub.nationtech.io/example/backend@sha256:... \
--wait
```
This application accepts registry digests and explicit `dev-*` tags. It rejects
`latest` and ordinary mutable tags.
## Watching and debugging
While `ship --wait` is running, the dashboard deployment page polls the current
summary every three seconds. It shows succeeded, pending, and failed targets,
plus the latest failing device, error, and timestamp. The CLI waits for the
status revision matching the object UID and generation returned by its apply,
so success from an older rollout cannot complete a newer pipeline run.
The initial dashboard view remains aggregate. Detailed per-device rollout state
and explicit canary-stage presentation are later work.

170
docs/guides/fleet-device-secrets.md
View File

@@ -1,113 +1,97 @@
# Fleet device secrets
# Fleet enrollment and device secrets
How fleet devices read their deployments' secrets — image-pull
credentials, application config, API keys — from OpenBao, and how you
control who reads what. Design reasoning lives in
[ADR-025](../adr/025-fleet-device-secret-access.md); this page is the
operating manual.
Enrollment gives a Linux device its Fleet identity and installs the agent. It
does not, by itself, authorize group placement or make application secrets
available.
## The architecture in one minute
## Enroll a device
A device proves *who it is* with the Zitadel machine key it already
uses for NATS. *What it may read* is decided by **group membership**,
which only admins can change. *Where it runs* is decided by labels.
The three never mix:
```
Admin Zitadel OpenBao
│ assign device to groups ──▶ device ∈ {edge-a, …} │
│ write secret ──────────────────────────────────────────────▶ secret/<fleet>/<deployment>/…
│ │
Deployment CR │
│ allowedGroups: [edge-a] ──▶ operator attaches policy ──────▶ group edge-a may read
│ targetSelector: labels ──▶ operator schedules within │ deployment-<name> subtree
│ authorized devices │
│ │
Device │
└─ machine key ─▶ Zitadel token (groups: [edge-a]) ─▶ JWT login ─▶ reads its secrets
```
Three rules to remember:
| Concept | Who sets it | What it controls |
|---|---|---|
| **Group** | Admin, in Zitadel | Security boundary: which deployments' secrets a device may read |
| **Label** | Device config (self-reported) | Placement only: which authorized devices actually run the workload — narrows, never widens |
| **`allowedGroups`** | Deployment author | Which groups may view (run + read secrets of) the deployment |
## Storing a secret for a deployment
Write it under the deployment's subtree of the fleet KV mount:
Use the `example_fleet_device_enroll` binary as the current enrollment driver.
It runs locally or targets a device over SSH:
```bash
bao kv put secret/<fleet-ns>/<deployment-name>/db-credentials \
username=app password=…
cargo run -p example_fleet_device_enroll -- \
--target ssh://pi@10.0.0.42 \
--issuer-url https://sso.example.com \
--audience <fleet-project-id> \
--nats-url wss://nats.example.com \
--admin-oidc-client-id <numeric-client-id> \
--agent-binary ./build/fleet-agent-aarch64 \
--device-id warehouse-042 \
--labels site=warehouse-east,arch=aarch64
```
Nothing else to configure — access follows from the deployment's
`allowedGroups` below. Every device in an allowed group can read the
whole `<deployment-name>/` subtree, whether or not the workload landed
on it; size your groups to the audience the secret may have.
The operator must authenticate to Zitadel through the browser flow or provide
`HARMONY_ZITADEL_ADMIN_TOKEN`. Enrollment then finds or creates the device
machine user, grants the requested project role, and mints a new JSON machine
key. Zitadel does not return an existing private key, so a repeated enrollment
mints another key.
## Authorizing and placing a deployment
`FleetDeviceSetupScore` installs Podman, the unprivileged `fleet-agent` user,
the root updater, systemd units, the agent binary, and its configuration. When a
machine key is supplied or minted, the Score writes it to
`/etc/fleet-agent/zitadel-key.json` with mode `0640`. The device cannot obtain
that long-lived key from NATS or OpenBao automatically.
```yaml
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: hello-web
spec:
allowedGroups: [edge-a] # security: who may view this deployment
targetSelector:
matchLabels:
hw: pi5 # placement: which authorized devices run it
score: …
The device ID must be a valid Kubernetes RFC 1123 subdomain. Labels are
self-reported placement data. Do not treat labels such as `group=production` as
authorization.
Verify the installation on the device and control plane:
```bash
systemctl status fleet-agent harmony-fleet-updater
journalctl -u fleet-agent -u harmony-fleet-updater
kubectl -n <namespace> get devices.fleet.nationtech.io
```
The operator schedules onto devices that are **members of an allowed
group AND match the selector**, then attaches the deployment's read
policy to each allowed group. Existing device sessions pick the grant
up immediately — rolling out a new deployment never touches logins,
tokens, or per-device configuration.
## Identity and tokens
## Managing device group membership
The device stores only its Zitadel machine key. On each NATS connection, the
agent signs a 60-second JWT assertion and exchanges it at Zitadel's token
endpoint. Production requires the returned `id_token`: it is a JWT that the NATS
callout and OpenBao can verify against Zitadel's JWKS. Zitadel's default
`access_token` is opaque and is not suitable for this path.
Until Zitadel's first-class groups are GA, a fleet group is a Zitadel
project role (one role per group, surfaced as a `groups` claim by the
fleet's token Action — applied by the fleet bootstrap, not by hand).
Grant or remove the role on the device's machine user to move it
between groups.
The `id_token` is cached in memory and renewed before expiry. No refresh token
is stored. Rotating a machine key requires installing the replacement on the
device and deleting the old key in Zitadel.
Membership changes take effect at the device's **next login, at most
one token TTL later (≈ 1 h)**. This is the only eventually-consistent
operation in the system — everything else binds immediately. If that
window matters, use the emergency levers below.
## Group placement and OpenBao
## Reading secrets on the device
The intended authorization rule is:
Nothing to do: the agent reads through `harmony_config`, which
authenticates to OpenBao with the machine keyfile automatically
(JWT-bearer rung of `OpenbaoSecretStore`). Tokens are short-lived
batch tokens; the store re-logins transparently on expiry.
```text
admin-managed group membership AND self-reported target labels
```
## Revocation runbook
`allowedGroups` names the authorized groups; `targetSelector` narrows placement
within those groups. An empty group list authorizes no device.
| Situation | Action | Effect |
|---|---|---|
| Device leaves a group (routine) | Remove the role/group in Zitadel | Converges at next login, ≤ 1 TTL |
| Device compromised | Disable its OpenBao identity entity, then rotate/delete its Zitadel machine key | Token dead at next request; no new logins |
| Secret/deployment compromised | Detach `deployment-<name>` policy from its groups (or rotate the secret) | All devices lose read immediately |
| Whole group compromised | Detach the group's policies | Immediate, group-wide |
The operator supports a Zitadel-backed group source and OpenBao policy grant
synchronization, but the public production composition does not configure
either one today. Without an explicit group source, deployments match no
devices. Without OpenBao administrator configuration, changing
`allowedGroups` does not create or attach OpenBao policies.
## Guarantees and limits
OpenBao use therefore requires an operator to configure all of the following:
- A device can only ever read secrets of deployments whose
`allowedGroups` intersect its groups. Labels, CR edits, or a
compromised agent cannot widen that — membership is signed into the
Zitadel token and validated against the project (`bound_audiences`),
so other projects' tokens are rejected outright.
- Visibility is **group-granular**: being scheduled is not required to
read. Finer isolation ⇒ finer groups.
- One Zitadel project + OpenBao pair (a *cell*) comfortably serves
~10⁵ devices; beyond that, shard fleets into cells
([ADR-025](../adr/025-fleet-device-secret-access.md), Rationale).
- the agent's OpenBao URL and secret prefix during device setup;
- Zitadel group roles and device grants;
- the operator group source;
- OpenBao JWT auth, deployment policies, and group-policy attachments;
- secret values under the configured deployment prefix.
Do not assume enrollment completed these steps. The E2E tests exercise explicit
group and grant setup, not the default production composition.
## Revocation
For a compromised device, remove or deactivate its Zitadel machine keys and
revoke any active OpenBao identity or token. Rotate secrets the device could
read. Group removal takes effect only after the device obtains a token carrying
the new claims; policy detachment or secret rotation is the immediate control.
See [ADR-025](../adr/025-fleet-device-secret-access.md) for the intended group
model. Treat its unwired portions as design until production composition adds
and verifies the required configuration.

215
docs/guides/fleet-manual-token-mint.md
View File

@@ -1,189 +1,126 @@
# Manual Zitadel token mint + NATS write
# Manually mint a Fleet token
Operator-side recipe for talking to a callout-protected NATS by
hand: sign a JWT-bearer assertion with a Zitadel machine user's
private key, exchange it for an access token, drive `nats` CLI
commands with the token. Useful for debugging the auth chain,
poking the desired-state KV without the operator running, and
validating that a deployed callout is actually accepting what
you think it should.
Use this procedure to test the Zitadel-to-NATS authentication chain. It signs a
JWT-bearer assertion with a machine key, exchanges it for an OIDC ID token, and
passes that ID token to the NATS CLI.
Read [fleet-zitadel-faq.md](./fleet-zitadel-faq.md) first for the
underlying mechanism (RFC 7523 JWT-bearer flow, why we sign
locally, what each claim means).
Do not use `access_token` from the response. Zitadel returns an opaque access
token by default; production Fleet requires the verifiable `id_token`.
## Inputs you need
## Inputs
Five strings:
Read the live values rather than relying on a local Zitadel cache:
| Input | Where to find it |
| --- | --- |
| `OIDC_ISSUER_URL` (the Zitadel base URL) | callout Deployment env: `kubectl exec -n fleet-system deploy/fleet-callout -- printenv OIDC_ISSUER_URL` |
| `project_id` (becomes the access token's `aud`) | callout Deployment env: `OIDC_AUDIENCE` |
| Machine user's `userId` | the JSON keyfile's `userId` field |
| Machine user's `keyId` | the JSON keyfile's `keyId` field |
| Private RSA key (PEM) | the JSON keyfile's `key` field |
| Input | Source |
|---|---|
| `OIDC_ISSUER_URL` | Callout environment |
| `PROJECT_ID` | Callout `OIDC_AUDIENCE` environment value |
| `USER_ID` | Machine key `userId` |
| `KEY_ID` | Machine key `keyId` |
| RSA private key | Machine key `key` |
Get the `fleet-ops` (admin role) JSON keyfile from the cache:
For a cached operator key:
```bash
jq -r '.machine_keys["fleet-ops"]' \
~/.local/share/harmony/zitadel/client-config.json \
> /tmp/fleet-ops.json
jq -r '.userId' /tmp/fleet-ops.json # → user_id
jq -r '.keyId' /tmp/fleet-ops.json # → key_id
jq -r '.key' /tmp/fleet-ops.json > /tmp/fleet-ops.pem
~/.local/share/harmony/zitadel/client-config.json > /tmp/fleet-ops.json
jq -r '.key' /tmp/fleet-ops.json > /tmp/fleet-ops.pem
```
The cache may drift from the deployed Zitadel state if Zitadel has
been re-seeded; **always pull `OIDC_AUDIENCE` from the running
callout**, not from the cache. The cache fix landed in commit
`f4d6fb94` but older entries can still trip you up.
Protect and remove these temporary files when finished.
## Mint script (PyJWT)
## Mint with PyJWT
Install `PyJWT`, not the unrelated `jwt` package.
```python
# pip install PyJWT requests ← MUST be PyJWT, not the `jwt` package.
# The two share `import jwt`; `jwt` (the package) refuses raw PEM
# strings and demands an AbstractJWKBase wrapper. PyJWT takes PEM
# directly. If you ever see `TypeError: key must be an instance of
# a class implements jwt.AbstractJWKBase`, you have the wrong one.
import json
import time
import jwt, time, requests
import jwt
import requests
# These come from the running callout + Zitadel. Don't reuse stale
# values from a checked-in note; verify against the live cluster.
OIDC_ISSUER_URL = "http://sso.fleet.local:8080"
PROJECT_ID = "371158654839160853" # = OIDC_AUDIENCE on callout
USER_ID = "..." # from machine keyfile
KEY_ID = "..." # from machine keyfile
key = open("/tmp/fleet-ops.pem").read()
OIDC_ISSUER_URL = "https://sso.example.com"
PROJECT_ID = "..." # OIDC_AUDIENCE from the running callout
machine_key = json.load(open("/tmp/fleet-ops.json"))
private_key = open("/tmp/fleet-ops.pem").read()
now = int(time.time())
assertion = jwt.encode(
{
"iss": USER_ID,
"sub": USER_ID,
"aud": OIDC_ISSUER_URL, # for Zitadel itself, NOT the project_id
"exp": now + 60, # Zitadel rejects exp - iat > 60s
"iss": machine_key["userId"],
"sub": machine_key["userId"],
"aud": OIDC_ISSUER_URL,
"iat": now,
"exp": now + 60,
},
key,
private_key,
algorithm="RS256",
headers={"kid": KEY_ID}, # PyJWT spelling — `headers=`, not `optional_headers=`
headers={"kid": machine_key["keyId"]},
)
r = requests.post(
f"{OIDC_ISSUER_URL}/oauth/v2/token",
response = requests.post(
f"{OIDC_ISSUER_URL.rstrip('/')}/oauth/v2/token",
data={
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": assertion,
# Three scopes:
# openid — base OIDC
# urn:zitadel:iam:org:projects:roles — PLURAL.
# Without this, Zitadel omits the role claim and the
# callout rejects with "no authorized role in token".
# urn:zitadel:iam:org:project:id:<id>:aud — singular.
# Tells Zitadel to put <id> into the access token's
# `aud` claim, which the callout's audience check
# compares against OIDC_AUDIENCE.
"assertion": assertion,
"scope": (
"openid "
"urn:zitadel:iam:org:projects:roles "
f"urn:zitadel:iam:org:project:id:{PROJECT_ID}:aud"
),
},
timeout=10,
)
r.raise_for_status()
token = r.json()["access_token"]
response.raise_for_status()
token = response.json()["id_token"]
# Sanity check — decode without verifying signature so you can see
# what Zitadel actually emitted. If anything below is wrong, the
# callout will reject your token.
print(jwt.decode(token, options={"verify_signature": False}))
print(token)
```
Expected decoded claims (the parts the callout will check):
The decoded ID token must have the exact issuer configured on the callout, the
Fleet project in `aud`, a `client_id`, and the Fleet project's role claim. The
callout accepts `fleet-admin` or `device`; it strips the configured `device-`
prefix from `client_id` for device identities.
| Claim | What it should be | Why |
| --- | --- | --- |
| `iss` | `OIDC_ISSUER_URL` (byte-equal) | Callout: `validation.set_issuer(&[&self.issuer_url])` |
| `aud` | `["<PROJECT_ID>"]` | Callout: `validation.set_audience(&[&self.audience])`; the array form is Zitadel's default |
| `exp` | ~now + 12h | Zitadel default access token TTL |
| `client_id` | the machine user's username (`fleet-ops`, `device-vm-device-00`, …) | Callout uses this as `device_id_claim` (with optional `DEVICE_ID_PREFIX_STRIP` applied) |
| `urn:zitadel:iam:org:project:<PROJECT_ID>:roles` | object with role names as keys (e.g. `{"fleet-admin": {"<orgId>": "<orgName>"}}`) | Callout uses this as `roles_claim` and admits the role if `fleet-admin` or `device` is present |
Decoding without signature verification is diagnostic only. The callout still
verifies the signature against Zitadel's JWKS.
If any of these is wrong, fix the script before bothering with NATS.
## Connect to NATS
## Drive NATS with the token
`nats --token=<bearer>` puts the value into the CONNECT frame's
`auth_token`, which is what the callout expects.
Capture the last output line from the script:
```bash
NATS_SERVER=192.168.122.1:30422 # libvirt host's port mapping
TOKEN=$(python3 mint.py | tail -1) # last line is the raw token
# Read everything (admin role allows >):
nats --server "$NATS_SERVER" --token "$TOKEN" kv ls device-info
nats --server "$NATS_SERVER" --token "$TOKEN" kv get device-info info.vm-device-00
# Write a desired state — agent's KV watcher fires within 1s,
# reconciler creates the podman container.
nats --server "$NATS_SERVER" --token "$TOKEN" \
kv put desired-state vm-device-00.hello-web '{
"name": "hello-web",
"type": "PodmanV0",
"data": {
"services": [{
"name": "testnginx",
"image": "docker.io/nginx:latest",
"ports": ["8080:80"]
}]
}
}'
TOKEN=$(python3 mint.py | tail -1)
nats --server wss://nats.example.com --token "$TOKEN" kv ls device-info
```
The exact JSON shape comes from
`harmony-reconciler-contracts/src/fleet.rs` — read that crate when
in doubt about field names, NOT this doc; this doc is a worked
example and may drift.
## Common failures and what they mean
| Symptom | Likely cause |
| --- | --- |
| `TypeError: key must be an instance of … AbstractJWKBase` | Wrong PyPI package. `pip uninstall jwt && pip install PyJWT`. |
| HTTP 400 from `/oauth/v2/token`: `"invalid_grant_type"` | Forgot the percent-encoded form encoding, OR `grant_type` value mistyped. The full URN is `urn:ietf:params:oauth:grant-type:jwt-bearer`. |
| HTTP 400: `"jwt: token is expired"` | Your assertion's `exp` is in the past. Wall-clock skew between your laptop and the cluster — sync NTP. |
| Token mints but no `urn:zitadel:…:roles` claim | Missing the **plural** `urn:zitadel:iam:org:projects:roles` in scope. |
| Token mints but `aud` is the issuer URL instead of the project id | Forgot the `urn:zitadel:iam:org:project:id:<id>:aud` scope. |
| NATS CLI: `nats: Authorization Violation` | Token is good but callout rejected it — check `kubectl logs -n fleet-system -l app=fleet-callout` for the actual reason. The most common ones are "InvalidAudience" (your `aud` ≠ deployed `OIDC_AUDIENCE`) and "no authorized role in token". |
| Callout log: `JWT validation failed: InvalidIssuer` | Trailing slash drift. `OIDC_ISSUER_URL=http://sso.fleet.local:8080/` ≠ `http://sso.fleet.local:8080`. Match exactly. |
When the callout rejects, **its log is the source of truth**, not
your decoded claims. The validation error includes which check
failed; work backwards from there.
## Rotating the deployed `OIDC_AUDIENCE`
If Zitadel was re-seeded and `OIDC_AUDIENCE` on the callout now
points at a non-existent project:
An administrator can inspect a device entry:
```bash
# 1. Confirm the live project id
oc -n zitadel exec -ti deploy/zitadel -- /bin/sh -c \
'curl -s -H "Authorization: Bearer $PAT" \
$ZITADEL_URL/management/v1/projects/_search \
| jq ".result[] | select(.name == \"fleet\") | .id"'
# 2. Re-run the bring-up — the live-query fix in f4d6fb94 will
# refresh OIDC_AUDIENCE on the next NatsAuthCalloutScore apply.
nats --server wss://nats.example.com --token "$TOKEN" \
kv get device-info info.<device-id>
```
The shape of `mint.py` doesn't change between regular operation
and post-recovery — you just plug in fresh values for
`OIDC_AUDIENCE` and `PROJECT_ID`.
Do not use manual KV writes as the deployment workflow. Apply a namespaced
Fleet `Deployment` resource so the operator owns target planning, rollout state,
and status.
## Failures
| Symptom | Check |
|---|---|
| Token response has no `id_token` | Include `openid`; confirm the API application permits an ID token |
| ID token has no project roles | Include the plural `urn:zitadel:iam:org:projects:roles` scope and verify the role grant |
| `InvalidAudience` | Use the live callout `OIDC_AUDIENCE` and include the project audience scope |
| `InvalidIssuer` | Match `OIDC_ISSUER_URL` exactly, including scheme, host, port, and trailing-slash behavior |
| Missing `client_id` | Confirm the token is an ID token minted for the Fleet API application |
| Assertion expired | Synchronize the client clock; the assertion window is 60 seconds |
| `AbstractJWKBase` type error | Remove the `jwt` package and install `PyJWT` |
The callout log contains the validation failure:
```bash
kubectl -n <namespace> logs deploy/fleet-callout
```

82
docs/guides/fleet-podman-deployments.md Normal file
View File

@@ -0,0 +1,82 @@
# Fleet Podman deployments
Fleet `Deployment` resources carry a `ReconcileScore`. `PodmanV0` supports one
optional init container followed by one or more long-running services.
```yaml
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: analysis-api
spec:
allowedGroups: [lab]
targetSelector:
matchLabels:
role: analyzer
rollout:
strategy: Immediate
score:
type: PodmanV0
data:
init_container:
name: analysis-migrate
image: registry.example.com/analysis-migrate@sha256:<digest>
ports: []
env: []
secret_env: []
volumes:
- host_path: /var/lib/analysis
container_path: /data
read_only: false
restart_policy: "no"
services:
- name: analysis-api
image: registry.example.com/analysis-api@sha256:<digest>
ports: ["8080:8080"]
env: []
secret_env: []
volumes:
- host_path: /var/lib/analysis
container_path: /data
read_only: false
restart_policy: unless-stopped
```
`Immediate` starts the rollout without operator approval. When any matched
device has the `canary=true` label, those devices must converge before the same
revision is released to the remaining frozen targets.
The target set and canary membership are frozen for the revision. A failed
canary prevents release to non-canaries. Fleet does not roll back canaries that
already converged, and it has no percentage, batch-size, concurrency, pause, or
manual-approval setting.
`allowedGroups` is enforced only when the operator has a group source. The
public production composition does not currently wire one, so it fails closed
and matches no devices until a private composition provides group membership.
The agent preflights names, ports, ownership, and images before changing the
deployment. It then runs `init_container` with restart policy `no` and waits for
it to exit. Exit code zero allows service reconciliation to continue. Any other
exit code marks the deployment failed and leaves services that have not yet
been reconciled untouched. The next reconciliation pass retries the failed init
container.
The completed init container remains in Podman as the durable completion
marker. Reconciliation skips it while the complete resolved score is unchanged.
Changing any score field, including a regular service field, replaces and
reruns the init container before reconciling services. Resolved secret values
are part of the score revision, but the agent does not watch the secret source
for changes. Init containers must be idempotent. Removing the Deployment removes
both the init container and its services.
Container replacement and Deployment removal ask Podman to stop each running
container with a 30-second timeout, then force-remove it. Fleet does not wait
indefinitely for graceful shutdown; applications must finish termination work
within that timeout.
`init_container` uses the same `PodmanService` fields as a service. Secret
references are resolved before Podman receives the definition. The runtime
always overrides its restart policy to `no`; setting another value does not
make an init container long-running. Its `ports` list must be empty because an
init container cannot publish host ports.

79
docs/guides/fleet-staging-install.md
View File

@@ -1,15 +1,16 @@
# Deploy Fleet to a remote cluster
# Fleet control-plane operations
Fleet uses the same lifecycle commands and Scores for every cluster. Remote
contexts currently use production-equivalent replication, TLS, exposure, and
credential behavior, including when the target cluster is used for staging.
Fleet uses the same lifecycle commands and Scores for local and remote clusters.
A remote context enables the production exposure and credential paths even when
the cluster is used for staging. The current operator deployment has one
replica; this guide does not imply active-active or failover support.
## Prerequisites
- Docker with Buildx, Helm, and kubectl are on `PATH`.
- The remote cluster has cert-manager and the CloudNativePG operator.
- The image registry contains credentials for `REGISTRY_USER` and
`REGISTRY_TOKEN`.
- Registry publication credentials are stored as `RegistryCredentials` in the
remote context's OpenBao namespace.
- The deploy identity can obtain the cluster kubeconfig through OpenBao.
- Public DNS points `zitadel`, `openbao`, and `nats` under the context domain at
the cluster ingress.
@@ -72,10 +73,12 @@ old releases. Disposable local and E2E clusters should be recreated instead.
## Provision the tenant
Run the tenant provisioning binary with a platform-admin context. It applies
`TenantScore`, creates the Fleet deployer ServiceAccount and RBAC, issues its
kubeconfig, and stores `ClusterAccess` through Harmony Config. The credential
store role must be able to write the tenant's OpenBao namespace.
Run the tenant provisioning binary with an explicit administrator kubeconfig.
It applies `TenantScore`, creates the Fleet deployer ServiceAccount and RBAC,
issues its namespace-scoped kubeconfig, and stores `ClusterAccess` through
Harmony Config. Do not store the administrator kubeconfig in OpenBao or expose
it to the tenant CI/CD identity. The provisioning token only needs write access
to the tenant's OpenBao namespace.
The deployer can create namespaced Roles and RoleBindings because Helm installs
the Fleet operator's runtime RBAC. Kubernetes prevents it from binding rights it
@@ -88,8 +91,8 @@ provisioning to create and store a replacement. This remains the rotation path
until short-lived TokenRequest brokerage is implemented.
```bash
cargo run --release --bin tenant-provision -- \
deploy --context platform-admin
export KUBECONFIG=/secure/path/platform-admin.kubeconfig
cargo run --release --bin tenant-provision
```
Fleet creates the immutable NATS callout credential Secret on first deploy and
@@ -104,17 +107,15 @@ public, so the remote context uses `image_pull_secret: None`.
references, then converges the Fleet Scores serially:
```bash
export REGISTRY_USER=...
export REGISTRY_TOKEN=...
export HARMONY_ZITADEL_KEY_JSON=...
cargo run -- ship --context customer-prod
```
These three environment variables are required today. The target CI model
keeps only `HARMONY_ZITADEL_KEY_JSON` in the runner and reads registry
credentials from OpenBao through `ConfigClient`; that publication work is not
implemented yet.
The runner receives only `HARMONY_ZITADEL_KEY_JSON`. Context resolution uses
that identity to load the cluster access and secret-class registry credentials
from OpenBao. `build` remains credential-free; `publish` and `ship` resolve the
full remote context.
For CI, the stages can run independently. Pass the digest references printed by
`build` into `publish` and `deploy`:
@@ -141,6 +142,48 @@ kubectl -n customer-fleet get deployments,pods,routes
kubectl -n customer-fleet get deployments.fleet.nationtech.io,devices.fleet.nationtech.io
```
For production, open `https://dashboard.<context-domain>`. Zitadel login must
return an ID token carrying `fleet-admin`. The dashboard shows live Device and
Deployment CR projections. It does not provide deployment creation, rollout
approval, task history, persistent alert acknowledgement, or historical trend
data.
## Troubleshooting
Start with the component that owns the failed boundary:
```bash
# Control-plane rollout and application logs
cargo run -- status --context customer-prod
cargo run -- logs --context customer-prod
kubectl -n customer-fleet get pods,events
# Operator reconciliation and dashboard
kubectl -n customer-fleet logs deploy/harmony-fleet-operator
# Device reconciliation and privileged updater
systemctl status fleet-agent harmony-fleet-updater
journalctl -u fleet-agent -u harmony-fleet-updater
podman ps -a
```
Use these symptoms to narrow the search:
| Symptom | Check |
|---|---|
| Deployment remains at zero targets | Production has no group source by default; inspect operator warnings and its group-source environment |
| Secret reference fails | Confirm device OpenBao configuration, JWT role, policy attachment, and secret path; production grant sync is not wired by default |
| Agent cannot connect to NATS | Check device time, Zitadel reachability, ID-token mint errors, DNS, TLS, and callout logs |
| Init container runs again | Any resolved score change changes its completion revision; keep init work idempotent |
| Container disappears after a slow stop | Fleet gives Podman 30 seconds, then force-removes the container |
| Dashboard alert returns after restart | Acknowledgements are in memory only |
| System upgrade reports `repair-required` | Inspect apt and dpkg directly on the device; Fleet does not retry or roll back packages |
The public production composition does not currently configure group placement
or OpenBao deployment grant synchronization. Do not diagnose their absence as
eventual convergence: supply the missing configuration in the private deploy
composition or manage those grants separately.
For local validation before a remote deploy:
```bash

78
docs/guides/fleet-tasks-upgrades.md Normal file
View File

@@ -0,0 +1,78 @@
# Fleet tasks and upgrades
Fleet has two distinct upgrade paths. `TaskRun` invokes the fixed operating
system upgrade executor. Setting `Device.spec.agentUpgrade` invokes the agent
binary upgrade protocol.
## System upgrade task
Create a namespaced `TaskRun`:
```yaml
apiVersion: fleet.nationtech.io/v1alpha1
kind: TaskRun
metadata:
name: upgrade-device-1
spec:
allowedGroups: [production]
targetSelector:
matchLabels:
device-id: device-1
deadlineSeconds: 21600
systemUpgradeV1: {}
```
Task placement uses the same group source as deployments. The public production
composition does not configure that source, so a task has no targets until a
private composition supplies group membership.
The operator freezes the matching target set. Each device must advertise the
`AptFullUpgradeV1` updater capability. The root updater then runs its compiled
sequence: package preflight, `apt-get update`, noninteractive full upgrade, dpkg
audit, reboot, and post-boot verification. Callers cannot supply package names,
repositories, commands, or reboot arguments.
If a matched target has `canary=true`, only canaries are released first. Every
canary must complete before the remaining frozen targets are released. A failed
or timed-out canary stops further release. Completed upgrades are not rolled
back.
The deadline prevents an expired intent from starting and determines when the
operator fails a run. It does not interrupt apt or dpkg after the updater has
accepted the attempt. An interrupted package transaction can enter
`repair-required`; repair then requires direct device access.
The executor exists in the agent and updater, but validation on a disposable
Debian VM and Raspberry Pi OS device remains listed as work in the current
design. Treat it as unproven on physical production devices until that run is
recorded.
Inspect progress with:
```bash
kubectl -n <namespace> get taskruns
kubectl -n <namespace> get taskrun <name> -o yaml
journalctl -u harmony-fleet-updater
```
Recurring schedules and dashboard task history are not implemented.
## Agent upgrade
An agent upgrade target contains a version, architecture, artifact URL, maximum
size, and SHA-256 digest. The operator writes a per-device attempt when desired
and reported versions differ.
The root updater downloads and verifies the candidate, runs its self-test and
probe, stops the old agent, switches the active symlink, and starts the new
agent. Readiness commits the change. Failure before commit restores the previous
agent executable; `rollback-failed` requires root repair.
This rollback covers only the executable. Configuration, local databases, and
formats changed by the candidate must remain readable by the previous release.
The root updater does not update itself. Devices installed before OCI artifact
support need `FleetDeviceSetupScore` run once before their first `oci://`
upgrade.
See [Fleet agent upgrades](../design/fleet-agent-upgrades.md) for the protocol,
timeouts, journal states, and repair conditions.

230
docs/guides/fleet-zitadel-faq.md
View File

@@ -1,185 +1,101 @@
# Fleet × Zitadel FAQ
# Fleet and Zitadel FAQ
Technical reference for the Zitadel setup behind the fleet
auth callout. Describes what exists, why it's that way, and where
each piece lives in the code.
This page describes the authentication path used by the public Fleet production
composition.
Code anchors:
- `examples/fleet_e2e_demo/src/lib.rs` — bring-up flow
- `harmony/src/modules/zitadel/setup.rs` — `ZitadelSetupScore`
- `harmony/src/modules/zitadel/mod.rs` — Helm install
- `nats/callout/src/handler.rs` — auth callout
- `fleet/harmony-fleet-agent/src/credentials.rs` — JWT-bearer mint
## What does Zitadel provision for Fleet?
---
`ZitadelSetupScore` creates a Fleet project, an API application for NATS, a web
PKCE application for the production dashboard, the `fleet-admin` and `device`
roles, and the operator machine identity. Device enrollment creates a separate
machine user and JSON key for each device.
## What is an "application" in Zitadel?
An application is an OIDC client configuration. Users belong to a Zitadel
organization; project role grants associate a user with roles in the Fleet
project. A project role is not a Zitadel administrator role.
An OIDC client config: `clientId`, allowed grant types, redirect
URIs (browser apps only), PKCE settings (browser apps only).
## How does a device authenticate?
Apps are not containers for users or roles — those live one
level up at the org. An app is the entry point a service uses to
delegate auth to Zitadel.
The agent keeps the JSON machine key at
`/etc/fleet-agent/zitadel-key.json`. On each NATS connection it:
The `nats` app is **API type**: JWT-bearer / client-credentials
only, no browser flow. Headless agents never see a login page.
The app's `clientId` is what tokens carry as `aud` and what the
auth callout validates against (`OIDC_AUDIENCE` env on the callout
Deployment).
1. signs a JWT assertion whose `iss` and `sub` are the machine user ID;
2. sends the assertion to `<issuer>/oauth/v2/token` with the RFC 7523
JWT-bearer grant;
3. requests `openid`, project roles, and the Fleet project audience;
4. takes `id_token` from the response and presents it as the NATS bearer token.
## Why are users and roles at org level instead of per-project?
Production requires `id_token`, not `access_token`. Zitadel access tokens are
opaque by default. The ID token is a JWT that the callout can verify through
Zitadel's JWKS and that contains the audience, `client_id`, and role claims used
by Fleet.
Roles are defined inside a project but are essentially labels —
strings + display names with no inherent permissions. Each app
enforces them in code (the callout maps `device` → a
permission template).
The assertion lasts 60 seconds. The ID token is cached in memory and replaced
before its JWT `exp`; no refresh token is stored. The machine key remains valid
until it expires or is removed in Zitadel.
Users live at org level so one identity can hold roles across
multiple projects in the same org and SSO between them. Role
grants are the join: "user X has roles \[A, B\] on project Y."
## How is a device ID derived?
The only privilege ladder Zitadel enforces directly is at the
instance/org level (IAM-Owner, Org-Owner). Project roles say
nothing about Zitadel admin rights.
Enrollment conventionally creates the username `device-<device-id>`. The
callout reads the ID token's `client_id` claim and strips `device-`. It validates
the result before inserting it into device-scoped NATS subjects.
## What is each service account for?
## What happens while a device is offline?
| User | Created by | Purpose |
| --- | --- | --- |
| `iam-admin` | Helm `FirstInstance.Org.Machine` | IAM-Owner. Its PAT (`iam-admin-pat` k8s Secret) drives the management API from `ZitadelSetupScore`. |
| `login-client` | Helm `FirstInstance.Org.LoginClient` | Internal — Zitadel's login UI pod uses it to call back into Zitadel. Don't touch. |
| `fleet-ops` | `fleet_e2e_demo` admin setup | `fleet-admin` role grant, JSON key, used by tests and admin tooling. |
| `device-vm-device-NN` | `fleet_e2e_demo::provision_device` | One per VM. JSON key copied to `/etc/fleet-agent/zitadel-key.json`. `device` role grant. |
| `ops-station`, `sensor-a`, `sensor-b`, `intruder` | `fleet_auth_callout` (separate example) | Leftovers from previous runs. Postgres survives cluster recreates. Harmless, deletable. |
The machine key remains on disk. Once the network returns, a NATS connection
attempt obtains a fresh ID token if the cached one is near expiry. The agent
cannot mint while Zitadel is unreachable, so reconnect waits until both Zitadel
and NATS are available.
The `device-` prefix on per-device usernames is intentional:
Zitadel emits the username verbatim in the access token's
`client_id` claim. The callout strips `device-` to recover the
bare device id used for NATS subject interpolation
(`DEVICE_ID_PREFIX_STRIP=device-` env var on the callout;
`nats/callout/src/zitadel.rs::extract_device_id`).
## Are machine keys equivalent to PATs?
## How does the agent authenticate? Are JWTs / refresh tokens cached?
Both permit impersonation if copied. A machine key signs short-lived assertions
sent to Zitadel, while a PAT is itself a bearer credential. Zitadel allows more
than one key per machine user, which permits installing a replacement before
removing the old key. Fleet does not currently use TPM-backed key storage.
On disk the agent keeps **only the JSON machine key** (RSA
private key) at `/etc/fleet-agent/zitadel-key.json`.
## Does enrollment make OpenBao secrets available?
It does NOT store:
- access tokens (in memory only)
- refresh tokens (the JWT-bearer flow has none — RFC 7523 is
stateless by design)
No. The same ID token can authenticate to an OpenBao JWT role, but enrollment
must also configure the agent's OpenBao endpoint. Group membership, OpenBao
policies, policy attachments, and secret values must exist separately.
On every NATS (re)connect, `credentials.rs::zitadel_mint`:
The operator has optional group-source and OpenBao grant-sync support. The
public production composition does not wire the required group source or
OpenBao administrator token, so those behaviors are disabled there.
1. Builds a JWT assertion with `exp = now + 60s`, signs it with
the RSA key
2. POSTs it to `<zitadel>/oauth/v2/token` with grant type
`urn:ietf:params:oauth:grant-type:jwt-bearer`
3. Receives an access token (~12h validity), caches it in memory
4. Re-mints when within 5min of expiry
(`TOKEN_REFRESH_LEEWAY_SECS`)
## How does dashboard login differ?
## What happens to an offline agent?
The dashboard uses an OIDC authorization-code flow with PKCE. It validates the
returned ID token, stores it in an encrypted private cookie, and requires the
`fleet-admin` role. Its Zitadel application is configured to include roles in
the ID token. See [Operator dashboard SSO](./operator-dashboard-sso.md).
| Time offline | Behavior |
| --- | --- |
| 0 – ~12 h | Cached access token still valid. Reconnects work transparently. |
| > ~12 h | Token expired. Agent enters reconnect loop until network returns, then mints fresh on first successful reach. |
## Why is the IAM administrator PAT in a Kubernetes Secret?
The RSA key never expires until rotated server-side.
`ZitadelSetupScore` uses the PAT for management API operations. Kubernetes
Secrets are not encrypted at rest unless the cluster enables encryption at
rest. Anyone who can read this Secret can act with its privileges. Moving this
bootstrap credential to a stronger storage path remains production-hardening
work.
## Where are the lifetimes set?
## Token reference
- **Access token TTL** — Zitadel UI: Org → Settings → OIDC
Settings → "Access Token Lifetime" (default 12 h).
- **Assertion TTL** — hardcoded 60 s in
`credentials.rs::ASSERTION_LIFETIME_SECS`. Zitadel rejects
assertions where `exp - iat > 60 s`; this is server-enforced,
not a knob.
- **Machine key TTL** — set when the key is created in
`harmony/src/modules/zitadel/setup.rs::create_machine_key`.
## Why is a JSON machine key more secure than a PAT?
Both are "if stolen, full impersonation" — the same blast radius.
The difference is in leak surface:
- **PAT**: a 60-char bearer string sent on every authenticated
request. Every log line, every env dump, every misrouted
request is a leak opportunity.
- **JSON key**: an RSA private key. Only ever signs short-lived
(60 s) assertions sent to one endpoint
(`<zitadel>/oauth/v2/token`). The bearer token NATS sees is
the access token — short-lived (12 h max), scoped, distinct
from the long-term secret. A full network capture of the
agent ↔ NATS traffic yields only access tokens that expire
within 12 h.
Plus: Zitadel allows multiple keys per machine user, so rotation
is zero-downtime (mint new → push to device → delete old). PATs
rotate one-at-a-time and are disruptive.
What this does not defend against: a fully compromised device
where the attacker reads the keyfile. That requires hardware
(TPM / secure element) and is out of scope.
## The machine keys expire in year 9999. Isn't that effectively forever?
Yes. Currently set in `ZitadelSetupScore::create_machine_key` as
a known-bad default chosen for demo convenience (re-running tests
shouldn't produce expired keys mid-run). Tracked as a known issue.
## Why is the IAM-Owner PAT stored as a plain k8s Secret?
K8s Secrets are base64-encoded, **not** encrypted at rest unless
etcd encryption-at-rest is explicitly enabled with a KMS provider.
Anyone with `get secrets` in the `zitadel` namespace effectively
has Zitadel admin.
The PAT exists because `ZitadelSetupScore` calls Zitadel's
management API (create project, role, machine user, mint key),
which requires IAM-Owner privileges. A PAT is the simplest
credential that survives across applies.
This is a known production-hardening gap. Harmony has the
`harmony_secret` crate (ADR-020) with OpenBao and local-encrypted-file
backends; the Score is currently wired against a k8s Secret only.
## What lifetime is set for the human admin password — why does the ConfigMap show one that doesn't work?
`ZitadelScore` regenerates a random admin password on every apply
and writes it to the rendered ConfigMap. Helm's `FirstInstance`
block only seeds Postgres on the **first** install against an
empty DB, so re-applies render a new ConfigMap password but leave
the original Postgres hash untouched. The displayed password is
stale on every apply after the first.
To recover access: use the `iam-admin-pat` to call Zitadel's
management API and reset the human admin's password directly.
Tracked as a known bug.
## Quick reference — tokens on the wire
| Token | Lives where | Lifetime | Signed by | Purpose |
| --- | --- | --- | --- | --- |
| **Assertion** | Agent memory, in-flight | 60 s | Agent (RSA key) | "I'm machine user X — give me an access token" |
| **Access token** | Agent memory + on-the-wire to NATS | ~12 h | Zitadel | "Zitadel says I'm device X with role `device`" |
| **NATS user JWT** | NATS server connection state | callout-defined (~30 s) | Auth callout (NKey) | "I have these permissions on these subjects" |
The agent only holds the RSA key on disk and the access token
in memory. The NATS user JWT is server-internal — agents don't
see it.
| Material | Storage | Purpose |
|---|---|---|
| Machine JSON key | Device filesystem | Signs JWT-bearer assertions |
| JWT assertion | Memory and token request | Proves possession of the machine key; valid for 60 seconds |
| OIDC ID token | Agent memory and NATS connection | Verifiable bearer used by the NATS callout and OpenBao JWT login |
| Zitadel access token | Token response only | Opaque by default; not used for production Fleet bearer authentication |
| NATS user JWT | NATS authorization exchange | Carries the permissions issued by the auth callout |
## Code map
| Topic | File |
| --- | --- |
| Helm install, masterkey, admin password | `harmony/src/modules/zitadel/mod.rs` |
| Project/role/machine user provisioning | `harmony/src/modules/zitadel/setup.rs` |
| Per-device machine user + key handoff | `examples/fleet_e2e_demo/src/lib.rs::provision_device` |
| JWT-bearer mint | `fleet/harmony-fleet-agent/src/credentials.rs::zitadel_mint` |
| Auth callout decision tree | `nats/callout/src/handler.rs::decide` |
| Per-device permission template | `nats/callout/src/permissions.rs::device_default` |
| End-to-end rehearsal runbook | `examples/fleet_e2e_demo/RUNBOOK.md` |
| Manual JWT-bearer mint + NATS write recipe | [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) |
|---|---|
| Fleet identity composition | `fleet/harmony-fleet-deploy/src/app.rs` |
| Machine-key token exchange | `harmony_zitadel_jwt/src/lib.rs` |
| Shared agent/operator credentials | `fleet/harmony-fleet-auth/src/credentials.rs` |
| Callout validation | `nats/callout/src/zitadel.rs` |
| Device enrollment | `fleet/harmony-fleet-deploy/src/device_setup.rs` |
| Manual diagnosis | [Manual token mint](./fleet-manual-token-mint.md) |

104
docs/guides/fleet.md Normal file
View File

@@ -0,0 +1,104 @@
# Harmony Fleet
Harmony Fleet deploys Podman workloads and runs bounded maintenance operations
on Linux devices outside a Kubernetes cluster. A Kubernetes operator holds the
desired state, while an agent on each device reconciles its local Podman runtime.
This fits small edge installations where devices may disconnect and reconnect.
Fleet is not a general Kubernetes distribution. The current workload contract is
`PodmanV0`, and the current maintenance task is an apt full upgrade for Debian and
Raspberry Pi OS.
## Data flow
```text
Deployment and TaskRun CRs
|
v
Kubernetes operator -----> NATS JetStream KV -----> device agent -----> Podman
^ ^ |
| | v
+------ CR status <-------+------------- device status
```
The operator watches namespaced `Deployment`, `Device`, and `TaskRun` resources.
It resolves targets, writes per-device intent to NATS, and aggregates reported
state. Agents do not receive Kubernetes credentials and do not call the
Kubernetes API.
Zitadel authenticates operators and devices. A device keeps a Zitadel machine
key and exchanges a signed assertion for an OIDC `id_token`. The NATS auth
callout validates that token and grants device-scoped subjects. OpenBao can use
the same verifiable `id_token` for its JWT login. Zitadel access tokens are
opaque by default and are not used as the production NATS or OpenBao bearer.
## Trust boundaries
- Kubernetes and the Fleet operator are the control plane. An operator
compromise can change workload and maintenance intent.
- NATS carries desired and reported state. The auth callout restricts a device
identity to its subjects, but NATS remains in the control-plane trust domain.
- The unprivileged agent controls workloads owned by the `fleet-agent` account.
- A root updater exposes fixed agent-upgrade and system-upgrade operations over a
Unix socket. It does not accept arbitrary commands, package names, or paths.
- Device labels are self-reported placement data, not authorization data.
- A copied machine key permits impersonation of that device until the key is
revoked. Hardware-backed key storage is not implemented.
## Current production composition
The production deploy composition installs PostgreSQL, Zitadel, NATS, the auth
callout, OpenBao, the operator, and the dashboard. It does **not** currently pass
a Zitadel group source or OpenBao administrator token to the operator. As a
result, `allowedGroups` placement fails closed and deployment-to-group OpenBao
grant synchronization is disabled unless a private deploy composition supplies
the corresponding operator configuration.
Enrollment installs a machine key on the device only when the enrollment
operator provides or mints one. It does not automatically make application
secrets available. OpenBao endpoint configuration, group placement, policies,
and secret data are separate setup steps.
## Dashboard
Production exposes the operator's web dashboard through Zitadel SSO and requires
the `fleet-admin` role. The live service reads Device and Deployment CRs and can:
- show current device liveness and deployment aggregate state;
- list device inventory and labels;
- derive alerts for stale devices and failing deployments;
- quarantine a device by adding the blacklist label;
- send an agent command from a device page.
Alert acknowledgements are held in operator memory and disappear after restart.
The displayed 24-hour trend is currently static presentation, not historical
telemetry. The dashboard does not create deployments, show `TaskRun` history, or
provide rollout controls.
For local UI work, `serve-web --mock` uses generated data and proves only the web
surface, not Fleet connectivity.
## Guide
1. [Enroll a device](./fleet-device-secrets.md).
2. [Deploy Podman workloads](./fleet-podman-deployments.md).
3. [Run tasks and upgrades](./fleet-tasks-upgrades.md).
4. [Deploy and operate the control plane](./fleet-staging-install.md).
5. Use the [Zitadel FAQ](./fleet-zitadel-faq.md) and [manual token
mint](./fleet-manual-token-mint.md) when diagnosing authentication.
## Limits
- The production operator is a single replica. Active-active reconciliation and
tested control-plane failover are not provided.
- There is no published or validated fleet-size target. Some controllers still
poll complete device sets.
- Deployment rollout supports one mode: immediate release, with an optional
`canary=true` gate. There are no percentages, batches, approval steps, or
workload rollback.
- System upgrades are direct `TaskRun` resources. Recurring schedules are not
implemented.
- Agent upgrades can restore the previous agent executable if activation fails.
Workload rollback and operating-system package rollback are not implemented.
- A device is trusted to report its own labels, inventory, workload status, and
task status.

376
docs/guides/harmony-auth-cli.md Normal file
View File

@@ -0,0 +1,376 @@
# Harmony Auth CLI
> **Status: read-only commands implemented.** Mutation commands follow the
> ADR-027 group migration.
`harmony-auth` inspects and manages the relationship between Zitadel
identities and OpenBao access. It presents tenants, identities, and Harmony
permissions first. JWT roles, policy names, subject claims, and HCL remain
available through advanced output.
The CLI is the preferred interface while the web UI matures. Both interfaces
use the same `harmony_auth` operations and return the same effective access.
## Mental model
The CLI has two main views:
- `identity`: who an identity is and what it can access
- `tenant`: who can access a tenant or project
Harmony permissions are the primary authorization vocabulary:
| Permission | Intended identity | Effect |
|---|---|---|
| `tenant-admin` | Human | Read, create, change, and delete secrets in a tenant or project |
| `cd-deployer` | Service account | Read deployment secrets in one project |
| `read-only` | Human or service account | Read secrets in a tenant or project |
Existing OpenBao policies that do not correspond to a Harmony assignment are
shown as imported access. The CLI does not rename, rewrite, or hide them.
## Command tree
```text
harmony-auth
├── connection check
├── identity list
├── identity show <subject-id>
├── tenant list
└── tenant show <tenant>
```
There are no flat aliases. `harmony-auth list` is not valid.
## Connection and credentials
Every command except `--help` and `--version` requires:
| Flag | Environment | Meaning |
|---|---|---|
| `--zitadel-url` | `ZITADEL_URL` | Zitadel base URL |
| `--openbao-url` | `OPENBAO_URL` | OpenBao base URL |
| none | `ZITADEL_PAT` | Zitadel service-account PAT |
| none | `OPENBAO_TOKEN` | Temporary OpenBao administrator token |
Secrets are environment-only because command-line arguments remain in shell
history and may be visible in the process list. Secret values never appear in
help output, normal output, JSON, or logs.
Example:
```sh
export ZITADEL_URL=https://sso.example.com
export ZITADEL_PAT=...
export OPENBAO_URL=https://secrets.example.com
export OPENBAO_TOKEN=...
harmony-auth connection check
```
The CLI does not persist profiles or credentials. Browser profile storage and
session credential refresh remain web UI concerns.
`connection check` attempts both backends even when one fails. It reports each
status without printing provider response bodies:
```text
Zitadel connected
OpenBao connected
```
## Identity commands
### List identities
```sh
harmony-auth identity list
harmony-auth identity list --search folk
harmony-auth identity list --kind human
harmony-auth identity list --tenant devsights
harmony-auth identity list --tenant devsights --kind service
```
Filters combine with AND semantics. `--search` matches display name, login, or
email. `--kind` accepts `human` or `service`. Tenant matching uses parsed
assignment and imported-policy scopes, not string-prefix matching.
Imported scopes are recognized only from wildcard roots:
- `<mount>/data/<tenant>/*` is tenant-wide.
- `<mount>/data/<tenant>/<project>/*` is project-specific.
- Exact secret paths and wildcard paths below a project are custom access. They
remain visible but do not affect tenant filters or summaries.
Tenant and project components must pass the same slug validation as managed
Harmony scopes. `harmony_auth` returns both the raw paths and parsed scopes;
frontends never infer scopes themselves.
Human output keeps the tenant visible:
```text
ACTIVE HUMAN Alice Example alice@example.com
subject 241696899342475267
devsights Tenant Admin
ACTIVE SERVICE Folk CD folk-cd
subject 241697058442100739
devsights/folk-timesheet CD Deployer
```
An identity without recognized access is still listed with `No access`.
### Show an identity
```sh
harmony-auth identity show 241696899342475267
```
Default output includes identity metadata, Harmony assignments, and imported
access:
```text
Alice Example
Subject: 241696899342475267
Login: alice@example.com
Kind: Human
Status: Active
Harmony permissions
019b... Tenant Admin devsights
Imported OpenBao access
legacy-folk-reader devsights/folk-timesheet Read secrets
```
Use `--advanced` to inspect implementation details:
```sh
harmony-auth identity show 241696899342475267 --advanced
```
Advanced output adds matching JWT roles, bound subject, audiences, attached
policy names, and the exact HCL returned by OpenBao. A built-in or inaccessible
policy remains listed with `Policy body unavailable`.
## Planned group-based mutations
The first release does not grant or revoke access. Existing per-subject JWT
roles are discovery input, not a writable authorization model.
ADR-027 makes Zitadel roles named `<tenant>:owner`, `<tenant>:deployer`, and
`<tenant>:viewer` authoritative. One shared OpenBao JWT role reads the `groups`
claim, and OpenBao external groups attach policies. Mutation commands ship only
after `harmony_auth` implements that model end to end.
### Grant a permission (planned)
```sh
harmony-auth identity grant 241696899342475267 \
--permission tenant-admin \
--tenant devsights
```
Grant is review-only by default:
```text
Plan
Identity: Alice Example (241696899342475267)
Permission: Tenant Admin
Scope: devsights
Effect: Read, create, change, and delete secrets
No changes applied. Re-run with --apply to continue.
```
Apply the reviewed request explicitly:
```sh
harmony-auth identity grant 241696899342475267 \
--permission tenant-admin \
--tenant devsights \
--apply
```
`cd-deployer` requires `--project`. Other permissions accept an optional
project. Permission applicability and scope validation come from
`harmony_auth`; the CLI does not duplicate those rules.
Applying a grant changes the Zitadel role assignment. It does not create or
edit a per-subject OpenBao JWT role. Applying the same grant twice must report
`changed: false`.
### Revoke a permission (planned)
The assignment ID comes from `identity show`:
```sh
harmony-auth identity revoke 241696899342475267 019b...
```
Revoke is also review-only by default. It shows identity, permission, scope,
and the warning that already-issued OpenBao tokens remain valid until expiry or
revocation. `--apply` performs the removal:
```sh
harmony-auth identity revoke 241696899342475267 019b... --apply
```
Imported per-subject access cannot be revoked through this command because it
is outside the ADR-027 group model. Its OpenBao policy name remains visible for
manual migration.
## Tenant commands
### List tenants
```sh
harmony-auth tenant list
```
Tenants and projects are discovered from managed assignments and recognized
OpenBao policy paths:
```text
TENANT PROJECT HUMANS SERVICES
devsights All projects 2 0
devsights folk-timesheet 1 1
detexion harmony-fleet 0 2
```
### Show a tenant
```sh
harmony-auth tenant show devsights
harmony-auth tenant show devsights --project folk-timesheet
```
Output lists matching scopes and identities with their access. With
`--project`, tenant-wide access and access to that exact project are included;
other projects are excluded:
```text
devsights/folk-timesheet
Alice Example Human Read-only
Folk CD Service CD Deployer
```
This is an authorization view, not a secret-value browser. Secret listing,
creation, update, and reveal are outside the first release.
## JSON output
Every implemented command accepts `--json`. JSON is written to stdout;
diagnostics and logs are written to stderr. The envelope is versioned:
```json
{
"schema_version": 1,
"command": "identity.list",
"result": {}
}
```
The first release uses these result shapes:
| Command | `result` fields |
|---|---|
| `connection check` | `zitadel: { connected }`, `openbao: { connected }` |
| `identity list` | `identities: [{ identity, access }]` |
| `identity show` | `identity`, `access` |
| `tenant list` | `tenants: [{ scope, humans, services }]` |
| `tenant show` | `tenant`, `project`, `identities: [{ identity, access }]` |
`identity` contains `subject_id`, `kind`, `display_name`, `login_name`,
`email`, and `active`. Identity kinds are `human` and `service`; `email` is a
string or `null`.
`scope` contains `tenant` and `project`, where `project` is a string or `null`.
An assignment contains `id`, `subject_id`, `permission`, `scope`,
`policy_name`, and `created_at`. IDs are UUID strings, timestamps are RFC 3339,
and permissions are `tenant_admin`, `cd_deployer`, or `read_only`.
`access` contains:
- `assignments`: assignment objects as defined above
- `imported`: objects with `role_name`, `policy_name`, `secret_paths`,
`scopes`, and `effect`
- `roles`: objects with `name`, `subject_id`, `bound_audiences`, and `policies`
Each role policy contains `name` and `body`. Policy bodies are strings only for
`identity show --advanced`; otherwise `body` is `null`. Imported
`secret_paths`, audiences, and scopes are arrays. `effect` is the
plain-language interpretation returned by `harmony_auth`.
`connection check` always returns both status objects and exits `1` when either
`connected` value is false. A top-level error is used only when the command
cannot produce those statuses.
Domain and backend errors use stdout when `--json` is active:
```json
{
"schema_version": 1,
"command": "identity.show",
"error": {
"kind": "not_found",
"message": "identity not found"
}
}
```
Error kinds are `not_found`, `invalid`, and `backend`. Clap usage errors remain
on stderr because command parsing fails before JSON dispatch.
Planned grant and revoke results will add `applied` and `changed` when those
commands are implemented. Their JSON schema is not frozen by this release.
## Exit and error behavior
| Exit | Meaning |
|---|---|
| `0` | Query completed, plan produced, or mutation applied/no-op |
| `2` | Invalid command, missing connection value, not found, or invalid input |
| `1` | Zitadel or OpenBao request failed |
Errors name the failed operation and backend but do not print credentials,
provider response bodies, or stack traces. `RUST_LOG=info` enables operation
logs on stderr.
## Architecture boundary
```text
harmony_auth_ui ─┐
├──> harmony_auth ──> Zitadel + OpenBao
harmony_auth_cli ─┘
```
`harmony_auth` owns:
- identities, scopes, permissions, assignments, roles, policies, and plans
- Zitadel identity discovery
- OpenBao role and policy discovery
- tenant and identity access queries
- permission applicability and scope validation
- assignment discovery and current legacy assignment operations used by the UI
- provider request construction and response interpretation
`harmony_auth_cli` owns:
- Clap arguments and environment mapping
- terminal and JSON rendering
- binary exit codes and logging setup
`harmony_auth_ui` owns HTTP routes, browser sessions, cookies, forms, HTML, CSS,
and browser JavaScript. Neither frontend may parse OpenBao policies, infer
tenants, reconcile JWT roles, or implement permission rules.
## First-release limits
- Authorization discovers existing per-subject JWT roles as imported access.
- Grant and revoke wait for Zitadel role and OpenBao external-group operations.
- The CLI does not create Zitadel identities.
- The CLI does not provide a generic OpenBao policy editor.
- Tenant administrators are not yet authenticated as constrained actors; the
supplied OpenBao token determines backend authority.
- The CLI does not store profiles or credentials.

41
docs/guides/operator-dashboard-sso.md
View File

@@ -5,21 +5,20 @@ public client). Distinct from the agent/callout machine auth
([fleet-zitadel-faq](./fleet-zitadel-faq.md)); the security rationale is in
[web-auth-security](./web-auth-security.md). Code: `harmony_zitadel_auth/`.
## Quickstart (staging)
## Deployment
1. **Zitadel app** — create a **Web** application, auth method **PKCE** (no client
secret), redirect URI `https://fleet-stg.<base>/auth/callback`, post-logout URI
`https://fleet-stg.<base>/`. Copy its **Client ID**.
2. **Seed config** in OpenBao (namespace `fleet-staging`) — the deploy derives every
host from `base_domain`, so you set only:
- `FleetDeployConfig.operator_oidc_client_id` = the Client ID
- `FleetDeployConfig.operator_trusted_audiences` = `["<Client ID>"]`
- `FleetDeploySecrets.operator_cookie_key_b64` = `openssl rand -base64 64`
3. **Deploy**: `./fleet/scripts/dev-deploy-operator.sh`
4. Open `https://fleet-stg.<base>/` → Zitadel login → back to the dashboard.
`FleetApp` declares a dedicated dashboard Web PKCE application with the
dashboard callback and logout URLs. The operator's Device Code application is
separate. `ZitadelSetupScore` reconciles both and publishes the dashboard client
ID. `FleetOperatorScore` builds `ZitadelAuthConfig` from that output and the
dashboard Ingress, then generates and retains the session cookie key in the
operator Secret. No client ID or cookie key is entered by hand.
`fleet_staging_install` already generates the cookie key, so a fresh install needs
only the Client ID + audiences.
The dashboard requires the exact Zitadel project role `fleet-admin`. Create the
role on the fleet project and grant it to each operator. Login automatically
requests `urn:zitadel:iam:org:project:roles`, so the project-level **Assert Roles
on Authentication** setting is not required. Users without the role receive a
403 response that retains the session cookie and includes a sign-out link.
## Local dev (`serve-web`)
@@ -39,17 +38,15 @@ on the app's **Development Mode** (Zitadel rejects non-HTTPS redirects otherwise
- **Cookie key** — `cookie_key_b64` must decode to ≥64 bytes, else the dashboard
refuses to start (`cookie_key_b64 must decode to at least 64 bytes`; reconcile
keeps running).
- **403 after login** — confirm the user has the exact `fleet-admin` project role
and that the aggregate roles claim is present in the ID token.
## Config reference
The operator reads `ZitadelAuthConfig` + `OperatorCookieKey` via ConfigClient. The
deploy derives `zitadel_base` / `base_url` / `logout_redirect_uri` from `base_domain`
(`https://sso-stg.<base>`, `https://fleet-stg.<base>`, `…/`) and fixes
`scope = openid profile email`; you supply `client_id`, `trusted_audiences`,
`cookie_key_b64`. All endpoints derive from `zitadel_base`:
The operator reads `ZitadelAuthConfig` and `OperatorCookieKey` through
ConfigClient. The deploy derives `zitadel_base`, `base_url`, client ID, trusted
audience, logout URI, and `scope = openid profile email`. The login flow adds the
aggregate roles scope once if it is absent. All endpoints derive from
`zitadel_base`:
`/.well-known/openid-configuration`, `/oauth/v2/authorize`, `/oauth/v2/token`,
`/oidc/v1/end_session`.
> The dashboard only checks that the user authenticated — no role gate yet
> ([web-auth-security](./web-auth-security.md) §3,
> [ROADMAP/09](../../ROADMAP/09-sso-config-hardening.md)).

22
examples/fleet_device_enroll/src/main.rs
View File

@@ -28,7 +28,7 @@ use harmony::inventory::Inventory;
use harmony::modules::fleet::ensure_fleet_ssh_keypair;
use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials};
use harmony_fleet_deploy::{
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
};
use harmony_types::id::Id;
@@ -52,6 +52,10 @@ use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
credentials inline (browser SSO or pre-acquired token)"
)]
struct Cli {
/// Apply a changed existing device configuration without prompting.
#[arg(long)]
yes: bool,
// ---- target ----------------------------------------------------------
/// Where to apply the score.
///
@@ -112,6 +116,14 @@ struct Cli {
#[arg(long)]
nats_url: Option<String>,
/// OpenBao endpoint used for deployment secrets and private image pulls.
#[arg(long, requires = "openbao_secret_prefix")]
openbao_url: Option<String>,
/// KV prefix containing this fleet's deployment and pull secrets.
#[arg(long, requires = "openbao_url")]
openbao_secret_prefix: Option<String>,
// ---- device identity -------------------------------------------------
/// Device id baked into the agent's TOML, the Zitadel machine
/// username (`device-<device_id>`), and the Kubernetes Device CR
@@ -302,9 +314,13 @@ async fn main() -> Result<()> {
auth,
agent_binary_path: agent_binary,
hosts_entries: vec![],
openbao: None,
openbao: cli
.openbao_url
.clone()
.zip(cli.openbao_secret_prefix.clone())
.map(|(url, secret_prefix)| DeviceOpenbao { url, secret_prefix }),
};
let setup_score = FleetDeviceSetupScore::new(setup_config);
let setup_score = FleetDeviceSetupScore::new(setup_config).overwrite_existing_config(cli.yes);
#[cfg(feature = "vm-rehearsal")]
if cli.vm_rehearsal {

9
examples/fleet_load_test/src/main.rs
View File

@@ -46,6 +46,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
const LOAD_TEST_GROUP: &str = "fleet-load-test";
#[derive(Parser, Debug, Clone)]
#[command(
name = "fleet_load_test",
@@ -406,7 +408,7 @@ async fn apply_one_cr(
let cr = Deployment::new(
&group.cr_name,
DeploymentSpec {
allowed_groups: None,
allowed_groups: vec![LOAD_TEST_GROUP.to_string()],
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,
@@ -416,9 +418,11 @@ async fn apply_one_cr(
// for each matched device; that's wire noise we accept
// as part of the realism.
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: group.cr_name.clone(),
image: "docker.io/library/nginx:alpine".to_string(),
image_pull_secret: None,
ports: vec!["8080:80".to_string()],
env: vec![],
secret_env: vec![],
@@ -468,6 +472,7 @@ async fn publish_one_info(bucket: kv::Store, device: DevicePlan) -> Result<()> {
device_id: Id::from(device.device_id.clone()),
labels: BTreeMap::from([("group".to_string(), device.cr_name.clone())]),
inventory: None,
updater: None,
updated_at: Utc::now(),
};
let key = device_info_key(&device.device_id);
@@ -495,6 +500,7 @@ async fn simulate_state_loop(
device_id: Id::from(device.device_id.clone()),
deployment: deployment.clone(),
phase,
rollout_revision: None,
last_event_at: Utc::now(),
last_error: matches!(phase, Phase::Failed)
.then(|| format!("synthetic failure @{}", device.device_id)),
@@ -529,6 +535,7 @@ async fn simulate_heartbeat_loop(
let hb = HeartbeatPayload {
device_id: Id::from(device.device_id.clone()),
at: Utc::now(),
agent_version: None,
};
if let Ok(payload) = serde_json::to_vec(&hb) {
if bucket.put(&hb_key, payload.into()).await.is_ok() {

8
examples/fleet_server_install/src/main.rs
View File

@@ -95,6 +95,9 @@ struct Cli {
/// `RUST_LOG` value injected into the operator pod's env.
#[arg(long, default_value = "info,kube_runtime=warn")]
log_level: String,
/// Static `device=group|group;...` membership for local testing.
#[arg(long)]
device_groups: Option<String>,
/// Hostname Zitadel should answer on. When set, Zitadel + its
/// PostgreSQL cluster are installed alongside the operator.
@@ -138,12 +141,15 @@ async fn main() -> Result<()> {
// NatsScore install creates. ClusterIP and LoadBalancer both
// expose the same `<release>.<namespace>:4222` for in-cluster
// callers.
let operator = FleetOperatorScore::new(&cli.operator_image)
let mut operator = FleetOperatorScore::new(&cli.operator_image)
.namespace(&cli.operator_namespace)
.release_name(&cli.operator_release)
.image_pull_policy(&cli.operator_image_pull_policy)
.messaging(&nats.client_ref())
.log_level(&cli.log_level);
if let Some(groups) = cli.device_groups {
operator = operator.device_groups(groups);
}
// FleetServerScore now takes NatsK8sScore (auth-callout-aware,
// OKD-Route-aware) — see `fleet_staging_install` for the

20
examples/fleet_typed_deploy/src/bin/tenant-provision.rs
View File

@@ -1,8 +1,20 @@
use example_fleet_typed_deploy::{credential_store, platform_context, tenant_config};
use harmony_fleet_deploy::provision_fleet_tenant_with_context;
use std::{env, path::PathBuf};
use anyhow::Context;
use example_fleet_typed_deploy::{credential_store, fleet_context, tenant_config};
use harmony_fleet_deploy::provision_fleet_tenant_with_kubeconfig;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
provision_fleet_tenant_with_context(platform_context()?, tenant_config(), credential_store()?)
.await
let kubeconfig = env::var_os("KUBECONFIG")
.map(PathBuf::from)
.context("set KUBECONFIG to your cluster-admin kubeconfig")?;
provision_fleet_tenant_with_kubeconfig(
fleet_context()?,
kubeconfig,
tenant_config(),
credential_store()?,
false,
)
.await
}

20
examples/fleet_typed_deploy/src/lib.rs
View File

@@ -18,26 +18,6 @@ pub fn fleet_context() -> anyhow::Result<Context> {
})
}
pub fn platform_context() -> anyhow::Result<Context> {
Ok(Context {
name: context_name!("platform-admin"),
namespace: "platform-system".parse()?,
spec: ContextSpec::Remote(RemoteContext {
registry: oci_registry!("registry.example.com"),
repository: oci_repository!("customer/fleet"),
domain: domain!("fleet.example.com"),
image_pull_secret: None,
access: OpenBaoClusterAccess {
namespace: openbao_namespace!("platform/admin"),
url: http_url!("https://secrets.example.com"),
role: "platform-admin".parse()?,
zitadel_url: http_url!("https://identity.example.com"),
zitadel_audience: "openbao".parse()?,
},
}),
})
}
pub fn tenant_config() -> TenantConfig {
TenantConfig {
id: "customer-fleet".into(),

10
examples/harmony_apply_deployment/Cargo.toml
View File

@@ -5,15 +5,17 @@ edition = "2024"
license.workspace = true
[[bin]]
name = "harmony_apply_deployment"
name = "harmony"
path = "src/main.rs"
[dependencies]
harmony = { path = "../../harmony" }
harmony_app = { path = "../../harmony_app" }
harmony_cli = { path = "../../harmony_cli" }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
kube = { workspace = true, features = ["runtime", "derive"] }
k8s-openapi = { workspace = true }
serde_json.workspace = true
async-trait.workspace = true
tokio.workspace = true
anyhow.workspace = true
clap.workspace = true

2
examples/harmony_apply_deployment/backend.Dockerfile Normal file
View File

@@ -0,0 +1,2 @@
FROM busybox:1.37
CMD ["sleep", "infinity"]

2
examples/harmony_apply_deployment/frontend.Dockerfile Normal file
View File

@@ -0,0 +1,2 @@
FROM busybox:1.37
CMD ["httpd", "-f", "-p", "8080"]

388
examples/harmony_apply_deployment/src/main.rs
View File

@@ -1,240 +1,168 @@
//! Typed-Rust applier for the harmony fleet `Deployment` CR.
//!
//! Builds a `Deployment` CR via the typed `DeploymentSpec` +
//! `PodmanV0Score` + `kube::Api`, then either applies it directly
//! through the kube client or prints it to stdout so the user can
//! pipe into `kubectl apply -f -`.
//!
//! The CRD is domain-agnostic — it's "declarative reconcile intent
//! for a set of devices matched by label selector," which is the
//! same shape whether the fleet is Pi podman, OKD clusters, or
//! KVM VMs. The name `harmony_apply_deployment` reflects that
//! (not `iot_`-anything), in line with the review call to position
//! the operator as a generic fleet/reconcile tool.
//!
//! The CRD types live in `harmony_fleet_operator`; the score types
//! live in `harmony_reconciler_contracts` (PodmanV0 being the first
//! reconciler variant — future variants drop in alongside).
//!
//! Typical demo-driver usage:
//!
//! # apply an nginx deployment
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:latest
//!
//! # print the CR JSON (lets the user kubectl-apply it manually)
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:latest --print | kubectl apply -f -
//!
//! # upgrade the same deployment to a newer image
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:1.26
//!
//! # delete the deployment
//! cargo run -q -p example_harmony_apply_deployment -- --delete
use anyhow::{Context, Result};
use clap::Parser;
use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
use harmony_reconciler_contracts::{
EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, VolumeMount,
use async_trait::async_trait;
use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology;
use harmony_app::{
AppContext, AppError, AppIdentity, Context, ContextCatalog, ContextSpec, HarmonyApp, ImageRefs,
ImageSpec, LocalContext, OpenBaoClusterAccess, Profile, RemoteContext,
};
use harmony_fleet_deploy::FleetDeploymentScore;
use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams};
use std::collections::BTreeMap;
#[derive(Parser, Debug)]
#[command(
name = "harmony_apply_deployment",
about = "Build + apply a harmony fleet Deployment CR from typed Rust (no yaml)"
)]
struct Cli {
/// Kubernetes namespace for the Deployment CR.
#[arg(long, default_value = "fleet-demo")]
namespace: String,
/// Deployment CR name. Also used as the KV key suffix and
/// podman container name on the device.
#[arg(long, default_value = "hello-world")]
name: String,
/// Shortcut: if set, picks a single device by id. Shorthand for
/// `--selector device-id=<target_device>` — the agent publishes
/// a `device-id=<id>` label on its DeviceInfo by default so this
/// works without any cluster-side label pre-wiring.
#[arg(long, default_value = "fleet-smoke-vm")]
target_device: String,
/// Repeatable `key=value` label selector. Takes precedence over
/// `--target-device` when provided. All pairs AND together.
#[arg(long = "selector", value_name = "KEY=VALUE")]
selectors: Vec<String>,
/// Container image to run.
#[arg(long, default_value = "docker.io/library/nginx:latest")]
image: String,
/// `host:container` port mapping exposed on the device.
#[arg(long, default_value = "8080:80")]
port: String,
/// Repeatable `KEY=VALUE` env var injected into the container.
#[arg(long = "env", value_name = "KEY=VALUE")]
envs: Vec<String>,
/// Repeatable bind-mount in `host_path:container_path[:ro]` form.
/// Append `:ro` for read-only.
#[arg(long = "volume", value_name = "HOST:CONTAINER[:ro]")]
volumes: Vec<String>,
/// Container restart policy.
#[arg(long, value_enum, default_value_t = CliRestart::UnlessStopped)]
restart: CliRestart,
/// Delete the Deployment CR instead of applying it.
#[arg(long)]
delete: bool,
/// Print the CR as JSON to stdout instead of applying it.
/// Useful for piping into `kubectl apply -f -`.
#[arg(long)]
print: bool,
struct ExampleApp;
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for ExampleApp {
fn identity(&self, ctx: &AppContext) -> AppIdentity {
AppIdentity {
name: "example-app".into(),
namespace: ctx.namespace().into(),
}
}
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
Ok(["frontend", "backend"]
.map(|name| ImageSpec {
name: name.into(),
image: ctx.image(name),
context: env!("CARGO_MANIFEST_DIR").into(),
dockerfile: format!("{}/{name}.Dockerfile", env!("CARGO_MANIFEST_DIR")).into(),
build_args: Vec::new(),
})
.into())
}
fn validate_deploy_images(&self, images: &ImageRefs) -> Result<(), AppError> {
if images
.iter()
.any(|(name, _)| !matches!(name, "frontend" | "backend"))
{
return Err(AppError::InvalidComposition(
"only frontend and backend images are accepted".into(),
));
}
for name in ["frontend", "backend"] {
let image = images.require(name)?;
let digest_pinned = harmony_app::is_digest_pinned(image);
let dev_tagged = !image.contains('@')
&& image.rsplit_once(':').is_some_and(|(repository, tag)| {
!repository.is_empty() && tag.starts_with("dev-")
});
if !digest_pinned && !dev_tagged {
return Err(AppError::InvalidComposition(format!(
"image '{name}' must be digest-pinned or use a dev-* tag"
)));
}
}
Ok(())
}
async fn scores(
&self,
ctx: &AppContext,
images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let services = ["frontend", "backend"]
.map(|name| {
Ok(PodmanService {
name: name.into(),
image: images.require(name)?.into(),
image_pull_secret: (ctx.profile() == Profile::Prod)
.then(|| "application-images".into()),
ports: (name == "frontend")
.then(|| "8080:8080".into())
.into_iter()
.collect(),
env: Vec::new(),
secret_env: Vec::new(),
volumes: Vec::new(),
restart_policy: Default::default(),
})
})
.into_iter()
.collect::<Result<Vec<_>, AppError>>()?;
let deployment = Deployment::new(
"example-app",
DeploymentSpec {
allowed_groups: vec!["application".into()],
target_selector: LabelSelector {
match_labels: Some([("application".into(), "example".into())].into()),
match_expressions: None,
},
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services,
}),
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
);
Ok(vec![Box::new(FleetDeploymentScore::new(
deployment,
ctx.namespace(),
))])
}
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
let cr = build_cr(&cli);
if cli.print {
println!("{}", serde_json::to_string_pretty(&cr)?);
return Ok(());
}
let client = Client::try_default()
.await
.context("building kube client (is KUBECONFIG set?)")?;
let api: Api<Deployment> = Api::namespaced(client, &cli.namespace);
if cli.delete {
match api.delete(&cli.name, &DeleteParams::default()).await {
Ok(_) => println!("deleted deployment '{}/{}'", cli.namespace, cli.name),
Err(kube::Error::Api(ae)) if ae.code == 404 => {
println!(
"deployment '{}/{}' not found (already gone)",
cli.namespace, cli.name
)
}
Err(e) => anyhow::bail!("delete failed: {e}"),
}
return Ok(());
}
// Server-side apply so repeated invocations (upgrades) patch
// the existing CR instead of erroring with "already exists."
let params = PatchParams::apply("harmony-apply-deployment").force();
let applied = api
.patch(&cli.name, &params, &Patch::Apply(&cr))
.await
.context("applying Deployment CR")?;
let meta = applied.metadata;
println!(
"applied deployment '{}/{}' (resourceVersion={}, image={})",
cli.namespace,
meta.name.as_deref().unwrap_or("?"),
meta.resource_version.as_deref().unwrap_or("?"),
cli.image,
);
Ok(())
}
/// Mirrors the contract `RestartPolicy` so we can keep the CLI
/// schema stable even if the underlying enum gains variants.
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
enum CliRestart {
No,
UnlessStopped,
OnFailure,
Always,
}
impl From<CliRestart> for RestartPolicy {
fn from(c: CliRestart) -> Self {
match c {
CliRestart::No => RestartPolicy::No,
CliRestart::UnlessStopped => RestartPolicy::UnlessStopped,
CliRestart::OnFailure => RestartPolicy::OnFailure,
CliRestart::Always => RestartPolicy::Always,
}
}
}
fn parse_env(s: &str) -> Result<(String, String)> {
let (k, v) = s
.split_once('=')
.ok_or_else(|| anyhow::anyhow!("--env expects KEY=VALUE, got {s:?}"))?;
Ok((k.to_string(), v.to_string()))
}
fn parse_volume(s: &str) -> Result<VolumeMount> {
let parts: Vec<&str> = s.split(':').collect();
let (host, cont, ro) = match parts.as_slice() {
[host, cont] => (host, cont, false),
[host, cont, mode] if *mode == "ro" => (host, cont, true),
[host, cont, mode] if *mode == "rw" => (host, cont, false),
_ => anyhow::bail!("--volume expects HOST:CONTAINER[:ro|rw], got {s:?}"),
};
Ok(VolumeMount {
host_path: host.to_string(),
container_path: cont.to_string(),
read_only: ro,
})
}
fn build_cr(cli: &Cli) -> Deployment {
let env: Vec<EnvVar> = cli
.envs
.iter()
.map(|s| EnvVar::from(parse_env(s).expect("--env validated")))
.collect();
let volumes: Vec<VolumeMount> = cli
.volumes
.iter()
.map(|s| parse_volume(s).expect("--volume validated"))
.collect();
let score = PodmanV0Score {
services: vec![PodmanService {
name: cli.name.clone(),
image: cli.image.clone(),
ports: vec![cli.port.clone()],
env,
secret_env: vec![],
volumes,
restart_policy: cli.restart.into(),
}],
};
let payload = ReconcileScore::PodmanV0(score);
let mut match_labels = BTreeMap::new();
if cli.selectors.is_empty() {
match_labels.insert("device-id".to_string(), cli.target_device.clone());
} else {
for kv in &cli.selectors {
let (k, v) = kv
.split_once('=')
.unwrap_or_else(|| panic!("--selector expects KEY=VALUE, got '{kv}'"));
match_labels.insert(k.to_string(), v.to_string());
}
}
Deployment::new(
&cli.name,
DeploymentSpec {
allowed_groups: None,
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,
async fn main() -> anyhow::Result<()> {
harmony_cli::app::app_main(
ExampleApp,
ContextCatalog::new([
Context {
name: "local".parse()?,
namespace: "example-app".parse()?,
spec: ContextSpec::Local(LocalContext::ManagedK3d),
},
score: payload,
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
Context {
name: "production".parse()?,
namespace: "example-app".parse()?,
spec: ContextSpec::Remote(RemoteContext {
registry: "registry.example.com".parse()?,
repository: "apps".parse()?,
domain: "example.com".parse()?,
image_pull_secret: None,
access: OpenBaoClusterAccess {
namespace: "platform/example-app".parse()?,
url: "https://secrets.example.com".parse()?,
role: "deployer".parse()?,
zitadel_url: "https://identity.example.com".parse()?,
zitadel_audience: "openbao".parse()?,
},
}),
},
},
])?,
)
.await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn standalone_deploy_allows_digests_and_dev_tags_only() {
let digest = "a".repeat(64);
assert!(
ExampleApp
.validate_deploy_images(&ImageRefs::new([
(
"frontend".into(),
format!("registry/frontend@sha256:{digest}")
),
("backend".into(), "registry/backend:dev-123".into()),
]))
.is_ok()
);
assert!(
ExampleApp
.validate_deploy_images(&ImageRefs::new([
("frontend".into(), "registry/frontend:latest".into()),
("backend".into(), "registry/backend:dev-123".into()),
]))
.is_err()
);
}
}

1
examples/harmony_sso/src/main.rs
View File

@@ -250,6 +250,7 @@ async fn main() -> anyhow::Result<()> {
openshift: false,
tls_issuer: None,
node_port: None,
create_namespace: true,
}
.interpret(&Inventory::autoload(), &topology)
.await

1
examples/monitoring_with_tenant/src/main.rs
View File

@@ -37,6 +37,7 @@ async fn main() {
memory_request_gb: 4.0,
memory_limit_gb: 4.0,
storage_total_gb: 10.0,
service_limit: 10,
},
network_policy: TenantNetworkPolicy::default(),
},

2
examples/openbao/src/main.rs
View File

@@ -95,6 +95,7 @@ async fn main() -> Result<()> {
openshift: cfg.openshift,
tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()),
node_port: None,
create_namespace: true,
};
// JWT auth composes in only when both issuer and audience are set; it
@@ -130,6 +131,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
users: vec![],
jwt_auth,
oidc_application: None,
endpoint: None,
};
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)];

32
fleet/README.md
View File

@@ -142,6 +142,38 @@ Run this from the private deploy repository whose binary calls
See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
### Releases
`scripts/release.sh` publishes immutable releases. Control-plane and agent
versions are independent; the operator and NATS callout share one control-plane
version.
```bash
# Publish either release independently.
fleet/scripts/release.sh --control-version 0.4.0 --publish-only
fleet/scripts/release.sh --agent-version 0.2.0 --publish-only
# Publish both, deploy through a private deploy crate, and upgrade one Device.
fleet/scripts/release.sh \
--control-version 0.4.0 \
--agent-version 0.2.0 \
--deploy-manifest /path/to/Cargo.toml \
--deploy-bin private-deploy \
--context production \
--namespace fleet \
--device device-1
```
Control-plane images publish to `hub.nationtech.io/harmony`. The agent is a
raw, single-layer OCI artifact and Devices receive a manifest-digest reference.
Registry pulls are anonymous, while publication uses `REGISTRY_USER` and
`REGISTRY_TOKEN`. Run the script with `--help` for its tool and deployment
inputs.
Before the first OCI-based upgrade, rerun `FleetDeviceSetupScore` for each
device to install an updater that understands `oci://` references. The updater
does not update itself. Direct `https://` agent artifacts remain supported.
### Connecting to the operator
The operator runs as a single-replica Deployment in the context namespace.

168
fleet/agent-reconciliation-and-upgrade-plan.md Normal file
View File

@@ -0,0 +1,168 @@
# Fleet agent reconciliation and upgrade plan
## Decisions
- One agent instance owns workload reconciliation for a device at a time.
- NATS desired state is authoritative. Podman labels are the durable observed
state. The agent rebuilds its in-memory state from both after every restart.
- Desired entries carry their JetStream revision. Older watch or snapshot data
never replaces a newer revision.
- KV watches accelerate convergence. A periodic full snapshot repairs missed
events and drives orphan cleanup.
- One worker serializes runtime mutations. New events replace older intent
before the next reconciliation pass.
- A deployment upgrade validates and pulls first, then sends the old container
its normal stop signal and waits without a deadline. The replacement does not
start until the old container exits. A stuck shutdown requires a separate,
explicit operator force-kill action; reconciliation never escalates to kill
on its own. The current agent still has a fixed 30-second stop timeout; this
decision describes the required replacement behavior.
- An existing container owned by another deployment or by a human is a
conflict. The agent reports it and does not delete the container.
- Agent upgrade never runs two workload reconcilers. The candidate runs only a
non-workload-mutating probe. A root-owned updater then switches the permanent
systemd service and rolls back if the new agent does not become ready.
Deployment rollout freezes the authorized devices matched for one revision. If
any frozen device has `canary=true`, only those devices receive the revision
first. Every canary must report successful deployment convergence for that
revision before the operator releases the remaining devices. A failed or missing
canary blocks the rollout. Without a canary, the operator releases the full
frozen set immediately.
Agents that predate rollout acknowledgements still accept and reconcile the
desired score, but cannot prove which revision reached `Running`. Upgrade agents
before enabling this operator rollout behavior; otherwise a legacy canary
remains pending and safely blocks the remainder.
## Deployment reconciliation
### Inputs and reconstruction
The reconciler owns one map from `DeploymentName` to the latest desired value.
A value is a valid unresolved score or an invalid payload with its error. At
startup and every periodic resync it replaces that map from the current
`desired-state.<device>.>` KV snapshot. Live puts and deletes update the same
map and wake the worker. A failed or incomplete snapshot cannot authorize
deletion. Because KV key enumeration and value reads are not atomic, a managed
deployment must be absent from two consecutive complete snapshots before it is
removed as an orphan. Any newer watch revision resets that absence count.
An explicit revisioned delete schedules removal immediately; the two-snapshot
rule applies only when reconstructing an absent key.
No local deployment database is needed. After a reboot or power loss, current
KV values reconstruct intent and Harmony-managed Podman labels reconstruct
observed state.
### Reconciliation pass
For one complete desired snapshot:
1. Validate every deployment. A duplicate service name or explicit host port
fails the conflicting deployments but does not block unrelated deployments.
2. Resolve secrets for the current score revision.
3. Ask the runtime to converge each valid deployment.
4. List Harmony-managed runtime deployments and remove deployments absent from
two complete snapshots. Cleanup runs before retrying a desired deployment
that needs the same service name.
5. Publish coarse `Phase` plus bounded `lastError`. Only an acknowledged phase
and error pair is deduplicated; failed status writes remain dirty.
The runtime convergence operation preflights the full score before stopping
anything. It validates ports and service names, pulls missing images, and checks
container ownership. If the score has an init container, the runtime runs it
with no restart policy and requires exit code zero before reconciling services.
Its exited container records completion for the resolved score. A score change
reruns initialization, so init containers must be idempotent. Preflight prevents
known destructive failures; it cannot make Podman operations transactional.
For each changed service the runtime performs stop, remove, create, start, and
inspect in that order. Successfully changed services remain changed if a later
service fails, and the next pass resumes from observed labels. Unchanged running
services are untouched. Missing or stopped desired services are recreated or
restarted. Services no longer in the score are removed with the same
graceful-stop behavior.
A malformed current payload remains present for orphan accounting, reports
`Failed`, and causes no workload mutation. The last valid workload is preserved
but no longer reported healthy until the payload is corrected or deleted.
If a newer revision arrives during a runtime call, that call may finish but its
result is not published. The worker immediately reconciles the latest revision;
it never interrupts one Podman operation halfway.
### Edge cases
The first implementation and its tests cover:
- new deployment;
- unchanged deployment as a no-op;
- upgrade with stop-before-replace ordering;
- preflight failure without stopping the old version;
- exited desired container reported as failed with its exit details; Podman
restart count is included for diagnosis but no time-window crashloop
classifier is introduced;
- online deletion;
- offline deletion followed by orphan cleanup when the device returns;
- agent restart and host reboot;
- power loss before stop, after stop, after remove, after create, and after
start, each repaired by the next full pass;
- service-name conflict between deployments;
- conflict with an unmanaged or differently owned container;
- desired container manually stopped or deleted;
- malformed desired payload replacing, rather than preserving, old intent;
- partial failure in a multi-service deployment;
- Podman unavailability and retry;
- failed state publication and retry;
- watch interruption or missed events repaired by full resync;
- failed and incomplete full snapshots;
- puts and deletes before, during, and after snapshot enumeration;
- invalid desired keys and failed state deletion;
- a newer desired revision arriving during reconciliation.
Secret values remain cached for one desired revision. Secret-only rotation is
out of scope until the secret source exposes a revision or watch contract.
## Agent upgrade
The signed authorization and probation design was replaced by stop-switch-start.
The maintained protocol is
[`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md);
the original implementation plan remains as a link-preserving pointer in
[`agent-upgrade-stop-switch-start-plan.md`](agent-upgrade-stop-switch-start-plan.md).
Deferred follow-up:
- retry and classify activation readiness failures before rollback or quarantine;
- add an isolated VM agent-upgrade E2E covering success and failure recovery;
- rerun `FleetDeviceSetupScore` to refresh the non-self-updating bootstrap updater,
then verify an anonymous pull from a digest-pinned OCI artifact and a
target/binary version mismatch. The mismatch must fail during candidate
self-test without stopping the active agent; updater child processes must not
emit stray systemd notifications or expected broken-pipe warnings;
- replace the operator's two-second full-device scan before fleet scale;
- add binary garbage collection when disk pressure makes retention relevant.
## Device commands
The dashboard sends bounded one-shot exec requests over Core NATS request/reply.
Exec is an administrative capability, not a sandbox: it runs as the
`fleet-agent` Unix account and can access that account's credentials and rootless
Podman runtime. Only a verified dashboard session with the exact `fleet-admin`
project role may use it.
Manual QA must cover stdout, stderr and non-zero exit status, timeout cleanup,
output truncation, an offline device, and denial for a dashboard user without the
`fleet-admin` role.
## Delivery checkpoints
1. Rework deployment reconciliation around a complete snapshot and a fakeable
runtime boundary. Add the full unit matrix and retain the existing VM E2E.
2. Update Podman convergence for ownership checks, preflight, graceful
stop-before-replace, inspection, and orphan inventory.
3. Amend ADR-022 and add attempt-scoped upgrade contracts.
4. Add the updater, probe mode, exclusive agent lock, durable transaction, and
operator coordination.
5. Run unit tests, compile environment-gated E2E, run focused local VM tests if
available without production credentials, and complete independent reviews.
6. Production QA is manual and stepwise: deployment recovery, bounded device
commands, bootstrap-updater diagnostics, then one agent upgrade attempt with
rollback rehearsed before any canary rollout.

18
fleet/agent-upgrade-stop-switch-start-plan.md Normal file
View File

@@ -0,0 +1,18 @@
# Fleet agent upgrade: stop, switch, start
## Status
Implemented. This plan replaced the signed two-step authorization protocol for
the IoT Podman fleet agent.
The maintained documents are:
- [`docs/adr/022-fleet-agent-upgrade.md`](../docs/adr/022-fleet-agent-upgrade.md)
for the decision and invariants;
- [`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md)
for the current protocol, recovery behavior, limits, and operational caveats;
- `harmony-reconciler-contracts/src/upgrade.rs` for wire fields and phases;
- `fleet/harmony-fleet-agent/src/updater.rs` for the local transaction.
This file is retained so links from the implementation work remain valid. It is
not an independent protocol specification.

9
fleet/harmony-fleet-agent/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "harmony-fleet-agent"
version = "0.1.0"
version = "0.1.2"
edition = "2024"
rust-version = "1.85"
@@ -15,7 +15,7 @@ futures-util = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2.workspace = true
tokio = { workspace = true }
tokio = { workspace = true, features = ["process"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
anyhow = { workspace = true }
@@ -23,3 +23,8 @@ clap = { workspace = true }
toml = { workspace = true }
thiserror = { workspace = true }
podman-api = "0.9"
sd-notify = "0.4"
reqwest.workspace = true
oci-client.workspace = true
uuid.workspace = true
fs2.workspace = true

2
fleet/harmony-fleet-agent/Dockerfile
View File

@@ -41,6 +41,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
COPY --from=builder /app/target/release/harmony-fleet-agent /usr/local/bin/harmony-fleet-agent
RUN install -d -o 65532 -g 65532 -m 0700 /run/harmony-fleet-agent
# Non-root runtime. 65532 is the `nonroot` UID convention from
# distroless. Pairs with `securityContext.runAsNonRoot: true` in
# whatever Pod spec the harness or production helm chart applies.

332
fleet/harmony-fleet-agent/src/command_server.rs
View File

@@ -1,10 +1,7 @@
//! Agent-side request/reply command server.
//!
//! Subscribes to `device-commands.<device_id>.>` and dispatches one
//! handler per verb. Single-shot replies for v1; streaming verbs
//! (logs, exec follow-up) will reuse this loop and write multiple
//! frames to the inbox, terminating with the `X-Harmony-Final`
//! header.
//! Subscribes to `device-commands.<device_id>.>` and returns one reply
//! for each ping or bounded exec request.
//!
//! Runs alongside the KV reconciler in the agent's top-level
//! `tokio::select!`. Independent of the podman runtime: when
@@ -12,23 +9,42 @@
//! the command server still runs (ping is useful for "is this device
//! online" health-checks regardless).
use std::io;
use std::os::unix::process::CommandExt;
use std::process::Stdio;
use std::sync::Arc;
use std::time::Instant;
use std::time::{Duration, Instant};
use async_nats::Client;
use async_nats::Subject;
use futures_util::StreamExt;
use harmony_reconciler_contracts::{
HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subscription,
CommandRequest, ExecReply, HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subject,
device_command_subscription,
};
use serde::Serialize;
use thiserror::Error;
use tokio::io::{AsyncRead, AsyncReadExt};
use tokio::sync::Semaphore;
const EXEC_MAX_COMMAND: usize = 16 * 1024;
const EXEC_MAX_OUTPUT: usize = 128 * 1024;
const EXEC_DEADLINE: Duration = Duration::from_secs(25);
const EXEC_CONCURRENCY: usize = 4;
const COMMAND_CONCURRENCY: usize = 32;
const SIGKILL: i32 = 9;
const ESRCH: i32 = 3;
unsafe extern "C" {
fn kill(pid: i32, signal: i32) -> i32;
}
pub struct CommandServer {
device_id: Id,
client: Client,
agent_version: &'static str,
started_at: Instant,
exec_permits: Semaphore,
}
impl CommandServer {
@@ -36,46 +52,39 @@ impl CommandServer {
Self {
device_id,
client,
agent_version: env!("CARGO_PKG_VERSION"),
agent_version: crate::VERSION,
started_at: Instant::now(),
exec_permits: Semaphore::new(EXEC_CONCURRENCY),
}
}
pub async fn run(self: Arc<Self>) -> Result<(), CommandServerError> {
let subject = device_command_subscription(&self.device_id.to_string());
tracing::info!(subject = %subject, "command server subscribing");
let mut sub = self.client.subscribe(subject.clone()).await.map_err(|e| {
let sub = self.client.subscribe(subject.clone()).await.map_err(|e| {
CommandServerError::Subscribe {
subject: subject.clone(),
source: e,
}
})?;
while let Some(msg) = sub.next().await {
sub.for_each_concurrent(COMMAND_CONCURRENCY, |msg| {
let me = self.clone();
tokio::spawn(async move {
async move {
match me.dispatch(msg).await {
Ok(()) => tracing::debug!("command handled"),
Err(e) => {
tracing::error!(command_error = %e, "failed to handle command")
}
};
});
}
}
})
.await;
tracing::warn!("command server subscription ended");
Ok(())
}
async fn dispatch(&self, msg: async_nats::Message) -> Result<(), CommandError> {
// Subject token after the device id is the verb. Pattern is
// `device-commands.<id>.<verb>` — we own both ends so this
// unwrap shape is safe under normal routing.
// FIXME do not unwrap here, we cannot affoard to crash an entire fleet because a verb is
// added or removed or format changed. Log an error and move on maybe we could list supported verbs.
let verb_token = if let Some(verb) = msg.subject.rsplit('.').next() {
verb
} else {
return Err(CommandError::InvalidFormat(msg.subject.to_string()));
};
let verb_token = msg.subject.rsplit('.').next().unwrap_or_default();
let request_id = msg
.headers
.as_ref()
@@ -96,9 +105,19 @@ impl CommandServer {
}
};
if verb_token == Verb::Ping.as_subject_token() {
let device_id = self.device_id.to_string();
if msg.subject.as_str() == device_command_subject(&device_id, Verb::Ping) {
self.reply_ping(reply_to).await?;
Ok(())
} else if msg.subject.as_str() == device_command_subject(&device_id, Verb::Exec) {
let reply = match serde_json::from_slice(&msg.payload) {
Ok(CommandRequest::Exec { command }) => {
run_shell(&command, EXEC_DEADLINE, &self.exec_permits).await
}
Ok(_) => exec_error("expected exec request body"),
Err(error) => exec_error(format!("invalid exec request: {error}")),
};
self.publish_reply(reply_to, &reply).await
} else {
tracing::warn!(verb = %verb_token, "unknown command verb");
Err(CommandError::UnknownVerb(verb_token.to_string()))
@@ -111,7 +130,15 @@ impl CommandServer {
agent_version: self.agent_version.to_string(),
uptime_s: self.started_at.elapsed().as_secs(),
};
let payload = serde_json::to_vec(&reply).map_err(CommandError::SerializeReply)?;
self.publish_reply(reply_to, &reply).await
}
async fn publish_reply(
&self,
reply_to: Subject,
reply: &impl Serialize,
) -> Result<(), CommandError> {
let payload = serde_json::to_vec(reply).map_err(CommandError::SerializeReply)?;
self.client
.publish(reply_to, payload.into())
.await
@@ -119,13 +146,164 @@ impl CommandServer {
}
}
async fn run_shell(command: &str, deadline: Duration, permits: &Semaphore) -> ExecReply {
if command.trim().is_empty() {
return exec_error("command must not be empty");
}
if command.len() > EXEC_MAX_COMMAND {
return exec_error(format!("command exceeds {EXEC_MAX_COMMAND} byte limit"));
}
let _permit = match permits.try_acquire() {
Ok(permit) => permit,
Err(_) => return exec_error("too many commands are already running"),
};
let mut command_process = std::process::Command::new("sh");
command_process
.arg("-c")
// The outer shell terminates same-group background jobs after the command.
.arg("sh -c \"$1\"; status=$?; trap '' TERM; kill -TERM 0; exit $status")
.arg("harmony-exec")
.arg(command)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
// A dedicated process group lets timeout cleanup reach shell descendants.
.process_group(0);
let mut child = match tokio::process::Command::from(command_process)
.kill_on_drop(true)
.spawn()
{
Ok(child) => child,
Err(error) => return exec_error(format!("failed to spawn shell: {error}")),
};
let Some(process_group) = child.id().and_then(|id| i32::try_from(id).ok()) else {
return exec_error("spawned shell has no process id");
};
let mut stdout = child.stdout.take().expect("piped stdout");
let mut stderr = child.stderr.take().expect("piped stderr");
let completed = tokio::time::timeout(
deadline,
collect_output(&mut child, &mut stdout, &mut stderr),
)
.await;
let (status, stdout, stderr, truncated) = match completed {
Ok(Ok(result)) => result,
outcome => {
// SAFETY: process_group is the positive PID returned for the child whose
// process group was set to that PID before exec.
if unsafe { kill(-process_group, SIGKILL) } == -1 {
tracing::warn!(error = %io::Error::last_os_error(), "failed to kill exec process group");
let _ = child.start_kill();
}
let _ = tokio::time::timeout(Duration::from_secs(1), child.wait()).await;
return match outcome {
Ok(Err(error)) => exec_error(format!("failed waiting for command output: {error}")),
Err(_) => exec_error(format!(
"command exceeded {}s deadline and was killed",
deadline.as_secs_f64()
)),
Ok(Ok(_)) => unreachable!(),
};
}
};
// The wrapper sends SIGTERM before exiting. Kill any same-group process
// that ignored it; ESRCH means the group is already gone.
let cleanup = unsafe { kill(-process_group, SIGKILL) };
if cleanup == -1 {
let error = io::Error::last_os_error();
if error.raw_os_error() != Some(ESRCH) {
tracing::warn!(%error, "failed to clean up exec process group");
}
}
let (stdout, stderr, utf8_truncated) = bounded_strings(stdout, stderr);
ExecReply {
exit_code: status.code().unwrap_or(-1),
stdout,
stderr,
truncated: truncated || utf8_truncated,
}
}
async fn collect_output(
child: &mut tokio::process::Child,
stdout: &mut (impl AsyncRead + Unpin),
stderr: &mut (impl AsyncRead + Unpin),
) -> io::Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>, bool)> {
let (mut stdout_open, mut stderr_open) = (true, true);
let (mut stdout_output, mut stderr_output) = (Vec::new(), Vec::new());
let (mut stdout_chunk, mut stderr_chunk) = ([0; 8192], [0; 8192]);
let (mut remaining, mut truncated) = (EXEC_MAX_OUTPUT, false);
let mut status = None;
while status.is_none() || stdout_open || stderr_open {
tokio::select! {
result = child.wait(), if status.is_none() => status = Some(result?),
result = stdout.read(&mut stdout_chunk), if stdout_open => {
let read = result?;
stdout_open = read != 0;
retain_output(&mut stdout_output, &stdout_chunk[..read], &mut remaining, &mut truncated);
}
result = stderr.read(&mut stderr_chunk), if stderr_open => {
let read = result?;
stderr_open = read != 0;
retain_output(&mut stderr_output, &stderr_chunk[..read], &mut remaining, &mut truncated);
}
}
}
Ok((
status.expect("loop waits for status"),
stdout_output,
stderr_output,
truncated,
))
}
fn retain_output(output: &mut Vec<u8>, chunk: &[u8], remaining: &mut usize, truncated: &mut bool) {
let retained = chunk.len().min(*remaining);
output.extend_from_slice(&chunk[..retained]);
*remaining -= retained;
*truncated |= retained < chunk.len();
}
fn bounded_strings(stdout: Vec<u8>, stderr: Vec<u8>) -> (String, String, bool) {
let mut left = EXEC_MAX_OUTPUT;
let mut truncated = false;
let mut convert = |bytes: Vec<u8>| {
let text = String::from_utf8_lossy(&bytes);
let end = text
.char_indices()
.map(|(index, _)| index)
.take_while(|index| *index <= left)
.last()
.unwrap_or(0);
let end = if text.len() <= left { text.len() } else { end };
truncated |= end < text.len();
left -= end;
text[..end].to_string()
};
let stdout = convert(stdout);
let stderr = convert(stderr);
(stdout, stderr, truncated)
}
fn exec_error(message: impl Into<String>) -> ExecReply {
ExecReply {
exit_code: -1,
stdout: String::new(),
stderr: message.into(),
truncated: false,
}
}
/// Failure modes the per-message dispatcher can report. Stays
/// `pub(crate)` for now — the run loop logs and continues on each
/// variant rather than surfacing them to a caller.
#[derive(Debug, Error, Serialize)]
pub(crate) enum CommandError {
#[error("invalid command subject: {0}")]
InvalidFormat(String),
#[error("unknown verb: {0}")]
UnknownVerb(String),
#[error("command message had no reply inbox")]
@@ -151,3 +329,103 @@ pub enum CommandServerError {
source: async_nats::SubscribeError,
},
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn exec_captures_stdout_and_nonzero_stderr() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
let reply = run_shell(
"printf output; printf error >&2; exit 7",
EXEC_DEADLINE,
&permits,
)
.await;
assert_eq!(reply.exit_code, 7);
assert_eq!(reply.stdout, "output");
assert_eq!(reply.stderr, "error");
assert!(!reply.truncated);
}
#[tokio::test]
async fn exec_rejects_empty_and_oversized_commands() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
for command in ["", " \t\n"] {
let reply = run_shell(command, EXEC_DEADLINE, &permits).await;
assert_eq!(reply.exit_code, -1);
assert!(reply.stderr.contains("empty"));
}
let oversized = "x".repeat(EXEC_MAX_COMMAND + 1);
let reply = run_shell(&oversized, EXEC_DEADLINE, &permits).await;
assert_eq!(reply.exit_code, -1);
assert!(reply.stderr.contains("limit"));
}
#[tokio::test]
async fn exec_caps_combined_output() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
let reply = run_shell(
"yes o | head -c 100000; yes e | head -c 100000 >&2",
EXEC_DEADLINE,
&permits,
)
.await;
assert_eq!(reply.exit_code, 0);
assert!(reply.truncated);
assert!(reply.stdout.len() + reply.stderr.len() <= EXEC_MAX_OUTPUT);
}
#[tokio::test]
async fn exec_timeout_kills_process_group() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
let marker = std::env::temp_dir().join(format!(
"harmony-exec-timeout-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let command = format!("(sleep 0.3; touch '{}') & wait", marker.to_string_lossy());
let reply = run_shell(&command, Duration::from_millis(100), &permits).await;
assert_eq!(reply.exit_code, -1);
assert!(reply.stderr.contains("deadline"));
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(!marker.exists(), "background process survived timeout");
}
#[tokio::test]
async fn exec_success_terminates_detached_same_group_processes() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
let marker = std::env::temp_dir().join(format!(
"harmony-exec-success-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let command = format!(
"(trap '' TERM; sleep 0.2; touch '{}') >/dev/null 2>&1 &",
marker.to_string_lossy()
);
let reply = run_shell(&command, EXEC_DEADLINE, &permits).await;
assert_eq!(reply.exit_code, 0);
tokio::time::sleep(Duration::from_millis(250)).await;
assert!(
!marker.exists(),
"background process survived successful exec"
);
}
#[tokio::test]
async fn exec_rejects_when_concurrency_is_exhausted() {
let permits = Semaphore::new(EXEC_CONCURRENCY);
let _held = permits.acquire_many(EXEC_CONCURRENCY as u32).await.unwrap();
let reply = run_shell("printf started", Duration::from_secs(2), &permits).await;
assert_eq!(reply.exit_code, -1);
assert!(reply.stderr.contains("already running"));
}
}

94
fleet/harmony-fleet-agent/src/fleet_publisher.rs
View File

@@ -3,15 +3,11 @@
//! Thin wrapper around three KV buckets: [`BUCKET_DEVICE_INFO`],
//! [`BUCKET_DEVICE_STATE`], [`BUCKET_DEVICE_HEARTBEAT`].
//!
//! Failure mode: log and swallow. The KV is the source of truth —
//! a dropped put gets corrected on the next reconcile transition
//! or operator watch reconnection.
use async_nats::jetstream::{self, kv};
use harmony_reconciler_contracts::{
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, device_heartbeat_key,
device_info_key, device_state_key,
DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, UpdaterCapabilities,
device_heartbeat_key, device_info_key, device_state_key,
};
use std::collections::BTreeMap;
@@ -27,37 +23,20 @@ pub struct FleetPublisher {
heartbeat_bucket: kv::Store,
}
#[async_trait::async_trait]
pub trait DeploymentStatePublisher: Send + Sync {
async fn write(&self, state: &DeploymentState) -> anyhow::Result<()>;
async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()>;
}
impl FleetPublisher {
/// Open every bucket the agent needs, creating those that don't
/// exist yet. Idempotent with operator-side creation.
/// Open the operator-owned buckets used by the agent.
pub async fn connect(client: async_nats::Client, device_id: Id) -> anyhow::Result<Self> {
let jetstream = jetstream::new(client.clone());
let info_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_INFO.to_string(),
// If this is as I think, it would be useful to keep a history of the last 10 device
// info, with a timestamp
history: 1,
..Default::default()
})
.await?;
let state_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_STATE.to_string(),
// If this is as I think, it would be useful to keep a history of the last 10 states
// a device had, with a timestamp
history: 1,
..Default::default()
})
.await?;
let heartbeat_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_HEARTBEAT.to_string(),
history: 1,
..Default::default()
})
.await?;
let info_bucket = jetstream.get_key_value(BUCKET_DEVICE_INFO).await?;
let state_bucket = jetstream.get_key_value(BUCKET_DEVICE_STATE).await?;
let heartbeat_bucket = jetstream.get_key_value(BUCKET_DEVICE_HEARTBEAT).await?;
Ok(Self {
device_id,
@@ -74,22 +53,19 @@ impl FleetPublisher {
&self,
labels: BTreeMap<String, String>,
inventory: Option<InventorySnapshot>,
) {
updater: Option<UpdaterCapabilities>,
) -> anyhow::Result<()> {
let info = DeviceInfo {
device_id: self.device_id.clone(),
labels,
inventory,
updater,
updated_at: chrono::Utc::now(),
};
let key = device_info_key(&self.device_id.to_string());
match serde_json::to_vec(&info) {
Ok(payload) => {
if let Err(e) = self.info_bucket.put(&key, payload.into()).await {
tracing::warn!(%key, error = %e, "publish_device_info: kv put failed");
}
}
Err(e) => tracing::warn!(error = %e, "publish_device_info: serialize failed"),
}
let payload = serde_json::to_vec(&info)?;
self.info_bucket.put(&key, payload.into()).await?;
Ok(())
}
/// Tiny liveness ping. Called every 30s.
@@ -97,6 +73,7 @@ impl FleetPublisher {
let hb = HeartbeatPayload {
device_id: self.device_id.clone(),
at: chrono::Utc::now(),
agent_version: Some(crate::VERSION.to_string()),
};
let key = device_heartbeat_key(&self.device_id.to_string());
match serde_json::to_vec(&hb) {
@@ -114,17 +91,12 @@ impl FleetPublisher {
/// bucket picks up this put and updates CR status counters.
/// Also fans out the same payload on `device-state.<device_id>`
/// for live observers that don't want to consume the KV stream.
pub async fn write_deployment_state(&self, state: &DeploymentState) {
pub async fn write_deployment_state(&self, state: &DeploymentState) -> anyhow::Result<()> {
let key = device_state_key(&self.device_id.to_string(), &state.deployment);
match serde_json::to_vec(state) {
Ok(payload) => {
if let Err(e) = self.state_bucket.put(&key, payload.clone().into()).await {
tracing::warn!(%key, error = %e, "write_deployment_state: kv put failed");
}
self.publish_direct_state(payload).await;
}
Err(e) => tracing::warn!(error = %e, "write_deployment_state: serialize failed"),
}
let payload = serde_json::to_vec(state)?;
self.state_bucket.put(&key, payload.clone().into()).await?;
self.publish_direct_state(payload).await;
Ok(())
}
/// Emit a tiny presence pulse on `device-state.<device_id>` so live
@@ -154,10 +126,20 @@ impl FleetPublisher {
/// Delete the authoritative current-phase entry, e.g. when the
/// Deployment CR is removed and the agent has torn down the
/// container.
pub async fn delete_deployment_state(&self, deployment: &DeploymentName) {
pub async fn delete_deployment_state(&self, deployment: &DeploymentName) -> anyhow::Result<()> {
let key = device_state_key(&self.device_id.to_string(), deployment);
if let Err(e) = self.state_bucket.delete(&key).await {
tracing::debug!(%key, error = %e, "delete_deployment_state: kv delete failed");
}
self.state_bucket.delete(&key).await?;
Ok(())
}
}
#[async_trait::async_trait]
impl DeploymentStatePublisher for FleetPublisher {
async fn write(&self, state: &DeploymentState) -> anyhow::Result<()> {
self.write_deployment_state(state).await
}
async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()> {
self.delete_deployment_state(deployment).await
}
}

509
fleet/harmony-fleet-agent/src/main.rs
View File

@@ -3,11 +3,15 @@ mod config;
mod fleet_publisher;
mod podman;
mod reconciler;
mod system_upgrade;
mod system_upgrade_service;
mod updater;
mod upgrade;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Error, Result};
use anyhow::{Context, Result};
use clap::Parser;
use config::AgentConfig;
use harmony_fleet_auth::{
@@ -18,20 +22,33 @@ use harmony_fleet_auth::{
type Creds = Arc<CredentialSource>;
use futures_util::StreamExt;
use harmony_reconciler_contracts::{
BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter,
BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE,
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, BUCKET_SYSTEM_UPGRADE_INTENT,
BUCKET_SYSTEM_UPGRADE_STATUS, Id, InventorySnapshot, agent_upgrade_intent_key,
agent_upgrade_status_key, desired_state_watch_filter, device_heartbeat_key, device_info_key,
};
use crate::command_server::CommandServer;
use crate::fleet_publisher::FleetPublisher;
use crate::podman::PodmanRuntime;
use crate::reconciler::Reconciler;
use crate::podman::{PodmanRuntime, SecretSource};
use crate::reconciler::{Reconciler, SnapshotEntry};
/// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
/// events are accelerators.
const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
const NATS_CONNECT_WINDOW: Duration = Duration::from_secs(3 * 60);
const NATS_CONNECT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15);
pub(crate) const VERSION: &str = match option_env!("HARMONY_FLEET_AGENT_VERSION") {
Some(version) => version,
None => env!("CARGO_PKG_VERSION"),
};
#[derive(Parser)]
#[command(name = "fleet-agent-v0", about = "IoT agent for Raspberry Pi devices")]
#[command(
name = "fleet-agent-v0",
version = VERSION,
about = "IoT agent for Raspberry Pi devices"
)]
struct Cli {
#[arg(
long,
@@ -41,85 +58,243 @@ struct Cli {
default_value = "/etc/fleet-agent/config.toml"
)]
config: std::path::PathBuf,
#[arg(long)]
self_test: bool,
#[arg(long, requires = "self_test", hide = true)]
expected_version: Option<String>,
#[arg(long)]
updater: bool,
#[arg(long, default_value = updater::DEFAULT_SOCKET)]
updater_socket: std::path::PathBuf,
}
fn acquire_process_lock() -> Result<std::fs::File> {
use fs2::FileExt;
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open("/run/harmony-fleet-agent/agent.lock")?;
file.try_lock_exclusive()
.context("another fleet agent instance is already active")?;
Ok(file)
}
async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result<async_nats::Client> {
let urls = &cfg.nats.urls;
tracing::info!(device_id = %cfg.agent.device_id, "connecting to NATS {urls:?}");
// The auth callback is invoked on every (re)connect, so a fresh
// Zitadel access token is minted automatically when the cached one
// is near-expiry — that's how we hold the "never lose connectivity"
// guarantee even across token rollovers and NATS pod restarts.
let client = connect_options_with_credentials(creds)
.ping_interval(Duration::from_secs(10))
// Surface async-nats's connection lifecycle in our logs. This
// is load-bearing for ops: a device that quietly disconnects
// is exactly the failure mode we promise won't happen, and
// operators need to see the reconnect attempts to debug.
.event_callback(|event| async move {
use async_nats::Event;
match event {
Event::Connected => tracing::info!("NATS connected"),
Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"),
Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"),
Event::SlowConsumer(sid) => {
tracing::warn!(sid = %sid, "NATS slow consumer")
let started = tokio::time::Instant::now();
let mut last_error = None;
let mut attempt = 0;
loop {
let remaining = NATS_CONNECT_WINDOW.saturating_sub(started.elapsed());
if remaining.is_zero() {
break;
}
attempt += 1;
// The callback mints a fresh token on every connect and reconnect.
let connect = connect_options_with_credentials(creds.clone())
.ping_interval(Duration::from_secs(10))
.connection_timeout(NATS_CONNECT_ATTEMPT_TIMEOUT)
.event_callback(|event| async move {
use async_nats::Event;
match event {
Event::Connected => tracing::info!("NATS connected"),
Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"),
Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"),
Event::SlowConsumer(sid) => {
tracing::warn!(sid = %sid, "NATS slow consumer")
}
Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"),
Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"),
Event::Closed => tracing::debug!("NATS connection closed"),
other => tracing::debug!(?other, "NATS event"),
}
Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"),
Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"),
Event::Closed => tracing::error!("NATS connection closed"),
other => tracing::debug!(?other, "NATS event"),
})
.connect(cfg.nats.urls.as_slice());
match tokio::time::timeout(NATS_CONNECT_ATTEMPT_TIMEOUT.min(remaining), connect).await {
Ok(Ok(client)) => {
tracing::info!(urls = ?cfg.nats.urls, "connected to NATS");
return Ok(client);
}
})
.connect(cfg.nats.urls.as_slice())
.await?;
tracing::info!(urls = ?cfg.nats.urls, "connected to NATS");
Ok(client)
Ok(Err(error)) => last_error = Some(error.to_string()),
Err(_) => last_error = Some("connection attempt timed out".to_string()),
}
tracing::warn!(attempt, error = %last_error.as_deref().unwrap(), "NATS connection failed; retrying");
tokio::time::sleep(
Duration::from_secs(5).min(NATS_CONNECT_WINDOW.saturating_sub(started.elapsed())),
)
.await;
}
anyhow::bail!(
"NATS connection failed after bounded retries: {}",
last_error.as_deref().unwrap_or("connection window expired")
)
}
async fn watch_desired_state(
async fn desired_state_store(
client: async_nats::Client,
device_id: Id,
reconciler: Arc<Reconciler>,
) -> Result<()> {
let jetstream = async_nats::jetstream::new(client);
let bucket = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DESIRED_STATE.to_string(),
) -> Result<async_nats::jetstream::kv::Store> {
Ok(async_nats::jetstream::new(client)
.get_key_value(BUCKET_DESIRED_STATE)
.await?)
}
async fn probe_services(
client: &async_nats::Client,
device_id: &Id,
updater_socket: &std::path::Path,
) -> Result<Vec<SnapshotEntry>> {
let jetstream = async_nats::jetstream::new(client.clone());
let desired = jetstream.get_key_value(BUCKET_DESIRED_STATE).await?;
let id = device_id.to_string();
for (bucket, key) in [
(BUCKET_AGENT_UPGRADE_INTENT, agent_upgrade_intent_key(&id)),
(BUCKET_AGENT_UPGRADE_STATUS, agent_upgrade_status_key(&id)),
(BUCKET_DEVICE_INFO, device_info_key(&id)),
(BUCKET_DEVICE_HEARTBEAT, device_heartbeat_key(&id)),
] {
jetstream.get_key_value(bucket).await?.get(key).await?;
}
jetstream
.get_key_value(BUCKET_DEVICE_STATE)
.await?
.status()
.await?;
for bucket in [BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS] {
jetstream.get_key_value(bucket).await?.status().await?;
}
let updater = updater::UpdaterClient::new(updater_socket);
tokio::time::timeout(Duration::from_secs(15), updater.status())
.await
.context("updater status probe timed out")??;
let capabilities = tokio::time::timeout(Duration::from_secs(15), updater.capabilities())
.await
.context("updater capabilities probe timed out")??;
if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 {
anyhow::bail!("updater does not support AptFullUpgradeV1");
}
load_desired_snapshot(&desired, device_id).await
}
async fn load_desired_snapshot(
bucket: &async_nats::jetstream::kv::Store,
device_id: &Id,
) -> Result<Vec<SnapshotEntry>> {
let prefix = format!("{device_id}.");
let filter = format!(
"{}{}",
bucket.prefix,
desired_state_watch_filter(&device_id.to_string())
);
let consumer = bucket
.stream
.create_consumer(async_nats::jetstream::consumer::pull::OrderedConfig {
filter_subject: filter,
deliver_policy: async_nats::jetstream::consumer::DeliverPolicy::LastPerSubject,
..Default::default()
})
.await?;
let pending = consumer.cached_info().num_pending;
if pending == 0 {
return Ok(Vec::new());
}
let mut entries = Vec::new();
let mut snapshot = consumer.messages().await?;
for _ in 0..pending {
let message = tokio::time::timeout(Duration::from_secs(10), snapshot.next())
.await
.context("timed out before desired-state snapshot completed")?
.context("desired-state snapshot ended before completion")??;
let info = message
.info()
.map_err(|error| anyhow::anyhow!(error.to_string()))?;
let key = message
.message
.subject
.strip_prefix(&bucket.prefix)
.context("desired-state subject has the wrong bucket prefix")?
.to_string();
let deleted = message.message.headers.as_ref().is_some_and(|headers| {
headers
.get("KV-Operation")
.is_some_and(|operation| matches!(operation.as_str(), "DEL" | "PURGE"))
|| headers.get("Nats-Marker-Reason").is_some()
});
if !deleted && key.starts_with(&prefix) {
entries.push(SnapshotEntry {
key,
revision: info.stream_sequence,
value: message.message.payload.to_vec(),
});
}
}
Ok(entries)
}
async fn watch_desired_state(
bucket: async_nats::jetstream::kv::Store,
device_id: Id,
reconciler: Arc<Reconciler>,
) -> Result<()> {
let key_filter = desired_state_watch_filter(&device_id.to_string());
tracing::info!(filter = %key_filter, "watching KV keys");
let mut watch = bucket.watch(&key_filter).await?;
while let Some(result) = watch.next().await {
let entry = match result {
Ok(e) => e,
Err(e) => {
tracing::warn!(error = %e, "watch error");
loop {
let mut watch = match bucket.watch(&key_filter).await {
Ok(watch) => watch,
Err(error) => {
tracing::warn!(%error, "desired-state watch start failed");
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
}
};
tracing::debug!(key = %entry.key, "bucket watch new value {entry:?}");
match entry.operation {
async_nats::jetstream::kv::Operation::Put => {
if let Err(e) = reconciler.apply(&entry.key, &entry.value).await {
tracing::warn!(key = %entry.key, error = %e, "apply failed");
while let Some(result) = watch.next().await {
let entry = match result {
Ok(entry) => entry,
Err(error) => {
tracing::warn!(%error, "desired-state watch failed; restarting");
break;
}
}
async_nats::jetstream::kv::Operation::Delete
| async_nats::jetstream::kv::Operation::Purge => {
if let Err(e) = reconciler.remove(&entry.key).await {
tracing::warn!(key = %entry.key, error = %e, "remove failed");
};
let result = match entry.operation {
async_nats::jetstream::kv::Operation::Put => {
reconciler
.put(&entry.key, entry.revision, &entry.value)
.await
}
async_nats::jetstream::kv::Operation::Delete
| async_nats::jetstream::kv::Operation::Purge => {
reconciler.delete(&entry.key, entry.revision).await
}
};
if let Err(error) = result {
tracing::warn!(key = %entry.key, %error, "desired-state event rejected");
}
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
async fn snapshot_loop(
bucket: async_nats::jetstream::kv::Store,
device_id: Id,
reconciler: Arc<Reconciler>,
) {
let mut interval = tokio::time::interval(RECONCILE_INTERVAL);
interval.tick().await;
loop {
interval.tick().await;
let generation = reconciler.generation().await;
match load_desired_snapshot(&bucket, &device_id).await {
Ok(snapshot) => {
if let Err(error) = reconciler.replace_snapshot(snapshot, generation).await {
tracing::warn!(%error, "desired-state snapshot rejected");
}
}
Err(error) => tracing::warn!(%error, "desired-state snapshot failed"),
}
}
Ok(())
}
/// Tiny liveness-only loop: push a `HeartbeatPayload` into the
@@ -158,7 +333,7 @@ fn local_inventory() -> InventorySnapshot {
.map(|n| n.get() as u32)
.unwrap_or(0),
memory_mb: sys_memory_total_mb().unwrap_or(0),
agent_version: env!("CARGO_PKG_VERSION").to_string(),
agent_version: VERSION.to_string(),
}
}
@@ -189,6 +364,18 @@ async fn main() -> Result<()> {
tracing_subscriber::fmt().with_env_filter(filter).init();
let cli = Cli::parse();
if cli.updater {
return updater::run_server(&cli.updater_socket).await;
}
if let Some(expected) = cli.expected_version.as_deref()
&& expected != VERSION
{
anyhow::bail!(
"candidate version mismatch: expected {expected}, binary reports {}",
VERSION
);
}
let _process_lock = (!cli.self_test).then(acquire_process_lock).transpose()?;
let cfg = config::load_config(&cli.config)?;
tracing::info!(
device_id = %cfg.agent.device_id,
@@ -242,7 +429,7 @@ async fn main() -> Result<()> {
)
.map_err(|e| anyhow::anyhow!("building OpenBao secret store: {e}"))?;
tracing::info!(url = %ob.url, prefix = %ob.secret_prefix, "OpenBao secret store ready");
Some(reconciler::SecretSource {
Some(SecretSource {
store: Arc::new(store),
prefix: ob.secret_prefix.clone(),
})
@@ -253,12 +440,22 @@ async fn main() -> Result<()> {
(None, _) => None,
};
let client = connect_nats(&cfg, creds).await.map_err(|e| {
let msg = format!("Nats connection FAILED : {e}");
tracing::error!(msg);
Error::msg(msg)
let client = connect_nats(&cfg, creds).await.map_err(|error| {
tracing::error!(%error, "NATS connection failed");
error
})?;
if cli.self_test {
let snapshot = probe_services(&client, &device_id, &cli.updater_socket).await?;
if let Some(topology) = topology {
let reconciler = Reconciler::new(device_id.clone(), topology, None, secrets);
let generation = reconciler.generation().await;
reconciler.replace_snapshot(snapshot, generation).await?;
}
tracing::info!(version = VERSION, "self-test ok");
return Ok(());
}
// Publish surface. Opens the three KV buckets (idempotent
// creates). Must be live before the reconciler starts so
// writes on the first desired-state KV watch land on the wire.
@@ -278,9 +475,29 @@ async fn main() -> Result<()> {
startup_labels
.entry("device-id".to_string())
.or_insert_with(|| device_id.to_string());
let updater_capabilities = if cli.updater_socket.exists() {
match updater::UpdaterClient::new(&cli.updater_socket)
.capabilities()
.await
{
Ok(capabilities) => Some(capabilities),
Err(error) if !cfg.agent.runtime_enabled => {
tracing::warn!(%error, "updater unavailable; capabilities omitted");
None
}
Err(error) => return Err(error.context("reading required updater capabilities")),
}
} else {
None
};
fleet
.publish_device_info(startup_labels, Some(inventory_snapshot.clone()))
.await;
.publish_device_info(
startup_labels,
Some(inventory_snapshot.clone()),
updater_capabilities,
)
.await
.context("publishing device registration")?;
// Reconciler exists only when a podman topology is available.
// Without it, the desired-state watch + periodic reconcile arms
@@ -295,6 +512,70 @@ async fn main() -> Result<()> {
))
});
let updater_socket = cli
.updater_socket
.to_str()
.context("non-UTF-8 updater socket")?;
let upgrade_service = if cli.updater_socket.exists() {
match upgrade::UpgradeService::connect(client.clone(), device_id.clone(), updater_socket)
.await
{
Ok(service) => Some(service),
Err(error) if !cfg.agent.runtime_enabled => {
tracing::warn!(%error, "updater unavailable; automatic agent upgrades disabled");
None
}
Err(error) => return Err(error.context("connecting required fleet updater")),
}
} else if cfg.agent.runtime_enabled {
anyhow::bail!(
"required fleet updater socket '{}' is unavailable",
cli.updater_socket.display()
);
} else {
tracing::warn!(
socket = %cli.updater_socket.display(),
"updater unavailable; automatic agent upgrades disabled"
);
None
};
let system_upgrade_service = if cli.updater_socket.exists() {
match system_upgrade_service::SystemUpgradeService::connect(
client.clone(),
device_id.clone(),
&cli.updater_socket,
)
.await
{
Ok(service) => Some(service),
Err(error) if !cfg.agent.runtime_enabled => {
tracing::warn!(%error, "updater unavailable; system upgrades disabled");
None
}
Err(error) => return Err(error.context("connecting required system upgrade relay")),
}
} else if cfg.agent.runtime_enabled {
anyhow::bail!(
"required fleet updater socket '{}' is unavailable",
cli.updater_socket.display()
);
} else {
None
};
let desired_bucket = desired_state_store(client.clone()).await?;
let snapshot = load_desired_snapshot(&desired_bucket, &device_id).await?;
if let Some(reconciler) = &reconciler {
let generation = reconciler.generation().await;
reconciler.replace_snapshot(snapshot, generation).await?;
}
if let Some(upgrade_service) = &upgrade_service {
upgrade_service.ensure_active_startup().await?;
}
sd_notify::notify(false, &[sd_notify::NotifyState::Ready])
.context("notifying systemd that initialization completed")?;
let command_server = Arc::new(CommandServer::new(device_id.clone(), client.clone()));
let ctrlc = async {
@@ -313,34 +594,96 @@ async fn main() -> Result<()> {
let watch: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
match reconciler.as_ref() {
Some(r) => Box::pin(watch_desired_state(
client.clone(),
Some(reconciler) => Box::pin(watch_desired_state(
desired_bucket.clone(),
device_id.clone(),
r.clone(),
reconciler.clone(),
)),
None => Box::pin(async {
std::future::pending::<()>().await;
Ok(())
}),
};
let reconcile: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> =
let snapshots: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> =
match reconciler.as_ref() {
Some(r) => Box::pin(r.clone().run_periodic(RECONCILE_INTERVAL)),
None => Box::pin(std::future::pending::<()>()),
Some(reconciler) => Box::pin(snapshot_loop(
desired_bucket,
device_id.clone(),
reconciler.clone(),
)),
_ => Box::pin(std::future::pending()),
};
let heartbeat = publish_heartbeat_loop(fleet);
let commands = command_server.run();
let mut worker = reconciler
.as_ref()
.map(|reconciler| tokio::spawn(reconciler.clone().run()));
let worker_finished = async {
match worker.as_mut() {
Some(worker) => worker.await.context("reconciler worker task"),
None => std::future::pending().await,
}
};
let upgrades: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
match upgrade_service.as_ref() {
Some(service) => Box::pin(service.clone().run()),
None => Box::pin(async {
std::future::pending::<()>().await;
Ok(())
}),
};
let system_upgrades: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
match system_upgrade_service {
Some(service) => Box::pin(service.run()),
None => Box::pin(std::future::pending()),
};
tokio::select! {
// Waiting on ctrlc in a select will automatically terminate other branches when
// ctrlc happens.
_ = ctrlc => {},
r = sigterm => { r?; }
r = watch => { r?; }
_ = reconcile => {}
_ = heartbeat => {}
r = commands => { r?; }
enum Shutdown {
Interrupt,
Terminate,
}
let signal = tokio::select! {
_ = ctrlc => Shutdown::Interrupt,
r = sigterm => { r?; Shutdown::Terminate }
r = watch => { r?; anyhow::bail!("desired-state watch exited unexpectedly") }
_ = snapshots => anyhow::bail!("desired-state snapshot loop exited unexpectedly"),
r = worker_finished => { r?; anyhow::bail!("reconciler worker exited unexpectedly") }
r = upgrades => { r?; anyhow::bail!("agent upgrade loop exited unexpectedly") }
r = system_upgrades => { r?; anyhow::bail!("system upgrade relay exited unexpectedly") }
_ = heartbeat => anyhow::bail!("heartbeat loop exited unexpectedly"),
r = commands => { r?; anyhow::bail!("command server exited unexpectedly") }
};
let drain_started = tokio::time::Instant::now();
if let Some(reconciler) = &reconciler {
reconciler.drain().await;
}
let drain_duration_ms = drain_started
.elapsed()
.as_millis()
.min(u128::from(u64::MAX)) as u64;
if let Some(worker) = &worker {
worker.abort();
}
let acknowledgement = if matches!(signal, Shutdown::Terminate)
&& let Some(upgrade_service) = &upgrade_service
{
upgrade_service
.acknowledge_shutdown(drain_duration_ms)
.await
.map(|_| ())
} else {
Ok(())
};
let flush = client
.flush()
.await
.context("flushing NATS during shutdown");
let stopping = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping])
.context("notifying systemd that shutdown started");
acknowledgement?;
flush?;
stopping?;
Ok(())
}

832
fleet/harmony-fleet-agent/src/podman.rs
View File

@@ -4,20 +4,63 @@ use std::time::Duration;
use anyhow::{Result, anyhow, bail};
use futures_util::StreamExt;
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy, VolumeMount};
use harmony_reconciler_contracts::{
DEVICE_PULL_SECRET_PATH, PodmanService, PodmanV0Score, RestartPolicy, VolumeMount,
};
use harmony_secret::SecretStore;
use oci_client::Reference;
use podman_api::Podman;
use podman_api::models::{ContainerMount, PortMapping};
use podman_api::models::{ContainerMount, ContainerStatus, PortMapping};
use podman_api::opts::{
ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts,
ContainerRestartPolicy, ContainerStopOpts, PullOpts,
ContainerRestartPolicy, ContainerStopOpts, ContainerWaitOpts, PullOpts, RegistryAuth,
};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::fmt;
use std::sync::Arc;
const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment";
const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256";
const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by";
const MANAGED_BY_VALUE: &str = "harmony";
const STOP_TIMEOUT: Duration = Duration::from_secs(300);
const STOP_TIMEOUT: Duration = Duration::from_secs(30);
#[async_trait::async_trait]
pub trait WorkloadRuntime: Send + Sync {
async fn reconcile(
&self,
deployment: &str,
score: &PodmanV0Score,
secrets: Option<&SecretSource>,
) -> Result<()>;
async fn remove_deployment(&self, deployment: &str) -> Result<()>;
async fn managed_deployments(&self) -> Result<HashSet<String>>;
}
#[derive(Clone)]
pub struct SecretSource {
pub store: Arc<dyn SecretStore>,
pub prefix: String,
}
#[derive(Deserialize)]
struct RegistryCredential {
registry: String,
username: String,
password: String,
}
impl fmt::Debug for RegistryCredential {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RegistryCredential")
.field("registry", &self.registry)
.field("username", &"[REDACTED]")
.field("password", &"[REDACTED]")
.finish()
}
}
pub struct PodmanRuntime {
podman: Podman,
@@ -42,93 +85,276 @@ impl PodmanRuntime {
Ok(())
}
pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> {
for service in &score.services {
self.ensure_service_running(service, deployment).await?;
}
pub async fn reconcile(
&self,
deployment: &str,
score: &PodmanV0Score,
secrets: Option<&SecretSource>,
) -> Result<()> {
score
.validate_init_container()
.map_err(anyhow::Error::msg)?;
score.image_pull_secrets().map_err(anyhow::Error::msg)?;
let desired = score
.services
.init_container
.iter()
.chain(&score.services)
.map(|service| service.name.as_str())
.collect::<HashSet<_>>();
for container in self.deployment_containers(deployment).await? {
let name = container_name(&container);
if !desired.contains(name.as_str()) {
self.remove_service(&name).await?;
}
let stale = self.preflight(deployment, score, &desired, secrets).await?;
if let Some(init) = &score.init_container {
self.ensure_init_completed(init, deployment, &score.revision())
.await?;
}
for name in stale {
self.remove_service(&name, deployment).await?;
log_action(deployment, &name, "container", "remove");
}
for service in &score.services {
self.ensure_service_running(service, deployment, secrets)
.await?;
}
for service in &score.services {
self.ensure_service_healthy(service).await?;
}
Ok(())
}
async fn preflight(
&self,
deployment: &str,
score: &PodmanV0Score,
desired: &HashSet<&str>,
secrets: Option<&SecretSource>,
) -> Result<Vec<String>> {
let mut names = HashSet::new();
let mut host_ports = HashSet::new();
let containers = self.all_containers().await?;
for service in score.init_container.iter().chain(&score.services) {
validate_service_name(&service.name)?;
if !names.insert(&service.name) {
bail!("duplicate service name '{}'", service.name);
}
for port in &service.ports {
let requested = parse_port_mapping(port)?;
let host_port = requested.host_port.unwrap_or_default();
if !host_ports.insert(host_port) {
bail!("host port {host_port} is requested more than once");
}
if containers.iter().any(|container| {
let name = container_name(container);
let removable_stale =
is_owned_by(container, deployment) && !desired.contains(name.as_str());
name != service.name
&& !removable_stale
&& container.ports.as_ref().is_some_and(|ports| {
ports
.iter()
.any(|port| port.host_port == requested.host_port)
})
}) {
bail!(
"host port {} is already in use",
requested.host_port.unwrap_or_default()
);
}
}
if let Some(existing) = self.get_by_name(&service.name).await? {
ensure_owned_by(&existing, deployment)?;
}
self.ensure_image_present(service, deployment, secrets)
.await?;
}
Ok(containers
.iter()
.filter(|container| {
is_owned_by(container, deployment)
&& !desired.contains(container_name(container).as_str())
})
.map(container_name)
.collect())
}
async fn ensure_init_completed(
&self,
init: &PodmanService,
deployment: &str,
score_hash: &str,
) -> Result<()> {
let existing = self.get_by_name(&init.name).await?;
let action = init_action(existing.as_ref(), score_hash);
if action == InitAction::Complete {
return Ok(());
}
if action == InitAction::Replace {
self.remove_service(&init.name, deployment).await?;
}
let container = match action {
InitAction::Create | InitAction::Replace => {
let created = self
.podman
.containers()
.create(
&container_create_opts(init, deployment, score_hash, RestartPolicy::No)?
.build(),
)
.await?;
self.podman.containers().get(created.id)
}
_ => self.podman.containers().get(&init.name),
};
if action != InitAction::Wait {
container.start(None).await?;
log_action(deployment, &init.name, "init", action.as_str());
} else {
tracing::info!(
%deployment,
service = %init.name,
role = "init",
"waiting for workload init"
);
}
container
.wait(
&ContainerWaitOpts::builder()
.conditions([ContainerStatus::Exited])
.build(),
)
.await?;
let exit_code = container
.inspect()
.await?
.state
.and_then(|state| state.exit_code)
.ok_or_else(|| anyhow!("init container '{}' has no exit code", init.name))?;
if exit_code != 0 {
bail!(
"init container '{}' failed with exit code {exit_code}",
init.name
);
}
tracing::info!(
%deployment,
service = %init.name,
role = "init",
"workload init completed"
);
Ok(())
}
pub async fn remove_deployment(&self, deployment: &str) -> Result<()> {
for container in self.deployment_containers(deployment).await? {
self.remove_service(&container_name(&container)).await?;
self.remove_service(&container_name(&container), deployment)
.await?;
}
Ok(())
}
pub async fn remove_service(&self, name: &str) -> Result<()> {
pub async fn managed_deployments(&self) -> Result<HashSet<String>> {
let opts = ContainerListOpts::builder()
.all(true)
.filter([ContainerListFilter::LabelKeyVal(
MANAGED_BY_LABEL.to_string(),
MANAGED_BY_VALUE.to_string(),
)])
.build();
Ok(self
.podman
.containers()
.list(&opts)
.await?
.into_iter()
.filter_map(|container| {
container
.labels
.and_then(|labels| labels.get(DEPLOYMENT_LABEL).cloned())
})
.collect())
}
async fn remove_service(&self, name: &str, deployment: &str) -> Result<()> {
let Some(existing) = self.get_by_name(name).await? else {
return Ok(());
};
ensure_owned_by(&existing, deployment)?;
let id = existing
.id
.clone()
.ok_or_else(|| anyhow!("container '{name}' has no id"))?;
let opts = ContainerStopOpts::builder()
.timeout(STOP_TIMEOUT.as_secs() as usize)
.build();
let container = self.podman.containers().get(name);
if container.exists().await.unwrap_or(false) {
let _ = container.stop(&opts).await;
let container = self.podman.containers().get(&id);
if existing.state.as_deref() == Some("running") {
container.stop(&opts).await?;
}
self.remove_container(name).await
self.remove_container(&id).await
}
async fn ensure_service_running(
&self,
service: &PodmanService,
deployment: &str,
secrets: Option<&SecretSource>,
) -> Result<()> {
let existing = self.get_by_name(&service.name).await?;
if let Some(existing) = existing.as_ref() {
if matches_spec(existing, service) {
if existing.state.as_deref() == Some("running") {
return Ok(());
}
let action = service_action(existing.as_ref(), service);
match action {
ServiceAction::Keep => return Ok(()),
ServiceAction::Start => {
let existing = existing
.as_ref()
.expect("start requires an existing container");
let id = existing.id.clone().unwrap_or_else(|| service.name.clone());
self.podman.containers().get(id).start(None).await?;
log_action(deployment, &service.name, "service", action.as_str());
return Ok(());
}
self.remove_container(&service.name).await?;
ServiceAction::Replace => {
self.remove_service(&service.name, deployment).await?;
}
ServiceAction::Create => {}
}
self.ensure_image_present(&service.image).await?;
let labels = HashMap::from([
(MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
(DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
(SPEC_LABEL.to_string(), spec_hash(service)?),
]);
let ports = service
.ports
.iter()
.map(|port| parse_port_mapping(port))
.collect::<Result<Vec<_>>>()?;
let env: HashMap<String, String> = service
.env
.iter()
.map(|env| (env.name.clone(), env.value.clone()))
.collect();
let mounts = service
.volumes
.iter()
.map(volume_to_mount)
.collect::<Vec<_>>();
let mut builder = ContainerCreateOpts::builder()
.name(&service.name)
.image(&service.image)
.labels(labels)
.portmappings(ports)
.env(env)
.restart_policy(map_restart_policy(service.restart_policy));
if !mounts.is_empty() {
builder = builder.mounts(mounts);
}
let created = self.podman.containers().create(&builder.build()).await?;
self.ensure_image_present(service, deployment, secrets)
.await?;
let created = self
.podman
.containers()
.create(
&container_create_opts(
service,
deployment,
&spec_hash(service)?,
service.restart_policy,
)?
.build(),
)
.await?;
self.podman.containers().get(created.id).start(None).await?;
log_action(deployment, &service.name, "service", action.as_str());
Ok(())
}
async fn ensure_service_healthy(&self, service: &PodmanService) -> Result<()> {
let inspected = self
.podman
.containers()
.get(&service.name)
.inspect()
.await?;
let state = inspected
.state
.ok_or_else(|| anyhow!("service '{}' has no runtime state", service.name))?;
let restarts = inspected.restart_count.unwrap_or_default();
if state.running != Some(true) {
bail!(
"service '{}' is not healthy: status={}, exit_code={}, restarts={restarts}",
service.name,
state.status.as_deref().unwrap_or("unknown"),
state.exit_code.unwrap_or_default(),
);
}
Ok(())
}
@@ -143,7 +369,12 @@ impl PodmanRuntime {
.list(&opts)
.await?
.into_iter()
.next())
.find(|container| container_name(container) == name))
}
async fn all_containers(&self) -> Result<Vec<podman_api::models::ListContainer>> {
let opts = ContainerListOpts::builder().all(true).build();
Ok(self.podman.containers().list(&opts).await?)
}
async fn deployment_containers(
@@ -152,25 +383,64 @@ impl PodmanRuntime {
) -> Result<Vec<podman_api::models::ListContainer>> {
let opts = ContainerListOpts::builder()
.all(true)
.filter([ContainerListFilter::LabelKeyVal(
DEPLOYMENT_LABEL.to_string(),
deployment.to_string(),
)])
.filter([
ContainerListFilter::LabelKeyVal(
MANAGED_BY_LABEL.to_string(),
MANAGED_BY_VALUE.to_string(),
),
ContainerListFilter::LabelKeyVal(
DEPLOYMENT_LABEL.to_string(),
deployment.to_string(),
),
])
.build();
Ok(self.podman.containers().list(&opts).await?)
}
async fn ensure_image_present(&self, image: &str) -> Result<()> {
async fn ensure_image_present(
&self,
service: &PodmanService,
deployment: &str,
secrets: Option<&SecretSource>,
) -> Result<()> {
let image = &service.image;
let images = self.podman.images();
if images.get(image).exists().await? {
return Ok(());
}
let mut pull = images.pull(&PullOpts::builder().reference(image).build());
let credential = match &service.image_pull_secret {
None => None,
Some(reference) => {
let source = secrets.ok_or_else(|| {
anyhow!(
"service '{}' references an image pull secret but this device has no secret store configured",
service.name
)
})?;
let namespace = format!("{}/{}", source.prefix, DEVICE_PULL_SECRET_PATH);
let bytes = source
.store
.get_raw(&namespace, reference)
.await
.map_err(|error| {
anyhow!("fetching image pull secret '{reference}': {error}")
})?;
Some(
serde_json::from_slice::<RegistryCredential>(&bytes).map_err(|error| {
anyhow!("parsing image pull secret '{reference}': {error}")
})?,
)
}
};
let opts = pull_opts(image, credential)?;
tracing::debug!(%deployment, service = %service.name, %image, "pulling workload image");
let mut pull = images.pull(&opts);
while let Some(event) = pull.next().await {
if let Some(error) = event?.error {
bail!("podman pull {image} failed: {error}");
}
}
tracing::info!(%deployment, service = %service.name, %image, "workload image pulled");
Ok(())
}
@@ -184,6 +454,66 @@ impl PodmanRuntime {
}
}
#[async_trait::async_trait]
impl WorkloadRuntime for PodmanRuntime {
async fn reconcile(
&self,
deployment: &str,
score: &PodmanV0Score,
secrets: Option<&SecretSource>,
) -> Result<()> {
self.reconcile(deployment, score, secrets).await
}
async fn remove_deployment(&self, deployment: &str) -> Result<()> {
self.remove_deployment(deployment).await
}
async fn managed_deployments(&self) -> Result<HashSet<String>> {
self.managed_deployments().await
}
}
fn registry_auth(image: &str, credential: RegistryCredential) -> Result<RegistryAuth> {
let reference: Reference = image
.parse()
.map_err(|error| anyhow!("invalid image reference '{image}': {error}"))?;
let expected = normalize_registry_host(&credential.registry)?;
let actual = reference.registry().to_ascii_lowercase();
if expected != actual {
bail!(
"image pull credential is scoped to registry '{}' but image '{}' uses registry '{}'",
credential.registry,
image,
reference.registry()
);
}
Ok(RegistryAuth::builder()
.username(credential.username)
.password(credential.password)
.server_address(reference.registry())
.build())
}
fn normalize_registry_host(host: &str) -> Result<String> {
if host.is_empty()
|| host != host.trim()
|| host.contains(['/', '@', '?', '#'])
|| host.ends_with(':')
{
bail!("invalid registry host '{host}'");
}
Ok(host.to_ascii_lowercase())
}
fn pull_opts(image: &str, credential: Option<RegistryCredential>) -> Result<PullOpts> {
let mut builder = PullOpts::builder().reference(image);
if let Some(credential) = credential {
builder.auth(registry_auth(image, credential)?);
}
Ok(builder.build())
}
fn default_user_socket() -> PathBuf {
if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") {
return PathBuf::from(format!("{dir}/podman/podman.sock"));
@@ -229,11 +559,185 @@ fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanSe
.is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected))
}
fn spec_hash(service: &PodmanService) -> Result<String> {
Ok(format!(
"{:x}",
Sha256::digest(serde_json::to_vec(service)?)
))
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum ServiceAction {
Keep,
Start,
Replace,
Create,
}
impl ServiceAction {
fn as_str(self) -> &'static str {
match self {
Self::Keep => "keep",
Self::Start => "start",
Self::Replace => "replace",
Self::Create => "create",
}
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum InitAction {
Complete,
Wait,
Start,
Replace,
Create,
}
impl InitAction {
fn as_str(self) -> &'static str {
match self {
Self::Complete => "complete",
Self::Wait => "wait",
Self::Start => "start",
Self::Replace => "replace",
Self::Create => "create",
}
}
}
fn log_action(deployment: &str, service: &str, role: &str, action: &str) {
tracing::info!(
%deployment,
%service,
%role,
%action,
"workload runtime action completed"
);
}
fn init_action(
observed: Option<&podman_api::models::ListContainer>,
score_hash: &str,
) -> InitAction {
match observed {
None => InitAction::Create,
Some(observed)
if observed
.labels
.as_ref()
.and_then(|labels| labels.get(SPEC_LABEL))
.is_none_or(|hash| hash != score_hash) =>
{
InitAction::Replace
}
Some(observed)
if observed.state.as_deref() == Some("exited") && observed.exit_code == Some(0) =>
{
InitAction::Complete
}
Some(observed)
if observed.state.as_deref() == Some("exited")
&& observed.exit_code.is_some_and(|code| code != 0) =>
{
// A fresh container prevents Podman's previous exit event from
// satisfying the retry's wait before the new process exits.
InitAction::Replace
}
Some(observed) if observed.state.as_deref() == Some("running") => InitAction::Wait,
Some(_) => InitAction::Start,
}
}
fn service_action(
observed: Option<&podman_api::models::ListContainer>,
service: &PodmanService,
) -> ServiceAction {
match observed {
None => ServiceAction::Create,
Some(observed) if !matches_spec(observed, service) => ServiceAction::Replace,
Some(observed) if observed.state.as_deref() == Some("running") => ServiceAction::Keep,
Some(_) => ServiceAction::Start,
}
}
fn ensure_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> Result<()> {
let labels = observed.labels.as_ref();
let managed = labels.and_then(|labels| labels.get(MANAGED_BY_LABEL));
let owner = labels.and_then(|labels| labels.get(DEPLOYMENT_LABEL));
if managed.map(String::as_str) != Some(MANAGED_BY_VALUE)
|| owner.map(String::as_str) != Some(deployment)
{
bail!(
"container '{}' conflicts with deployment '{deployment}' (owner: {})",
container_name(observed),
owner.map(String::as_str).unwrap_or("unmanaged")
);
}
Ok(())
}
fn is_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> bool {
let labels = observed.labels.as_ref();
labels
.and_then(|labels| labels.get(MANAGED_BY_LABEL))
.map(String::as_str)
== Some(MANAGED_BY_VALUE)
&& labels
.and_then(|labels| labels.get(DEPLOYMENT_LABEL))
.map(String::as_str)
== Some(deployment)
}
fn validate_service_name(name: &str) -> Result<()> {
if name.is_empty()
|| !name
.chars()
.all(|character| character.is_ascii_alphanumeric() || "_.-".contains(character))
|| !name
.chars()
.next()
.is_some_and(|character| character.is_ascii_alphanumeric())
{
bail!("invalid service name '{name}'");
}
Ok(())
}
fn spec_hash(value: &impl serde::Serialize) -> Result<String> {
Ok(format!("{:x}", Sha256::digest(serde_json::to_vec(value)?)))
}
fn container_create_opts(
service: &PodmanService,
deployment: &str,
spec_hash: &str,
restart_policy: RestartPolicy,
) -> Result<podman_api::opts::ContainerCreateOptsBuilder> {
let labels = HashMap::from([
(MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
(DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
(SPEC_LABEL.to_string(), spec_hash.to_string()),
]);
let ports = service
.ports
.iter()
.map(|port| parse_port_mapping(port))
.collect::<Result<Vec<_>>>()?;
let env = service
.env
.iter()
.map(|env| (env.name.clone(), env.value.clone()))
.collect::<HashMap<_, _>>();
let mounts = service
.volumes
.iter()
.map(volume_to_mount)
.collect::<Vec<_>>();
let mut builder = ContainerCreateOpts::builder()
.name(&service.name)
.image(&service.image)
.labels(labels)
.portmappings(ports)
.env(env)
.restart_policy(map_restart_policy(restart_policy));
if !mounts.is_empty() {
builder = builder.mounts(mounts);
}
Ok(builder)
}
fn container_name(container: &podman_api::models::ListContainer) -> String {
@@ -282,6 +786,7 @@ mod tests {
PodmanService {
name: "web".into(),
image: "nginx:latest".into(),
image_pull_secret: None,
ports: vec!["8080:80".into()],
env: vec![EnvVar::new("MODE", "prod")],
secret_env: vec![],
@@ -290,6 +795,19 @@ mod tests {
}
}
fn observed(
name: &str,
state: &str,
labels: HashMap<String, String>,
) -> podman_api::models::ListContainer {
serde_json::from_value(serde_json::json!({
"Names": [name],
"State": state,
"Labels": labels,
}))
.unwrap()
}
#[test]
fn spec_hash_is_stable_and_covers_desired_state() {
let original = service();
@@ -299,4 +817,178 @@ mod tests {
changed.env[0].value = "dev".into();
assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap());
}
#[test]
fn replacement_is_selected_before_create_for_changed_spec() {
let original = service();
let observed = observed(
&original.name,
"running",
HashMap::from([
(MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()),
(DEPLOYMENT_LABEL.into(), "deployment-a".into()),
(SPEC_LABEL.into(), spec_hash(&original).unwrap()),
]),
);
let mut changed = original;
changed.image = "nginx:new".into();
assert_eq!(
service_action(Some(&observed), &changed),
ServiceAction::Replace
);
assert_eq!(STOP_TIMEOUT, Duration::from_secs(30));
}
#[test]
fn ownership_requires_both_harmony_and_deployment_labels() {
let mut observed = observed("web", "running", HashMap::new());
assert!(ensure_owned_by(&observed, "deployment-a").is_err());
observed.labels = Some(HashMap::from([
(MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()),
(DEPLOYMENT_LABEL.into(), "deployment-a".into()),
]));
assert!(ensure_owned_by(&observed, "deployment-a").is_ok());
assert!(ensure_owned_by(&observed, "deployment-b").is_err());
}
#[test]
fn non_running_service_is_restarted_regardless_of_exit_code() {
let service = service();
let observed: podman_api::models::ListContainer =
serde_json::from_value(serde_json::json!({
"Names": [service.name],
"State": "exited",
"Exited": true,
"ExitCode": 42,
"Labels": { SPEC_LABEL: spec_hash(&service).unwrap() },
}))
.unwrap();
assert_eq!(
service_action(Some(&observed), &service),
ServiceAction::Start
);
let stopped: podman_api::models::ListContainer =
serde_json::from_value(serde_json::json!({
"Names": [service.name],
"State": "exited",
"Exited": true,
"ExitCode": 0,
"Labels": { SPEC_LABEL: spec_hash(&service).unwrap() },
}))
.unwrap();
assert_eq!(
service_action(Some(&stopped), &service),
ServiceAction::Start
);
}
#[test]
fn init_completion_is_reused_only_for_the_same_score() {
let completed = observed(
"init",
"exited",
HashMap::from([(SPEC_LABEL.into(), "score-a".into())]),
);
let mut completed: podman_api::models::ListContainer = completed;
completed.exit_code = Some(0);
assert_eq!(
init_action(Some(&completed), "score-a"),
InitAction::Complete
);
assert_eq!(
init_action(Some(&completed), "score-b"),
InitAction::Replace
);
completed.exit_code = Some(1);
assert_eq!(
init_action(Some(&completed), "score-a"),
InitAction::Replace
);
}
#[test]
fn invalid_service_names_are_rejected() {
assert!(validate_service_name("web-1").is_ok());
assert!(validate_service_name("").is_err());
assert!(validate_service_name("/web").is_err());
}
#[test]
fn pull_options_are_anonymous_without_credentials_and_scoped_to_image_registry() {
let anonymous = pull_opts("docker.io/library/nginx:latest", None).unwrap();
assert_eq!(
anonymous.serialize().unwrap(),
"reference=docker.io%2Flibrary%2Fnginx%3Alatest"
);
let authenticated = pull_opts(
"registry.example:5000/team/app:v1",
Some(RegistryCredential {
registry: "registry.example:5000".into(),
username: "device".into(),
password: "pull-token".into(),
}),
)
.unwrap();
assert_eq!(
authenticated.serialize().unwrap(),
"reference=registry.example%3A5000%2Fteam%2Fapp%3Av1"
);
let auth = registry_auth(
"registry.example:5000/team/app:v1",
RegistryCredential {
registry: "REGISTRY.EXAMPLE:5000".into(),
username: "device".into(),
password: "pull-token".into(),
},
)
.unwrap();
let RegistryAuth::Password { server_address, .. } = auth else {
panic!("expected password registry auth")
};
assert_eq!(server_address.as_deref(), Some("registry.example:5000"));
let error = registry_auth(
"other.example:5000/team/app:v1",
RegistryCredential {
registry: "registry.example:5000".into(),
username: "device".into(),
password: "pull-token".into(),
},
)
.unwrap_err();
assert!(error.to_string().contains("scoped to registry"));
let error = registry_auth(
"registry.example/team/app:v1",
RegistryCredential {
registry: "registry.example:5000".into(),
username: "device".into(),
password: "pull-token".into(),
},
)
.unwrap_err();
assert!(error.to_string().contains("scoped to registry"));
}
#[test]
fn registry_credentials_are_redacted() {
let credential = RegistryCredential {
registry: "registry.example".into(),
username: "device-user".into(),
password: "super-secret".into(),
};
let debug = format!("{credential:?}");
assert!(!debug.contains("device-user"));
assert!(!debug.contains("super-secret"));
assert_eq!(
debug,
"RegistryCredential { registry: \"registry.example\", username: \"[REDACTED]\", password: \"[REDACTED]\" }"
);
}
}

1445
fleet/harmony-fleet-agent/src/reconciler.rs
View File

File diff suppressed because it is too large Load Diff

661
fleet/harmony-fleet-agent/src/system_upgrade.rs Normal file
View File

@@ -0,0 +1,661 @@
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use harmony_reconciler_contracts::{SystemUpgradeAttempt, SystemUpgradePhase, SystemUpgradeStatus};
use serde::{Deserialize, Serialize};
use crate::updater::{bounded_error, safe_token, sync_directory};
const JOURNAL_DIR: &str = "/var/lib/harmony-fleet-updater/system-upgrades";
const BOOT_ID: &str = "/proc/sys/kernel/random/boot_id";
const MAX_REBOOT_REQUESTS: u8 = 3;
const TERMINAL_REPLAY_DAYS: i64 = 2;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct CommandSpec {
pub program: &'static str,
pub args: &'static [&'static str],
pub env: &'static [(&'static str, &'static str)],
}
const AUDIT: CommandSpec = CommandSpec {
program: "/usr/bin/dpkg",
args: &["--audit"],
env: &[],
};
const CHECK: CommandSpec = CommandSpec {
program: "/usr/bin/apt-get",
args: &["-o", "DPkg::Lock::Timeout=300", "check"],
env: &[],
};
const UPDATE: CommandSpec = CommandSpec {
program: "/usr/bin/apt-get",
args: &[
"-o",
"DPkg::Lock::Timeout=300",
"-o",
"APT::Update::Error-Mode=any",
"-o",
"Acquire::AllowInsecureRepositories=false",
"-o",
"Acquire::AllowDowngradeToInsecureRepositories=false",
"-o",
"Acquire::AllowWeakRepositories=false",
"update",
],
env: &[],
};
const FULL_UPGRADE: CommandSpec = CommandSpec {
program: "/usr/bin/apt-get",
args: &[
"-o",
"DPkg::Lock::Timeout=300",
"-o",
"Dpkg::Use-Pty=0",
"-o",
"Dpkg::Options::=--force-confold",
"-o",
"APT::Get::AllowUnauthenticated=false",
"-y",
"full-upgrade",
],
env: &[
("DEBIAN_FRONTEND", "noninteractive"),
("APT_LISTCHANGES_FRONTEND", "none"),
("NEEDRESTART_MODE", "a"),
],
};
const REBOOT: CommandSpec = CommandSpec {
program: "/bin/systemctl",
args: &["reboot", "--no-wall"],
env: &[],
};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(super) struct Journal {
attempt_id: String,
run_uid: String,
device_id: harmony_reconciler_contracts::Id,
attempt_digest: String,
phase: SystemUpgradePhase,
started_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
error: Option<String>,
boot_id: Option<String>,
reboot_requests: u8,
}
impl Journal {
pub(super) fn status(&self) -> SystemUpgradeStatus {
SystemUpgradeStatus {
attempt_id: self.attempt_id.clone(),
run_uid: self.run_uid.clone(),
phase: self.phase,
started_at: self.started_at,
updated_at: self.updated_at,
post_completion_heartbeat_at: None,
error: self.error.clone(),
}
}
fn transition(&mut self, phase: SystemUpgradePhase, error: Option<String>) {
self.phase = phase;
self.error = error;
self.updated_at = Utc::now();
}
fn validate(&self, expected_attempt_id: &str) -> Result<()> {
if self.attempt_id != expected_attempt_id
|| uuid::Uuid::parse_str(&self.attempt_id).is_err()
|| !safe_token(&self.run_uid)
|| !safe_token(&self.device_id.to_string())
|| self.attempt_digest.len() != 64
|| !self.attempt_digest.chars().all(|c| c.is_ascii_hexdigit())
|| self.reboot_requests > MAX_REBOOT_REQUESTS
|| self.phase == SystemUpgradePhase::Blocked
|| matches!(
self.phase,
SystemUpgradePhase::Rebooting | SystemUpgradePhase::Verifying
) && self.boot_id.as_deref().is_none_or(str::is_empty)
{
bail!("invalid system upgrade journal");
}
Ok(())
}
}
pub(super) enum Acceptance {
Existing(SystemUpgradeStatus),
New(Journal),
}
pub(super) fn accept(
attempt: &SystemUpgradeAttempt,
existing: Option<&Journal>,
now: DateTime<Utc>,
) -> Result<Acceptance> {
validate(attempt)?;
if let Some(journal) = existing {
if journal.attempt_digest != attempt.digest() {
bail!("system upgrade attempt id was reused with different content");
}
return Ok(Acceptance::Existing(journal.status()));
}
if attempt.expires_at <= now {
bail!("system upgrade attempt has expired");
}
Ok(Acceptance::New(Journal {
attempt_id: attempt.attempt_id.clone(),
run_uid: attempt.run_uid.clone(),
device_id: attempt.device_id.clone(),
attempt_digest: attempt.digest(),
phase: SystemUpgradePhase::Preflight,
started_at: now,
updated_at: now,
error: None,
boot_id: None,
reboot_requests: 0,
}))
}
fn validate(attempt: &SystemUpgradeAttempt) -> Result<()> {
uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid attempt id")?;
if !safe_token(&attempt.run_uid) {
bail!("invalid run uid");
}
if !safe_token(&attempt.device_id.to_string()) {
bail!("invalid device id");
}
Ok(())
}
pub(super) async fn read(attempt_id: &str) -> Result<Option<Journal>> {
uuid::Uuid::parse_str(attempt_id).context("invalid attempt id")?;
let path = journal_path(attempt_id);
match tokio::fs::read(&path).await {
Ok(bytes) => {
let journal: Journal = serde_json::from_slice(&bytes)
.with_context(|| format!("reading system upgrade journal {}", path.display()))?;
journal.validate(attempt_id)?;
Ok(Some(journal))
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error.into()),
}
}
pub(super) async fn write(journal: &Journal) -> Result<()> {
let path = journal_path(&journal.attempt_id);
let parent = path
.parent()
.context("system upgrade journal has no parent")?;
tokio::fs::create_dir_all(parent).await?;
let temporary = path.with_extension("tmp");
let mut file = tokio::fs::File::create(&temporary).await?;
use tokio::io::AsyncWriteExt;
file.write_all(&serde_json::to_vec(journal)?).await?;
file.sync_all().await?;
tokio::fs::rename(&temporary, &path).await?;
sync_directory(parent)?;
if journal.phase != SystemUpgradePhase::Preflight {
tracing::info!(
attempt_id = %journal.attempt_id,
run_uid = %journal.run_uid,
phase = ?journal.phase,
reboot_requests = journal.reboot_requests,
"system upgrade state persisted"
);
}
Ok(())
}
pub(super) async fn recover_active() -> Result<Option<Journal>> {
let mut entries = match tokio::fs::read_dir(JOURNAL_DIR).await {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error.into()),
};
let mut active = None;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.extension().and_then(|value| value.to_str()) != Some("json") {
continue;
}
let journal: Journal = serde_json::from_slice(&tokio::fs::read(&path).await?)
.with_context(|| format!("corrupt system upgrade journal {}", path.display()))?;
let attempt_id = path
.file_stem()
.and_then(|value| value.to_str())
.context("invalid system upgrade journal filename")?;
journal.validate(attempt_id)?;
if !retain_journal(&journal, Utc::now()) {
tokio::fs::remove_file(path).await?;
continue;
}
if !journal.phase.is_terminal() && active.replace(journal).is_some() {
bail!("multiple active system upgrade journals");
}
}
Ok(active)
}
fn retain_journal(journal: &Journal, now: DateTime<Utc>) -> bool {
journal.phase == SystemUpgradePhase::RepairRequired
|| !journal.phase.is_terminal()
|| journal.updated_at + chrono::Duration::days(TERMINAL_REPLAY_DAYS) >= now
}
pub(super) async fn run(mut journal: Journal) -> Result<SystemUpgradeStatus> {
let action = recovery_action(&journal, &current_boot_id()?);
if journal.phase == SystemUpgradePhase::Rebooting {
tracing::info!(
attempt_id = %journal.attempt_id,
run_uid = %journal.run_uid,
?action,
reboot_requests = journal.reboot_requests,
"recovering system upgrade after reboot request"
);
}
let result = match action {
RecoveryAction::Apply => apply(&mut journal).await,
RecoveryAction::Repair => {
let audit = run_command(&AUDIT, true).await;
let message = match audit {
Ok(()) => "system upgrade was interrupted while applying packages".into(),
Err(error) => format!("system upgrade was interrupted; {error}"),
};
journal.transition(SystemUpgradePhase::RepairRequired, Some(message));
write(&journal).await
}
RecoveryAction::Reboot => request_reboot(&mut journal).await,
RecoveryAction::Verify => verify(&mut journal).await,
RecoveryAction::None => Ok(()),
};
if let Err(error) = result {
if journal.phase == SystemUpgradePhase::Rebooting {
journal.error = Some(bounded_error(&error.to_string()));
journal.updated_at = Utc::now();
write(&journal).await?;
} else if !journal.phase.is_terminal() {
let phase = if journal.phase == SystemUpgradePhase::Applying {
SystemUpgradePhase::RepairRequired
} else {
SystemUpgradePhase::Failed
};
journal.transition(phase, Some(bounded_error(&error.to_string())));
write(&journal).await?;
}
return Err(error);
}
Ok(journal.status())
}
async fn apply(journal: &mut Journal) -> Result<()> {
run_command(&AUDIT, true).await?;
run_command(&CHECK, false).await?;
validate_sources().await?;
run_command(&UPDATE, false).await?;
journal.transition(SystemUpgradePhase::Applying, None);
write(journal).await?;
let upgrade = run_command(&FULL_UPGRADE, false).await;
let health = match run_command(&AUDIT, true).await {
Ok(()) => run_command(&CHECK, false).await,
error => error,
};
if let Err(error) = health {
journal.transition(
SystemUpgradePhase::RepairRequired,
Some(bounded_error(&error.to_string())),
);
write(journal).await?;
return Ok(());
}
if let Err(error) = upgrade {
journal.transition(
SystemUpgradePhase::Failed,
Some(bounded_error(&error.to_string())),
);
write(journal).await?;
return Ok(());
}
journal.boot_id = Some(current_boot_id()?);
journal.transition(SystemUpgradePhase::Rebooting, None);
request_reboot(journal).await
}
async fn validate_sources() -> Result<()> {
let mut paths = vec![PathBuf::from("/etc/apt/sources.list")];
match tokio::fs::read_dir("/etc/apt/sources.list.d").await {
Ok(mut entries) => {
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if matches!(
path.extension().and_then(|extension| extension.to_str()),
Some("list" | "sources")
) {
paths.push(path);
}
}
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
for path in paths {
let contents = match tokio::fs::read_to_string(&path).await {
Ok(contents) => contents,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error.into()),
};
if source_disables_authentication(&contents) {
bail!(
"apt source {} disables repository authentication",
path.display()
);
}
}
Ok(())
}
fn source_disables_authentication(contents: &str) -> bool {
contents.lines().any(|line| {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
return false;
}
let lower = line.to_ascii_lowercase();
let options = lower
.strip_prefix("deb ")
.or_else(|| lower.strip_prefix("deb-src "))
.and_then(|line| line.strip_prefix('['))
.and_then(|line| line.split_once(']').map(|(options, _)| options));
if let Some(options) = options {
let options = options.replace(" =", "=").replace("= ", "=");
return options.split_ascii_whitespace().any(|option| {
matches!(
option.split_once('='),
Some((
"trusted" | "allow-insecure" | "allow-weak",
"yes" | "true" | "1"
))
)
});
}
matches!(
lower
.split_once(':')
.map(|(key, value)| (key.trim(), value.trim())),
Some((
"trusted" | "allow-insecure" | "allow-weak",
"yes" | "true" | "1"
))
)
})
}
async fn request_reboot(journal: &mut Journal) -> Result<()> {
while journal.reboot_requests < MAX_REBOOT_REQUESTS {
journal.reboot_requests += 1;
write(journal).await?;
match run_command(&REBOOT, false).await {
Ok(()) => tokio::time::sleep(std::time::Duration::from_secs(30)).await,
Err(error) => {
journal.error = Some(bounded_error(&error.to_string()));
journal.updated_at = Utc::now();
write(journal).await?;
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
}
journal.transition(
SystemUpgradePhase::Failed,
Some("reboot did not change the boot id".into()),
);
write(journal).await?;
Ok(())
}
async fn verify(journal: &mut Journal) -> Result<()> {
journal.transition(SystemUpgradePhase::Verifying, None);
write(journal).await?;
let result = run_command(&AUDIT, true).await;
let result = match result {
Ok(()) => run_command(&CHECK, false).await,
error => error,
};
if let Err(error) = result {
journal.transition(
SystemUpgradePhase::RepairRequired,
Some(bounded_error(&error.to_string())),
);
return write(journal).await;
}
journal.transition(SystemUpgradePhase::Complete, None);
write(journal).await
}
async fn run_command(spec: &CommandSpec, require_empty_stdout: bool) -> Result<()> {
let mut command = tokio::process::Command::new(spec.program);
command
.env_remove("NOTIFY_SOCKET")
.envs(spec.env.iter().copied())
.args(spec.args)
.kill_on_drop(true);
let output = command.output().await?;
if !output.status.success() {
bail!("{} failed: {}", spec.program, output.status);
}
if require_empty_stdout && !output.stdout.iter().all(u8::is_ascii_whitespace) {
bail!("dpkg audit reported incomplete package state");
}
Ok(())
}
fn current_boot_id() -> Result<String> {
Ok(std::fs::read_to_string(BOOT_ID)?.trim().to_string())
}
#[derive(Debug, PartialEq, Eq)]
enum RecoveryAction {
Apply,
Repair,
Reboot,
Verify,
None,
}
fn recovery_action(journal: &Journal, boot_id: &str) -> RecoveryAction {
match journal.phase {
SystemUpgradePhase::Preflight => RecoveryAction::Apply,
SystemUpgradePhase::Applying => RecoveryAction::Repair,
SystemUpgradePhase::Rebooting if journal.boot_id.as_deref() == Some(boot_id) => {
RecoveryAction::Reboot
}
SystemUpgradePhase::Rebooting => RecoveryAction::Verify,
SystemUpgradePhase::Verifying => RecoveryAction::Verify,
_ => RecoveryAction::None,
}
}
fn journal_path(attempt_id: &str) -> PathBuf {
Path::new(JOURNAL_DIR).join(format!("{attempt_id}.json"))
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::TimeDelta;
use harmony_reconciler_contracts::Id;
fn attempt() -> SystemUpgradeAttempt {
SystemUpgradeAttempt {
attempt_id: uuid::Uuid::new_v4().to_string(),
run_uid: "run-1".into(),
device_id: Id::from("device-1"),
expires_at: Utc::now() + TimeDelta::minutes(5),
}
}
fn journal(phase: SystemUpgradePhase) -> Journal {
let attempt = attempt();
let Acceptance::New(mut journal) = accept(&attempt, None, Utc::now()).unwrap() else {
unreachable!()
};
journal.phase = phase;
journal.boot_id = Some("old-boot".into());
journal
}
#[test]
fn validates_expiry_and_duplicate_digest() {
let now = Utc::now();
let original = attempt();
let Acceptance::New(journal) = accept(&original, None, now).unwrap() else {
unreachable!()
};
assert!(matches!(
accept(&original, Some(&journal), now + TimeDelta::hours(1)).unwrap(),
Acceptance::Existing(_)
));
let mut changed = original.clone();
changed.run_uid = "changed".into();
assert!(accept(&changed, Some(&journal), now).is_err());
let mut expired = attempt();
expired.expires_at = now;
assert!(accept(&expired, None, now).is_err());
expired.expires_at = now + TimeDelta::minutes(1);
expired.attempt_id = "invalid".into();
assert!(accept(&expired, None, now).is_err());
expired.attempt_id = uuid::Uuid::new_v4().to_string();
expired.run_uid = "unsafe/run".into();
assert!(accept(&expired, None, now).is_err());
}
#[test]
fn command_policy_is_exact() {
assert_eq!(
AUDIT,
CommandSpec {
program: "/usr/bin/dpkg",
args: &["--audit"],
env: &[]
}
);
assert_eq!(CHECK.args, &["-o", "DPkg::Lock::Timeout=300", "check"]);
assert_eq!(
UPDATE.args,
&[
"-o",
"DPkg::Lock::Timeout=300",
"-o",
"APT::Update::Error-Mode=any",
"-o",
"Acquire::AllowInsecureRepositories=false",
"-o",
"Acquire::AllowDowngradeToInsecureRepositories=false",
"-o",
"Acquire::AllowWeakRepositories=false",
"update"
]
);
assert_eq!(
FULL_UPGRADE.args,
&[
"-o",
"DPkg::Lock::Timeout=300",
"-o",
"Dpkg::Use-Pty=0",
"-o",
"Dpkg::Options::=--force-confold",
"-o",
"APT::Get::AllowUnauthenticated=false",
"-y",
"full-upgrade"
]
);
assert_eq!(
FULL_UPGRADE.env,
&[
("DEBIAN_FRONTEND", "noninteractive"),
("APT_LISTCHANGES_FRONTEND", "none"),
("NEEDRESTART_MODE", "a")
]
);
assert_eq!(
REBOOT,
CommandSpec {
program: "/bin/systemctl",
args: &["reboot", "--no-wall"],
env: &[]
}
);
}
#[test]
fn apt_sources_cannot_disable_authentication() {
for source in [
"deb [trusted=yes] https://example.invalid stable main",
"deb [allow-insecure = yes] https://example.invalid stable main",
"deb [allow-weak=yes] https://example.invalid stable main",
"Types: deb\nTrusted: yes\nURIs: https://example.invalid",
"Types: deb\nAllow-Insecure: true\nURIs: https://example.invalid",
"Types: deb\nAllow-Weak: 1\nURIs: https://example.invalid",
] {
assert!(
source_disables_authentication(source),
"accepted insecure source: {source}"
);
}
assert!(!source_disables_authentication(
"deb [signed-by=/etc/apt/keyrings/vendor.gpg] https://example.invalid stable main"
));
assert!(!source_disables_authentication(
"deb https://trusted=yes.example.invalid stable main"
));
assert!(!source_disables_authentication(
"# deb [trusted=yes] disabled"
));
}
#[test]
fn recovery_never_reapplies_an_interrupted_upgrade() {
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Preflight), "old-boot"),
RecoveryAction::Apply
);
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Applying), "old-boot"),
RecoveryAction::Repair
);
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Rebooting), "old-boot"),
RecoveryAction::Reboot
);
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Rebooting), "new-boot"),
RecoveryAction::Verify
);
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Verifying), "new-boot"),
RecoveryAction::Verify
);
assert_eq!(
recovery_action(&journal(SystemUpgradePhase::Complete), "new-boot"),
RecoveryAction::None
);
}
#[test]
fn terminal_journals_expire_but_repair_state_remains() {
let now = Utc::now();
let mut complete = journal(SystemUpgradePhase::Complete);
complete.updated_at = now - chrono::Duration::days(TERMINAL_REPLAY_DAYS + 1);
assert!(!retain_journal(&complete, now));
let mut repair = complete;
repair.phase = SystemUpgradePhase::RepairRequired;
assert!(retain_journal(&repair, now));
}
}

325
fleet/harmony-fleet-agent/src/system_upgrade_service.rs Normal file
View File

@@ -0,0 +1,325 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use async_nats::jetstream::kv::{Operation, Store};
use chrono::Utc;
use futures_util::StreamExt;
use harmony_reconciler_contracts::{
BUCKET_DEVICE_HEARTBEAT, BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS,
HeartbeatPayload, Id, SystemUpgradeAttempt, SystemUpgradePhase, SystemUpgradeStatus,
device_heartbeat_key, system_upgrade_intent_key, system_upgrade_intent_watch_filter,
system_upgrade_status_key,
};
use crate::updater::{UpdaterClient, safe_token};
const RETRY_INTERVAL: Duration = Duration::from_secs(1);
#[derive(Clone)]
pub struct SystemUpgradeService {
device_id: Id,
intent: Store,
status: Store,
heartbeat: Store,
updater: UpdaterClient,
}
impl SystemUpgradeService {
pub async fn connect(
client: async_nats::Client,
device_id: Id,
updater_socket: &std::path::Path,
) -> Result<Self> {
let jetstream = async_nats::jetstream::new(client);
let intent = jetstream
.get_key_value(BUCKET_SYSTEM_UPGRADE_INTENT)
.await?;
let status = jetstream
.get_key_value(BUCKET_SYSTEM_UPGRADE_STATUS)
.await?;
let heartbeat = jetstream.get_key_value(BUCKET_DEVICE_HEARTBEAT).await?;
let updater = UpdaterClient::new(updater_socket);
let capabilities = updater.capabilities().await?;
if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 {
bail!("updater does not support AptFullUpgradeV1");
}
Ok(Self {
device_id,
intent,
status,
heartbeat,
updater,
})
}
pub async fn run(self) -> Result<()> {
let filter = system_upgrade_intent_watch_filter(&self.device_id.to_string());
loop {
let mut intents = match self.intent.watch_with_history(&filter).await {
Ok(intents) => intents,
Err(error) => {
tracing::warn!(%error, "system upgrade intent watch start failed");
tokio::time::sleep(RETRY_INTERVAL).await;
continue;
}
};
while let Some(entry) = intents.next().await {
match entry {
Ok(entry) if entry.operation == Operation::Put => {
let result = serde_json::from_slice(&entry.value)
.context("decoding system upgrade attempt")
.and_then(|attempt| {
validate_intent(&self.device_id, &entry.key, &attempt)
.map(|()| attempt)
});
match result {
Ok(attempt) => self.relay(attempt).await,
Err(error) => {
tracing::warn!(key = %entry.key, %error, "system upgrade attempt rejected")
}
}
}
Ok(_) => {}
Err(error) => {
tracing::warn!(%error, "system upgrade intent watch failed; restarting");
break;
}
}
}
tokio::time::sleep(RETRY_INTERVAL).await;
}
}
async fn relay(&self, attempt: SystemUpgradeAttempt) {
let mut blocked_since = None;
let mut blocked_published = false;
let recovered = match self
.updater
.system_upgrade_status(&attempt.attempt_id)
.await
{
Ok(Some(status))
if status.attempt_id == attempt.attempt_id && status.run_uid == attempt.run_uid =>
{
Some(status)
}
Ok(Some(_)) => {
tracing::warn!(attempt_id = %attempt.attempt_id, "updater status does not match system upgrade intent");
return;
}
Ok(None) | Err(_) => None,
};
let accepted = if let Some(status) = recovered {
status
} else {
loop {
if blocked_since.is_some() && attempt.expires_at <= Utc::now() {
return;
}
match self.updater.start_system_upgrade(&attempt).await {
Ok(status) => break status,
Err(error)
if error
.to_string()
.contains("another upgrade is already in progress") =>
{
let now = Utc::now();
let started_at = *blocked_since.get_or_insert(now);
let status = SystemUpgradeStatus {
attempt_id: attempt.attempt_id.clone(),
run_uid: attempt.run_uid.clone(),
phase: SystemUpgradePhase::Blocked,
started_at,
updated_at: now,
post_completion_heartbeat_at: None,
error: Some(error.to_string()),
};
match self.publish(&attempt, status).await {
Ok(()) if !blocked_published => {
tracing::info!(
attempt_id = %attempt.attempt_id,
run_uid = %attempt.run_uid,
"system upgrade blocked status published"
);
blocked_published = true;
}
Ok(()) => {}
Err(error) => {
tracing::warn!(%error, "publishing blocked system upgrade status failed")
}
}
if attempt.expires_at <= Utc::now() {
return;
}
tokio::time::sleep(RETRY_INTERVAL).await;
}
Err(error) if temporary_updater_error(&error) => {
tracing::warn!(%error, "updater unavailable during system upgrade submit; retrying");
if attempt.expires_at <= Utc::now() {
return;
}
tokio::time::sleep(RETRY_INTERVAL).await;
}
Err(error) => {
tracing::warn!(%error, "system upgrade submit failed");
return;
}
}
}
};
let accepted = self.attach_heartbeat(accepted).await;
let accepted_published = match self.publish(&attempt, accepted.clone()).await {
Ok(()) => true,
Err(error) => {
tracing::warn!(%error, "publishing accepted system upgrade status failed");
false
}
};
if terminal_relay_complete(&accepted) && accepted_published {
return;
}
let mut last = accepted_published.then_some(accepted);
loop {
match self
.updater
.system_upgrade_status(&attempt.attempt_id)
.await
{
Ok(Some(status)) => {
let status = self.attach_heartbeat(status).await;
if last.as_ref() == Some(&status) {
tokio::time::sleep(RETRY_INTERVAL).await;
continue;
}
let terminal = terminal_relay_complete(&status);
let published = match self.publish(&attempt, status.clone()).await {
Ok(()) => {
last = Some(status);
true
}
Err(error) => {
tracing::warn!(%error, "publishing system upgrade status failed");
false
}
};
if terminal && published {
return;
}
}
Ok(_) => {}
Err(error) => {
tracing::warn!(%error, "updater system upgrade status unavailable; retrying")
}
}
tokio::time::sleep(RETRY_INTERVAL).await;
}
}
async fn attach_heartbeat(&self, mut status: SystemUpgradeStatus) -> SystemUpgradeStatus {
if status.phase != SystemUpgradePhase::Complete
|| status.post_completion_heartbeat_at.is_some()
{
return status;
}
let key = device_heartbeat_key(&self.device_id.to_string());
let heartbeat = match self.heartbeat.entry(&key).await {
Ok(Some(entry)) if entry.operation == Operation::Put => {
serde_json::from_slice::<HeartbeatPayload>(&entry.value).ok()
}
_ => None,
};
if let Some(heartbeat) = heartbeat
&& heartbeat.device_id == self.device_id
&& heartbeat.at > status.updated_at
{
status.post_completion_heartbeat_at = Some(heartbeat.at);
}
status
}
async fn publish(
&self,
attempt: &SystemUpgradeAttempt,
status: SystemUpgradeStatus,
) -> Result<()> {
if status.attempt_id != attempt.attempt_id || status.run_uid != attempt.run_uid {
bail!("updater system upgrade status does not match current attempt");
}
let key = system_upgrade_status_key(&self.device_id.to_string(), &status.run_uid);
self.status
.put(&key, serde_json::to_vec(&status)?.into())
.await?;
if status.phase != SystemUpgradePhase::Blocked {
tracing::info!(
attempt_id = %status.attempt_id,
run_uid = %status.run_uid,
phase = ?status.phase,
heartbeat_confirmed = status.post_completion_heartbeat_at.is_some(),
"system upgrade status published"
);
}
Ok(())
}
}
fn terminal_relay_complete(status: &SystemUpgradeStatus) -> bool {
status.phase.is_terminal()
&& (status.phase != SystemUpgradePhase::Complete
|| status.post_completion_heartbeat_at.is_some())
}
fn temporary_updater_error(error: &anyhow::Error) -> bool {
error.downcast_ref::<std::io::Error>().is_some()
|| error.to_string().starts_with("updater response timed out")
}
fn validate_intent(device_id: &Id, key: &str, attempt: &SystemUpgradeAttempt) -> Result<()> {
uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid system upgrade attempt id")?;
if &attempt.device_id != device_id {
bail!("system upgrade attempt targets another device");
}
if !safe_token(&attempt.run_uid) {
bail!("invalid system upgrade run uid");
}
if key != system_upgrade_intent_key(&device_id.to_string(), &attempt.run_uid) {
bail!("system upgrade intent key does not match its device and run");
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::TimeDelta;
fn attempt(device_id: &str) -> SystemUpgradeAttempt {
SystemUpgradeAttempt {
attempt_id: uuid::Uuid::new_v4().to_string(),
run_uid: "run-1".into(),
device_id: Id::from(device_id),
expires_at: Utc::now() + TimeDelta::minutes(5),
}
}
#[test]
fn rejects_wrong_device_and_bad_keys() {
let device_id = Id::from("device-1");
let wrong = attempt("device-2");
assert!(validate_intent(&device_id, "device-1.run-1", &wrong).is_err());
let invalid = attempt("device-1");
assert!(validate_intent(&device_id, "device-1.wrong", &invalid).is_err());
}
#[test]
fn intent_and_status_keys_map_to_the_attempt_run() {
let device_id = Id::from("device-1");
let attempt = attempt("device-1");
assert!(validate_intent(&device_id, "device-1.run-1", &attempt).is_ok());
assert!(validate_intent(&device_id, "device-1.other", &attempt).is_err());
assert_eq!(
system_upgrade_status_key(&device_id.to_string(), &attempt.run_uid),
"device-1.run-1"
);
}
}

1210
fleet/harmony-fleet-agent/src/updater.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1008
fleet/harmony-fleet-agent/src/upgrade.rs Normal file
View File

File diff suppressed because it is too large Load Diff

6
fleet/harmony-fleet-deploy/Cargo.toml
View File

@@ -20,6 +20,10 @@ path = "src/main.rs"
name = "harmony-fleet-crds-deploy"
path = "src/bin/harmony-fleet-crds-deploy.rs"
[[bin]]
name = "harmony-fleet-release"
path = "src/bin/harmony-fleet-release.rs"
[dependencies]
harmony = { path = "../../harmony", features = ["podman"] }
harmony_cli = { path = "../../harmony_cli" }
@@ -29,6 +33,7 @@ harmony_types = { path = "../../harmony_types" }
harmony_macros = { path = "../../harmony_macros" }
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony-fleet-operator = { path = "../harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
anyhow = { workspace = true }
@@ -40,6 +45,7 @@ kube = { workspace = true, features = ["runtime", "derive"] }
log = { workspace = true }
env_logger = { workspace = true }
non-blank-string-rs = "1"
oci-client.workspace = true
inquire.workspace = true
schemars = "0.8"
serde = { workspace = true }

295
fleet/harmony-fleet-deploy/src/app.rs
View File

@@ -22,11 +22,18 @@ const PROJECT: &str = "fleet";
const ADMIN_ROLE: &str = "fleet-admin";
const DEVICE_ROLE: &str = "device";
const OPERATOR_APP: &str = "harmony-fleet-operator";
const DASHBOARD_APP: &str = "harmony-fleet-dashboard";
const OPERATOR_USER: &str = "fleet-operator";
const NATS_ACCOUNT: &str = "FLEET";
pub struct FleetApp;
impl FleetApp {
pub fn official_images(tag: &str) -> Vec<ImageSpec> {
fleet_images(|name| format!("hub.nationtech.io/harmony/{name}:{tag}"))
}
}
#[async_trait]
impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
fn identity(&self, ctx: &AppContext) -> AppIdentity {
@@ -37,22 +44,7 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
}
fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
Ok(vec![
ImageSpec {
name: "operator".to_string(),
image: ctx.image("harmony-fleet-operator"),
context: ".".into(),
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
build_args: Vec::new(),
},
ImageSpec {
name: "callout".to_string(),
image: ctx.image("harmony-nats-callout"),
context: ".".into(),
dockerfile: "nats/callout/Dockerfile".into(),
build_args: Vec::new(),
},
])
Ok(fleet_images(|name| ctx.image(name)))
}
async fn scores(
@@ -60,6 +52,12 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
ctx: &AppContext,
images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let operator_image = images
.get("operator")
.map_or_else(|| ctx.image("harmony-fleet-operator"), str::to_owned);
let callout_image = images
.get("callout")
.map_or_else(|| ctx.image("harmony-nats-callout"), str::to_owned);
let namespace = ctx.namespace();
let image_pull_secret = ctx.image_pull_secret();
@@ -73,16 +71,38 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
zitadel = zitadel.http(Some(8080));
}
let provider = zitadel.provider_ref();
let dashboard_host =
(ctx.profile() == Profile::Prod).then(|| ctx.service_host("dashboard"));
let identity = ZitadelSetupScore::for_provider(&provider, namespace, namespace)
.application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
.api_application(PROJECT, "nats")
.role(PROJECT, ADMIN_ROLE, "Fleet Admin")
.role(PROJECT, DEVICE_ROLE, "Device")
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
.port_forward("zitadel")
.groups_claim();
.machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE]);
let identity = if let Some(host) = &dashboard_host {
identity.application(
PROJECT,
DASHBOARD_APP,
ZitadelAppType::WebPkce {
redirect_uris: vec![format!("https://{host}/auth/callback")],
post_logout_redirect_uris: vec![format!("https://{host}/")],
id_token_role_assertion: true,
},
)
} else {
identity
};
let identity = if ctx.profile() == Profile::Local {
identity.port_forward("zitadel")
} else {
identity
}
.groups_claim();
let application = identity.application_ref(OPERATOR_APP);
let dashboard_application = dashboard_host
.as_ref()
.map(|_| identity.application_ref(DASHBOARD_APP));
let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
let credentials =
@@ -97,26 +117,30 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
// I feel like this should not be a standalone nats score but rather be configuration passed
// to the main nats score that is installing the nats cluster
let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
.with_jetstream_size("2Gi");
.with_jetstream_size("2Gi")
.create_namespace(false);
let nats = if ctx.profile() == Profile::Prod {
nats.websocket(ctx.service_host("nats"), "letsencrypt-prod")
} else {
nats
};
let account = nats.account_ref();
let callout =
let mut callout =
NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
.credentials(&credentials.credentials_ref())
.with_oidc(&provider, &application)
.image(images.require("callout")?)
.image(callout_image)
.image_pull_secret(image_pull_secret.clone())
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE)
.device_id_claim("client_id");
callout.device_id_prefix_strip = "device-".to_string();
let nats = nats.with_auth_callout(&callout.auth_callout_ref());
let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao"));
let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao"))
.create_namespace(false);
if ctx.profile() == Profile::Prod {
openbao.openshift = true;
openbao = openbao.tls("letsencrypt-prod");
}
let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
@@ -124,12 +148,26 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
&application,
OpenbaoJwtAuth::oidc("fleet-device"),
);
let openbao_setup = if ctx.profile() == Profile::Prod {
openbao_setup.endpoint(format!("https://{}", ctx.service_host("openbao")))
} else {
openbao_setup
};
let operator = FleetOperatorScore::new(images.require("operator")?)
let operator = FleetOperatorScore::new(operator_image)
.namespace(namespace)
.image_pull_secret(image_pull_secret)
.messaging(&nats.client_ref())
.identity(&provider, &application, &operator_identity);
let operator = if let Some((host, dashboard_application)) =
dashboard_host.zip(dashboard_application)
{
operator
.ingress(host, Some("letsencrypt-prod".to_string()))
.web_auth(&dashboard_application)
} else {
operator
};
Ok(vec![
Box::new(postgres),
@@ -145,6 +183,25 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
}
}
fn fleet_images(image: impl Fn(&str) -> String) -> Vec<ImageSpec> {
vec![
ImageSpec {
name: "operator".to_string(),
image: image("harmony-fleet-operator"),
context: ".".into(),
dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
build_args: Vec::new(),
},
ImageSpec {
name: "callout".to_string(),
image: image("harmony-nats-callout"),
context: ".".into(),
dockerfile: "nats/callout/Dockerfile".into(),
build_args: Vec::new(),
},
]
}
pub struct FleetCrdsApp;
#[async_trait]
@@ -167,31 +224,35 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetCrdsApp {
pub struct FleetTenantProvisionApp {
tenant: TenantConfig,
credential_store: Arc<ConfigClient>,
credential_store: TenantCredentialStore,
allow_insecure_source: bool,
}
enum TenantCredentialStore {
Client(Arc<ConfigClient>),
OpenBao(OpenBaoClusterAccess),
}
impl FleetTenantProvisionApp {
pub fn new(tenant: TenantConfig, credential_store: Arc<ConfigClient>) -> Self {
Self {
tenant,
credential_store,
credential_store: TenantCredentialStore::Client(credential_store),
allow_insecure_source: false,
}
}
pub async fn from_openbao(
tenant: TenantConfig,
credential_store: &OpenBaoClusterAccess,
) -> Result<Self> {
let source = harmony_config::openbao_source(
credential_store.namespace.as_ref(),
Some(credential_store.url.to_string()),
Some(credential_store.zitadel_url.to_string()),
Some(credential_store.zitadel_audience.to_string()),
Some(credential_store.role.to_string()),
)
.await
.ok_or_else(|| anyhow::anyhow!("tenant credential store is unavailable"))?;
Ok(Self::new(tenant, Arc::new(ConfigClient::new(vec![source]))))
pub fn from_openbao(tenant: TenantConfig, credential_store: OpenBaoClusterAccess) -> Self {
Self {
tenant,
credential_store: TenantCredentialStore::OpenBao(credential_store),
allow_insecure_source: false,
}
}
pub fn allow_insecure_source(mut self) -> Self {
self.allow_insecure_source = true;
self
}
}
@@ -209,6 +270,21 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetTenantProvisionApp {
_ctx: &AppContext,
_images: &ImageRefs,
) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
let credential_store = match &self.credential_store {
TenantCredentialStore::Client(client) => client.clone(),
TenantCredentialStore::OpenBao(store) => {
let source = harmony_config::openbao_source(
store.namespace.as_ref(),
Some(store.url.to_string()),
Some(store.zitadel_url.to_string()),
Some(store.zitadel_audience.to_string()),
Some(store.role.to_string()),
)
.await
.ok_or_else(|| AppError::Deploy("tenant credential store is unavailable".into()))?;
Arc::new(ConfigClient::new(vec![source]))
}
};
let namespace = self
.tenant
.name
@@ -224,7 +300,8 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetTenantProvisionApp {
.parse()
.expect("static Kubernetes name is valid"),
fleet_deployer_rules(),
self.credential_store.clone(),
credential_store,
self.allow_insecure_source,
)),
])
}
@@ -301,6 +378,12 @@ fn fleet_deployer_rules() -> Vec<PolicyRule> {
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["route.openshift.io".to_string()]),
resources: Some(vec!["routes/custom-host".to_string()]),
verbs: vec!["create".to_string()],
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["policy".to_string()]),
resources: Some(vec!["poddisruptionbudgets".to_string()]),
@@ -335,15 +418,122 @@ fn fleet_deployer_rules() -> Vec<PolicyRule> {
verbs: verbs(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["fleet.nationtech.io".to_string()]),
resources: Some(vec!["taskruns".to_string()]),
verbs: ["get", "list", "watch"].map(String::from).to_vec(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec!["fleet.nationtech.io".to_string()]),
resources: Some(vec!["taskruns/status".to_string()]),
verbs: ["get", "update", "patch"].map(String::from).to_vec(),
..Default::default()
},
]
}
#[cfg(test)]
mod tenant_tests {
mod tests {
use super::*;
use harmony::topology::tenant::TenantNetworkPolicy;
use harmony_types::id::Id;
fn prod_context() -> harmony_app::Context {
harmony_app::Context {
name: "prod".parse().unwrap(),
namespace: "fleet".parse().unwrap(),
spec: harmony_app::ContextSpec::Remote(harmony_app::RemoteContext {
registry: "hub.nationtech.io".parse().unwrap(),
repository: "harmony".parse().unwrap(),
domain: "fleet.example.com".parse().unwrap(),
image_pull_secret: None,
access: OpenBaoClusterAccess {
namespace: "customer/fleet".parse().unwrap(),
url: "https://secrets.example.com".parse().unwrap(),
role: "fleet-deployer".parse().unwrap(),
zitadel_url: "https://identity.example.com".parse().unwrap(),
zitadel_audience: "openbao".parse().unwrap(),
},
}),
}
}
async fn serialized_fleet(context: harmony_app::Context, images: ImageRefs) -> String {
let context = AppContext::load_metadata(&context, "test", None);
FleetApp
.scores(&context, &images)
.await
.unwrap()
.iter()
.map(|score| serde_json::to_string(&score.serialize()).unwrap())
.collect()
}
#[test]
fn official_images_share_the_release_tag() {
let images = FleetApp::official_images("0.0.6");
assert_eq!(images.len(), 2);
assert!(images.iter().all(
|image| image.image.starts_with("hub.nationtech.io/harmony/")
&& image.image.ends_with(":0.0.6")
));
}
#[tokio::test]
async fn fleet_uses_supplied_images() {
let serialized = serialized_fleet(
prod_context(),
ImageRefs::new([
(
"operator".to_string(),
"registry.example/operator@sha256:operator".to_string(),
),
(
"callout".to_string(),
"registry.example/callout@sha256:callout".to_string(),
),
]),
)
.await;
assert!(serialized.contains("registry.example/operator@sha256:operator"));
assert!(serialized.contains("registry.example/callout@sha256:callout"));
assert!(!serialized.contains("\"port_forward_service\":\"zitadel\""));
assert!(serialized.contains("https://openbao.fleet.example.com"));
assert!(serialized.contains("dashboard.fleet.example.com"));
assert!(serialized.contains("https://dashboard.fleet.example.com/auth/callback"));
assert!(
serialized
.contains("\"app_name\":\"harmony-fleet-operator\",\"app_type\":\"DeviceCode\"")
);
assert!(serialized.contains("\"app_name\":\"harmony-fleet-dashboard\""));
assert!(serialized.contains("zitadel-harmony-fleet-dashboard-oidc"));
}
#[tokio::test]
async fn fleet_deploy_uses_the_context_release_tag() {
let serialized = serialized_fleet(prod_context(), ImageRefs::default()).await;
assert!(serialized.contains("hub.nationtech.io/harmony/harmony-fleet-operator:test"));
assert!(serialized.contains("hub.nationtech.io/harmony/harmony-nats-callout:test"));
}
#[tokio::test]
async fn local_fleet_uses_zitadel_port_forward() {
let serialized = serialized_fleet(
harmony_app::Context {
name: "local".parse().unwrap(),
namespace: "fleet".parse().unwrap(),
spec: harmony_app::ContextSpec::Local(harmony_app::LocalContext::ManagedK3d),
},
ImageRefs::default(),
)
.await;
assert!(serialized.contains("\"port_forward_service\":\"zitadel\""));
}
#[test]
fn deployer_permissions_exclude_cluster_resources() {
let rules = fleet_deployer_rules();
@@ -355,6 +545,17 @@ mod tenant_tests {
rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()])
&& rule.resources.as_deref() == Some(&["routes".to_string()])
}));
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&[String::new()])
&& rule.resources.as_deref()
== Some(&["pods/exec".to_string(), "pods/portforward".to_string()])
&& rule.verbs == ["create".to_string()]
}));
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()])
&& rule.resources.as_deref() == Some(&["routes/custom-host".to_string()])
&& rule.verbs == ["create".to_string()]
}));
assert!(rules.iter().all(|rule| {
!rule.resources.as_ref().is_some_and(|resources| {
resources
@@ -362,6 +563,16 @@ mod tenant_tests {
.any(|resource| resource == "customresourcedefinitions")
})
}));
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()])
&& rule.resources.as_deref() == Some(&["taskruns".to_string()])
&& rule.verbs == ["get", "list", "watch"].map(String::from)
}));
assert!(rules.iter().any(|rule| {
rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()])
&& rule.resources.as_deref() == Some(&["taskruns/status".to_string()])
&& rule.verbs == ["get", "update", "patch"].map(String::from)
}));
}
#[tokio::test]

83
fleet/harmony-fleet-deploy/src/bin/harmony-fleet-example-deployment.rs Normal file
View File

@@ -0,0 +1,83 @@
use clap::Parser;
use harmony::{inventory::Inventory, topology::K8sAnywhereTopology};
use harmony_fleet_deploy::FleetDeploymentScore;
use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::{Api, Client, api::DeleteParams};
#[derive(Parser)]
struct Args {
#[arg(long, default_value = "fleet-demo")]
namespace: String,
#[arg(long, default_value = "hello-world")]
name: String,
#[arg(long)]
target_device: String,
#[arg(long = "allowed-group", default_value = "application")]
allowed_groups: Vec<String>,
#[arg(long, default_value = "docker.io/library/nginx:latest")]
image: String,
#[arg(long, default_value = "8080:80")]
port: String,
#[arg(long)]
delete: bool,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let args = Args::parse();
if args.delete {
let deployments: Api<Deployment> =
Api::namespaced(Client::try_default().await?, &args.namespace);
deployments
.delete(&args.name, &DeleteParams::default())
.await?;
return Ok(());
}
let deployment = Deployment::new(
&args.name,
DeploymentSpec {
allowed_groups: args.allowed_groups,
target_selector: LabelSelector {
match_labels: Some([("device-id".into(), args.target_device)].into()),
match_expressions: None,
},
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: args.name.clone(),
image: args.image,
image_pull_secret: None,
ports: vec![args.port],
env: Vec::new(),
secret_env: Vec::new(),
volumes: Vec::new(),
restart_policy: Default::default(),
}],
}),
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
);
harmony_cli::run(
Inventory::autoload(),
K8sAnywhereTopology::from_env(),
vec![Box::new(FleetDeploymentScore::new(
deployment,
args.namespace,
))],
Some(harmony_cli::Args {
yes: true,
filter: None,
interactive: false,
all: true,
number: 0,
list: false,
}),
)
.await
.map_err(|error| anyhow::anyhow!(error.to_string()))
}

194
fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs Normal file
View File

@@ -0,0 +1,194 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use anyhow::{Context, bail};
use clap::{Parser, Subcommand};
use harmony_app::{PublicationTopology, publish::build_images};
use harmony_fleet_deploy::FleetApp;
use harmony_reconciler_contracts::upgrade::{AGENT_OCI_ARTIFACT_TYPE, AGENT_OCI_LAYER_MEDIA_TYPE};
use oci_client::annotations::ORG_OPENCONTAINERS_IMAGE_VERSION;
use oci_client::client::{Config, ImageLayer};
use oci_client::errors::{OciDistributionError, OciErrorCode};
use oci_client::manifest::OciImageManifest;
use oci_client::secrets::RegistryAuth;
use oci_client::{Client, Reference};
#[derive(Parser)]
struct Args {
#[command(subcommand)]
release: Release,
}
#[derive(Subcommand)]
enum Release {
Control {
#[arg(long)]
version: String,
#[arg(long)]
push: bool,
},
Agent {
#[arg(long)]
binary: PathBuf,
#[arg(long)]
reference: String,
#[arg(long)]
version: String,
},
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
harmony_cli::cli_logger::init();
match Args::parse().release {
Release::Control { version, push } => publish_control(&version, push).await,
Release::Agent {
binary,
reference,
version,
} => publish_agent(binary, &reference, &version).await,
}
}
async fn publish_control(version: &str, push: bool) -> anyhow::Result<()> {
let images = FleetApp::official_images(version);
let registry = PublicationTopology::Registry {
registry: "hub.nationtech.io".to_string(),
};
if push {
let client = Client::default();
let auth = registry_auth()?;
let mut existing = Vec::new();
for image in &images {
let reference = Reference::try_from(image.image.as_str())?;
match client.pull_manifest(&reference, &auth).await {
Ok((_, digest)) => existing.push((
image.name.clone(),
format!(
"{}/{}@{digest}",
reference.registry(),
reference.repository()
),
)),
Err(error) if manifest_is_missing(&error) => {}
Err(error) => return Err(error).context("checking control-plane release"),
}
}
if existing.len() == images.len() {
for (name, image) in existing {
println!("{name}={image}");
}
return Ok(());
}
if !existing.is_empty() {
bail!("control-plane release is incomplete; refusing to overwrite existing tags");
}
}
let refs = build_images(&images, &registry)?;
let refs = if push {
let credentials = registry_credentials()?;
harmony_app::publish::publish_images(&images, &refs, &registry, Some(&credentials))?
} else {
refs
};
for (name, image) in refs.iter() {
println!("{name}={image}");
}
Ok(())
}
async fn publish_agent(binary: PathBuf, reference: &str, version: &str) -> anyhow::Result<()> {
let reference = Reference::try_from(reference).context("invalid agent OCI reference")?;
if reference.tag().is_none() || reference.digest().is_some() {
bail!("agent publication requires a tag reference");
}
let auth = registry_auth()?;
let client = Client::default();
let layer = ImageLayer::new(
std::fs::read(&binary)
.with_context(|| format!("reading agent binary {}", binary.display()))?,
AGENT_OCI_LAYER_MEDIA_TYPE.to_string(),
None,
);
match client.pull_manifest(&reference, &auth).await {
Ok((oci_client::manifest::OciManifest::Image(manifest), digest))
if manifest.artifact_type.as_deref() == Some(AGENT_OCI_ARTIFACT_TYPE)
&& manifest
.annotations
.as_ref()
.and_then(|annotations| annotations.get(ORG_OPENCONTAINERS_IMAGE_VERSION))
.map(String::as_str)
== Some(version)
&& matches!(manifest.layers.as_slice(), [existing]
if existing.media_type == AGENT_OCI_LAYER_MEDIA_TYPE
&& existing.digest == layer.sha256_digest()) =>
{
println!(
"agent=oci://{}/{}@{digest}",
reference.registry(),
reference.repository()
);
return Ok(());
}
Ok(_) => bail!(
"agent artifact tag exists with different content: {}",
reference.whole()
),
Err(error) if manifest_is_missing(&error) => {}
Err(error) => return Err(error).context("checking agent artifact tag"),
}
let config = Config::new(
b"{}".to_vec(),
"application/vnd.oci.empty.v1+json".to_string(),
None,
);
let mut annotations = BTreeMap::new();
annotations.insert(
ORG_OPENCONTAINERS_IMAGE_VERSION.to_string(),
version.to_string(),
);
let mut manifest =
OciImageManifest::build(std::slice::from_ref(&layer), &config, Some(annotations));
manifest.artifact_type = Some(AGENT_OCI_ARTIFACT_TYPE.to_string());
client
.push(&reference, &[layer], config, &auth, Some(manifest))
.await
.context("publishing agent OCI artifact")?;
let (_, digest) = client
.pull_manifest(&reference, &auth)
.await
.context("resolving published agent manifest")?;
println!(
"agent=oci://{}/{}@{digest}",
reference.registry(),
reference.repository()
);
Ok(())
}
fn registry_auth() -> anyhow::Result<RegistryAuth> {
let credentials = registry_credentials()?;
Ok(RegistryAuth::Basic(credentials.username, credentials.token))
}
fn registry_credentials() -> anyhow::Result<harmony_app::RegistryCredentials> {
Ok(harmony_app::RegistryCredentials {
username: std::env::var("REGISTRY_USER").context("REGISTRY_USER is required")?,
token: std::env::var("REGISTRY_TOKEN").context("REGISTRY_TOKEN is required")?,
})
}
fn manifest_is_missing(error: &OciDistributionError) -> bool {
matches!(error, OciDistributionError::ImageManifestNotFoundError(_))
|| matches!(
error,
OciDistributionError::RegistryError { envelope, .. }
if envelope.errors.iter().any(|error| matches!(
&error.code,
OciErrorCode::ManifestUnknown
| OciErrorCode::NameUnknown
| OciErrorCode::NotFound
))
)
}

210
fleet/harmony-fleet-deploy/src/deployment.rs Normal file
View File

@@ -0,0 +1,210 @@
use std::time::Duration;
use async_trait::async_trait;
use harmony::data::Version;
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use harmony::inventory::Inventory;
use harmony::score::Score;
use harmony::topology::{K8sclient, Topology};
use harmony_fleet_operator::{Deployment, DeploymentStatus};
use harmony_types::id::Id;
use kube::ResourceExt;
use serde::Serialize;
const ROLLOUT_TIMEOUT: Duration = Duration::from_secs(900);
#[derive(Debug, Clone, Serialize)]
pub struct FleetDeploymentScore {
pub deployment: Deployment,
pub namespace: String,
}
impl FleetDeploymentScore {
pub fn new(deployment: Deployment, namespace: impl Into<String>) -> Self {
Self {
deployment,
namespace: namespace.into(),
}
}
}
impl<T: Topology + K8sclient> Score<T> for FleetDeploymentScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(FleetDeploymentInterpret(self.clone()))
}
fn name(&self) -> String {
format!("{} [FleetDeploymentScore]", self.deployment.name_any())
}
}
#[derive(Debug)]
struct FleetDeploymentInterpret(FleetDeploymentScore);
#[async_trait]
impl<T: Topology + K8sclient> Interpret<T> for FleetDeploymentInterpret {
async fn execute(
&self,
_inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let client = topology.k8s_client().await.map_err(InterpretError::new)?;
let applied = client
.apply(&self.0.deployment, Some(&self.0.namespace))
.await
.map_err(|error| InterpretError::new(format!("apply Fleet Deployment: {error}")))?;
let revision = applied.rollout_revision().ok_or_else(|| {
InterpretError::new("applied Fleet Deployment has no UID or generation".to_string())
})?;
let name = applied.name_any();
let mut last_observation = "operator has not reported rollout status".to_string();
let completed = tokio::time::timeout(ROLLOUT_TIMEOUT, async {
loop {
let current = client
.get_resource::<Deployment>(&name, Some(&self.0.namespace))
.await
.map_err(|error| {
InterpretError::new(format!("read Fleet Deployment status: {error}"))
})?
.ok_or_else(|| {
InterpretError::new("Fleet Deployment disappeared".to_string())
})?;
if let Some(aggregate) = current
.status
.as_ref()
.filter(|status| status.rollout_revision.as_deref() == Some(&revision))
.and_then(|status| status.aggregate.as_ref())
{
last_observation = rollout_observation(aggregate);
}
match rollout_result(current.status.as_ref(), &revision) {
Ok(Some(message)) => return Ok(message),
Ok(None) => {}
Err(error) => return Err(InterpretError::new(error)),
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await
.map_err(|_| {
InterpretError::new(format!(
"Fleet Deployment {}/{} did not complete within 15 minutes: {last_observation}",
self.0.namespace, name
))
})??;
Ok(Outcome::success(completed))
}
fn get_name(&self) -> InterpretName {
InterpretName::Custom("FleetDeploymentInterpret")
}
fn get_version(&self) -> Version {
Version::from(env!("CARGO_PKG_VERSION")).expect("package version")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::QUEUED
}
fn get_children(&self) -> Vec<Id> {
Vec::new()
}
}
fn rollout_result(
status: Option<&DeploymentStatus>,
revision: &str,
) -> Result<Option<String>, String> {
let Some(status) = status.filter(|status| status.rollout_revision.as_deref() == Some(revision))
else {
return Ok(None);
};
let Some(aggregate) = &status.aggregate else {
return Ok(None);
};
if aggregate.matched_device_count == 0 {
return Err("rollout matched no devices".to_string());
}
if aggregate.failed > 0 {
let mut message = format!("rollout failed: {}", rollout_observation(aggregate));
if let Some(error) = &aggregate.last_error {
message.push_str(&format!(
"; last error from {} at {}: {}",
error.device_id, error.at, error.message
));
}
return Err(message);
}
if aggregate.pending > 0 {
return Ok(None);
}
Ok(Some(format!(
"Fleet rollout complete: {} target(s), {} succeeded",
aggregate.matched_device_count, aggregate.succeeded
)))
}
fn rollout_observation(aggregate: &harmony_fleet_operator::DeploymentAggregate) -> String {
format!(
"{} succeeded, {} pending, {} failed across {} target(s)",
aggregate.succeeded, aggregate.pending, aggregate.failed, aggregate.matched_device_count,
)
}
#[cfg(test)]
mod tests {
use harmony_fleet_operator::{AggregateLastError, DeploymentAggregate};
use super::*;
fn status(revision: &str, matched: u32, failed: u32, pending: u32) -> DeploymentStatus {
DeploymentStatus {
rollout_revision: Some(revision.to_string()),
aggregate: Some(DeploymentAggregate {
matched_device_count: matched,
succeeded: matched - failed - pending,
failed,
pending,
last_error: (failed > 0).then(|| AggregateLastError {
device_id: "device-1".into(),
message: "container failed".into(),
at: "2026-01-01T00:00:00Z".into(),
}),
}),
}
}
#[test]
fn rollout_requires_the_exact_revision() {
assert_eq!(
rollout_result(Some(&status("uid:1", 1, 0, 0)), "uid:2"),
Ok(None)
);
}
#[test]
fn zero_targets_fail_the_waited_deployment() {
assert_eq!(
rollout_result(Some(&status("uid:1", 0, 0, 0)), "uid:1"),
Err("rollout matched no devices".into())
);
}
#[test]
fn rollout_failure_reports_last_error() {
assert_eq!(
rollout_result(Some(&status("uid:1", 1, 1, 0)), "uid:1"),
Err("rollout failed: 0 succeeded, 0 pending, 1 failed across 1 target(s); last error from device-1 at 2026-01-01T00:00:00Z: container failed".into())
);
}
#[test]
fn rollout_observation_preserves_current_counts() {
let status = status("uid:1", 5, 0, 2);
assert_eq!(
rollout_observation(status.aggregate.as_ref().unwrap()),
"3 succeeded, 2 pending, 0 failed across 5 target(s)"
);
}
}

131
fleet/harmony-fleet-deploy/src/device_setup.rs
View File

@@ -290,18 +290,48 @@ impl FleetDeviceSetupConfig {
Description=IoT Agent (Harmony)
After=network-online.target
Wants=network-online.target
Requires=harmony-fleet-updater.service
After=harmony-fleet-updater.service
[Service]
Type=simple
Type=notify
NotifyAccess=main
User=fleet-agent
RuntimeDirectory=harmony-fleet-agent
RuntimeDirectoryMode=0700
Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml
Environment=RUST_LOG=info
ExecStart=/usr/local/bin/fleet-agent
TimeoutStartSec=4min
TimeoutStopSec=60s
Restart=on-failure
RestartSec=5
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target
"#
}
pub fn render_updater_systemd_unit(&self) -> &'static str {
r#"[Unit]
Description=Harmony Fleet Agent Updater
Before=fleet-agent.service
[Service]
Type=notify
NotifyAccess=main
User=root
Group=fleet-agent
RuntimeDirectory=harmony-fleet-updater
RuntimeDirectoryMode=0750
StateDirectory=harmony-fleet-updater
StateDirectoryMode=0700
ExecStart=/usr/lib/harmony-fleet/fleet-agent-bootstrap --updater
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
"#
@@ -428,11 +458,21 @@ async fn resolve_zitadel_enroll(
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FleetDeviceSetupScore {
pub config: FleetDeviceSetupConfig,
#[serde(skip)]
overwrite_existing_config: bool,
}
impl FleetDeviceSetupScore {
pub fn new(config: FleetDeviceSetupConfig) -> Self {
Self { config }
Self {
config,
overwrite_existing_config: false,
}
}
pub fn overwrite_existing_config(mut self, overwrite: bool) -> Self {
self.overwrite_existing_config = overwrite;
self
}
}
@@ -444,6 +484,7 @@ impl<T: Topology + LinuxHostConfiguration> Score<T> for FleetDeviceSetupScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(FleetDeviceSetupInterpret {
config: self.config.clone(),
overwrite_existing_config: self.overwrite_existing_config,
version: Version::from("0.1.0").expect("static version"),
status: InterpretStatus::QUEUED,
})
@@ -453,6 +494,7 @@ impl<T: Topology + LinuxHostConfiguration> Score<T> for FleetDeviceSetupScore {
#[derive(Debug)]
struct FleetDeviceSetupInterpret {
config: FleetDeviceSetupConfig,
overwrite_existing_config: bool,
version: Version,
status: InterpretStatus,
}
@@ -543,13 +585,14 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
}
}
}
let confirmed = inquire::Confirm::new(
"Device already has /etc/fleet-agent/config.toml with different content.\n \
Overwrite it and apply the new config?",
)
.with_default(false)
.prompt()
.map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
let confirmed = self.overwrite_existing_config
|| inquire::Confirm::new(
"Device already has /etc/fleet-agent/config.toml with different content.\n \
Overwrite it and apply the new config?",
)
.with_default(false)
.prompt()
.map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
if !confirmed {
return Err(InterpretError::new(
"User aborted: refused to overwrite existing config".to_string(),
@@ -596,8 +639,10 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
}
// 1. Dependencies.
info!("[{tag}] Step 2/7 — ensuring system packages: podman, systemd-container");
for pkg in ["podman", "systemd-container"] {
info!(
"[{tag}] Step 2/7 — ensuring system packages: podman, systemd-container, systemd-timesyncd"
);
for pkg in ["podman", "systemd-container", "systemd-timesyncd"] {
let r = PackageInstaller::ensure_package(topology, pkg)
.await
.map_err(wrap)?;
@@ -605,6 +650,12 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
change_count += 1;
}
}
let r = SystemdManager::ensure_system_unit_active(topology, "systemd-timesyncd")
.await
.map_err(wrap)?;
if r.changed {
change_count += 1;
}
// 2. fleet-agent user. Not `--system`: Ubuntu's useradd skips
// subuid/subgid auto-allocation for system users on the
@@ -651,19 +702,20 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
change_count += 1;
}
// 4. Binary. Ship via ansible's native copy-from-local-file
// 4. Bootstrap binary. The root updater owns the active symlink;
// normal agent upgrades never replace this privileged helper.
// path (`FileSource::LocalPath`). Ansible handles binary
// content over SFTP and reports `changed: true` only when the
// remote file actually differs from the local one — so
// re-running this Score without a new binary is a true NOOP.
info!(
"[{tag}] Step 5/7 — uploading agent binary {} -> /usr/local/bin/fleet-agent",
"[{tag}] Step 5/8 — uploading agent bootstrap {}",
cfg.agent_binary_path.display()
);
let binary_r = FileDelivery::ensure_file(
topology,
&FileSpec {
path: "/usr/local/bin/fleet-agent".to_string(),
path: "/usr/lib/harmony-fleet/fleet-agent-bootstrap".to_string(),
source: FileSource::LocalPath(cfg.agent_binary_path.clone()),
owner: Some("root".to_string()),
group: Some("root".to_string()),
@@ -729,8 +781,33 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
change_count += 1;
}
// 6. systemd unit for the agent itself.
info!("[{tag}] Step 7/7 — installing fleet-agent.service");
// 6. Root updater must be active before the agent. On first
// install it creates /usr/local/bin/fleet-agent as an atomic
// symlink to the bootstrap binary.
info!("[{tag}] Step 7/8 — installing harmony-fleet-updater.service");
let updater_unit = SystemdUnitSpec {
name: "harmony-fleet-updater".to_string(),
unit_content: cfg.render_updater_systemd_unit().to_string(),
scope: SystemdScope::System,
start_immediately: true,
};
let updater_unit_r = SystemdManager::ensure_systemd_unit(topology, &updater_unit)
.await
.map_err(wrap)?;
if updater_unit_r.changed {
change_count += 1;
}
if binary_r.changed || updater_unit_r.changed {
SystemdManager::restart_service(
topology,
"harmony-fleet-updater",
SystemdScope::System,
)
.await
.map_err(wrap)?;
}
info!("[{tag}] Step 8/8 — installing fleet-agent.service");
let unit = SystemdUnitSpec {
name: "fleet-agent".to_string(),
unit_content: cfg.render_systemd_unit().to_string(),
@@ -745,7 +822,8 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
}
// 7. Restart the agent iff anything that affects it changed.
let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed || key_r;
let needs_restart =
toml_r.changed || unit_r.changed || updater_unit_r.changed || binary_r.changed || key_r;
let service_state = if needs_restart {
info!("[{tag}] 🔄 Restarting fleet-agent (config/binary/unit changed)");
SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System)
@@ -1109,4 +1187,23 @@ mod tests {
let toml = cfg.render_toml();
assert!(toml.contains("danger_accept_invalid_certs = true"));
}
#[test]
fn systemd_service_is_ready_only_after_agent_notification() {
let config = base_config(BTreeMap::new());
let unit = config.render_systemd_unit();
assert!(unit.contains("Type=notify\n"));
assert!(unit.contains("NotifyAccess=main\n"));
assert!(unit.contains("TimeoutStartSec=4min\n"));
assert!(unit.contains("TimeoutStopSec=60s\n"));
assert!(unit.contains("Requires=harmony-fleet-updater.service\n"));
assert!(unit.contains("RuntimeDirectory=harmony-fleet-agent\n"));
let updater = config.render_updater_systemd_unit();
assert!(updater.contains("User=root\n"));
assert!(updater.contains("Type=notify\n"));
assert!(updater.contains("Group=fleet-agent\n"));
assert!(updater.contains("RuntimeDirectoryMode=0750\n"));
assert!(updater.contains("fleet-agent-bootstrap --updater\n"));
}
}

36
fleet/harmony-fleet-deploy/src/lib.rs
View File

@@ -12,11 +12,13 @@
pub mod agent;
mod app;
mod deployment;
mod device_setup;
pub mod operator;
pub use agent::{FleetAgentScore, PodTarget};
pub use app::{FleetApp, FleetCrdsApp, FleetTenantProvisionApp};
pub use deployment::FleetDeploymentScore;
pub use device_setup::{
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
HostsEntry, merge_hosts_file,
@@ -33,12 +35,44 @@ pub async fn deploy_fleet_crds_with_context(context: harmony_app::Context) -> an
harmony_cli::app::app_main(FleetCrdsApp, contexts).await
}
pub async fn deploy_fleet_crds_with_kubeconfig(
context: harmony_app::Context,
kubeconfig: std::path::PathBuf,
) -> anyhow::Result<()> {
deploy_with_kubeconfig(&FleetCrdsApp, context, kubeconfig).await
}
pub async fn provision_fleet_tenant_with_context(
context: harmony_app::Context,
tenant: harmony::topology::tenant::TenantConfig,
credential_store: harmony_app::OpenBaoClusterAccess,
) -> anyhow::Result<()> {
let app = FleetTenantProvisionApp::from_openbao(tenant, &credential_store).await?;
let app = FleetTenantProvisionApp::from_openbao(tenant, credential_store);
let contexts = harmony_app::ContextCatalog::new([context])?;
harmony_cli::app::app_main(app, contexts).await
}
pub async fn provision_fleet_tenant_with_kubeconfig(
context: harmony_app::Context,
kubeconfig: std::path::PathBuf,
tenant: harmony::topology::tenant::TenantConfig,
credential_store: harmony_app::OpenBaoClusterAccess,
allow_insecure_source: bool,
) -> anyhow::Result<()> {
let mut app = FleetTenantProvisionApp::from_openbao(tenant, credential_store);
if allow_insecure_source {
app = app.allow_insecure_source();
}
deploy_with_kubeconfig(&app, context, kubeconfig).await
}
async fn deploy_with_kubeconfig(
app: &dyn harmony_app::HarmonyApp<harmony::topology::K8sAnywhereTopology>,
context: harmony_app::Context,
kubeconfig: std::path::PathBuf,
) -> anyhow::Result<()> {
harmony_cli::cli_logger::init();
let ctx = harmony_app::AppContext::from_kubeconfig(&context, "bootstrap", kubeconfig)?;
harmony_app::deploy(app, ctx.topology(), &ctx).await?;
Ok(())
}

114
fleet/harmony-fleet-deploy/src/operator/chart.rs
View File

@@ -67,8 +67,7 @@ pub struct ChartOptions {
/// at `…/harmony-fleet-operator-chart:<tag>` matching the image tag.
pub chart_version: Option<String>,
/// JSON of the dashboard's `ZitadelAuthConfig`, stored in the
/// operator Secret under [`ENV_WEB_AUTH_CONFIG`]. `None` leaves the
/// dashboard unauthenticated (dev/e2e).
/// operator Secret under [`ENV_WEB_AUTH_CONFIG`].
pub web_auth_config_json: Option<String>,
/// JSON of the dashboard's `OperatorCookieKey`, stored under
/// [`ENV_WEB_COOKIE_KEY`].
@@ -76,6 +75,7 @@ pub struct ChartOptions {
pub identity: Option<OperatorIdentityRefs>,
pub identity_version: Option<String>,
pub image_pull_secret: Option<K8sName>,
pub device_groups: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
@@ -110,6 +110,7 @@ impl Default for ChartOptions {
identity: None,
identity_version: None,
image_pull_secret: None,
device_groups: None,
}
}
}
@@ -183,12 +184,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
/// (with the JSON keyfile inlined under `key_json`). Returns `None`
/// when no credentials are configured (no-auth dev mode).
pub fn operator_secret(opts: &ChartOptions) -> Option<Secret> {
let creds = opts.credentials.as_ref()?;
if opts.credentials.is_none()
&& opts.web_auth_config_json.is_none()
&& opts.web_cookie_key_json.is_none()
{
return None;
}
let mut data: BTreeMap<String, ByteString> = BTreeMap::new();
data.insert(
SECRET_KEY_CREDENTIALS_TOML.to_string(),
ByteString(creds.credentials_toml.as_bytes().to_vec()),
);
if let Some(creds) = &opts.credentials {
data.insert(
SECRET_KEY_CREDENTIALS_TOML.to_string(),
ByteString(creds.credentials_toml.as_bytes().to_vec()),
);
}
// Dashboard auth config + cookie key (when configured) ride in the
// same Secret; the Deployment sources them as HARMONY_CONFIG_* env
// for the operator's ConfigClient.
@@ -282,7 +290,7 @@ fn role() -> Role {
// Device liveness: the device-status reconciler patches the
// status subresource — a distinct RBAC resource from `devices`.
PolicyRule {
api_groups: Some(vec![group]),
api_groups: Some(vec![group.clone()]),
resources: Some(vec!["devices/status".to_string()]),
verbs: vec!["get", "update", "patch"]
.into_iter()
@@ -290,6 +298,24 @@ fn role() -> Role {
.collect(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec![group.clone()]),
resources: Some(vec!["taskruns".to_string()]),
verbs: vec!["get", "list", "watch", "patch"]
.into_iter()
.map(String::from)
.collect(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec![group]),
resources: Some(vec!["taskruns/status".to_string()]),
verbs: vec!["get", "update", "patch"]
.into_iter()
.map(String::from)
.collect(),
..Default::default()
},
]),
}
}
@@ -456,8 +482,7 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
env.push(secret_env(ENV_WEB_AUTH_CONFIG));
env.push(secret_env(ENV_WEB_COOKIE_KEY));
// Secret-grant sync (OpenBao) + the device-group scheduling gate
// (Zitadel role grants) — ADR-025. All optional: absent, the
// operator logs and runs ungated/without grant sync.
// (Zitadel role grants) — ADR-025. Missing group data fails closed.
for name in [
"OPENBAO_URL",
"OPENBAO_TOKEN",
@@ -467,6 +492,13 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
] {
env.push(secret_env(name));
}
if let Some(groups) = &opts.device_groups {
env.push(EnvVar {
name: "FLEET_DEVICE_GROUPS".to_string(),
value: Some(groups.clone()),
..Default::default()
});
}
// Namespace deliberately omitted — same rationale as the
// ServiceAccount: helm fills in the release namespace at install
@@ -641,6 +673,27 @@ mod tests {
};
assert!(grants_patch("deployments/status"));
assert!(grants_patch("devices/status"));
assert!(grants_patch("taskruns/status"));
}
#[test]
fn role_grants_only_required_taskrun_access() {
let rules = role().rules.unwrap();
assert!(rules.iter().any(|rule| {
rule.resources.as_deref() == Some(&["taskruns".to_string()])
&& rule.verbs == ["get", "list", "watch", "patch"].map(String::from)
}));
assert!(rules.iter().any(|rule| {
rule.resources.as_deref() == Some(&["taskruns/status".to_string()])
&& rule.verbs == ["get", "update", "patch"].map(String::from)
}));
assert!(rules.iter().all(|rule| {
!rule.resources.as_ref().is_some_and(|resources| {
resources.iter().any(|resource| {
resource.contains("schedule") || resource == "taskruns/finalizers"
})
})
}));
}
#[test]
@@ -699,6 +752,32 @@ mod tests {
);
}
#[test]
fn deployment_injects_static_device_groups() {
let deployment = operator_deployment(&ChartOptions {
device_groups: Some("pi-01=edge-a".to_string()),
..Default::default()
});
let env = deployment
.spec
.unwrap()
.template
.spec
.unwrap()
.containers
.into_iter()
.next()
.unwrap()
.env
.unwrap();
assert_eq!(
env.iter()
.find(|env| env.name == "FLEET_DEVICE_GROUPS")
.and_then(|env| env.value.as_deref()),
Some("pi-01=edge-a")
);
}
// The chart bakes these env names at publish time; the operator's
// ConfigClient derives them from the struct names at runtime. Lock
// them together so a rename can't silently desync the two.
@@ -715,4 +794,19 @@ mod tests {
format!("HARMONY_CONFIG_{}", OperatorCookieKey::KEY)
);
}
#[test]
fn web_auth_does_not_require_static_nats_credentials() {
let secret = operator_secret(&ChartOptions {
web_auth_config_json: Some("auth".to_string()),
web_cookie_key_json: Some("cookie".to_string()),
..Default::default()
})
.expect("web auth requires an operator Secret");
let data = secret.data.unwrap();
assert_eq!(data[ENV_WEB_AUTH_CONFIG].0, b"auth");
assert_eq!(data[ENV_WEB_COOKIE_KEY].0, b"cookie");
assert!(!data.contains_key(SECRET_KEY_CREDENTIALS_TOML));
}
}

342
fleet/harmony-fleet-deploy/src/operator/score.rs
View File

@@ -41,7 +41,7 @@ use harmony::modules::nats::NatsClientRef;
use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, Topology};
use harmony_fleet_operator::{Deployment, Device};
use harmony_fleet_operator::{Deployment, Device, TaskRun};
use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
@@ -56,7 +56,7 @@ use crate::operator::chart::{
pub struct FleetCrdsScore;
fn fleet_crds() -> Vec<CustomResourceDefinition> {
vec![Deployment::crd(), Device::crd()]
vec![Deployment::crd(), Device::crd(), TaskRun::crd()]
}
impl<T: Topology + K8sclient> Score<T> for FleetCrdsScore {
@@ -163,19 +163,12 @@ pub struct FleetOperatorScore {
/// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no
/// host) serves plain HTTP — the right default on issuer-less k3d.
pub cluster_issuer: Option<String>,
/// Dashboard SSO config + cookie key, baked into the operator Secret
/// for the pod's `ConfigClient` to read. `None` leaves the dashboard
/// unauthenticated (dev/e2e).
pub web_auth: Option<WebAuth>,
/// Dashboard Web PKCE application used to derive SSO configuration.
pub web_auth: Option<OidcApplicationRef>,
pub identity: Option<OperatorIdentityRefs>,
pub image_pull_secret: Option<K8sName>,
}
/// The dashboard's auth inputs the operator reads via `ConfigClient`.
#[derive(Debug, Clone, Serialize)]
pub struct WebAuth {
pub config: harmony_zitadel_auth::ZitadelAuthConfig,
pub cookie: harmony_zitadel_auth::OperatorCookieKey,
/// Static `device=group|group;...` membership for dev and tests.
pub device_groups: Option<String>,
}
impl FleetOperatorScore {
@@ -198,6 +191,7 @@ impl FleetOperatorScore {
web_auth: None,
identity: None,
image_pull_secret: None,
device_groups: None,
}
}
@@ -210,14 +204,9 @@ impl FleetOperatorScore {
self
}
/// Configure dashboard SSO: the `ZitadelAuthConfig` + cookie key are
/// baked into the operator Secret for the pod's `ConfigClient`.
pub fn web_auth(
mut self,
config: harmony_zitadel_auth::ZitadelAuthConfig,
cookie: harmony_zitadel_auth::OperatorCookieKey,
) -> Self {
self.web_auth = Some(WebAuth { config, cookie });
/// Configure dashboard SSO from the declared identity and Ingress.
pub fn web_auth(mut self, application: &OidcApplicationRef) -> Self {
self.web_auth = Some(application.clone());
self
}
@@ -260,6 +249,11 @@ impl FleetOperatorScore {
self
}
pub fn device_groups(mut self, groups: impl Into<String>) -> Self {
self.device_groups = Some(groups.into());
self
}
/// Set the operator's NATS auth-callout credentials (zitadel-jwt
/// `[credentials]` TOML). Applied as the operator Secret before the
/// helm install — including on the published-chart (CD) path.
@@ -296,9 +290,11 @@ pub struct FleetOperatorInterpret {
async fn smoke_test_operator<T: K8sclient>(
namespace: &str,
expected_config_hash: &str,
require_dashboard: bool,
topology: &T,
) -> Result<(), InterpretError> {
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
let mut last_observation = "no operator pod observed".to_string();
tokio::time::timeout(Duration::from_secs(180), async {
loop {
let pods = k8s
@@ -310,32 +306,62 @@ async fn smoke_test_operator<T: K8sclient>(
),
)
.await;
if let Ok(pods) = pods {
for pod in pods.items {
let has_expected_config = pod
.metadata
.annotations
.as_ref()
.and_then(|annotations| {
annotations.get("harmony.nationtech.io/config-hash")
})
.is_some_and(|hash| hash == expected_config_hash);
if let Some(name) = pod.metadata.name
&& has_expected_config
&& k8s
.pod_logs(namespace, &name, Some(100))
.await
.is_ok_and(|logs| logs.contains("KV bucket ready"))
{
return;
match pods {
Ok(pods) if pods.items.is_empty() => {
last_observation = "no operator pod matched the release label".to_string();
}
Ok(pods) => {
for pod in pods.items {
let name = pod.metadata.name.unwrap_or_default();
let has_expected_config = pod
.metadata
.annotations
.as_ref()
.and_then(|annotations| {
annotations.get("harmony.nationtech.io/config-hash")
})
.is_some_and(|hash| hash == expected_config_hash);
if !has_expected_config {
last_observation = format!("pod {name} has stale configuration");
continue;
}
match k8s.pod_logs(namespace, &name, Some(100)).await {
Ok(logs)
if logs.contains("KV bucket ready")
&& (!require_dashboard
|| logs.contains(
"fleet operator web frontend listening",
)) =>
{
return;
}
Ok(logs) => {
last_observation = if !logs.contains("KV bucket ready") {
format!("pod {name} logs lack `KV bucket ready`")
} else {
format!(
"pod {name} logs lack `fleet operator web frontend listening`"
)
};
}
Err(error) => {
last_observation =
format!("reading pod {name} logs failed: {error}");
}
}
}
}
Err(error) => last_observation = format!("listing operator pods failed: {error}"),
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await
.map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string()))
.map_err(|_| {
InterpretError::new(format!(
"operator did not become ready: {last_observation}"
))
})
}
#[async_trait]
@@ -360,6 +386,14 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
self.score.namespace
)));
}
if let Some(web_auth) = &self.score.web_auth
&& web_auth.namespace() != self.score.namespace
{
return Err(InterpretError::new(format!(
"dashboard application output must be in namespace '{}'",
self.score.namespace
)));
}
let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
k8s.ensure_namespace(&self.score.namespace)
.await
@@ -369,99 +403,119 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
self.score.namespace
))
})?;
let identity_version = if let Some(identity) = &self.score.identity {
Some(
tokio::time::timeout(Duration::from_secs(180), async {
loop {
let application = k8s
.get_resource::<ConfigMap>(
identity.application.config_map_name(),
Some(identity.application.namespace()),
)
.await
.ok()
.flatten();
let machine = k8s
.get_resource::<Secret>(
identity.machine.secret_name(),
Some(identity.machine.namespace()),
)
.await
.ok()
.flatten();
if let (Some(application), Some(machine)) = (application, machine)
&& let Some(project_id) = application
.data
.as_ref()
.and_then(|data| data.get(identity.application.project_id_key()))
{
return format!(
let (identity_version, web_client_id) = if let Some(identity) = &self.score.identity {
let (version, client_id) = tokio::time::timeout(Duration::from_secs(180), async {
loop {
let application = k8s
.get_resource::<ConfigMap>(
identity.application.config_map_name(),
Some(identity.application.namespace()),
)
.await
.ok()
.flatten();
let machine = k8s
.get_resource::<Secret>(
identity.machine.secret_name(),
Some(identity.machine.namespace()),
)
.await
.ok()
.flatten();
let web_client_id = if let Some(web_auth) = &self.score.web_auth {
k8s.get_resource::<ConfigMap>(
web_auth.config_map_name(),
Some(web_auth.namespace()),
)
.await
.ok()
.flatten()
.and_then(|config_map| config_map.data)
.and_then(|data| data.get(web_auth.client_id_key()).cloned())
.map(Some)
} else {
Some(None)
};
if let (Some(application), Some(machine)) = (application, machine)
&& let Some(data) = application.data.as_ref()
&& let Some(project_id) = data.get(identity.application.project_id_key())
&& let Some(web_client_id) = web_client_id
{
return (
format!(
"{}:{}",
project_id,
machine.metadata.resource_version.unwrap_or_default()
);
}
tokio::time::sleep(Duration::from_secs(2)).await;
),
web_client_id,
);
}
})
.await
.map_err(|_| {
InterpretError::new("timed out waiting for operator identity refs".to_string())
})?,
)
tokio::time::sleep(Duration::from_secs(2)).await;
}
})
.await
.map_err(|_| {
InterpretError::new("timed out waiting for operator identity refs".to_string())
})?;
(Some(version), client_id)
} else {
None
(None, None)
};
let credentials = self.score.credentials.clone();
// Apply the credentials Secret BEFORE the helm install (the
// chart's Deployment references it via secretKeyRef). Applied
// directly, not via the chart — it's environment-specific. The
// published-chart CD path runs without credentials today, so
// this is a no-op there.
let (web_auth_config_json, web_cookie_key_json) = match &self.score.web_auth {
Some(w) => (
Some(serde_json::to_string(&w.config).map_err(|e| {
// Apply environment-specific credentials before Helm creates the pod.
// Keeping the Secret outside the chart avoids competing field owners.
let (web_auth_config_json, web_cookie_key_json) = if self.score.web_auth.is_some() {
let identity = self.score.identity.as_ref().ok_or_else(|| {
InterpretError::new("dashboard web auth requires an operator identity".to_string())
})?;
let host = self.score.operator_ui_host.as_ref().ok_or_else(|| {
InterpretError::new("dashboard web auth requires an Ingress".to_string())
})?;
let client_id = web_client_id.as_ref().expect("identity resolved above");
let scheme = if self.score.cluster_issuer.is_some() {
"https"
} else {
"http"
};
let base_url = format!("{scheme}://{host}");
let config = harmony_zitadel_auth::ZitadelAuthConfig {
zitadel_base: identity.provider.issuer(),
base_url: base_url.clone(),
client_id: client_id.clone(),
scope: "openid profile email".to_string(),
trusted_audiences: vec![client_id.clone()],
logout_redirect_uri: format!("{base_url}/"),
};
let existing_secret = k8s
.get_resource::<Secret>(chart::SECRET_NAME, Some(&self.score.namespace))
.await
.map_err(|e| InterpretError::new(format!("read operator Secret: {e}")))?;
let cookie = existing_secret
.as_ref()
.and_then(|secret| secret.data.as_ref())
.and_then(|data| data.get(chart::ENV_WEB_COOKIE_KEY))
.map(|value| {
serde_json::from_slice::<harmony_zitadel_auth::OperatorCookieKey>(&value.0)
.map_err(|e| {
InterpretError::new(format!("parse existing OperatorCookieKey: {e}"))
})
})
.transpose()?
.unwrap_or_else(harmony_zitadel_auth::OperatorCookieKey::generate);
(
Some(serde_json::to_string(&config).map_err(|e| {
InterpretError::new(format!("serialize ZitadelAuthConfig: {e}"))
})?),
Some(serde_json::to_string(&w.cookie).map_err(|e| {
Some(serde_json::to_string(&cookie).map_err(|e| {
InterpretError::new(format!("serialize OperatorCookieKey: {e}"))
})?),
),
None => (None, None),
)
} else {
(None, None)
};
let expected_config_hash = chart::config_hash(&ChartOptions {
credentials: credentials.clone(),
web_auth_config_json: web_auth_config_json.clone(),
web_cookie_key_json: web_cookie_key_json.clone(),
identity: self.score.identity.clone(),
identity_version: identity_version.clone(),
image_pull_secret: self.score.image_pull_secret.clone(),
..ChartOptions::default()
});
if let Some(creds) = &credentials
&& let Some(secret) = operator_secret(&ChartOptions {
credentials: Some(creds.clone()),
web_auth_config_json: web_auth_config_json.clone(),
web_cookie_key_json: web_cookie_key_json.clone(),
identity: self.score.identity.clone(),
identity_version: identity_version.clone(),
image_pull_secret: self.score.image_pull_secret.clone(),
..ChartOptions::default()
})
{
info!(
"Applying operator credentials Secret '{}' in {}",
chart::SECRET_NAME,
self.score.namespace
);
K8sResourceScore::single(secret, Some(self.score.namespace.clone()))
.interpret(inventory, topology)
.await?;
}
let tmp = tempfile::tempdir()
.map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?;
let chart_path = build_chart(&ChartOptions {
let chart_options = ChartOptions {
output_dir: tmp.path().to_path_buf(),
image: self.score.image.clone(),
image_pull_policy: self.score.image_pull_policy.clone(),
@@ -474,8 +528,22 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
identity: self.score.identity.clone(),
identity_version,
image_pull_secret: self.score.image_pull_secret.clone(),
})
.map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
device_groups: self.score.device_groups.clone(),
};
let expected_config_hash = chart::config_hash(&chart_options);
if let Some(secret) = operator_secret(&chart_options) {
info!(
"Applying operator credentials Secret '{}' in {}",
chart::SECRET_NAME,
self.score.namespace
);
K8sResourceScore::single(secret, Some(self.score.namespace.clone()))
.interpret(inventory, topology)
.await?;
}
let chart_path = build_chart(&chart_options)
.map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
let chart_path_str = chart_path
.to_str()
.ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?;
@@ -507,10 +575,6 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
))
})?;
if credentials.is_some() || self.score.identity.is_some() {
smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?;
}
// Expose the UI. Applied after the chart so the backing Service
// (shipped in the chart) exists. Skipped when no host is set —
// dev/e2e harnesses keep the operator cluster-internal.
@@ -545,6 +609,31 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
details.push(format!("operator UI: {scheme}://{host}"));
}
if credentials.is_some() || self.score.identity.is_some() {
smoke_test_operator(
&self.score.namespace,
&expected_config_hash,
self.score.web_auth.is_some(),
topology,
)
.await?;
}
info!(
r#"
===== FLEET VERIFICATION =====
kubectl -n {namespace} get \
pods,statefulsets,deployments,pvc,services,routes,certificates
kubectl -n {namespace} logs \
deploy/{release} --tail=200 | grep 'KV bucket ready'
kubectl -n {namespace} get resourcequota
=============================="#,
namespace = self.score.namespace,
release = self.score.release_name,
);
Ok(Outcome::success_with_details(helm_outcome.message, details))
}
@@ -577,7 +666,8 @@ mod tests {
#[test]
fn fleet_crds_are_namespaced_resources() {
let crds = fleet_crds();
assert_eq!(crds.len(), 2);
assert_eq!(crds.len(), 3);
assert!(crds.iter().any(|crd| crd.spec.names.kind == "TaskRun"));
for crd in crds {
assert_eq!(
crd.spec.scope,

4
fleet/harmony-fleet-e2e/Cargo.toml
View File

@@ -14,6 +14,10 @@ path = "src/lib.rs"
name = "ping"
path = "tests/ping.rs"
[[test]]
name = "exec"
path = "tests/exec.rs"
[[test]]
name = "operator"
path = "tests/operator.rs"

1
fleet/harmony-fleet-e2e/src/images.rs
View File

@@ -106,6 +106,7 @@ async fn build_and_load_binary_image(
format!(
r#"FROM docker.io/library/archlinux:base
COPY target/release/{crate_name} /usr/local/bin/{crate_name}
RUN if [ "{crate_name}" = "harmony-fleet-agent" ]; then install -d -o 65532 -g 65532 -m 0700 /run/harmony-fleet-agent; fi
USER 65532:65532
ENTRYPOINT ["/usr/local/bin/{crate_name}"]
"#

10
fleet/harmony-fleet-e2e/src/stack.rs
View File

@@ -4,7 +4,7 @@
//! same `*Score` types production uses. The only thing this module
//! owns is the *plumbing* around them: ensure a k3d cluster exists,
//! sideload the agent image into it, mint a per-bring-up namespace,
//! compose [`NatsScore`] + [`FleetAgentScore`] against the test
//! compose [`NatsScore`] + [`FleetOperatorScore`] + [`FleetAgentScore`] against the test
//! Topology, wait for pods to be Ready, and hand the test an admin
//! NATS client.
//!
@@ -85,6 +85,7 @@ pub struct StackOptions {
pub log_level: String,
pub deploy_agent: bool,
pub deploy_operator: bool,
pub device_groups: Option<String>,
}
impl Default for StackOptions {
@@ -96,7 +97,8 @@ impl Default for StackOptions {
operator_image: OPERATOR_IMAGE_TAG.to_string(),
log_level: "info".to_string(),
deploy_agent: true,
deploy_operator: false,
deploy_operator: true,
device_groups: None,
}
}
}
@@ -107,6 +109,7 @@ impl StackOptions {
pub fn infra_only() -> Self {
Self {
num_devices: 0,
deploy_operator: false,
..Self::default()
}
}
@@ -313,6 +316,9 @@ impl Stack {
.namespace(namespace.clone())
.messaging(&nats_ref)
.log_level(opts.log_level.clone());
if let Some(groups) = &opts.device_groups {
operator = operator.device_groups(groups);
}
if let Some(handles) = callout_handles.as_ref() {
operator = operator.identity(

39
fleet/harmony-fleet-e2e/tests/exec.rs Normal file
View File

@@ -0,0 +1,39 @@
//! Operator-to-agent exec over Core NATS request/reply.
use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_fleet_operator::commands::{CommandError, FleetCommandsClient};
const E2E_ENV: &str = "HARMONY_FLEET_E2E";
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn operator_can_execute_a_bounded_agent_command() -> anyhow::Result<()> {
if !matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true")) {
eprintln!("skipping {E2E_ENV}-gated exec e2e test");
return Ok(());
}
let stack = shared_stack(StackOptions::default()).await?;
let client = FleetCommandsClient::new(stack.nats_client.clone());
let reply = client
.exec(
&stack.device_ids[0],
"printf stdout; printf stderr >&2; exit 7",
)
.await?;
assert_eq!(reply.exit_code, 7);
assert_eq!(reply.stdout, "stdout");
assert_eq!(reply.stderr, "stderr");
assert!(!reply.truncated);
let delayed = client
.exec(&stack.device_ids[0], "sleep 11; printf delayed")
.await?;
assert_eq!(delayed.stdout, "delayed");
assert!(matches!(
client.exec("missing-device", "true").await,
Err(CommandError::DeviceOffline)
));
Ok(())
}

10
fleet/harmony-fleet-e2e/tests/openbao_groups.rs
View File

@@ -109,7 +109,7 @@ async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> {
FLEET_NS.to_string(),
);
grants
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()], vec![])])
.await?;
// Device in edge-a: login binds the group, secret readable, batch token.
@@ -126,12 +126,14 @@ async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> {
// Detach binds at request time: the in-hand token loses access with
// no re-login involved.
grants.set_deployment_groups(&[(dn("web"), vec![])]).await?;
grants
.set_deployment_groups(&[(dn("web"), vec![], vec![])])
.await?;
bao.assert_denied(&login_a.token, "web/config").await?;
// Re-attach binds at request time too.
grants
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()], vec![])])
.await?;
bao.assert_can_read(&login_a.token, "web/config").await?;
@@ -169,6 +171,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
openshift: false,
tls_issuer: None,
node_port: None,
create_namespace: true,
}),
Box::new(OpenbaoSetupScore {
instance: instance.clone(),
@@ -188,6 +191,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
max_ttl: "1h".to_string(),
}),
oidc_application: None,
endpoint: None,
}),
];
for score in scores {

2
fleet/harmony-fleet-e2e/tests/openbao_policy.rs
View File

@@ -90,6 +90,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
openshift: false,
tls_issuer: None,
node_port: None,
create_namespace: true,
}),
Box::new(OpenbaoSetupScore {
instance: instance.clone(),
@@ -105,6 +106,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
}],
jwt_auth: None,
oidc_application: None,
endpoint: None,
}),
];

79
fleet/harmony-fleet-e2e/tests/operator.rs
View File

@@ -11,7 +11,7 @@ use k8s_openapi::api::authorization::v1::{
use k8s_openapi::api::core::v1::Namespace;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client;
use kube::api::{Api, DeleteParams, ObjectMeta, PostParams};
use kube::api::{Api, DeleteParams, ObjectMeta, Patch, PatchParams, PostParams};
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -51,9 +51,13 @@ async fn operator_writes_desired_state_for_matching_device() -> anyhow::Result<(
let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
create_device(&devices, "desired-state-device").await?;
create_device(&devices, "unauthorized-device").await?;
create_fleet_deployment(&deployments, "desired-state-test").await?;
wait_for_desired_state_entry(&stack, "desired-state-device", "desired-state-test", true)
.await?;
tokio::time::sleep(Duration::from_secs(2)).await;
wait_for_desired_state_entry(&stack, "unauthorized-device", "desired-state-test", false)
.await?;
Ok(())
}
@@ -85,6 +89,40 @@ async fn operator_deletes_desired_state_when_deployment_is_deleted() -> anyhow::
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn operator_deletes_desired_state_when_selector_stops_matching() -> anyhow::Result<()> {
if !e2e_enabled() {
skip_e2e();
return Ok(());
}
let stack = operator_stack().await?;
let client = Client::try_default().await?;
let devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
create_device(&devices, "retarget-device").await?;
create_fleet_deployment(&deployments, "retarget-test").await?;
wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", true).await?;
deployments
.patch(
"retarget-test",
&PatchParams::default(),
&Patch::Merge(serde_json::json!({
"spec": {
"targetSelector": {
"matchLabels": { "device-id": "missing" }
}
}
})),
)
.await?;
wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", false).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
if !e2e_enabled() {
@@ -121,15 +159,12 @@ async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
create_device(&devices, "other-tenant-device").await?;
create_fleet_deployment(&deployments, "other-tenant-deployment").await?;
let sentinel = format!(
"isolation-sentinel-{}",
&uuid::Uuid::new_v4().simple().to_string()[..8]
);
let sentinel = "isolation-sentinel";
let tenant_devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
let tenant_deployments: Api<Deployment> = Api::namespaced(client.clone(), &stack.namespace);
create_device(&tenant_devices, &sentinel).await?;
create_fleet_deployment(&tenant_deployments, &sentinel).await?;
wait_for_desired_state_entry(&stack, &sentinel, &sentinel, true).await?;
create_device(&tenant_devices, sentinel).await?;
create_fleet_deployment(&tenant_deployments, sentinel).await?;
wait_for_desired_state_entry(&stack, sentinel, sentinel, true).await?;
let deployment = deployments.get("other-tenant-deployment").await?;
assert!(
@@ -173,10 +208,10 @@ async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
.await?;
assert!(!review.status.is_some_and(|status| status.allowed));
tenant_deployments
.delete(&sentinel, &DeleteParams::default())
.delete(sentinel, &DeleteParams::default())
.await?;
tenant_devices
.delete(&sentinel, &DeleteParams::default())
.delete(sentinel, &DeleteParams::default())
.await?;
namespaces
.delete(&namespace, &DeleteParams::default())
@@ -196,6 +231,17 @@ async fn operator_stack() -> anyhow::Result<Arc<harmony_fleet_e2e::Stack>> {
let stack = shared_stack(StackOptions {
deploy_agent: false,
deploy_operator: true,
device_groups: Some(
[
"desired-state-device",
"cleanup-device",
"retarget-device",
"other-tenant-device",
"isolation-sentinel",
]
.map(|device| format!("{device}=e2e"))
.join(";"),
),
..StackOptions::default()
})
.await?;
@@ -219,7 +265,14 @@ async fn fleet_deployments(namespace: &str) -> anyhow::Result<Api<Deployment>> {
}
async fn create_device(devices: &Api<Device>, name: &str) -> anyhow::Result<()> {
let device = Device::new(name, DeviceSpec { inventory: None });
let device = Device::new(
name,
DeviceSpec {
inventory: None,
updater: None,
agent_upgrade: None,
},
);
devices.create(&PostParams::default(), &device).await?;
Ok(())
}
@@ -231,12 +284,14 @@ async fn create_fleet_deployment(deployments: &Api<Deployment>, name: &str) -> a
..Default::default()
},
spec: DeploymentSpec {
allowed_groups: None,
allowed_groups: vec!["e2e".to_string()],
target_selector: LabelSelector::default(),
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: "hello".to_string(),
image: "docker.io/library/hello-world:latest".to_string(),
image_pull_secret: None,
ports: vec![],
env: vec![],
secret_env: vec![],

2
fleet/harmony-fleet-e2e/tests/ping.rs
View File

@@ -64,7 +64,7 @@ async fn operator_can_ping_agent() -> anyhow::Result<()> {
);
assert!(
!reply.agent_version.is_empty(),
"agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)"
"agent_version must be non-empty"
);
Ok(())
}

34
fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs
View File

@@ -91,6 +91,18 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
"sudo -iu fleet-agent podman ps must show our service, got:\n{}",
ps.stdout,
);
let init = device
.ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.State.Status}} {{.State.ExitCode}}'")
.await?
.into_successful()
.map_err(|e| anyhow::anyhow!("init container inspect failed: {e}"))?;
assert_eq!(init.stdout.trim(), "exited 0");
let init_id = device
.ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.Id}}'")
.await?
.into_successful()
.map_err(|e| anyhow::anyhow!("init container ID inspect failed: {e}"))?
.stdout;
// ---- phase 2: upgrade ----
tracing::info!(
@@ -132,6 +144,13 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
let upgraded_init_id = device
.ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.Id}}'")
.await?
.into_successful()
.map_err(|e| anyhow::anyhow!("upgraded init container ID inspect failed: {e}"))?
.stdout;
assert_ne!(init_id.trim(), upgraded_init_id.trim());
// ---- phase 3: delete ----
tracing::info!(
@@ -162,8 +181,8 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
.into_successful()
.map_err(|e| anyhow::anyhow!("final podman ps failed: {e}"))?;
assert!(
!ps_final.stdout.contains("hello-web-svc"),
"container hello-web-svc still present after delete:\n{}",
!ps_final.stdout.contains("hello-web-svc") && !ps_final.stdout.contains("hello-web-init"),
"deployment containers still present after delete:\n{}",
ps_final.stdout,
);
@@ -172,6 +191,16 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
fn podman_score(image_tag: &str) -> PodmanV0Score {
PodmanV0Score {
init_container: Some(PodmanService {
name: "hello-web-init".to_string(),
image: "docker.io/library/hello-world:latest".to_string(),
image_pull_secret: None,
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::No,
}),
services: vec![PodmanService {
name: "hello-web-svc".to_string(),
// Pin upstream to docker.io/library so the VM doesn't
@@ -179,6 +208,7 @@ fn podman_score(image_tag: &str) -> PodmanV0Score {
// image ships with. nginx:alpine is multi-arch and the
// smallest battle-tested long-running ARM image.
image: format!("docker.io/library/{image_tag}"),
image_pull_secret: None,
ports: vec![],
env: vec![],
secret_env: vec![],

2
fleet/harmony-fleet-e2e/tests/vm_isolation.rs
View File

@@ -83,9 +83,11 @@ async fn agent_ignores_other_devices_keys() -> anyhow::Result<()> {
let admin = AdminKv::connect(&stack.infra.nats_client).await?;
let score = PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: "intruder-svc".to_string(),
image: "docker.io/library/nginx:alpine".to_string(),
image_pull_secret: None,
ports: vec![],
env: vec![],
secret_env: vec![],

2
fleet/harmony-fleet-e2e/tests/vm_ping.rs
View File

@@ -61,7 +61,7 @@ async fn agent_on_vm_replies_to_ping() -> anyhow::Result<()> {
);
assert!(
!reply.agent_version.is_empty(),
"agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)",
"agent_version must be non-empty",
);
Ok(())
}

6
fleet/harmony-fleet-e2e/tests/vm_secret_access.rs
View File

@@ -59,9 +59,11 @@ fn callout_stack_options() -> VmStackOptions {
fn secret_score(service: &str, spec_marker: &str) -> PodmanV0Score {
PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: service.to_string(),
image: "docker.io/library/nginx:alpine".to_string(),
image_pull_secret: None,
ports: vec![],
// Changing the marker changes the serialized spec, which
// is how the tests force a re-apply (and with it a fresh
@@ -186,7 +188,7 @@ async fn device_reads_deployment_secret_via_sso_groups() -> anyhow::Result<()> {
bao.put_secret(&deployment, "db_password", &secret_value)
.await?;
bao.grants()
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()], vec![])])
.await?;
admin
@@ -256,7 +258,7 @@ async fn removed_policy_denies_secret_fetch() -> anyhow::Result<()> {
bao.put_secret(&deployment, "db_password", "soon-revoked")
.await?;
bao.grants()
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()], vec![])])
.await?;
// Sanity: with the grant in place the deployment converges.

2
fleet/harmony-fleet-operator/Cargo.toml
View File

@@ -36,8 +36,8 @@ futures-util = { workspace = true }
thiserror.workspace = true
async-trait.workspace = true
url.workspace = true
base64.workspace = true
reqwest.workspace = true
uuid.workspace = true
axum = { version = "0.8", optional = true }
axum-extra = { version = "0.10", features = ["cookie", "cookie-private"], optional = true }

39
fleet/harmony-fleet-operator/README.md
View File

@@ -33,17 +33,11 @@ SSR + HTMX), so the runtime/macro footprint was dead weight. Maud is a
compile-time HTML macro that produces a `Markup` value — smaller dep tree,
faster compiles, same Rust-flavored ergonomics.
**Why HTMX + xterm.js for interactivity?** A real terminal needs xterm.js in
the browser regardless; once that JS exists, HTMX (~14 KB) is a rounding
error and lets every other interaction stay declarative in markup
(`hx-post`, `hx-target`, `hx-swap`).
**Why everything bundled?** The operator already ships as a single
container. Tailwind CSS, HTMX, and the HTMX SSE extension are all embedded
via `include_bytes!` so air-gapped clusters get the dashboard with nothing
extra to mount. The only build-time external is the standalone `tailwindcss`
v4 CLI — missing-CLI degrades gracefully (warning + empty embedded CSS); the
dev workflow uses `--css-from` instead anyway.
container. Tailwind CSS, HTMX, and the small CSRF helper are embedded so
air-gapped clusters need nothing extra to mount. The only build-time external
is the standalone `tailwindcss` v4 CLI. A missing CLI produces a warning and
empty embedded CSS; local development uses `--css-from` instead.
### Running it locally (mock data, no NATS, no kube)
@@ -76,9 +70,8 @@ Open <http://localhost:18080>.
`--mock` uses [`MockFleetService`](src/service/mock.rs), an in-memory
seeded dataset (10 fake devices in mixed states, 4 deployments). You can
click "Blacklist" on a row and the row will swap in place to reflect the
new status — this exercises the same `FleetService` API the real impl
will satisfy. No NATS, no Kubernetes cluster needed.
blacklist a device and see its updated detail page. This exercises the same
`FleetService` API as production without NATS or a Kubernetes cluster.
#### Iteration cost
@@ -120,21 +113,21 @@ fleet/harmony-fleet-operator/
│ ├── assets.rs ← embedded Tailwind/HTMX bytes
│ └── views/
│ ├── dashboard.rs
│ ├── devices.rs ← also exposes `row()` for HTMX swaps
│ ├── devices.rs
│ └── deployments.rs
├── style/
│ └── input.css ← Tailwind v4 entry point
└── vendor/
├── htmx.min.js ← HTMX v2.0.9
└── htmx-ext-sse.js ← SSE extension (used by future log-tail views)
├── app.js ← CSRF header helper
└── htmx.min.js ← HTMX v2.0.9
```
### What's deferred
- **Real `FleetService` impl** (wraps the kube client + NATS KV the
reconcilers already use). `serve-web` without `--mock` currently errors
out.
- **Zitadel SSO + admin-role check.** v1 assumes an oauth2-proxy fronts the
dashboard at the cluster edge.
- **Live log tail** (SSE-based, HTMX `sse-swap`) — the wiring is in place.
- **Interactive shell** (xterm.js + axum WS + portable-pty) — separate design.
- **`fleet-admin` authorization.** Zitadel login is implemented, but any
authenticated tenant user can currently reach privileged dashboard routes.
- **Live log tail.** Add a typed, authorized agent transport before exposing it.
- **Device commands.** Define a bounded typed protocol and authorization before
adding command controls.
- **Persistent alert state and receivers.** Per-alert acknowledgement currently
lives in operator memory; receiver configuration remains deployment-time.

36
fleet/harmony-fleet-operator/src/access.rs
View File

@@ -12,6 +12,8 @@ pub struct StaticDeviceGroups {
}
impl StaticDeviceGroups {
/// Parse `device=group|group;...`. The reserved `*` device supplies
/// membership to every device in local test and load-test setups.
pub fn parse(spec: &str) -> Self {
let map = spec
.split(';')
@@ -44,7 +46,7 @@ impl DeviceGroupSource for StaticDeviceGroups {
#[derive(Debug, Default)]
pub struct InMemoryDeploymentSecretGrants {
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
grants: Mutex<BTreeMap<DeploymentName, (BTreeSet<String>, BTreeSet<String>)>>,
}
impl InMemoryDeploymentSecretGrants {
@@ -57,7 +59,16 @@ impl InMemoryDeploymentSecretGrants {
.lock()
.unwrap()
.get(deployment)
.map(|groups| groups.iter().cloned().collect())
.map(|(groups, _)| groups.iter().cloned().collect())
.unwrap_or_default()
}
pub fn pull_secrets_for(&self, deployment: &DeploymentName) -> Vec<String> {
self.grants
.lock()
.unwrap()
.get(deployment)
.map(|(_, references)| references.iter().cloned().collect())
.unwrap_or_default()
}
}
@@ -66,14 +77,20 @@ impl InMemoryDeploymentSecretGrants {
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
grants: &[(DeploymentName, Vec<String>, Vec<String>)],
) -> Result<(), SecretAccessError> {
let mut current = self.grants.lock().unwrap();
for (deployment, groups) in grants {
for (deployment, groups, image_pull_secrets) in grants {
if groups.is_empty() {
current.remove(deployment);
} else {
current.insert(deployment.clone(), groups.iter().cloned().collect());
current.insert(
deployment.clone(),
(
groups.iter().cloned().collect(),
image_pull_secrets.iter().cloned().collect(),
),
);
}
}
Ok(())
@@ -86,10 +103,11 @@ mod tests {
#[tokio::test]
async fn static_groups_parse_membership() {
let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b; *=load ;;junk");
let groups = source.device_groups().await.unwrap();
assert_eq!(groups.len(), 2);
assert_eq!(groups.len(), 3);
assert_eq!(groups["pi-01"].len(), 2);
assert!(groups["*"].contains("load"));
}
#[tokio::test]
@@ -97,12 +115,12 @@ mod tests {
let grants = InMemoryDeploymentSecretGrants::new();
let deployment = DeploymentName::try_new("web").unwrap();
grants
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()])])
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()], vec![])])
.await
.unwrap();
assert_eq!(grants.groups_for(&deployment), vec!["edge-a"]);
grants
.set_deployment_groups(&[(deployment.clone(), vec![])])
.set_deployment_groups(&[(deployment.clone(), vec![], vec![])])
.await
.unwrap();
assert!(grants.groups_for(&deployment).is_empty());

218
fleet/harmony-fleet-operator/src/agent_upgrade.rs Normal file
View File

@@ -0,0 +1,218 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use harmony_reconciler_contracts::{
AgentUpgradeAttempt, AgentUpgradeStatus, BUCKET_AGENT_UPGRADE_INTENT,
BUCKET_AGENT_UPGRADE_STATUS, Id, agent_upgrade_intent_key, agent_upgrade_status_key,
};
use kube::api::{Api, ListParams, Patch, PatchParams};
use kube::{Client, ResourceExt};
use serde_json::json;
use crate::crd::{
AgentUpgradeTarget, Device, DeviceUpgradeJournalRef, DeviceUpgradeStatus,
DeviceUpgradeTransition,
};
pub async fn run(
client: Client,
namespace: &str,
jetstream: async_nats::jetstream::Context,
) -> Result<()> {
let intents = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_AGENT_UPGRADE_INTENT.into(),
..Default::default()
})
.await?;
let statuses = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_AGENT_UPGRADE_STATUS.into(),
..Default::default()
})
.await?;
let devices: Api<Device> = Api::namespaced(client, namespace);
// TODO: Replace full-device polling with watch-driven work before fleet scale.
let mut ticker = tokio::time::interval(Duration::from_secs(2));
loop {
ticker.tick().await;
for device in devices.list(&ListParams::default()).await?.items {
if let Err(error) = reconcile_device(&devices, &intents, &statuses, device).await {
tracing::warn!(device = %error.0, error = %error.1, "agent upgrade reconcile failed");
}
}
}
}
async fn reconcile_device(
devices: &Api<Device>,
intents: &async_nats::jetstream::kv::Store,
statuses: &async_nats::jetstream::kv::Store,
device: Device,
) -> std::result::Result<(), (String, anyhow::Error)> {
let id = device.name_any();
reconcile_device_inner(devices, intents, statuses, device)
.await
.map_err(|error| (id, error))
}
async fn reconcile_device_inner(
devices: &Api<Device>,
intents: &async_nats::jetstream::kv::Store,
statuses: &async_nats::jetstream::kv::Store,
device: Device,
) -> Result<()> {
let id = device.name_any();
let status = statuses
.get(agent_upgrade_status_key(&id))
.await?
.map(|bytes| serde_json::from_slice::<AgentUpgradeStatus>(&bytes))
.transpose()?;
if let Some(status) = status.as_ref() {
let phase = serde_json::to_value(status.phase)?
.as_str()
.context("upgrade phase did not serialize as a string")?
.to_string();
let reflected = DeviceUpgradeStatus {
attempt_id: status.attempt_id.clone(),
target_version: status.target_version.clone(),
phase,
started_at: status.started_at.to_rfc3339(),
updated_at: status.updated_at.to_rfc3339(),
reason: status.reason.map(|reason| {
serde_json::to_value(reason)
.expect("upgrade reason is serializable")
.as_str()
.expect("upgrade reason serializes as a string")
.to_string()
}),
detail: status.detail.clone(),
error: status.error.clone(),
drain_duration_ms: status.drain_duration_ms,
boot_id: status.boot_id.clone(),
invocation_id: status.invocation_id.clone(),
journal: status
.journal
.as_ref()
.map(|journal| DeviceUpgradeJournalRef {
unit: journal.unit.clone(),
since: journal.since.to_rfc3339(),
}),
transitions: status
.transitions
.iter()
.map(|transition| DeviceUpgradeTransition {
phase: serde_json::to_value(transition.phase)
.expect("upgrade phase is serializable")
.as_str()
.expect("upgrade phase serializes as a string")
.to_string(),
entered_at: transition.entered_at.to_rfc3339(),
exited_at: transition.exited_at.map(|at| at.to_rfc3339()),
duration_ms: transition.duration_ms,
})
.collect(),
};
if device
.status
.as_ref()
.and_then(|value| value.agent_upgrade.as_ref())
!= Some(&reflected)
{
devices
.patch_status(
&id,
&PatchParams::default(),
&Patch::Merge(&json!({ "status": { "agentUpgrade": reflected } })),
)
.await?;
}
}
let Some(target) = device.spec.agent_upgrade.as_ref() else {
return Ok(());
};
let current_version = device
.status
.as_ref()
.and_then(|status| status.current_version.as_deref())
.context("device has not reported an agent version")?;
if current_version == target.version {
return Ok(());
}
let intent_key = agent_upgrade_intent_key(&id);
let existing_entry = intents.entry(&intent_key).await?;
let existing = existing_entry
.as_ref()
.filter(|entry| entry.operation == async_nats::jetstream::kv::Operation::Put)
.map(|entry| serde_json::from_slice::<AgentUpgradeAttempt>(&entry.value))
.transpose()?;
match existing {
Some(attempt) if attempt_matches(&attempt, target) => {}
Some(attempt)
if status.as_ref().is_some_and(|status| {
status.attempt_id == attempt.attempt_id && !status.phase.is_terminal()
}) =>
{
bail!("attempt '{}' is still active", attempt.attempt_id)
}
_ => {
let attempt = AgentUpgradeAttempt {
attempt_id: uuid::Uuid::new_v4().to_string(),
device_id: Id::from(id.clone()),
target_version: target.version.clone(),
architecture: target.architecture.clone(),
artifact_url: target.artifact_url.clone(),
max_bytes: target.max_bytes,
sha256: target.sha256.clone(),
};
let value = serde_json::to_vec(&attempt)?.into();
if let Some(entry) = existing_entry {
intents.update(&intent_key, value, entry.revision).await?;
} else {
intents.create(&intent_key, value).await?;
}
}
}
Ok(())
}
fn attempt_matches(attempt: &AgentUpgradeAttempt, target: &AgentUpgradeTarget) -> bool {
attempt.target_version == target.version
&& attempt.architecture == target.architecture
&& attempt.artifact_url == target.artifact_url
&& attempt.max_bytes == target.max_bytes
&& attempt.sha256 == target.sha256
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn release_metadata_must_match_the_existing_attempt() {
let target = AgentUpgradeTarget {
version: "0.2.0".into(),
architecture: "aarch64".into(),
artifact_url: "https://example.invalid/agent".into(),
max_bytes: 10,
sha256: "digest".into(),
};
let attempt = AgentUpgradeAttempt {
attempt_id: uuid::Uuid::new_v4().to_string(),
device_id: Id::from("device"),
target_version: target.version.clone(),
architecture: target.architecture.clone(),
artifact_url: target.artifact_url.clone(),
max_bytes: target.max_bytes,
sha256: target.sha256.clone(),
};
assert!(attempt_matches(&attempt, &target));
let mut changed = target;
changed.artifact_url = "https://example.invalid/repacked-agent".into();
assert!(!attempt_matches(&attempt, &changed));
}
}

27
fleet/harmony-fleet-operator/src/commands.rs
View File

@@ -15,12 +15,15 @@ use std::time::Duration;
use async_nats::Client;
use async_nats::error::Error as NatsError;
use harmony_reconciler_contracts::{PingReply, Verb, device_command_subject};
use harmony_reconciler_contracts::{
CommandRequest, ExecReply, HDR_REQUEST_ID, PingReply, Verb, device_command_subject,
};
/// Default reply timeout for a single-shot command. 5 s is plenty for
/// a healthy device on a LAN; offline devices get short-circuited
/// earlier by NATS's `no_responders` reply.
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
pub const EXEC_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Debug, thiserror::Error)]
pub enum CommandError {
@@ -76,6 +79,28 @@ impl FleetCommandsClient {
let reply: PingReply = serde_json::from_slice(&resp.payload)?;
Ok(reply)
}
pub async fn exec(&self, device_id: &str, command: &str) -> Result<ExecReply, CommandError> {
let subject = device_command_subject(device_id, Verb::Exec);
let payload = serde_json::to_vec(&CommandRequest::Exec {
command: command.to_string(),
})
.expect("command request is serializable");
let mut headers = async_nats::HeaderMap::new();
headers.insert(HDR_REQUEST_ID, uuid::Uuid::new_v4().to_string());
let response = self
.nc
.send_request(
subject,
async_nats::Request::new()
.headers(headers)
.payload(payload.into())
.timeout(Some(EXEC_TIMEOUT)),
)
.await
.map_err(|error| map_request_error(error, EXEC_TIMEOUT))?;
Ok(serde_json::from_slice(&response.payload)?)
}
}
/// Map an async-nats `RequestError` to our typed surface. The `kind`

30
fleet/harmony-fleet-operator/src/controller.rs
View File

@@ -34,6 +34,7 @@ use kube::runtime::watcher::Config as WatcherConfig;
use kube::{Api, Client, ResourceExt};
use harmony_fleet_operator::Deployment;
use harmony_fleet_operator::rollout;
const FINALIZER: &str = "fleet.nationtech.io/finalizer";
@@ -52,11 +53,21 @@ pub enum Error {
pub struct Context {
pub client: Client,
pub kv: Store,
pub rollout_plans: Store,
}
pub async fn run(client: Client, namespace: &str, kv: Store) -> anyhow::Result<()> {
pub async fn run(
client: Client,
namespace: &str,
kv: Store,
rollout_plans: Store,
) -> anyhow::Result<()> {
let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
let ctx = Arc::new(Context { client, kv });
let ctx = Arc::new(Context {
client,
kv,
rollout_plans,
});
tracing::info!("starting Deployment controller");
kube::runtime::controller::Controller::new(api, WatcherConfig::default())
@@ -87,7 +98,7 @@ async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Er
// its own kube watch and writes KV entries for matching
// devices. Long requeue so we're not pointlessly polling.
FinalizerEvent::Apply(_) => Ok(Action::requeue(Duration::from_secs(300))),
FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv).await,
FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv, &ctx.rollout_plans).await,
}
})
.await
@@ -101,11 +112,17 @@ async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Er
})
}
async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
async fn cleanup(obj: Arc<Deployment>, kv: &Store, plans: &Store) -> Result<Action, Error> {
let name = obj.name_any();
let deployment_name =
DeploymentName::try_new(&name).map_err(|e| Error::InvalidName(name, e.to_string()))?;
let suffix = format!(".{}", deployment_name.as_str());
let plan_key = obj.metadata.uid.as_deref().map(rollout::deployment_key);
if let Some(key) = plan_key.as_deref() {
rollout::close(plans, key)
.await
.map_err(|error| Error::Kv(format!("closing rollout plan: {error}")))?;
}
let mut removed = 0u64;
let mut keys = kv
@@ -121,6 +138,11 @@ async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
removed += 1;
}
}
if let Some(key) = plan_key.as_deref() {
rollout::delete(plans, key)
.await
.map_err(|error| Error::Kv(format!("deleting rollout plan: {error}")))?;
}
tracing::info!(
deployment = %deployment_name,
removed,

163
fleet/harmony-fleet-operator/src/crd.rs
View File

@@ -1,4 +1,4 @@
use harmony_reconciler_contracts::InventorySnapshot;
use harmony_reconciler_contracts::{InventorySnapshot, UpdaterCapabilities};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::CustomResource;
use schemars::JsonSchema;
@@ -10,7 +10,7 @@ pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileSc
/// to the pattern K8s itself uses for DaemonSet nodeSelector, Service
/// pod selector, etc. The operator resolves the selector against
/// `Device` CRs at reconcile time; no list of device ids on spec.
#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[derive(CustomResource, Serialize, Clone, Debug, JsonSchema)]
#[kube(
group = "fleet.nationtech.io",
version = "v1alpha1",
@@ -22,21 +22,41 @@ pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileSc
)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentSpec {
/// Device groups allowed to *view* this deployment — run it and
/// read its secrets (ADR-025). Membership is admin-managed identity
/// (Zitadel role grants), never device-reported labels: labels can
/// narrow placement below, but only groups grant. Absent = no group
/// gating and no secret access — the selector alone places a
/// secretless workload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub allowed_groups: Option<Vec<String>>,
/// Device groups allowed to run this deployment and read its secrets
/// (ADR-025). Membership is admin-managed identity, never
/// device-reported labels. Empty means no device is authorized.
pub allowed_groups: Vec<String>,
/// Which devices this deployment targets, *within* the allowed
/// groups. Matches against `Device.metadata.labels`.
pub target_selector: LabelSelector,
pub score: ReconcileScore,
/// Retained for existing manifests. Rollouts begin immediately, with
/// `canary=true` targets gated before the remaining frozen targets.
pub rollout: Rollout,
}
impl<'de> Deserialize<'de> for DeploymentSpec {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct StoredSpec {
#[serde(default)]
allowed_groups: Vec<String>,
target_selector: LabelSelector,
score: ReconcileScore,
rollout: Rollout,
}
let stored = StoredSpec::deserialize(deserializer)?;
Ok(Self {
allowed_groups: stored.allowed_groups,
target_selector: stored.target_selector,
score: stored.score,
rollout: stored.rollout,
})
}
}
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
pub struct Rollout {
pub strategy: RolloutStrategy,
@@ -50,6 +70,9 @@ pub enum RolloutStrategy {
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentStatus {
/// Exact `<metadata.uid>:<metadata.generation>` represented by `aggregate`.
#[serde(skip_serializing_if = "Option::is_none")]
pub rollout_revision: Option<String>,
/// Per-deployment rollup. Present once the aggregator has
/// evaluated the selector at least once.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -60,21 +83,18 @@ pub struct DeploymentStatus {
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentAggregate {
/// How many Device CRs currently match `spec.targetSelector`.
/// The three phase counters below sum to this; targeted-but-
/// unreported devices are folded into `pending`.
/// Number of devices in the frozen rollout plan. The three phase counters
/// below sum to this; unreleased devices and devices without state are
/// folded into `pending`.
pub matched_device_count: u32,
pub succeeded: u32,
pub failed: u32,
pub pending: u32,
/// Device id of the most recent device reporting a failure, with
/// its short error message. Cleared when that device transitions
/// back to Running.
#[serde(skip_serializing_if = "Option::is_none")]
/// Most recent reported failure details, when available.
pub last_error: Option<AggregateLastError>,
}
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct AggregateLastError {
pub device_id: String,
@@ -82,6 +102,16 @@ pub struct AggregateLastError {
pub at: String,
}
impl Deployment {
pub fn rollout_revision(&self) -> Option<String> {
self.metadata
.uid
.as_deref()
.zip(self.metadata.generation)
.map(|(uid, generation)| format!("{uid}:{generation}"))
}
}
/// A physical/virtual device registered with a tenant's fleet.
///
/// Created by the operator from `DeviceInfo` entries in the NATS
@@ -90,11 +120,8 @@ pub struct AggregateLastError {
/// reflects it here.
///
/// `metadata.labels` carries the device's routing labels. `spec.
/// inventory` holds the hardware/OS snapshot. No status subresource
/// today — liveness is queried from the NATS `device-heartbeat`
/// bucket directly; when a CR-side reflection (Reachable / Stale
/// conditions) becomes useful, it'll land with its own reconciler
/// rather than sitting here as speculative surface.
/// inventory` holds the hardware/OS snapshot. The status subresource
/// reflects heartbeat liveness and the running agent version.
#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[kube(
group = "fleet.nationtech.io",
@@ -111,20 +138,70 @@ pub struct DeviceSpec {
/// Rarely changes after first publish.
#[serde(skip_serializing_if = "Option::is_none")]
pub inventory: Option<InventorySnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub updater: Option<UpdaterCapabilities>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub agent_upgrade: Option<AgentUpgradeTarget>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct AgentUpgradeTarget {
pub version: String,
pub architecture: String,
pub artifact_url: String,
pub max_bytes: u64,
pub sha256: String,
}
/// Operator-maintained liveness reflection of the NATS
/// `device-heartbeat` bucket onto the CR, so `kubectl get devices` and
/// the dashboard see reachability without reading NATS. Written by the
/// device-status reconciler.
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeviceStatus {
/// RFC 3339 timestamp of the last heartbeat seen. `None` until the
/// device has pinged at least once.
#[serde(skip_serializing_if = "Option::is_none")]
pub last_heartbeat: Option<String>,
pub reachability: Reachability,
pub current_version: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub agent_upgrade: Option<DeviceUpgradeStatus>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeviceUpgradeStatus {
pub attempt_id: String,
pub target_version: String,
pub phase: String,
pub started_at: String,
pub updated_at: String,
pub reason: Option<String>,
pub detail: Option<String>,
pub error: Option<String>,
pub drain_duration_ms: Option<u64>,
pub boot_id: Option<String>,
pub invocation_id: Option<String>,
pub journal: Option<DeviceUpgradeJournalRef>,
pub transitions: Vec<DeviceUpgradeTransition>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeviceUpgradeTransition {
pub phase: String,
pub entered_at: String,
pub exited_at: Option<String>,
pub duration_ms: Option<u64>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeviceUpgradeJournalRef {
pub unit: String,
pub since: String,
}
/// Coarse liveness derived from heartbeat freshness. Failing/Pending
@@ -145,10 +222,44 @@ pub enum Reachability {
mod tests {
use kube::CustomResourceExt;
use super::Device;
use super::{Deployment, DeploymentAggregate, Device};
#[test]
fn device_is_namespaced() {
assert_eq!(Device::crd().spec.scope, "Namespaced");
}
#[test]
fn deployment_requires_allowed_groups() {
let crd = serde_json::to_value(Deployment::crd()).unwrap();
let required = crd
.pointer("/spec/versions/0/schema/openAPIV3Schema/properties/spec/required")
.and_then(serde_json::Value::as_array)
.unwrap();
assert!(required.iter().any(|field| field == "allowedGroups"));
}
#[test]
fn legacy_deployment_without_allowed_groups_fails_closed() {
let deployment: Deployment = serde_json::from_value(serde_json::json!({
"apiVersion": "fleet.nationtech.io/v1alpha1",
"kind": "Deployment",
"metadata": { "name": "legacy" },
"spec": {
"targetSelector": {},
"score": { "type": "PodmanV0", "data": { "services": [] } },
"rollout": { "strategy": "Immediate" }
}
}))
.unwrap();
assert!(deployment.spec.allowed_groups.is_empty());
}
#[test]
fn aggregate_serializes_cleared_error() {
let value = serde_json::to_value(DeploymentAggregate::default()).unwrap();
assert_eq!(value["lastError"], serde_json::Value::Null);
}
}

49
fleet/harmony-fleet-operator/src/device_reconciler.rs
View File

@@ -13,7 +13,7 @@
use anyhow::Result;
use async_nats::jetstream::kv::{Operation, Store};
use futures_util::StreamExt;
use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo};
use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo, device_info_key};
use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams};
use std::collections::BTreeMap;
@@ -62,6 +62,10 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()>
continue;
}
};
if !info_key_matches(&entry.key, &info) {
tracing::warn!(key = %entry.key, device = %info.device_id, "device-reconciler: key does not match payload device_id");
continue;
}
if let Err(e) = upsert_device(&devices, namespace, &info).await {
tracing::warn!(
device = %info.device_id,
@@ -85,12 +89,7 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()>
async fn upsert_device(api: &Api<Device>, namespace: &str, info: &DeviceInfo) -> Result<()> {
let name = info.device_id.to_string();
let mut device = Device::new(
&name,
DeviceSpec {
inventory: info.inventory.clone(),
},
);
let mut device = device_from_info(info);
device.metadata.namespace = Some(namespace.to_string());
device.metadata.labels = Some(clean_labels(&info.labels));
@@ -104,6 +103,21 @@ async fn upsert_device(api: &Api<Device>, namespace: &str, info: &DeviceInfo) ->
Ok(())
}
fn device_from_info(info: &DeviceInfo) -> Device {
Device::new(
&info.device_id.to_string(),
DeviceSpec {
inventory: info.inventory.clone(),
updater: info.updater.clone(),
agent_upgrade: None,
},
)
}
fn info_key_matches(key: &str, info: &DeviceInfo) -> bool {
key == device_info_key(&info.device_id.to_string())
}
async fn delete_device(api: &Api<Device>, name: &str) -> Result<()> {
match api.delete(name, &DeleteParams::default()).await {
Ok(_) => {
@@ -149,6 +163,9 @@ fn is_label_value(s: &str) -> bool {
#[cfg(test)]
mod tests {
use chrono::Utc;
use harmony_reconciler_contracts::{Id, UpdaterCapabilities};
use super::*;
#[test]
@@ -167,4 +184,22 @@ mod tests {
assert!(!is_label_value("has space"));
assert!(!is_label_value(&"x".repeat(64)));
}
#[test]
fn device_info_identity_and_capability_are_reflected() {
let info = DeviceInfo {
device_id: Id::from("device-1"),
labels: BTreeMap::new(),
inventory: None,
updater: Some(UpdaterCapabilities {
protocol: 1,
apt_full_upgrade_v1: true,
}),
updated_at: Utc::now(),
};
assert!(info_key_matches("info.device-1", &info));
assert!(!info_key_matches("info.device-2", &info));
assert_eq!(device_from_info(&info).spec.updater, info.updater);
}
}

175
fleet/harmony-fleet-operator/src/device_status.rs
View File

@@ -14,7 +14,7 @@
use std::collections::HashMap;
use std::time::Duration;
use anyhow::Result;
use anyhow::{Context, Result, bail};
use async_nats::jetstream::kv::{Operation, Store};
use chrono::{DateTime, Utc};
use futures_util::StreamExt;
@@ -24,7 +24,54 @@ use kube::{Client, ResourceExt};
use serde_json::json;
use tokio::sync::Mutex;
use crate::crd::{Device, Reachability};
use crate::crd::{Device, DeviceStatus, Reachability};
#[derive(Clone, Debug, PartialEq, Eq)]
struct ObservedHeartbeat {
received_at: DateTime<Utc>,
agent_version: Option<String>,
}
fn heartbeat_status(heartbeat: Option<ObservedHeartbeat>, now: DateTime<Utc>) -> DeviceStatus {
match heartbeat {
Some(heartbeat) => DeviceStatus {
last_heartbeat: Some(heartbeat.received_at.to_rfc3339()),
reachability: reachability(heartbeat.received_at, now),
current_version: heartbeat.agent_version,
agent_upgrade: None,
},
None => DeviceStatus {
last_heartbeat: None,
reachability: Reachability::Unknown,
current_version: None,
agent_upgrade: None,
},
}
}
fn observe_heartbeat(
key: &str,
payload: &[u8],
received_at: DateTime<Utc>,
) -> Result<(String, ObservedHeartbeat)> {
let device_id = key
.strip_prefix("heartbeat.")
.context("heartbeat key has no heartbeat. prefix")?;
let heartbeat: HeartbeatPayload = serde_json::from_slice(payload)?;
if heartbeat.device_id.to_string() != device_id {
bail!(
"heartbeat payload device {} does not match authorized key {device_id}",
heartbeat.device_id
);
}
Ok((
device_id.to_string(),
ObservedHeartbeat {
received_at,
agent_version: heartbeat.agent_version,
},
))
}
/// A device with no heartbeat within this window is `Stale`. Agents
/// ping every 30 s, so this tolerates ~2 missed pings.
@@ -44,7 +91,7 @@ pub async fn run(
})
.await?;
let heartbeats: Mutex<HashMap<String, DateTime<Utc>>> = Mutex::new(HashMap::new());
let heartbeats: Mutex<HashMap<String, Option<ObservedHeartbeat>>> = Mutex::new(HashMap::new());
let devices: Api<Device> = Api::namespaced(client, namespace);
tokio::try_join!(
@@ -56,7 +103,7 @@ pub async fn run(
async fn watch_heartbeats(
bucket: &Store,
heartbeats: &Mutex<HashMap<String, DateTime<Utc>>>,
heartbeats: &Mutex<HashMap<String, Option<ObservedHeartbeat>>>,
) -> Result<()> {
let mut watch = bucket.watch_with_history(">").await?;
tracing::info!("device-status: watching device-heartbeat KV");
@@ -70,46 +117,63 @@ async fn watch_heartbeats(
};
match entry.operation {
Operation::Put => {
if let Ok(hb) = serde_json::from_slice::<HeartbeatPayload>(&entry.value) {
heartbeats
.lock()
.await
.insert(hb.device_id.to_string(), hb.at);
let Some(received_at) = DateTime::from_timestamp(
entry.created.unix_timestamp(),
entry.created.nanosecond(),
) else {
tracing::warn!(key = %entry.key, "device-status: invalid NATS timestamp");
continue;
};
match observe_heartbeat(&entry.key, &entry.value, received_at) {
Ok((device_id, heartbeat)) => {
heartbeats.lock().await.insert(device_id, Some(heartbeat));
}
Err(error) => {
tracing::warn!(key = %entry.key, %error, "device-status: invalid heartbeat")
}
}
}
Operation::Delete | Operation::Purge => {
if let Some(id) = entry.key.strip_prefix("heartbeat.") {
heartbeats.lock().await.remove(id);
heartbeats.lock().await.insert(id.to_string(), None);
}
}
}
}
Ok(())
bail!("device-status: heartbeat watch ended")
}
async fn patch_loop(
devices: &Api<Device>,
heartbeats: &Mutex<HashMap<String, DateTime<Utc>>>,
heartbeats: &Mutex<HashMap<String, Option<ObservedHeartbeat>>>,
) -> Result<()> {
// Last status written per device, to skip no-op patches.
let mut applied: HashMap<String, (Reachability, DateTime<Utc>)> = HashMap::new();
let mut applied: HashMap<String, DeviceStatus> = HashMap::new();
let mut ticker = tokio::time::interval(TICK);
loop {
ticker.tick().await;
let snapshot: Vec<(String, DateTime<Utc>)> = heartbeats
let snapshot: Vec<(String, Option<ObservedHeartbeat>)> = heartbeats
.lock()
.await
.iter()
.map(|(k, v)| (k.clone(), *v))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
let now = Utc::now();
for (id, at) in snapshot {
let reachability = reachability(at, now);
if applied.get(&id) == Some(&(reachability, at)) {
for (id, heartbeat) in snapshot {
let status = heartbeat_status(heartbeat, now);
if applied.get(&id) == Some(&status) {
continue;
}
if patch_status(devices, &id, reachability, at).await {
applied.insert(id, (reachability, at));
if patch_status(devices, &id, &status).await {
if status.reachability == Reachability::Unknown {
let mut heartbeats = heartbeats.lock().await;
if matches!(heartbeats.get(&id), Some(None)) {
heartbeats.remove(&id);
applied.remove(&id);
continue;
}
}
applied.insert(id, status);
}
}
}
@@ -128,29 +192,19 @@ fn reachability(last_heartbeat: DateTime<Utc>, now: DateTime<Utc>) -> Reachabili
/// Returns whether the patch succeeded (so we only cache applied state
/// on success and retry next tick otherwise).
async fn patch_status(
devices: &Api<Device>,
id: &str,
reachability: Reachability,
last_heartbeat: DateTime<Utc>,
) -> bool {
let status = json!({
"status": {
"lastHeartbeat": last_heartbeat.to_rfc3339(),
"reachability": reachability,
}
});
async fn patch_status(devices: &Api<Device>, id: &str, status: &DeviceStatus) -> bool {
let patch = json!({ "status": status });
match devices
.patch_status(id, &PatchParams::default(), &Patch::Merge(&status))
.patch_status(id, &PatchParams::default(), &Patch::Merge(&patch))
.await
{
Ok(d) => {
tracing::debug!(device = %d.name_any(), ?reachability, "device-status: patched");
tracing::debug!(device = %d.name_any(), reachability = ?status.reachability, "device-status: patched");
true
}
// A heartbeat can outrace the Device CR's creation by the
// device-reconciler; skip this tick and retry on the next.
Err(kube::Error::Api(ae)) if ae.code == 404 => false,
// Retry a heartbeat that outraced CR creation, but a tombstone
// for an absent Device is already converged.
Err(kube::Error::Api(ae)) if ae.code == 404 => status.reachability == Reachability::Unknown,
Err(e) => {
tracing::warn!(%id, error = %e, "device-status: patch failed");
false
@@ -161,6 +215,7 @@ async fn patch_status(
#[cfg(test)]
mod tests {
use super::*;
use harmony_reconciler_contracts::Id;
#[test]
fn reachable_within_window_stale_after() {
@@ -174,4 +229,50 @@ mod tests {
Reachability::Stale
);
}
#[test]
fn observation_uses_nats_time_and_authorized_key_identity() {
let device_time = Utc::now() - chrono::Duration::hours(2);
let server_time = Utc::now();
let payload = serde_json::to_vec(&HeartbeatPayload {
device_id: Id::from("device-1".to_string()),
at: device_time,
agent_version: Some("1.2.3".to_string()),
})
.unwrap();
let (device_id, observed) =
observe_heartbeat("heartbeat.device-1", &payload, server_time).unwrap();
assert_eq!(device_id, "device-1");
assert_eq!(observed.received_at, server_time);
assert_eq!(observed.agent_version.as_deref(), Some("1.2.3"));
assert_eq!(
reachability(observed.received_at, server_time),
Reachability::Reachable
);
assert!(observe_heartbeat("heartbeat.other-device", &payload, server_time).is_err());
}
#[test]
fn unknown_version_clears_previous_status_value() {
let status = DeviceStatus {
last_heartbeat: Some(Utc::now().to_rfc3339()),
reachability: Reachability::Reachable,
current_version: None,
agent_upgrade: None,
};
assert!(serde_json::to_value(status).unwrap()["currentVersion"].is_null());
}
#[test]
fn missing_heartbeat_clears_status() {
let status = heartbeat_status(None, Utc::now());
assert_eq!(status.reachability, Reachability::Unknown);
assert_eq!(status.last_heartbeat, None);
assert_eq!(status.current_version, None);
let json = serde_json::to_value(status).unwrap();
assert!(json["lastHeartbeat"].is_null());
assert!(json["currentVersion"].is_null());
}
}

1458
fleet/harmony-fleet-operator/src/fleet_aggregator.rs
View File

File diff suppressed because it is too large Load Diff

5
fleet/harmony-fleet-operator/src/frontend/assets.rs
View File

@@ -2,10 +2,9 @@
//!
//! Tailwind CSS is built by `build.rs` into `$OUT_DIR/tailwind.css`
//! (empty if the CLI was unavailable — dev uses `--css-from` instead).
//! HTMX and its SSE extension are vendored under `vendor/` so the
//! container ships with no external script dependencies.
//! HTMX is vendored under `vendor/` so the container ships with no
//! external script dependencies.
pub const TAILWIND_CSS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/tailwind.css"));
pub const HTMX_JS: &[u8] = include_bytes!("../../vendor/htmx.min.js");
pub const HTMX_SSE_JS: &[u8] = include_bytes!("../../vendor/htmx-ext-sse.js");
pub const APP_JS: &[u8] = include_bytes!("../../vendor/app.js");

38
fleet/harmony-fleet-operator/src/frontend/layout.rs
View File

@@ -8,7 +8,6 @@ const ICON_DASHBOARD: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="1
const ICON_DEVICES: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="3" width="20" height="14" rx="2"/><line x1="8" y1="21" x2="16" y2="21"/><line x1="12" y1="17" x2="12" y2="21"/></svg>"#;
const ICON_DEPLOY: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>"#;
const ICON_BELL: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M6 8a6 6 0 0 1 12 0c0 7 3 9 3 9H3s3-2 3-9"/><path d="M10.3 21a1.94 1.94 0 0 0 3.4 0"/></svg>"#;
const ICON_COG: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"/></svg>"#;
const ICON_LOGOUT: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M9 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h4"/><polyline points="16 17 21 12 16 7"/><line x1="21" y1="12" x2="9" y2="12"/></svg>"#;
const ICON_BRAND: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><path d="M4 4v16M20 4v16M4 12h16"/></svg>"#;
@@ -30,13 +29,12 @@ pub fn page(
title { (title) " — Harmony Fleet" }
link rel="stylesheet" href="/static/tailwind.css";
script src="/static/htmx.min.js" defer {}
script src="/static/htmx-ext-sse.js" defer {}
script src="/static/app.js" defer {}
@if live_reload {
script { (PreEscaped(LIVE_RELOAD_JS)) }
}
}
body class="min-h-screen" hx-ext="sse" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" {
body class="min-h-screen" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" {
div class="flex h-screen overflow-hidden" style="background:var(--bg)" {
(sidebar(current_path, session, unacked_alerts))
main class="flex-1 min-w-0 flex flex-col overflow-hidden" {
@@ -44,7 +42,6 @@ pub fn page(
div class="flex-1 overflow-y-auto grid-bg" { (content) }
}
}
div id="modal-root" {}
}
}
}
@@ -55,22 +52,21 @@ fn sidebar(
session: Option<&DashboardSession>,
unacked_alerts: usize,
) -> Markup {
let nav_items: [(&str, &str, &str, usize); 5] = [
let nav_items: [(&str, &str, &str, usize); 4] = [
("/", ICON_DASHBOARD, "Dashboard", 0),
("/devices", ICON_DEVICES, "Devices", 0),
("/deployments", ICON_DEPLOY, "Deployments", 0),
("/alerts", ICON_BELL, "Alerts", unacked_alerts),
("/settings", ICON_COG, "Settings", 0),
];
html! {
aside class="shrink-0 flex flex-col border-r w-[224px]" style="border-color:var(--border); background:var(--bg)" {
div class="flex items-center justify-between px-4 py-4 border-b" style="border-color:var(--border)" {
aside class="shrink-0 flex flex-col border-r w-14 sm:w-[224px]" style="border-color:var(--border); background:var(--bg)" {
div class="flex items-center justify-center sm:justify-between px-2 sm:px-4 py-4 border-b" style="border-color:var(--border)" {
div class="flex items-center gap-2" {
div class="relative w-6 h-6 rounded-md flex items-center justify-center" style="background:var(--accent); color:#0c0c0c" {
(PreEscaped(ICON_BRAND))
}
span class="text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" }
span class="hidden sm:inline text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" }
}
}
@@ -79,7 +75,9 @@ fn sidebar(
@let active = is_active(current_path, href);
a
href=(*href)
class={"group w-full flex items-center gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative "
title=(*label)
aria-label=(*label)
class={"group w-full flex items-center justify-center sm:justify-start gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative "
(if active { "text-slate-100 font-medium" } else { "text-slate-400 hover:text-slate-100" })}
style={(if active { "background:rgba(148,163,184,0.06)" } else { "background:transparent" })}
{
@@ -89,9 +87,9 @@ fn sidebar(
span class={(if active { "text-slate-100" } else { "text-slate-500 group-hover:text-slate-300" })} {
(PreEscaped(icon))
}
span class="flex-1 text-left" { (label) }
span class="hidden sm:inline flex-1 text-left" { (label) }
@if *badge > 0 {
span class="inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" {
span class="hidden sm:inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" {
(badge)
}
}
@@ -100,9 +98,12 @@ fn sidebar(
}
@if let Some(s) = session {
div class="border-t p-3" style="border-color:var(--border)" {
div class="hidden sm:block border-t p-3" style="border-color:var(--border)" {
(user_footer(s))
}
a href="/logout" class="sm:hidden border-t py-3 flex justify-center text-slate-500 hover:text-rose-400" style="border-color:var(--border)" title="Log out" {
(PreEscaped(ICON_LOGOUT))
}
}
}
}
@@ -170,14 +171,14 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup {
div class="flex items-center gap-2" {
div class="relative" {
input
class="input w-64"
class="hidden sm:block input w-40 lg:w-64"
type="text"
name="search"
placeholder="Search devices, deployments\u{2026}"
hx-get="/devices/search"
hx-get="/devices"
hx-trigger="keyup changed delay:300ms"
hx-target="#device-table-wrapper"
hx-swap="innerHTML";
hx-target="body"
hx-push-url="true";
}
a href="/alerts" class="relative btn btn-ghost py-1.5" {
(PreEscaped(ICON_BELL))
@@ -188,9 +189,6 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup {
}
}
}
a href="/settings" class="btn btn-ghost py-1.5" title="Settings" {
(PreEscaped(ICON_COG))
}
}
}
}

226
fleet/harmony-fleet-operator/src/frontend/server.rs
View File

@@ -19,17 +19,18 @@ use maud::Markup;
use serde::Deserialize;
use tokio_stream::StreamExt;
use super::assets::{APP_JS, HTMX_JS, HTMX_SSE_JS, TAILWIND_CSS};
use super::assets::{APP_JS, HTMX_JS, TAILWIND_CSS};
use super::layout::page;
use super::views::{
alerts as alerts_view, dashboard as dashboard_view, deployments as deployments_view,
devices as devices_view, settings as settings_view,
devices as devices_view,
};
use crate::frontend::auth::{self, DASHBOARD_SESSION_COOKIE, DashboardSession, JwksCache};
use crate::service::FleetService;
use harmony_zitadel_auth::ZitadelAuthConfig;
pub const DEFAULT_PORT: u16 = 18080;
const REQUIRED_ROLE: &str = "fleet-admin";
#[derive(Clone)]
pub struct AppState {
@@ -89,9 +90,9 @@ pub fn router(state: AppState) -> Router {
let public_routes = Router::new()
.route("/login", get(auth::login_handler))
.route("/auth/callback", get(auth::callback_handler))
.route("/logout", get(auth::logout_handler))
.route("/static/tailwind.css", get(tailwind_css))
.route("/static/htmx.min.js", get(htmx_js))
.route("/static/htmx-ext-sse.js", get(htmx_sse_js))
.route("/static/app.js", get(app_js));
let private_routes = Router::new()
@@ -99,10 +100,7 @@ pub fn router(state: AppState) -> Router {
.route("/", get(dashboard_handler))
// Devices
.route("/devices", get(devices_handler))
.route("/devices/search", get(devices_search_handler))
.route("/devices/{id}/blacklist", post(blacklist_handler))
.route("/devices/{id}/logs", get(device_logs_handler))
.route("/devices/{id}/logs/stream", get(device_logs_stream_handler))
.route("/devices/{id}/exec", post(device_exec_handler))
// Device detail
.route("/device/{id}", get(device_detail_handler))
@@ -112,11 +110,6 @@ pub fn router(state: AppState) -> Router {
// Alerts
.route("/alerts", get(alerts_handler))
.route("/alerts/{id}/ack", post(ack_alert_handler))
// Settings
.route("/settings", get(settings_handler))
.route("/settings/toggle/{key}", post(settings_toggle_handler))
// Logout
.route("/logout", get(auth::logout_handler))
.route_layer(middleware::from_fn_with_state(state.clone(), csrf_protect))
.route_layer(middleware::from_fn_with_state(state.clone(), require_auth));
@@ -145,6 +138,13 @@ async fn require_auth(
match state.jwks.verify(cookie.value(), &state.config).await {
Ok(session) => {
if !session.has_role(REQUIRED_ROLE) {
tracing::warn!(
roles = ?session.roles,
"dashboard access denied: missing {REQUIRED_ROLE} role"
);
return forbidden_response();
}
req.extensions_mut().insert(session);
next.run(req).await
}
@@ -156,6 +156,27 @@ async fn require_auth(
}
}
fn forbidden_response() -> Response {
let body = maud::html! {
(maud::DOCTYPE)
html lang="en" {
head {
meta charset="utf-8";
meta name="viewport" content="width=device-width, initial-scale=1";
title { "Access denied - Harmony Fleet" }
}
body {
main {
h1 { "Access denied" }
p { "The " code { (REQUIRED_ROLE) } " role is required to use this dashboard." }
p { "Ask your administrator to grant the role, or " a href="/logout" { "sign out" } "." }
}
}
}
};
(StatusCode::FORBIDDEN, body).into_response()
}
async fn csrf_protect(State(state): State<AppState>, req: Request<Body>, next: Next) -> Response {
if !is_mutating_method(req.method()) {
return next.run(req).await;
@@ -371,44 +392,11 @@ async fn devices_handler(
))
}
async fn devices_search_handler(
State(s): State<AppState>,
Query(q): Query<DevicesQuery>,
) -> Result<Markup, AppError> {
let status = q.status.as_deref().and_then(parse_device_status);
let devices = s
.fleet
.filtered_devices(
status,
q.deployment.clone(),
q.region.clone(),
q.search.clone(),
)
.await?;
Ok(devices_view::page(
&devices,
&[],
&[],
status,
q.deployment.as_deref(),
q.region.as_deref(),
q.search.as_deref(),
))
}
// ── Device detail ──────────────────────────────────────────────────────
#[derive(Deserialize, Default)]
struct DeviceDetailQuery {
tab: Option<String>,
}
async fn device_detail_handler(
State(s): State<AppState>,
Path(id): Path<String>,
Query(q): Query<DeviceDetailQuery>,
session: Option<Extension<DashboardSession>>,
) -> Result<Markup, AppError> {
let device = s
@@ -423,18 +411,6 @@ async fn device_detail_handler(
None
};
let tab = q.tab.as_deref().unwrap_or("overview");
// Tab click (HTMX): return the tabs block (bar + content) so the
// active highlight re-renders with the content.
if q.tab.is_some() {
return Ok(devices_view::device_tabs(
&device,
deployment_version.as_deref(),
tab,
));
}
let unacked = s
.fleet
.list_alerts()
@@ -453,6 +429,60 @@ async fn device_detail_handler(
))
}
#[derive(Deserialize)]
struct ExecForm {
command: String,
}
async fn device_exec_handler(
State(s): State<AppState>,
Path(id): Path<String>,
Extension(session): Extension<DashboardSession>,
Form(form): Form<ExecForm>,
) -> Result<Markup, AppError> {
if form.command.trim().is_empty() {
return Ok(devices_view::command_output(
&form.command,
"Command failed: command must not be empty",
));
}
if form.command.len() > 16 * 1024 {
return Ok(devices_view::command_output(
"",
"Command failed: command exceeds 16384 byte limit",
));
}
tracing::info!(
operator = %session.subject,
device = %id,
command_bytes = form.command.len(),
"dashboard device command requested"
);
let started = tokio::time::Instant::now();
let output = match s.fleet.run_command(&id, &form.command).await {
Ok(output) => {
tracing::info!(
operator = %session.subject,
device = %id,
duration_ms = started.elapsed().as_millis(),
"dashboard device command completed"
);
output
}
Err(error) => {
tracing::warn!(
operator = %session.subject,
device = %id,
duration_ms = started.elapsed().as_millis(),
%error,
"dashboard device command failed"
);
format!("Command failed: {error}")
}
};
Ok(devices_view::command_output(&form.command, &output))
}
// ── Deployments ────────────────────────────────────────────────────────
async fn deployments_handler(
@@ -483,6 +513,8 @@ async fn deployments_handler(
#[derive(Deserialize, Default)]
struct DeploymentQuery {
tab: Option<String>,
#[serde(default)]
summary: bool,
}
async fn deployment_handler(
@@ -497,6 +529,10 @@ async fn deployment_handler(
.await?
.ok_or_else(|| anyhow::anyhow!("deployment not found: {id}"))?;
if q.summary {
return Ok(deployments_view::summary(&deployment));
}
let devices = s.fleet.get_deployment_devices(&id).await?;
let tab = q.tab.as_deref().unwrap_or("overview");
let unacked = s
@@ -561,79 +597,14 @@ async fn ack_alert_handler(
Ok(alerts_view::alert_row(alert))
}
// ── Settings ───────────────────────────────────────────────────────────
async fn settings_handler(
State(s): State<AppState>,
session: Option<Extension<DashboardSession>>,
) -> Result<Markup, AppError> {
let unacked = s
.fleet
.list_alerts()
.await?
.iter()
.filter(|a| !a.acked)
.count();
Ok(page(
"Settings",
s.live_reload,
"/settings",
session.as_ref().map(|e| &e.0),
unacked,
settings_view::page(),
))
}
async fn settings_toggle_handler(Path(_key): Path<String>) -> Result<Markup, AppError> {
// In a real app this would toggle a notification channel.
// For the mock, we return the same static content.
Ok(settings_view::page())
}
// ── Device logs ────────────────────────────────────────────────────────
async fn device_logs_handler(Path(id): Path<String>) -> Result<Markup, AppError> {
Ok(devices_view::logs_modal(&id))
}
async fn device_logs_stream_handler(
Path(_id): Path<String>,
) -> Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>> {
// One honest notice, then keep-alive. Real agent-log streaming (over
// NATS) is pending — don't fabricate log lines.
let html = r#"<div class="px-0 py-px italic text-slate-600">— live agent log streaming is not implemented yet —</div>"#;
let stream = futures_util::stream::once(async move {
Ok::<_, Infallible>(Event::default().event("log").data(html))
});
Sse::new(stream).keep_alive(KeepAlive::new().interval(Duration::from_secs(15)))
}
// ── Run command ────────────────────────────────────────────────────────
#[derive(Deserialize)]
struct ExecForm {
command: String,
}
async fn device_exec_handler(
State(s): State<AppState>,
Path(id): Path<String>,
Form(form): Form<ExecForm>,
) -> Result<Markup, AppError> {
let output = s.fleet.run_command(&id, &form.command).await?;
Ok(devices_view::command_output(&form.command, &output))
}
// ── Blacklist ──────────────────────────────────────────────────────────
async fn blacklist_handler(
State(s): State<AppState>,
Path(id): Path<String>,
) -> Result<Markup, AppError> {
let updated = s.fleet.blacklist_device(&id).await?;
Ok(devices_view::row(&updated))
) -> Result<Response, AppError> {
s.fleet.blacklist_device(&id).await?;
Ok(Redirect::to(&format!("/device/{id}")).into_response())
}
// ── Helpers ────────────────────────────────────────────────────────────
@@ -670,13 +641,6 @@ async fn htmx_js() -> Response {
static_response(HTMX_JS.to_vec(), "application/javascript; charset=utf-8")
}
async fn htmx_sse_js() -> Response {
static_response(
HTMX_SSE_JS.to_vec(),
"application/javascript; charset=utf-8",
)
}
async fn app_js() -> Response {
static_response(APP_JS.to_vec(), "application/javascript; charset=utf-8")
}

2
fleet/harmony-fleet-operator/src/frontend/views/alerts.rs
View File

@@ -10,8 +10,6 @@ pub fn page(alerts: &[Alert]) -> Markup {
div class="flex items-center gap-2" {
h2 class="text-[15px] font-semibold text-slate-200" { "Alerts" }
span class="text-[11px] text-slate-500" { "\u{b7} " (unacked) " unacked" }
div class="flex-1" {}
button class="btn btn-ghost" { "Ack all" }
}
div class="card card-flush" {
table class="tbl" {

94
fleet/harmony-fleet-operator/src/frontend/views/deployments.rs
View File

@@ -74,6 +74,20 @@ fn deployment_card(d: &DeploymentDetail) -> Markup {
// ── Deployment detail page ─────────────────────────────────────────────
pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup {
html! {
div class="p-6 space-y-4" {
(summary(deployment))
// The whole tabs block re-renders on switch so the active
// highlight follows the content.
div id="dep-tabs" {
(tabs_and_content(deployment, devices, "overview"))
}
}
}
}
pub fn summary(deployment: &DeploymentDetail) -> Markup {
let pct = if deployment.target > 0 {
((deployment.healthy as f64 / deployment.target as f64) * 100.0).round() as u32
} else {
@@ -81,9 +95,13 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
};
html! {
div class="p-6 space-y-4" {
// Header
div class="card p-5" {
div
id="deployment-summary"
class="card p-5"
hx-get={"/deployment/" (&deployment.name) "?summary=true"}
hx-trigger="every 3s"
hx-target="this"
hx-swap="outerHTML" {
div class="flex items-start justify-between gap-6" {
div class="min-w-0" {
div class="flex items-center gap-3 flex-wrap" {
@@ -102,6 +120,18 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
}
}
@if let Some(error) = &deployment.last_error {
div class="mt-5 rounded border px-4 py-3" style="border-color:var(--bad); background:rgba(251,113,133,0.08)" {
div class="flex items-center gap-2 text-[11px] uppercase tracking-wider" style="color:var(--bad)" {
span { "Latest rollout error" }
span class="text-slate-600" { "\u{b7}" }
a href={"/device/" (&error.device)} class="font-mono normal-case tracking-normal hover:underline" { (&error.device) }
span class="text-slate-500 normal-case tracking-normal tabular-nums" { (&error.at) }
}
p class="mt-1 text-[13px] text-slate-200" { (&error.message) }
}
}
// Rollout progress
div class="mt-5" {
div {
@@ -140,13 +170,6 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
}
}
}
}
// The whole tabs block re-renders on switch so the active
// highlight follows the content.
div id="dep-tabs" {
(tabs_and_content(deployment, devices, "overview"))
}
}
}
}
@@ -211,7 +234,7 @@ fn devices_tab(devices: &[DeviceDetail]) -> Markup {
td { (badges::device_status(d.status)) }
td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } }
td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" {
@if let Some(inv) = &d.inventory { (&inv.agent_version) } @else { "\u{2014}" }
@if let Some(version) = &d.current_version { (version) } @else { "\u{2014}" }
} }
td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } }
}
@@ -251,7 +274,7 @@ fn per_device_grid(devices: &[DeviceDetail]) -> Markup {
html! {
div class="card" {
div class="flex items-center justify-between px-4 py-3 border-b" style="border-color:var(--border)" {
span class="section-title" { "Per-device rollout" }
span class="section-title" { "Device reachability" }
span class="text-[10px] text-slate-600 font-mono" { (devices.len()) " devices" }
}
div class="p-4 grid grid-cols-10 gap-1.5" {
@@ -266,10 +289,9 @@ fn per_device_grid(devices: &[DeviceDetail]) -> Markup {
}
}
div class="border-t px-4 py-2.5 flex items-center gap-3 text-[10px] text-slate-500 font-mono" style="border-color:var(--border)" {
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--ok)" {} " healthy" }
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--warn)" {} " pending" }
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--bad)" {} " failing" }
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--ok)" {} " reachable" }
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:rgba(251,113,133,0.6)" {} " stale" }
span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:#475569" {} " unknown or blacklisted" }
}
}
}
@@ -315,3 +337,45 @@ fn time_ago(minutes: i64) -> String {
format!("{}d ago", minutes / (60 * 24))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::service::{DeploymentError, DeploymentStatus};
#[test]
fn summary_polls_itself_and_shows_actionable_error() {
let deployment = DeploymentDetail {
name: "edge".into(),
rollout_revision: "deployment-uid:2".into(),
version: "1.2.3".into(),
status: DeploymentStatus::Failing,
target: 2,
healthy: 1,
failing: 1,
pending: 0,
updated_at: "2026-07-27 10:15".into(),
last_error: Some(DeploymentError {
device: "pi-09".into(),
message: "image pull denied".into(),
at: "2026-07-27T10:15:00Z".into(),
}),
};
let html = summary(&deployment).into_string();
assert!(html.contains("hx-get=\"/deployment/edge?summary=true\""));
assert!(html.contains("hx-trigger=\"every 3s\""));
assert!(html.contains("hx-swap=\"outerHTML\""));
assert!(html.contains("href=\"/device/pi-09\""));
assert!(html.contains("image pull denied"));
assert!(html.contains("2026-07-27T10:15:00Z"));
}
#[test]
fn overview_describes_device_reachability() {
let html = per_device_grid(&[]).into_string();
assert!(html.contains("Device reachability"));
assert!(html.contains("reachable"));
assert!(!html.contains("Per-device rollout"));
}
}

286
fleet/harmony-fleet-operator/src/frontend/views/devices.rs
View File

@@ -6,13 +6,8 @@ use crate::service::{DeviceDetail, DeviceStatus};
// ── Inline icons ────────────────────────────────────────────────────────
const ICON_SEARCH: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>"#;
const ICON_CHEVRON_DOWN: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg>"#;
const ICON_POWER: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18.36 6.64a9 9 0 1 1-12.73 0"/><line x1="12" y1="2" x2="12" y2="12"/></svg>"#;
const ICON_PAUSE: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="6" y="4" width="4" height="16"/><rect x="14" y="4" width="4" height="16"/></svg>"#;
const ICON_BAN: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="4.93" y1="4.93" x2="19.07" y2="19.07"/></svg>"#;
const ICON_EXPAND: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 3 21 3 21 9"/><polyline points="9 21 3 21 3 15"/><line x1="21" y1="3" x2="14" y2="10"/><line x1="3" y1="21" x2="10" y2="14"/></svg>"#;
const ICON_EXTERNAL: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"/><polyline points="15 3 21 3 21 9"/><line x1="10" y1="14" x2="21" y2="3"/></svg>"#;
const ICON_REFRESH: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10"/><path d="M20.49 15a9 9 0 0 1-14.85 3.36L1 14"/></svg>"#;
const ICON_COPY: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>"#;
// ── Devices list page ──────────────────────────────────────────────────
@@ -172,7 +167,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
div class="p-6 space-y-4" {
// Header
div class="card p-5" {
div class="flex items-start justify-between gap-6" {
div class="flex flex-col sm:flex-row sm:items-start justify-between gap-4 sm:gap-6" {
div class="min-w-0" {
div class="flex items-center gap-3 flex-wrap" {
h1 class="text-[22px] font-semibold font-mono text-slate-50 truncate whitespace-nowrap" { (&device.id) }
@@ -189,10 +184,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
span { span class="text-slate-600" { "Last ping" } " " span class="text-slate-300 tabular-nums" { (time_ago(device.minutes_ago)) } }
}
}
div class="flex items-center gap-2 shrink-0" {
button class="btn btn-ghost" { (PreEscaped(ICON_REFRESH)) " Reconcile" }
button class="btn btn-ghost" { (PreEscaped(ICON_POWER)) " Restart" }
button class="btn btn-ghost" { (PreEscaped(ICON_PAUSE)) " Suspend" }
div class="flex items-center gap-2 shrink-0 self-start" {
@if device.status != DeviceStatus::Blacklisted {
button
class="btn btn-danger"
@@ -206,80 +198,28 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
}
}
// The whole tabs block re-renders on switch so the active
// highlight follows (only the content swapping would leave it
// stuck on Overview).
div id="device-tabs" {
(device_tabs(device, deployment_version, "overview"))
}
}
}
}
/// Tab bar (active highlighted) + the active tab's content. Swapped as a
/// unit into `#device-tabs`.
pub fn device_tabs(
device: &DeviceDetail,
deployment_version: Option<&str>,
active: &str,
) -> Markup {
let content = match active {
"logs" => logs_tab(device),
"command" => command_tab(&device.id),
_ => overview_tab(device, deployment_version),
};
html! {
div class="flex items-center gap-1 border-b" style="border-color:var(--border)" {
(tab_button(&device.id, "Overview", "overview", active == "overview"))
(tab_button(&device.id, "Logs", "logs", active == "logs"))
(tab_button(&device.id, "Run command", "command", active == "command"))
div class="flex-1" {}
button
class="btn btn-ghost mb-1"
hx-get={"/devices/" (device.id) "/logs"}
hx-target="#modal-root"
hx-swap="innerHTML" {
(PreEscaped(ICON_EXPAND)) " Pop-out logs"
}
}
div { (content) }
}
}
fn tab_button(device_id: &str, label: &str, tab: &str, active: bool) -> Markup {
html! {
button
class={"px-3 py-2 text-[13px] font-medium relative "
(if active { "text-slate-100" } else { "text-slate-500 hover:text-slate-300" })}
hx-get={"/device/" (device_id) "?tab=" (tab)}
hx-target="#device-tabs"
hx-swap="innerHTML" {
(label)
@if active {
span class="absolute left-0 right-0 -bottom-px h-0.5" style="background:var(--accent)" {}
}
(overview_tab(device, deployment_version))
(command_panel(device))
}
}
}
fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup {
html! {
div class="grid grid-cols-12 gap-4" {
// Device info + current deployment
div class="col-span-12 lg:col-span-5 space-y-4" {
div class="space-y-4" {
div class="card p-5" {
div class="section-title mb-3" { "Device info" }
(definition("Device ID", &device.id, true, true))
(definition("Region", &device.region, true, false))
(definition("Last ping", &time_ago(device.minutes_ago), false, false))
(definition("Device ID", &device.id, true))
(definition("Region", &device.region, true))
(definition("Last ping", &time_ago(device.minutes_ago), false))
(definition("Agent", agent_version(device), true))
@if let Some(inv) = &device.inventory {
(definition("Hostname", &inv.hostname, true, false))
(definition("Arch", &inv.arch, true, false))
(definition("OS", &inv.os, false, false))
(definition("Kernel", &inv.kernel, true, false))
(definition("CPU cores", &inv.cpu_cores.to_string(), false, false))
(definition("Memory", &format!("{} MB", inv.memory_mb), false, false))
(definition("Agent", &inv.agent_version, true, false))
(definition("Hostname", &inv.hostname, true))
(definition("Arch", &inv.arch, true))
(definition("OS", &inv.os, false))
(definition("Kernel", &inv.kernel, true))
(definition("CPU cores", &inv.cpu_cores.to_string(), false))
(definition("Memory", &format!("{} MB", inv.memory_mb), false))
} @else {
div class="text-[12px] text-slate-500 mt-2" { "No inventory reported yet" }
}
@@ -310,181 +250,48 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark
div class="text-[12px] text-slate-500" { "No deployment assigned" }
}
}
}
// Recent logs (live stream)
div class="col-span-12 lg:col-span-7" {
div class="card overflow-hidden" {
div class="flex items-center justify-between px-4 py-3 border-b" style="border-color:var(--border)" {
span class="section-title" { "Recent logs" }
button
class="text-[11px] text-slate-400 hover:text-slate-100 flex items-center gap-1"
hx-get={"/devices/" (device.id) "/logs"}
hx-target="#modal-root"
hx-swap="innerHTML" {
"Pop out " (PreEscaped(ICON_EXTERNAL))
}
}
(log_stream(&device.id, "260px"))
}
}
}
}
}
fn logs_tab(device: &DeviceDetail) -> Markup {
fn command_panel(device: &DeviceDetail) -> Markup {
html! {
div class="card overflow-hidden mt-4" {
div class="flex items-center gap-2 px-4 py-2.5 border-b" style="border-color:var(--border)" {
span class="relative flex w-1.5 h-1.5" {
span class="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60" style="background:var(--accent)" {}
span class="relative inline-flex w-1.5 h-1.5 rounded-full" style="background:var(--accent)" {}
}
span class="text-[11px] font-mono text-slate-400" { "streaming" }
span class="text-[11px] text-slate-600 font-mono" { "\u{b7} live" }
}
(log_stream(&device.id, "520px"))
}
}
}
/// Shared SSE log console. The stream endpoint is the seam the real
/// agent-log transport plugs into; the markup is identical wherever a
/// device's logs are shown (overview card, logs tab, pop-out modal).
fn log_stream(device_id: &str, height: &str) -> Markup {
html! {
div
class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto"
style={"background:#050608; height:" (height)}
hx-ext="sse"
sse-connect={"/devices/" (device_id) "/logs/stream"}
sse-swap="log"
hx-swap="beforeend" {
div class="px-0 py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" }
}
}
}
/// One-shot "run a shell command on the device" panel. Submits the
/// command to the [`run_command`](crate::service::FleetService::run_command)
/// seam and appends the response to the console. Live streaming and a
/// full TTY are later refinements.
fn command_tab(device_id: &str) -> Markup {
html! {
div class="card overflow-hidden mt-4" {
div class="card p-5" {
div class="section-title mb-3" { "Run command" }
form
class="flex items-center gap-2 px-4 py-3 border-b"
style="border-color:var(--border)"
hx-post={"/devices/" (device_id) "/exec"}
hx-target="#exec-output"
hx-swap="beforeend"
"hx-on::after-request"="this.reset()" {
span class="font-mono text-slate-500 text-[13px]" { "$" }
class="flex gap-2"
hx-post={"/devices/" (device.id) "/exec"}
hx-target="#command-output"
hx-swap="innerHTML" {
input
class="input flex-1 font-mono text-[12px]"
class="flex-1 rounded border px-3 py-2 font-mono text-[12px] text-slate-100"
style="background:#050608; border-color:var(--border)"
type="text"
name="command"
placeholder="e.g. systemctl status harmony-agent"
maxlength="16384"
autocomplete="off"
placeholder="uname -a"
required;
button class="btn btn-primary" type="submit" { "Run" }
button class="btn" type="submit" { "Run" }
}
div
id="exec-output"
class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto"
style="background:#050608; height:440px" {
div class="px-0 py-px italic text-slate-700" {
"\u{2014} commands run here are sent to the device; output appears below \u{2014}"
}
pre id="command-output" class="mt-3 min-h-16 overflow-auto whitespace-pre-wrap rounded p-3 font-mono text-[11.5px] text-slate-300" style="background:#050608" {
"Command output appears here."
}
}
}
}
/// Markup for one command's response, appended to the exec console.
pub fn command_output(command: &str, output: &str) -> Markup {
html! {
div class="py-1 border-t" style="border-color:var(--border)" {
div class="text-slate-300" { span class="text-slate-500" { "$ " } (command) }
pre class="text-slate-400 whitespace-pre-wrap mt-0.5" { (output) }
}
}
}
// ── Logs modal (SSE streaming) ─────────────────────────────────────────
pub fn logs_modal(device_id: &str) -> Markup {
html! {
dialog
id="device-logs-modal"
class="m-auto grid grid-rows-[auto_1fr] h-[88vh] w-[min(96vw,82rem)] overflow-hidden rounded-none border-t-2 border-x-0 border-b-0 p-0 text-slate-100 shadow-[0_32px_64px_rgba(0,0,0,0.9),0_0_0_1px_rgba(148,163,184,0.06)] backdrop:bg-black/85"
style="border-color:var(--accent); background:#080a0c"
{
div class="flex items-center justify-between border-b px-5 py-3" style="background:#0c1018; border-color:var(--border)" {
div class="flex items-center gap-3" {
span class="relative flex h-1.5 w-1.5 shrink-0" {
span class="absolute inline-flex h-full w-full animate-ping rounded-full bg-orange-400 opacity-60" {}
span class="relative inline-flex h-1.5 w-1.5 rounded-full" style="background:var(--accent)" {}
}
code class="text-sm font-medium text-slate-100" { (device_id) }
span class="text-[10px] font-semibold uppercase tracking-[0.15em] text-orange-500/60" { "\u{b7} logs" }
}
form method="dialog" {
button
type="submit"
class="flex items-center gap-1.5 text-slate-500 transition-colors hover:text-slate-200"
aria-label="Close"
{
kbd class="rounded border bg-slate-800/60 px-1.5 py-0.5 font-mono text-[10px] text-slate-400" style="border-color:var(--border-strong)" { "esc" }
span class="text-xs" { "close" }
}
}
}
div
class="overflow-y-auto py-3 font-mono text-[11.5px] leading-6 px-5"
style="background:#050608"
hx-ext="sse"
sse-connect={"/devices/" (device_id) "/logs/stream"}
sse-swap="log"
hx-swap="beforeend" {
div class="py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" }
}
}
}
}
// ── Row (for blacklist response) ───────────────────────────────────────
pub fn row(d: &DeviceDetail) -> Markup {
html! {
tr id={"device-" (d.id)} hx-get={"/device/" (d.id)} hx-target="body" hx-push-url="true" class="cursor-pointer" {
td {
span class="font-mono text-slate-100 hover:text-(--accent-fg) hover:underline underline-offset-2 whitespace-nowrap" {
(&d.id)
}
}
td { (badges::device_status(d.status)) }
td {
@if let Some(dep) = &d.deployment { span class="font-mono text-[12px] text-slate-300 whitespace-nowrap" { (dep) } }
@else { span class="text-slate-700" { "\u{2014}" } }
}
td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } }
td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" { (agent_version(d)) } }
td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } }
}
span class="text-slate-500" { "$ " (command) "\n" }
(output)
}
}
// ── Helpers ────────────────────────────────────────────────────────────
/// Agent version from the device's inventory, or an em-dash placeholder
/// when the agent hasn't reported inventory yet.
fn agent_version(d: &DeviceDetail) -> &str {
d.inventory
.as_ref()
.map(|i| i.agent_version.as_str())
.unwrap_or("\u{2014}")
d.current_version.as_deref().unwrap_or("\u{2014}")
}
fn time_ago(minutes: i64) -> String {
@@ -499,15 +306,12 @@ fn time_ago(minutes: i64) -> String {
}
}
fn definition(label: &str, value: &str, mono: bool, copyable: bool) -> Markup {
fn definition(label: &str, value: &str, mono: bool) -> Markup {
html! {
div class="flex items-center justify-between py-1.5 text-[12px] border-b last:border-b-0" style="border-color:var(--border)" {
span class="text-slate-500" { (label) }
span class={(if mono { "font-mono whitespace-nowrap" } else { "" }) " text-slate-200 flex items-center gap-1.5"} {
(value)
@if copyable {
button class="text-slate-600 hover:text-slate-300" title="Copy" { (PreEscaped(ICON_COPY)) }
}
}
}
}
@@ -527,6 +331,7 @@ mod tests {
deployment: Some("edge-gateway".into()),
region: "eu-paris-1".into(),
tags: vec!["prod".into()],
current_version: Some("v1.2.3".into()),
inventory: Some(crate::service::InventorySnapshot {
hostname: "hf-edge-001".into(),
arch: "aarch64".into(),
@@ -540,28 +345,15 @@ mod tests {
}
#[test]
fn overview_shows_device_info_not_removed_mock() {
fn detail_shows_command_form_and_only_implemented_device_features() {
let html = detail(&sample(), Some("v2.14.1")).into_string();
assert!(html.contains("Device info"));
assert!(html.contains("v1.2.3"), "agent version from inventory");
assert!(html.contains("v1.2.3"), "agent version from heartbeat");
assert!(html.contains("aarch64"));
assert!(html.contains("Run command"));
// Removed mock surfaces must be gone.
assert!(!html.contains("MAC"));
assert!(!html.contains("triggered reconcile"));
assert!(!html.contains("Deployment history"));
}
#[test]
fn command_tab_posts_to_exec_seam() {
let html = device_tabs(&sample(), None, "command").into_string();
assert!(html.contains("/devices/hf-edge-001/exec"));
assert!(html.contains(r#"name="command""#));
}
#[test]
fn logs_tab_connects_to_stream() {
let html = device_tabs(&sample(), None, "logs").into_string();
assert!(html.contains("/devices/hf-edge-001/logs/stream"));
for unsupported in ["Reconcile", "Restart", "Suspend", "Recent logs"] {
assert!(!html.contains(unsupported));
}
}
}

1
fleet/harmony-fleet-operator/src/frontend/views/mod.rs
View File

@@ -3,4 +3,3 @@ pub mod badges;
pub mod dashboard;
pub mod deployments;
pub mod devices;
pub mod settings;

74
fleet/harmony-fleet-operator/src/frontend/views/settings.rs
View File

@@ -1,74 +0,0 @@
use maud::{Markup, PreEscaped, html};
pub fn page() -> Markup {
html! {
div class="p-6 max-w-3xl space-y-4" {
div {
h2 class="text-[15px] font-semibold text-slate-200" { "Notification channels" }
p class="text-[12px] text-slate-500 mt-1" { "Where alerts get delivered when something needs your attention." }
}
(channel_row("Email", "email", "alerts@example.com", true))
(channel_row("Slack", "slack", "#fleet-alerts", true))
(channel_row("Discord", "discord", "https://discord.com/api/webhooks/\u{2026}", false))
(channel_row("SMS", "sms", "+1 555 010 0001", true))
}
}
}
fn channel_row(name: &str, key: &str, placeholder: &str, enabled: bool) -> Markup {
let enabled_val = if enabled {
"var(--ok)"
} else {
"rgba(148,163,184,0.2)"
};
let translate = if enabled { "18px" } else { "2px" };
let display_val = if enabled { placeholder } else { "disabled" };
html! {
div class="card p-5" {
div class="flex items-center justify-between" {
div class="flex items-center gap-3" {
span class="inline-flex items-center justify-center w-9 h-9 rounded-md" style="background:var(--bg-elev-2); color:var(--accent-fg)" {
(PreEscaped(channel_icon(key)))
}
div {
div class="text-[14px] text-slate-100 font-medium" { (name) }
div class="text-[11px] text-slate-500" { (display_val) }
}
}
button
class="relative w-9 h-5 rounded-full transition-colors"
style={"background:" (enabled_val)}
hx-post={"/settings/toggle/" (key)}
hx-target="closest .card"
hx-swap="outerHTML" {
span class="absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform" style={"transform:translateX(" (translate) ")"} {}
}
}
div class={(if enabled { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3" } else { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3 max-h-0 opacity-0 overflow-hidden" })} {
div {
label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Destination" }
input class="input mt-1 w-full" style="padding-left:10px" type="text" placeholder=(placeholder) value=(placeholder) {}
}
div {
label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Notify on" }
div class="mt-1 flex gap-1.5" {
span class="chip active" { "critical" }
span class="chip active" { "warning" }
span class="chip" { "info" }
}
}
}
}
}
}
fn channel_icon(key: &str) -> String {
match key {
"email" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 4h16c1.1 0 2 .9 2 2v12c0 1.1-.9 2-2 2H4c-1.1 0-2-.9-2-2V6c0-1.1.9-2 2-2z"/><polyline points="22,6 12,13 2,6"/></svg>"#.to_string(),
"slack" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="currentColor" stroke="none"><path d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"/></svg>"#.to_string(),
"discord" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="currentColor" stroke="none"><path d="M20.317 4.37a19.79 19.79 0 0 0-4.885-1.515.07.07 0 0 0-.07.035c-.21.375-.444.864-.608 1.25a18.27 18.27 0 0 0-5.487 0 12.64 12.64 0 0 0-.617-1.25.077.077 0 0 0-.07-.035 19.74 19.74 0 0 0-4.885 1.515.07.07 0 0 0-.032.027C.533 9.046-.32 13.58.099 18.057c.001.01.008.02.018.027a19.9 19.9 0 0 0 5.993 3.03.078.078 0 0 0 .084-.028c.462-.63.873-1.295 1.226-1.994a.076.076 0 0 0-.041-.106 13.107 13.107 0 0 1-1.872-.892.077.077 0 0 1-.008-.128 10.2 10.2 0 0 0 .372-.292.074.074 0 0 1 .077-.01c3.928 1.793 8.18 1.793 12.062 0a.074.074 0 0 1 .078.01c.12.098.246.198.373.292a.077.077 0 0 1-.006.127 12.299 12.299 0 0 1-1.873.892.077.077 0 0 0-.041.107c.36.698.772 1.362 1.225 1.993a.076.076 0 0 0 .084.028 19.84 19.84 0 0 0 6.002-3.03.077.077 0 0 0 .018-.026c.5-5.177-.838-9.674-3.548-13.66a.061.061 0 0 0-.031-.029zM8.02 15.33c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.956-2.419 2.157-2.419 1.21 0 2.176 1.095 2.157 2.42 0 1.333-.956 2.418-2.157 2.418zm7.975 0c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.955-2.419 2.157-2.419 1.21 0 2.176 1.095 2.157 2.42 0 1.333-.946 2.418-2.157 2.418z"/></svg>"#.to_string(),
"sms" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>"#.to_string(),
_ => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/></svg>"#.to_string(),
}
}

93
fleet/harmony-fleet-operator/src/group_cache.rs Normal file
View File

@@ -0,0 +1,93 @@
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use chrono::{DateTime, Utc};
use harmony_reconciler_contracts::DeviceGroupSource;
use tokio::sync::RwLock;
pub type DeviceGroups = HashMap<String, HashSet<String>>;
#[derive(Default)]
struct CachedGroups {
groups: Option<Arc<DeviceGroups>>,
updated_at: Option<DateTime<Utc>>,
}
/// Failed refreshes retain the last successful snapshot; before the first
/// success there is no snapshot, so startup remains fail-closed.
pub struct DeviceGroupCache {
source: Arc<dyn DeviceGroupSource>,
cached: RwLock<CachedGroups>,
}
impl DeviceGroupCache {
pub fn new(source: Arc<dyn DeviceGroupSource>) -> Self {
Self {
source,
cached: RwLock::new(CachedGroups::default()),
}
}
pub async fn refresh(&self) -> Option<Arc<DeviceGroups>> {
match self.source.device_groups().await {
Ok(groups) => {
let groups = Arc::new(groups);
let mut cached = self.cached.write().await;
cached.groups = Some(groups.clone());
cached.updated_at = Some(Utc::now());
Some(groups)
}
Err(error) => {
let cached = self.cached.read().await;
let age_seconds = cached
.updated_at
.map(|updated_at| Utc::now().signed_duration_since(updated_at).num_seconds());
tracing::warn!(
%error,
?age_seconds,
"device group refresh failed; retaining last known state"
);
cached.groups.clone()
}
}
}
pub async fn snapshot(&self) -> Option<Arc<DeviceGroups>> {
self.cached.read().await.groups.clone()
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use async_trait::async_trait;
use harmony_reconciler_contracts::GroupSourceError;
use super::*;
struct OneSuccess(AtomicUsize);
#[async_trait]
impl DeviceGroupSource for OneSuccess {
async fn device_groups(&self) -> Result<DeviceGroups, GroupSourceError> {
if self.0.fetch_add(1, Ordering::Relaxed) == 0 {
Ok(HashMap::from([(
"device-1".into(),
HashSet::from(["edge-a".into()]),
)]))
} else {
Err(GroupSourceError::Source("unavailable".into()))
}
}
}
#[tokio::test]
async fn failed_refresh_retains_last_snapshot() {
let cache = DeviceGroupCache::new(Arc::new(OneSuccess(AtomicUsize::new(0))));
let first = cache.refresh().await.unwrap();
let stale = cache.refresh().await.unwrap();
assert!(Arc::ptr_eq(&first, &stale));
}
}

11
fleet/harmony-fleet-operator/src/lib.rs
View File

@@ -10,13 +10,20 @@
//! `harmony-fleet-deploy` when installing the operator.
pub mod access;
pub mod agent_upgrade;
pub mod commands;
pub mod crd;
pub mod device_reconciler;
pub mod device_status;
pub mod fleet_aggregator;
pub mod group_cache;
pub mod rollout;
pub mod task;
pub mod task_run_controller;
pub use crd::{
AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
AgentUpgradeTarget, AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec,
DeploymentStatus, Device, DeviceSpec, DeviceStatus, DeviceUpgradeStatus, Reachability, Rollout,
RolloutStrategy,
};
pub use task::{SystemUpgradeTaskV1, TaskRun, TaskRunPhase, TaskRunSpec, TaskRunStatus};

79
fleet/harmony-fleet-operator/src/main.rs
View File

@@ -6,7 +6,10 @@ mod frontend;
mod service;
use harmony_fleet_operator::access::StaticDeviceGroups;
use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
use harmony_fleet_operator::group_cache::DeviceGroupCache;
use harmony_fleet_operator::{
agent_upgrade, device_reconciler, device_status, fleet_aggregator, task_run_controller,
};
use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
use harmony_secret::OpenBaoDeploymentSecretGrants;
use harmony_zitadel_auth::ZitadelDeviceGroups;
@@ -159,33 +162,27 @@ async fn main() -> Result<()> {
addr,
css_from,
live_reload,
} => serve_web(mock, addr, css_from, live_reload, &cli.tenant_namespace).await,
} => serve_web(mock, addr, css_from, live_reload).await,
}
}
/// `serve-web` subcommand: dashboard on its own (mock data, or the live
/// CR-reading service). The deployed operator instead serves the
/// dashboard alongside the reconcile loop — see [`spawn_dashboard`].
/// `serve-web` subcommand for frontend iteration with mock data. The
/// live dashboard runs alongside the reconcile loop; see [`spawn_dashboard`].
#[cfg(feature = "web-frontend")]
async fn serve_web(
mock: bool,
addr: std::net::SocketAddr,
css_from: Option<PathBuf>,
live_reload: bool,
tenant_namespace: &str,
) -> Result<()> {
use std::sync::Arc;
use service::{FleetService, mock::MockFleetService, real::RealFleetService};
use service::{FleetService, mock::MockFleetService};
let fleet: Arc<dyn FleetService> = if mock {
Arc::new(MockFleetService::default())
} else {
Arc::new(RealFleetService::new(
Client::try_default().await?,
tenant_namespace,
))
};
if !mock {
anyhow::bail!("live dashboard runs in-process with the operator; use serve-web --mock");
}
let fleet: Arc<dyn FleetService> = Arc::new(MockFleetService::default());
serve_dashboard(fleet, addr, css_from, live_reload).await
}
@@ -244,7 +241,12 @@ async fn serve_dashboard(
/// (e.g. Zitadel not yet reachable for JWKS) is logged but never tears
/// down reconcile — the read UI is best-effort, the controller is not.
#[cfg(feature = "web-frontend")]
fn spawn_dashboard(client: Client, tenant_namespace: &str) {
fn spawn_dashboard(
client: Client,
tenant_namespace: &str,
commands: harmony_fleet_operator::commands::FleetCommandsClient,
fleet_state: fleet_aggregator::SharedFleetState,
) {
use std::net::SocketAddr;
use std::sync::Arc;
@@ -253,7 +255,12 @@ fn spawn_dashboard(client: Client, tenant_namespace: &str) {
let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT));
let tenant_namespace = tenant_namespace.to_string();
tokio::spawn(async move {
let fleet = Arc::new(RealFleetService::new(client, tenant_namespace));
let fleet = Arc::new(RealFleetService::new(
client,
tenant_namespace,
commands,
fleet_state,
));
if let Err(e) = serve_dashboard(fleet, addr, None, false).await {
tracing::error!(error = %e, "dashboard server exited; reconcile continues");
}
@@ -270,7 +277,7 @@ async fn run(
) -> Result<()> {
let nats = connect_with_retry(nats_url, credentials_toml).await?;
tracing::info!(url = %nats_url, "connected to NATS");
let js = jetstream::new(nats);
let js = jetstream::new(nats.clone());
let desired_state_kv = js
.create_key_value(jetstream::kv::Config {
bucket: bucket.to_string(),
@@ -304,7 +311,7 @@ async fn run(
};
// Group membership (the scheduling gate): Zitadel role grants in
// prod, a static map for dev/e2e, or ungated when neither is set.
// prod or a static map for dev/e2e. Missing membership fails closed.
let group_source: Option<Arc<dyn DeviceGroupSource>> = match (
std::env::var("ZITADEL_URL"),
std::env::var("ZITADEL_PAT"),
@@ -322,17 +329,27 @@ async fn run(
_ => {
tracing::warn!(
"no device-group source (ZITADEL_URL/PAT/PROJECT_ID or FLEET_DEVICE_GROUPS); \
deployments with allowedGroups will match no devices"
deployments will match no devices"
);
None
}
};
// Serve the read-only dashboard in the same process (best-effort;
// it reads CRs only, no NATS). Built only with the web-frontend
// feature; absent from the lean reconcile-only image.
let group_cache = group_source.map(|source| Arc::new(DeviceGroupCache::new(source)));
let fleet_state = Arc::new(tokio::sync::Mutex::new(
fleet_aggregator::FleetState::default(),
));
let rollout_plans = harmony_fleet_operator::rollout::open(&js).await?;
// Dashboard associations use the aggregator's authorization decision;
// interactive commands use NATS request/reply.
#[cfg(feature = "web-frontend")]
spawn_dashboard(client.clone(), tenant_namespace);
spawn_dashboard(
client.clone(),
tenant_namespace,
harmony_fleet_operator::commands::FleetCommandsClient::new(nats),
fleet_state.clone(),
);
// Concurrent tasks:
// controller — CR validation + finalizer-cleanup
@@ -344,10 +361,20 @@ async fn run(
let ctl_client = client.clone();
let dr_client = client.clone();
let dr_js = js.clone();
let ds_client = client.clone();
let ds_js = js.clone();
let upgrade_client = client.clone();
let upgrade_js = js.clone();
let task_client = client.clone();
let task_js = js.clone();
let task_group_cache = group_cache.clone();
tokio::select! {
r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r,
r = controller::run(ctl_client, tenant_namespace, desired_state_kv, rollout_plans.clone()) => r,
r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r,
r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r,
r = device_status::run(ds_client, tenant_namespace, ds_js) => r,
r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js) => r,
r = task_run_controller::run(task_client, tenant_namespace, task_js, task_group_cache, rollout_plans.clone()) => r,
r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_cache, fleet_state, rollout_plans) => r,
}
}

331
fleet/harmony-fleet-operator/src/rollout.rs Normal file
View File

@@ -0,0 +1,331 @@
use std::collections::BTreeMap;
use anyhow::{Context, Result};
use async_nats::jetstream::kv::{Operation, Store};
use serde::{Deserialize, Serialize};
use crate::AggregateLastError;
const BUCKET: &str = "rollout-plan";
const MAX_BYTES: i64 = 64 * 1024 * 1024;
const MAX_PLAN_BYTES: i32 = 1024 * 1024;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Target {
pub device_id: String,
pub canary: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Plan {
pub revision: String,
pub targets: Vec<Target>,
#[serde(
default,
alias = "task_failures",
skip_serializing_if = "Vec::is_empty"
)]
pub failures: Vec<Failure>,
#[serde(default)]
pub closed: bool,
}
impl<'de> Deserialize<'de> for Plan {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
struct StoredPlan {
revision: String,
targets: Vec<Target>,
#[serde(default, alias = "task_failures")]
failures: Vec<Failure>,
#[serde(default)]
failed: Vec<String>,
#[serde(default)]
closed: bool,
}
let mut stored = StoredPlan::deserialize(deserializer)?;
for device_id in stored.failed {
if stored
.failures
.iter()
.all(|failure| failure.error.device_id != device_id)
{
stored.failures.push(Failure {
reason: "legacy rollout failure".into(),
error: AggregateLastError {
device_id,
message: "rollout failed before failure details were recorded".into(),
at: String::new(),
},
});
}
}
Ok(Self {
revision: stored.revision,
targets: stored.targets,
failures: stored.failures,
closed: stored.closed,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Failure {
pub reason: String,
pub error: AggregateLastError,
}
impl Plan {
pub fn new(revision: String, mut targets: Vec<Target>) -> Self {
targets.sort_by(|a, b| a.device_id.cmp(&b.device_id));
Self {
revision,
targets,
failures: Vec::new(),
closed: false,
}
}
pub fn has_canaries(&self) -> bool {
self.targets.iter().any(|target| target.canary)
}
pub fn failure(&self, device_id: &str) -> Option<&Failure> {
self.failures
.iter()
.find(|failure| failure.error.device_id == device_id)
}
}
pub async fn record_failures(
store: &Store,
key: &str,
revision: &str,
failures: &[Failure],
) -> Result<Plan> {
update_plan(store, key, revision, |plan| {
plan.failures.extend_from_slice(failures);
plan.failures
.sort_by(|a, b| a.error.device_id.cmp(&b.error.device_id));
plan.failures
.dedup_by(|a, b| a.error.device_id == b.error.device_id);
})
.await
}
async fn update_plan(
store: &Store,
key: &str,
revision: &str,
update: impl FnOnce(&mut Plan),
) -> Result<Plan> {
let entry = store
.entry(key)
.await?
.filter(|entry| entry.operation == Operation::Put)
.with_context(|| format!("rollout plan '{key}' missing"))?;
let mut plan: Plan = serde_json::from_slice(&entry.value)
.with_context(|| format!("invalid rollout plan '{key}'"))?;
anyhow::ensure!(plan.revision == revision, "rollout plan revision changed");
anyhow::ensure!(!plan.closed, "rollout plan '{key}' is closed");
update(&mut plan);
let value = serde_json::to_vec(&plan)?;
anyhow::ensure!(
value.len() <= MAX_PLAN_BYTES as usize,
"rollout plan '{key}' progress is too large"
);
store.update(key, value.into(), entry.revision).await?;
Ok(plan)
}
pub fn is_canary(labels: &BTreeMap<String, String>) -> bool {
labels.get("canary").map(String::as_str) == Some("true")
}
pub async fn open(jetstream: &async_nats::jetstream::Context) -> Result<Store> {
Ok(jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET.into(),
history: 1,
max_bytes: MAX_BYTES,
max_value_size: MAX_PLAN_BYTES,
..Default::default()
})
.await?)
}
pub async fn freeze(store: &Store, key: &str, desired: &Plan) -> Result<Plan> {
let value = serde_json::to_vec(desired)?;
anyhow::ensure!(
value.len() <= MAX_PLAN_BYTES as usize / 2,
"rollout plan '{key}' is too large"
);
let existing = store.entry(key).await?;
if let Some(entry) = existing.filter(|entry| entry.operation == Operation::Put) {
let plan: Plan = serde_json::from_slice(&entry.value)
.with_context(|| format!("invalid rollout plan '{key}'"))?;
anyhow::ensure!(!plan.closed, "rollout plan '{key}' is closed");
if plan.revision == desired.revision {
return Ok(plan);
}
if newer_generation(&plan.revision, &desired.revision) {
return Ok(plan);
}
store.update(key, value.into(), entry.revision).await?;
} else {
store.create(key, value.into()).await?;
}
Ok(desired.clone())
}
pub async fn close(store: &Store, key: &str) -> Result<Option<Plan>> {
loop {
if let Some(entry) = store
.entry(key)
.await?
.filter(|entry| entry.operation == Operation::Put)
{
let mut plan: Plan = serde_json::from_slice(&entry.value)
.with_context(|| format!("invalid rollout plan '{key}'"))?;
if !plan.closed {
plan.closed = true;
store
.update(key, serde_json::to_vec(&plan)?.into(), entry.revision)
.await?;
}
return Ok(Some(plan));
}
let mut sentinel = Plan::new(String::new(), Vec::new());
sentinel.closed = true;
match store
.create(key, serde_json::to_vec(&sentinel)?.into())
.await
{
Ok(_) => return Ok(Some(sentinel)),
Err(error) => {
if store.entry(key).await?.is_some() {
continue;
}
return Err(error.into());
}
}
}
}
fn newer_generation(existing: &str, desired: &str) -> bool {
let Some((existing_uid, existing_generation)) = existing.rsplit_once(':') else {
return false;
};
let Some((desired_uid, desired_generation)) = desired.rsplit_once(':') else {
return false;
};
existing_uid == desired_uid
&& matches!(
(existing_generation.parse::<u64>(), desired_generation.parse::<u64>()),
(Ok(existing), Ok(desired)) if existing > desired
)
}
pub async fn load(store: &Store, key: &str) -> Result<Option<Plan>> {
store
.entry(key)
.await?
.filter(|entry| entry.operation == Operation::Put)
.map(|entry| serde_json::from_slice(&entry.value).context("invalid rollout plan"))
.transpose()
}
pub async fn delete(store: &Store, key: &str) -> Result<()> {
if store
.entry(key)
.await?
.is_some_and(|entry| entry.operation == Operation::Put)
{
store.delete(key).await?;
}
Ok(())
}
pub fn deployment_key(uid: &str) -> String {
format!("deployment.{uid}")
}
pub fn task_key(uid: &str) -> String {
format!("task.{uid}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn plan_sorts_and_detects_canaries() {
let plan = Plan::new(
"revision".into(),
vec![
Target {
device_id: "b".into(),
canary: false,
},
Target {
device_id: "a".into(),
canary: true,
},
],
);
assert_eq!(plan.targets[0].device_id, "a");
assert!(plan.has_canaries());
}
#[test]
fn task_failure_checkpoint_round_trips_exactly() {
let mut plan = Plan::new("revision".into(), Vec::new());
plan.failures.push(Failure {
reason: "RepairRequired".into(),
error: AggregateLastError {
device_id: "device-1".into(),
message: "dpkg requires manual repair".into(),
at: "2026-07-24T12:00:00+00:00".into(),
},
});
let encoded = serde_json::to_vec(&plan).unwrap();
assert_eq!(serde_json::from_slice::<Plan>(&encoded).unwrap(), plan);
let legacy = serde_json::json!({
"revision": "revision",
"targets": [],
"task_failures": [{
"reason": "RepairRequired",
"error": {
"deviceId": "device-1",
"message": "dpkg requires manual repair",
"at": "2026-07-24T12:00:00+00:00"
}
}],
"closed": false
});
assert_eq!(serde_json::from_value::<Plan>(legacy).unwrap(), plan);
}
#[test]
fn legacy_failed_devices_remain_failed() {
let plan: Plan = serde_json::from_value(serde_json::json!({
"revision": "revision",
"targets": [],
"failed": ["device-1"]
}))
.unwrap();
assert!(plan.failure("device-1").is_some());
}
#[test]
fn deployment_generation_cannot_move_backwards() {
assert!(newer_generation("uid:3", "uid:2"));
assert!(!newer_generation("uid:2", "uid:3"));
assert!(!newer_generation("other:3", "uid:2"));
assert!(!newer_generation("task-uid", "task-uid"));
}
}

39
fleet/harmony-fleet-operator/src/service/mock.rs
View File

@@ -4,8 +4,8 @@ use std::sync::Mutex;
use async_trait::async_trait;
use super::{
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
DeviceStatus, FleetService, InventorySnapshot,
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentError, DeploymentStatus,
DeviceDetail, DeviceStatus, FleetService, InventorySnapshot,
};
pub struct MockFleetService {
@@ -168,6 +168,7 @@ fn seed_devices() -> Vec<DeviceDetail> {
deployment,
region,
tags,
current_version: inventory.as_ref().map(|i| i.agent_version.clone()),
inventory,
});
}
@@ -190,6 +191,7 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
vec![
DeploymentDetail {
name: "edge-gateway".into(),
rollout_revision: "edge-gateway:1".into(),
version: "v2.14.1".into(),
status: DeploymentStatus::Active,
target: 32,
@@ -197,9 +199,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 0,
pending: 1,
updated_at: "2026-05-19 04:12".into(),
last_error: None,
},
DeploymentDetail {
name: "sensor-firmware".into(),
rollout_revision: "sensor-firmware:1".into(),
version: "v0.9.3".into(),
status: DeploymentStatus::Rolling,
target: 41,
@@ -207,9 +211,15 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 1,
pending: 12,
updated_at: "2026-05-19 06:48".into(),
last_error: Some(DeploymentError {
device: "hf-sensor-042".into(),
message: "image pull timed out".into(),
at: "2026-05-19 06:48".into(),
}),
},
DeploymentDetail {
name: "ingest-pipeline".into(),
rollout_revision: "ingest-pipeline:1".into(),
version: "v1.7.0".into(),
status: DeploymentStatus::Active,
target: 8,
@@ -217,9 +227,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 0,
pending: 0,
updated_at: "2026-05-15 11:30".into(),
last_error: None,
},
DeploymentDetail {
name: "control-plane".into(),
rollout_revision: "control-plane:1".into(),
version: "v3.2.0".into(),
status: DeploymentStatus::Failing,
target: 6,
@@ -227,9 +239,15 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 2,
pending: 1,
updated_at: "2026-05-19 07:01".into(),
last_error: Some(DeploymentError {
device: "hf-gw-018".into(),
message: "service failed its health check".into(),
at: "2026-05-19 07:01".into(),
}),
},
DeploymentDetail {
name: "telemetry-collector".into(),
rollout_revision: "telemetry-collector:1".into(),
version: "v0.4.12".into(),
status: DeploymentStatus::Active,
target: 12,
@@ -237,9 +255,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 0,
pending: 0,
updated_at: "2026-05-12 09:22".into(),
last_error: None,
},
DeploymentDetail {
name: "gateway-proxy".into(),
rollout_revision: "gateway-proxy:1".into(),
version: "v1.0.5".into(),
status: DeploymentStatus::Paused,
target: 4,
@@ -247,9 +267,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 0,
pending: 4,
updated_at: "2026-05-18 18:14".into(),
last_error: None,
},
DeploymentDetail {
name: "media-relay".into(),
rollout_revision: "media-relay:1".into(),
version: "v2.0.0-rc.3".into(),
status: DeploymentStatus::Rolling,
target: 9,
@@ -257,6 +279,7 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
failing: 0,
pending: 4,
updated_at: "2026-05-19 06:55".into(),
last_error: None,
},
]
}
@@ -429,6 +452,10 @@ impl FleetService for MockFleetService {
Ok(dev.clone())
}
async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
Ok(format!("[{device_id}] {command}\n[exit 0]"))
}
async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>> {
Ok(self.alerts.lock().unwrap().clone())
}
@@ -443,14 +470,6 @@ impl FleetService for MockFleetService {
}
}
async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
// Seam only: the real impl publishes the command to the device
// and streams stdout/stderr back.
Ok(format!(
"$ {command}\n[{device_id}] command transport not yet implemented"
))
}
async fn filtered_devices(
&self,
status: Option<DeviceStatus>,

16
fleet/harmony-fleet-operator/src/service/mod.rs
View File

@@ -16,6 +16,7 @@ pub trait FleetService: Send + Sync + 'static {
async fn get_deployment(&self, name: &str) -> anyhow::Result<Option<DeploymentDetail>>;
async fn get_deployment_devices(&self, name: &str) -> anyhow::Result<Vec<DeviceDetail>>;
async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail>;
async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String>;
async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>>;
async fn ack_alert(&self, id: &str) -> anyhow::Result<bool>;
async fn filtered_devices(
@@ -25,10 +26,6 @@ pub trait FleetService: Send + Sync + 'static {
region: Option<String>,
search: Option<String>,
) -> anyhow::Result<Vec<DeviceDetail>>;
/// Send a one-shot shell command to a device for administrative
/// access. Returns the (eventual) output; streaming back live is a
/// later refinement. The device round-trip is not wired yet.
async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String>;
}
// ── Device ─────────────────────────────────────────────────────────────
@@ -42,6 +39,8 @@ pub struct DeviceDetail {
pub deployment: Option<String>,
pub region: String,
pub tags: Vec<String>,
/// Running version from the latest heartbeat.
pub current_version: Option<String>,
/// Hardware/OS facts from the agent. `None` until the first
/// post-enrollment publish (mirrors `DeviceInfo.inventory`).
pub inventory: Option<InventorySnapshot>,
@@ -76,6 +75,7 @@ impl DeviceStatus {
#[derive(Debug, Clone, Serialize)]
pub struct DeploymentDetail {
pub name: String,
pub rollout_revision: String,
pub version: String,
pub status: DeploymentStatus,
pub target: u32,
@@ -83,6 +83,14 @@ pub struct DeploymentDetail {
pub failing: u32,
pub pending: u32,
pub updated_at: String,
pub last_error: Option<DeploymentError>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DeploymentError {
pub device: String,
pub message: String,
pub at: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]

359
fleet/harmony-fleet-operator/src/service/real.rs
View File

@@ -1,11 +1,10 @@
//! Live [`FleetService`] as a read-only projection of Kubernetes CRs.
//! Live [`FleetService`] backed by Kubernetes CRs and the NATS command channel.
//!
//! The operator is the write side: `device_reconciler` materializes
//! `Device` CRs (labels + inventory), `device_status` reflects liveness
//! onto `Device.status`, and `fleet_aggregator` writes
//! `Deployment.status.aggregate`. This dashboard is the read side — it
//! only reads those CRs and projects them to view DTOs. No NATS: the
//! CR is the single contract between the two sides.
//! onto `Device.status`, and `fleet_aggregator` owns target eligibility
//! and writes `Deployment.status.aggregate`. This dashboard projects
//! those sources to view DTOs.
use std::collections::{BTreeMap, HashSet};
use std::sync::Mutex;
@@ -17,15 +16,17 @@ use kube::api::{Api, ListParams, Patch, PatchParams};
use kube::{Client, ResourceExt};
use serde_json::json;
use harmony_fleet_operator::commands::FleetCommandsClient;
use harmony_fleet_operator::crd::{
Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability,
Deployment as DeploymentCr, DeploymentAggregate, Device as DeviceCr,
DeviceStatus as DeviceLiveness, Reachability,
};
use harmony_fleet_operator::fleet_aggregator::selector_matches;
use harmony_reconciler_contracts::ReconcileScore;
use harmony_fleet_operator::fleet_aggregator::SharedFleetState;
use harmony_reconciler_contracts::{ExecReply, ReconcileScore};
use super::{
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
DeviceStatus, FleetService,
Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentError, DeploymentStatus,
DeviceDetail, DeviceStatus, FleetService,
};
/// Label the operator UI sets to quarantine a device.
@@ -36,16 +37,25 @@ const REGION_LABEL: &str = "region";
pub struct RealFleetService {
kube: Client,
namespace: String,
commands: FleetCommandsClient,
fleet_state: SharedFleetState,
/// In-memory ack set. Alerts are derived from live CR state and
/// have no store of their own, so acks don't survive a restart.
acked_alerts: Mutex<HashSet<String>>,
}
impl RealFleetService {
pub fn new(kube: Client, namespace: impl Into<String>) -> Self {
pub fn new(
kube: Client,
namespace: impl Into<String>,
commands: FleetCommandsClient,
fleet_state: SharedFleetState,
) -> Self {
Self {
kube,
namespace: namespace.into(),
commands,
fleet_state,
acked_alerts: Mutex::new(HashSet::new()),
}
}
@@ -61,13 +71,23 @@ impl RealFleetService {
}
async fn devices(&self) -> anyhow::Result<Vec<DeviceDetail>> {
let deployments = self.deployment_crs().await?;
let eligible = self
.fleet_state
.lock()
.await
.eligible_deployments_by_device();
let now = Utc::now();
let mut devices: Vec<DeviceDetail> = self
.device_crs()
.await?
.iter()
.map(|cr| map_device(cr, &deployments, now))
.map(|cr| {
let deployment = eligible
.get(&cr.name_any())
.and_then(|deployments| deployments.first())
.cloned();
map_device(cr, deployment, now)
})
.collect();
devices.sort_by(|a, b| a.id.cmp(&b.id));
Ok(devices)
@@ -85,7 +105,25 @@ impl RealFleetService {
}
}
fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime<Utc>) -> DeviceDetail {
fn format_exec(reply: ExecReply) -> String {
let mut output = reply.stdout;
if !reply.stderr.is_empty() {
if !output.is_empty() && !output.ends_with('\n') {
output.push('\n');
}
output.push_str(&reply.stderr);
}
if !output.is_empty() && !output.ends_with('\n') {
output.push('\n');
}
output.push_str(&format!("[exit {}]", reply.exit_code));
if reply.truncated {
output.push_str(" [output truncated]");
}
output
}
fn map_device(cr: &DeviceCr, deployment: Option<String>, now: DateTime<Utc>) -> DeviceDetail {
let labels = cr.metadata.labels.clone().unwrap_or_default();
let blacklisted = labels.get(BLACKLIST_LABEL).map(String::as_str) == Some("true");
let last_seen = cr
@@ -102,12 +140,13 @@ fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime<Utc>) -
status: device_status(blacklisted, cr.status.as_ref()),
last_seen,
minutes_ago: (now - last_seen).num_minutes().max(0),
deployment: primary_deployment(&labels, deployments),
deployment,
region: labels
.get(REGION_LABEL)
.cloned()
.unwrap_or_else(|| "\u{2014}".to_string()),
tags: tags_from_labels(&labels),
current_version: cr.status.as_ref().and_then(|s| s.current_version.clone()),
inventory: cr.spec.inventory.clone(),
}
}
@@ -126,22 +165,6 @@ fn device_status(blacklisted: bool, liveness: Option<&DeviceLiveness>) -> Device
}
}
/// First deployment (by name) whose selector matches the device — the
/// canonical [`selector_matches`] over CR labels, the same matcher the
/// aggregator uses. No reconstruction.
fn primary_deployment(
labels: &BTreeMap<String, String>,
deployments: &[DeploymentCr],
) -> Option<String> {
let mut matched: Vec<String> = deployments
.iter()
.filter(|d| selector_matches(&d.spec.target_selector, labels))
.map(ResourceExt::name_any)
.collect();
matched.sort();
matched.into_iter().next()
}
/// Routing labels rendered as `k=v` chips, minus internal keys.
fn tags_from_labels(labels: &BTreeMap<String, String>) -> Vec<String> {
labels
@@ -152,20 +175,31 @@ fn tags_from_labels(labels: &BTreeMap<String, String>) -> Vec<String> {
}
fn map_deployment(cr: &DeploymentCr) -> DeploymentDetail {
let current_revision = cr.rollout_revision();
let agg = cr
.status
.as_ref()
.filter(|status| {
current_revision
.as_deref()
.is_some_and(|revision| status.rollout_revision.as_deref() == Some(revision))
})
.and_then(|s| s.aggregate.clone())
.unwrap_or_default();
let status = if agg.failed > 0 {
DeploymentStatus::Failing
} else if agg.pending > 0 {
DeploymentStatus::Rolling
} else {
DeploymentStatus::Active
};
.map(|agg| {
let status = if agg.failed > 0 || agg.matched_device_count == 0 {
DeploymentStatus::Failing
} else if agg.pending > 0 {
DeploymentStatus::Rolling
} else {
DeploymentStatus::Active
};
(agg, status)
});
let (agg, status) =
agg.unwrap_or_else(|| (DeploymentAggregate::default(), DeploymentStatus::Rolling));
DeploymentDetail {
name: cr.name_any(),
rollout_revision: current_revision.unwrap_or_default(),
version: deployment_version(&cr.spec.score),
status,
target: agg.matched_device_count,
@@ -176,6 +210,11 @@ fn map_deployment(cr: &DeploymentCr) -> DeploymentDetail {
.creation_timestamp()
.map(|t| t.0.format("%Y-%m-%d %H:%M").to_string())
.unwrap_or_else(|| "\u{2014}".to_string()),
last_error: agg.last_error.map(|error| DeploymentError {
device: error.device_id,
message: error.message,
at: error.at,
}),
}
}
@@ -203,16 +242,40 @@ fn derive_alerts(
) -> Vec<Alert> {
let mut alerts = Vec::new();
for d in deployments {
if d.failing > 0 {
let id = format!("dep:{}:failing", d.name);
if d.status == DeploymentStatus::Failing {
let id = format!("dep:{}:{}:failing", d.name, d.rollout_revision);
let (title, device, at) = if d.target == 0 {
(
format!("{} matched no devices", d.name),
None,
String::new(),
)
} else {
d.last_error.as_ref().map_or_else(
|| {
(
format!("{} has {} failing device(s)", d.name, d.failing),
None,
String::new(),
)
},
|error| {
(
format!("{}: {}", error.device, error.message),
Some(error.device.clone()),
error.at.clone(),
)
},
)
};
alerts.push(Alert {
acked: acked.contains(&id),
id,
severity: AlertSeverity::Critical,
title: format!("{} has {} failing device(s)", d.name, d.failing),
title,
deployment: Some(d.name.clone()),
device: None,
at: String::new(),
device,
at,
});
}
}
@@ -312,21 +375,26 @@ impl FleetService for RealFleetService {
}
async fn get_deployment_devices(&self, name: &str) -> anyhow::Result<Vec<DeviceDetail>> {
let deployments = self.deployment_crs().await?;
let Some(cr) = deployments.iter().find(|c| c.name_any() == name) else {
return Ok(Vec::new());
};
let selector = cr.spec.target_selector.clone();
let eligible = self
.fleet_state
.lock()
.await
.eligible_deployments_by_device();
let now = Utc::now();
Ok(self
let mut devices: Vec<_> = self
.device_crs()
.await?
.iter()
.filter(|dev| {
selector_matches(&selector, &dev.metadata.labels.clone().unwrap_or_default())
.filter_map(|device| {
let deployments = eligible.get(&device.name_any())?;
deployments
.iter()
.any(|deployment| deployment == name)
.then(|| map_device(device, deployments.first().cloned(), now))
})
.map(|dev| map_device(dev, &deployments, now))
.collect())
.collect();
devices.sort_by(|a, b| a.id.cmp(&b.id));
Ok(devices)
}
async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail> {
@@ -340,6 +408,15 @@ impl FleetService for RealFleetService {
.ok_or_else(|| anyhow::anyhow!("device {id} not found after blacklist"))
}
async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
Ok(format_exec(
self.commands
.exec(device_id, command)
.await
.with_context(|| format!("running command on device {device_id}"))?,
))
}
async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>> {
let devices = self.devices().await?;
let deployments = self.deployments().await?;
@@ -351,14 +428,6 @@ impl FleetService for RealFleetService {
Ok(self.acked_alerts.lock().unwrap().insert(id.to_string()))
}
async fn run_command(&self, _device_id: &str, command: &str) -> anyhow::Result<String> {
// Seam only: the device round-trip (publish to the agent over
// NATS, stream stdout/stderr back) needs agent-side support.
Ok(format!(
"$ {command}\n[device command transport not implemented yet]"
))
}
async fn filtered_devices(
&self,
status: Option<DeviceStatus>,
@@ -367,15 +436,15 @@ impl FleetService for RealFleetService {
search: Option<String>,
) -> anyhow::Result<Vec<DeviceDetail>> {
let search = search.map(|s| s.to_lowercase());
Ok(self
.devices()
.await?
let devices = if let Some(deployment) = deployment.as_deref() {
self.get_deployment_devices(deployment).await?
} else {
self.devices().await?
};
Ok(devices
.into_iter()
.filter(|d| {
status.is_none_or(|s| d.status == s)
&& deployment
.as_deref()
.is_none_or(|dep| d.deployment.as_deref() == Some(dep))
&& region.as_deref().is_none_or(|r| d.region == r)
&& search.as_deref().is_none_or(|q| {
d.id.to_lowercase().contains(q)
@@ -392,12 +461,19 @@ impl FleetService for RealFleetService {
#[cfg(test)]
mod tests {
use super::*;
use harmony_fleet_operator::crd::{
AggregateLastError, DeploymentAggregate, DeploymentSpec,
DeploymentStatus as DeploymentCrStatus, Device, DeviceSpec, Rollout, RolloutStrategy,
};
use harmony_reconciler_contracts::{PodmanService, PodmanV0Score};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
fn liveness(r: Reachability) -> DeviceLiveness {
DeviceLiveness {
last_heartbeat: None,
reachability: r,
current_version: None,
agent_upgrade: None,
}
}
@@ -418,12 +494,44 @@ mod tests {
assert_eq!(device_status(false, None), DeviceStatus::Unknown);
}
#[test]
fn device_mapping_uses_authoritative_deployment() {
let device = Device::new(
"pi-01",
DeviceSpec {
inventory: None,
updater: None,
agent_upgrade: None,
},
);
assert_eq!(
map_device(&device, Some("web".into()), Utc::now()).deployment,
Some("web".into())
);
assert_eq!(map_device(&device, None, Utc::now()).deployment, None);
}
#[test]
fn exec_output_includes_stderr_exit_and_truncation() {
assert_eq!(
format_exec(ExecReply {
exit_code: 7,
stdout: "out".into(),
stderr: "err".into(),
truncated: true,
}),
"out\nerr\n[exit 7] [output truncated]"
);
}
#[test]
fn version_from_image_tag() {
let score = ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![PodmanService {
name: "web".into(),
image: "nginx:1.25".into(),
image_pull_secret: None,
ports: vec![],
env: vec![],
secret_env: vec![],
@@ -434,6 +542,91 @@ mod tests {
assert_eq!(deployment_version(&score), "1.25");
}
#[test]
fn deployment_mapping_projects_last_error() {
let mut deployment = DeploymentCr::new(
"edge",
DeploymentSpec {
allowed_groups: vec![],
target_selector: LabelSelector::default(),
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![],
}),
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
);
deployment.metadata.uid = Some("deployment-uid".into());
deployment.metadata.generation = Some(2);
deployment.status = Some(DeploymentCrStatus {
rollout_revision: Some("deployment-uid:2".into()),
aggregate: Some(DeploymentAggregate {
matched_device_count: 1,
failed: 1,
last_error: Some(AggregateLastError {
device_id: "pi-09".into(),
message: "image pull denied".into(),
at: "2026-07-27T10:15:00Z".into(),
}),
..Default::default()
}),
});
assert_eq!(
map_deployment(&deployment).last_error,
Some(DeploymentError {
device: "pi-09".into(),
message: "image pull denied".into(),
at: "2026-07-27T10:15:00Z".into(),
})
);
assert_eq!(
map_deployment(&deployment).status,
DeploymentStatus::Failing
);
}
#[test]
fn deployment_mapping_ignores_absent_or_stale_aggregate() {
let mut deployment = DeploymentCr::new(
"edge",
DeploymentSpec {
allowed_groups: vec![],
target_selector: LabelSelector::default(),
score: ReconcileScore::PodmanV0(PodmanV0Score {
init_container: None,
services: vec![],
}),
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
);
deployment.metadata.uid = Some("deployment-uid".into());
deployment.metadata.generation = Some(2);
deployment.status = Some(DeploymentCrStatus {
rollout_revision: Some("deployment-uid:1".into()),
aggregate: Some(DeploymentAggregate {
matched_device_count: 3,
succeeded: 3,
..Default::default()
}),
});
let detail = map_deployment(&deployment);
assert_eq!(detail.rollout_revision, "deployment-uid:2");
assert_eq!(detail.status, DeploymentStatus::Rolling);
assert_eq!((detail.target, detail.healthy), (0, 0));
deployment.status = None;
assert_eq!(
map_deployment(&deployment).status,
DeploymentStatus::Rolling
);
}
#[test]
fn alerts_from_failing_and_stale() {
let devices = vec![DeviceDetail {
@@ -444,10 +637,12 @@ mod tests {
deployment: None,
region: "\u{2014}".into(),
tags: vec![],
current_version: None,
inventory: None,
}];
let deployments = vec![DeploymentDetail {
name: "edge".into(),
rollout_revision: "deployment-uid:2".into(),
version: "1.0".into(),
status: DeploymentStatus::Failing,
target: 3,
@@ -455,10 +650,34 @@ mod tests {
failing: 2,
pending: 0,
updated_at: "\u{2014}".into(),
last_error: Some(DeploymentError {
device: "pi-10".into(),
message: "image pull denied".into(),
at: "2026-07-27T10:15:00Z".into(),
}),
}];
let alerts = derive_alerts(&devices, &deployments, &HashSet::new());
let alerts = derive_alerts(
&devices,
&deployments,
&HashSet::from(["dep:edge:deployment-uid:1:failing".into()]),
);
assert_eq!(alerts.len(), 2);
assert!(alerts.iter().any(|a| a.severity == AlertSeverity::Critical));
let deployment_alert = alerts
.iter()
.find(|a| a.severity == AlertSeverity::Critical)
.unwrap();
assert_eq!(deployment_alert.device.as_deref(), Some("pi-10"));
assert_eq!(deployment_alert.title, "pi-10: image pull denied");
assert_eq!(deployment_alert.at, "2026-07-27T10:15:00Z");
assert_eq!(deployment_alert.id, "dep:edge:deployment-uid:2:failing");
assert!(!deployment_alert.acked);
assert!(alerts.iter().any(|a| a.severity == AlertSeverity::Warning));
let mut no_targets = deployments[0].clone();
no_targets.target = 0;
no_targets.failing = 0;
no_targets.last_error = None;
let alerts = derive_alerts(&[], &[no_targets], &HashSet::new());
assert_eq!(alerts[0].title, "edge matched no devices");
}
}

108
fleet/harmony-fleet-operator/src/task.rs Normal file
View File

@@ -0,0 +1,108 @@
use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, Time};
use kube::{CustomResource, KubeSchema};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::AggregateLastError;
#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, KubeSchema)]
#[kube(
group = "fleet.nationtech.io",
version = "v1alpha1",
kind = "TaskRun",
plural = "taskruns",
shortname = "fleettask",
namespaced,
status = "TaskRunStatus",
validation = Rule::new("self.spec == oldSelf.spec").message("TaskRun spec is immutable")
)]
#[serde(rename_all = "camelCase")]
pub struct TaskRunSpec {
#[schemars(length(min = 1))]
pub allowed_groups: Vec<String>,
pub target_selector: LabelSelector,
#[schemars(range(min = 60, max = 86400))]
pub deadline_seconds: u32,
pub system_upgrade_v1: SystemUpgradeTaskV1,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)]
pub struct SystemUpgradeTaskV1 {}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, JsonSchema)]
pub enum TaskRunPhase {
Planning,
Running,
Complete,
Failed,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct TaskRunStatus {
pub phase: TaskRunPhase,
pub target_count: u32,
pub succeeded: u32,
pub failed: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub start_time: Option<Time>,
#[serde(skip_serializing_if = "Option::is_none")]
pub completion_time: Option<Time>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(length(max = 128))]
pub reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_error: Option<AggregateLastError>,
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use kube::CustomResourceExt;
use super::*;
fn system_upgrade_run() -> TaskRun {
TaskRun::new(
"upgrade-device-1",
TaskRunSpec {
allowed_groups: vec!["production".into()],
target_selector: LabelSelector {
match_labels: Some(BTreeMap::from([("device-id".into(), "device-1".into())])),
match_expressions: None,
},
deadline_seconds: 21_600,
system_upgrade_v1: SystemUpgradeTaskV1 {},
},
)
}
#[test]
fn system_upgrade_has_no_operator_control_over_apt() {
let value = serde_json::to_value(system_upgrade_run()).unwrap();
assert_eq!(
value.pointer("/spec/systemUpgradeV1").unwrap(),
&serde_json::json!({})
);
}
#[test]
fn task_run_crd_is_namespaced_and_immutable() {
let crd = TaskRun::crd();
assert_eq!(crd.spec.scope, "Namespaced");
assert!(
crd.spec.versions[0]
.subresources
.as_ref()
.unwrap()
.status
.is_some()
);
let encoded = serde_json::to_string(&crd).unwrap();
assert!(encoded.contains("self.spec == oldSelf.spec"));
assert!(encoded.contains("minItems"));
assert!(encoded.contains("maximum"));
}
}

843
fleet/harmony-fleet-operator/src/task_run_controller.rs Normal file
View File

@@ -0,0 +1,843 @@
use std::collections::HashSet;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use async_nats::jetstream::kv::{Operation, Store};
use chrono::Utc;
use harmony_reconciler_contracts::{
BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS, Id, SystemUpgradeAttempt,
SystemUpgradePhase, SystemUpgradeStatus, system_upgrade_intent_key, system_upgrade_status_key,
};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
use kube::api::{Api, ListParams, Patch, PatchParams};
use kube::{Client, ResourceExt};
use crate::crd::{AggregateLastError, Device};
use crate::fleet_aggregator::{device_eligible, group_allows};
use crate::group_cache::DeviceGroupCache;
use crate::rollout::{self, Plan, Target};
use crate::task::{TaskRun, TaskRunPhase, TaskRunStatus};
const POLL_INTERVAL: Duration = Duration::from_secs(2);
const MAX_ERROR_CHARS: usize = 512;
const TASK_BUCKET_MAX_BYTES: i64 = 64 * 1024 * 1024;
const FINALIZER: &str = "fleet.nationtech.io/task-run-finalizer";
pub async fn run(
client: Client,
namespace: &str,
jetstream: async_nats::jetstream::Context,
group_cache: Option<Arc<DeviceGroupCache>>,
rollout_plans: Store,
) -> Result<()> {
let intents = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_SYSTEM_UPGRADE_INTENT.into(),
history: 1,
max_age: Duration::from_secs(2 * 24 * 60 * 60),
max_bytes: TASK_BUCKET_MAX_BYTES,
max_value_size: 16 * 1024,
..Default::default()
})
.await?;
let statuses = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_SYSTEM_UPGRADE_STATUS.into(),
history: 1,
max_age: Duration::from_secs(30 * 24 * 60 * 60),
max_bytes: TASK_BUCKET_MAX_BYTES,
max_value_size: 16 * 1024,
..Default::default()
})
.await?;
let runs: Api<TaskRun> = Api::namespaced(client.clone(), namespace);
let devices: Api<Device> = Api::namespaced(client, namespace);
let mut ticker = tokio::time::interval(POLL_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
for task_run in runs.list(&ListParams::default()).await?.items {
let name = task_run.name_any();
if let Err(error) = reconcile(
&runs,
&devices,
&intents,
&statuses,
group_cache.as_ref(),
&rollout_plans,
task_run,
)
.await
{
tracing::warn!(task_run = %name, error = %error, "TaskRun reconcile failed");
}
}
}
}
async fn reconcile(
runs: &Api<TaskRun>,
devices: &Api<Device>,
intents: &Store,
statuses: &Store,
group_cache: Option<&Arc<DeviceGroupCache>>,
rollout_plans: &Store,
task_run: TaskRun,
) -> Result<()> {
let run_uid = task_run
.metadata
.uid
.as_deref()
.context("TaskRun metadata.uid missing")?;
let plan_key = rollout::task_key(run_uid);
if task_run.metadata.deletion_timestamp.is_some() {
if let Some(plan) = rollout::close(rollout_plans, &plan_key).await? {
cleanup_intents(intents, &plan, run_uid).await?;
rollout::delete(rollout_plans, &plan_key).await?;
}
patch_finalizer(runs, &task_run, false).await?;
return Ok(());
}
if !task_run
.finalizers()
.iter()
.any(|finalizer| finalizer == FINALIZER)
{
patch_finalizer(runs, &task_run, true).await?;
return Ok(());
}
if task_run
.status
.as_ref()
.is_some_and(|status| matches!(status.phase, TaskRunPhase::Complete | TaskRunPhase::Failed))
{
if let Some(plan) = rollout::close(rollout_plans, &plan_key).await? {
cleanup_intents(intents, &plan, run_uid).await?;
rollout::delete(rollout_plans, &plan_key).await?;
}
return Ok(());
}
if task_run.status.is_none()
|| task_run
.status
.as_ref()
.is_some_and(|status| status.phase == TaskRunPhase::Planning)
{
let Some(cache) = group_cache else {
return patch_failed(runs, &task_run, 0, 0, None, "NoTargets", None).await;
};
let groups = cache
.snapshot()
.await
.context("device group cache is not ready")?;
let eligible = devices
.list(&ListParams::default())
.await?
.items
.into_iter()
.filter(|device| {
supports_system_upgrade(device)
&& device_eligible(
&task_run.spec.allowed_groups,
&task_run.spec.target_selector,
&device.metadata.labels.clone().unwrap_or_default(),
groups.get(&device.name_any()),
groups.get("*"),
)
})
.map(|device| Target {
canary: device
.metadata
.labels
.as_ref()
.is_some_and(rollout::is_canary),
device_id: device.name_any(),
})
.collect::<Vec<_>>();
if eligible.is_empty() {
return patch_failed(runs, &task_run, 0, 0, None, "NoTargets", None).await;
}
let plan = rollout::freeze(
rollout_plans,
&plan_key,
&Plan::new(run_uid.to_string(), eligible),
)
.await?;
let start = Time(Utc::now());
patch_status(
runs,
&task_run,
&TaskRunStatus {
phase: TaskRunPhase::Running,
target_count: plan.targets.len().try_into().unwrap_or(u32::MAX),
succeeded: 0,
failed: 0,
start_time: Some(start),
completion_time: None,
reason: None,
last_error: None,
},
)
.await?;
return Ok(());
}
let status = task_run.status.as_ref().context("TaskRun status missing")?;
let Some(start) = status.start_time.as_ref() else {
return patch_failed(
runs,
&task_run,
status.target_count,
0,
None,
"InvalidPlan",
None,
)
.await;
};
let Some(plan) = rollout::load(rollout_plans, &plan_key).await? else {
return patch_failed(
runs,
&task_run,
status.target_count,
0,
Some(start),
"InvalidPlan",
None,
)
.await;
};
if plan.revision != run_uid {
return patch_failed(
runs,
&task_run,
status.target_count,
0,
Some(start),
"InvalidPlan",
None,
)
.await;
}
let expires_at = start.0 + chrono::Duration::seconds(task_run.spec.deadline_seconds.into());
reconcile_running(
runs,
devices,
intents,
statuses,
rollout_plans,
group_cache,
&task_run,
&plan,
&plan_key,
run_uid,
start,
expires_at,
)
.await
}
fn supports_system_upgrade(device: &Device) -> bool {
device
.spec
.updater
.as_ref()
.is_some_and(|capabilities| capabilities.protocol == 1 && capabilities.apt_full_upgrade_v1)
}
fn authorized_for_release(
device: &Device,
allowed_groups: &[String],
device_groups: Option<&HashSet<String>>,
default_groups: Option<&HashSet<String>>,
) -> bool {
supports_system_upgrade(device)
&& (group_allows(allowed_groups, device_groups)
|| group_allows(allowed_groups, default_groups))
}
#[derive(Debug)]
enum TargetOutcome {
Pending,
Succeeded,
Failed {
reason: String,
error: AggregateLastError,
},
}
#[allow(clippy::too_many_arguments)]
async fn reconcile_running(
runs: &Api<TaskRun>,
devices: &Api<Device>,
intents: &Store,
statuses: &Store,
rollout_plans: &Store,
group_cache: Option<&Arc<DeviceGroupCache>>,
task_run: &TaskRun,
plan: &Plan,
plan_key: &str,
run_uid: &str,
start: &Time,
expires_at: chrono::DateTime<Utc>,
) -> Result<()> {
if plan.closed {
return Ok(());
}
let persisted_failures = plan
.failures
.iter()
.map(|failure| (failure.error.device_id.as_str(), failure))
.collect::<std::collections::HashMap<_, _>>();
let mut outcomes = Vec::with_capacity(plan.targets.len());
for target in &plan.targets {
outcomes.push(
if let Some(failure) = persisted_failures.get(target.device_id.as_str()) {
TargetOutcome::Failed {
reason: failure.reason.clone(),
error: failure.error.clone(),
}
} else {
inspect_target(statuses, &target.device_id, run_uid, expires_at).await?
},
);
}
let canaries_succeeded = plan
.targets
.iter()
.zip(&outcomes)
.filter(|(target, _)| target.canary)
.all(|(_, outcome)| matches!(outcome, TargetOutcome::Succeeded));
let has_canaries = plan.has_canaries();
let groups = if outcomes
.iter()
.any(|outcome| matches!(outcome, TargetOutcome::Pending))
{
Some(
group_cache
.context("device group cache is unavailable")?
.snapshot()
.await
.context("device group cache is not ready")?,
)
} else {
None
};
for (target, outcome) in plan.targets.iter().zip(&mut outcomes) {
let released = !has_canaries || target.canary || canaries_succeeded;
if !released || !matches!(outcome, TargetOutcome::Pending) || Utc::now() >= expires_at {
continue;
}
let attempt = attempt(&target.device_id, run_uid, expires_at);
let key = system_upgrade_intent_key(&target.device_id, run_uid);
match intent_matches(intents, &key, &attempt).await? {
Some(true) => continue,
Some(false) => {
*outcome = failed_outcome(&target.device_id, "AttemptMismatch", None, Utc::now());
continue;
}
None => {}
}
let current_device = devices.get_opt(&target.device_id).await?;
let still_eligible = groups.as_ref().is_some_and(|groups| {
current_device.as_ref().is_some_and(|device| {
authorized_for_release(
device,
&task_run.spec.allowed_groups,
groups.get(&target.device_id),
groups.get("*"),
)
})
});
if !still_eligible {
*outcome = failed_outcome(&target.device_id, "TargetRevoked", None, Utc::now());
continue;
}
if let Err(create_error) = intents
.create(&key, serde_json::to_vec(&attempt)?.into())
.await
{
match intent_matches(intents, &key, &attempt).await? {
Some(true) => {}
Some(false) => {
*outcome =
failed_outcome(&target.device_id, "AttemptMismatch", None, Utc::now());
}
None => return Err(create_error.into()),
}
}
}
let newly_failed = plan
.targets
.iter()
.zip(&outcomes)
.filter(|(target, outcome)| {
matches!(outcome, TargetOutcome::Failed { .. })
&& !persisted_failures.contains_key(target.device_id.as_str())
})
.filter_map(|(_, outcome)| match outcome {
TargetOutcome::Failed { reason, error } => Some(rollout::Failure {
reason: reason.clone(),
error: error.clone(),
}),
_ => None,
})
.collect::<Vec<_>>();
if !newly_failed.is_empty() {
rollout::record_failures(rollout_plans, plan_key, run_uid, &newly_failed).await?;
}
let succeeded = outcomes
.iter()
.filter(|outcome| matches!(outcome, TargetOutcome::Succeeded))
.count() as u32;
let known_failed = outcomes
.iter()
.filter(|outcome| matches!(outcome, TargetOutcome::Failed { .. }))
.count() as u32;
let deadline = Utc::now() >= expires_at;
let canaries_finished = plan
.targets
.iter()
.zip(&outcomes)
.filter(|(target, _)| target.canary)
.all(|(_, outcome)| !matches!(outcome, TargetOutcome::Pending));
let canary_failed =
plan.targets.iter().zip(&outcomes).any(|(target, outcome)| {
target.canary && matches!(outcome, TargetOutcome::Failed { .. })
});
let all_finished = outcomes
.iter()
.all(|outcome| !matches!(outcome, TargetOutcome::Pending));
let (phase, failed) = run_result(
plan.targets.len() as u32,
succeeded,
known_failed,
canary_failed,
canaries_finished,
all_finished,
deadline,
);
let last_error = outcomes
.iter()
.filter_map(|outcome| match outcome {
TargetOutcome::Failed { error, .. } => Some(error),
_ => None,
})
.max_by(|a, b| a.at.cmp(&b.at))
.cloned()
.or_else(|| {
task_run
.status
.as_ref()
.and_then(|status| status.last_error.clone())
});
let reason = if phase == TaskRunPhase::Failed && deadline {
Some("DeadlineExceeded".to_string())
} else {
outcomes.iter().find_map(|outcome| match outcome {
TargetOutcome::Failed { reason, .. } => Some(reason.clone()),
_ => None,
})
};
let new_status = TaskRunStatus {
phase,
target_count: plan.targets.len().try_into().unwrap_or(u32::MAX),
succeeded,
failed,
start_time: Some(start.clone()),
completion_time: (phase != TaskRunPhase::Running).then(|| Time(Utc::now())),
reason,
last_error,
};
if task_run.status.as_ref() == Some(&new_status) {
return Ok(());
}
patch_status(runs, task_run, &new_status).await
}
fn run_result(
target_count: u32,
succeeded: u32,
known_failed: u32,
canary_failed: bool,
canaries_finished: bool,
all_finished: bool,
deadline: bool,
) -> (TaskRunPhase, u32) {
if succeeded == target_count {
(TaskRunPhase::Complete, 0)
} else if deadline
|| canary_failed && canaries_finished
|| !canary_failed && known_failed > 0 && all_finished
{
(TaskRunPhase::Failed, target_count - succeeded)
} else {
(TaskRunPhase::Running, known_failed)
}
}
async fn inspect_target(
statuses: &Store,
device_id: &str,
run_uid: &str,
expires_at: chrono::DateTime<Utc>,
) -> Result<TargetOutcome> {
let attempt = attempt(device_id, run_uid, expires_at);
let Some(entry) = statuses
.entry(&system_upgrade_status_key(device_id, run_uid))
.await?
.filter(|entry| entry.operation == Operation::Put)
else {
return Ok(TargetOutcome::Pending);
};
let Some(received_at) = chrono::DateTime::from_timestamp(
entry.created.unix_timestamp(),
entry.created.nanosecond(),
) else {
return Ok(failed_outcome(
device_id,
"StatusMismatch",
Some("NATS returned an invalid status timestamp"),
Utc::now(),
));
};
let status = match serde_json::from_slice::<SystemUpgradeStatus>(&entry.value) {
Ok(status) if status_matches(&status, &attempt) => status,
Ok(_) => {
return Ok(failed_outcome(
device_id,
"StatusMismatch",
None,
received_at,
));
}
Err(error) => {
return Ok(failed_outcome(
device_id,
"StatusMismatch",
Some(&error.to_string()),
received_at,
));
}
};
match status.phase {
SystemUpgradePhase::Complete if completion_verified(&status, expires_at) => {
Ok(TargetOutcome::Succeeded)
}
SystemUpgradePhase::Complete => Ok(TargetOutcome::Pending),
SystemUpgradePhase::Failed | SystemUpgradePhase::RepairRequired => {
let reason = if received_at > expires_at {
"DeadlineExceeded".to_string()
} else {
format!("{:?}", status.phase)
};
Ok(failed_outcome(
device_id,
&reason,
status.error.as_deref(),
status.updated_at,
))
}
_ => Ok(TargetOutcome::Pending),
}
}
fn attempt(
device_id: &str,
run_uid: &str,
expires_at: chrono::DateTime<Utc>,
) -> SystemUpgradeAttempt {
SystemUpgradeAttempt {
attempt_id: run_uid.to_string(),
run_uid: run_uid.to_string(),
device_id: Id::from(device_id),
expires_at,
}
}
fn failed_outcome(
device_id: &str,
reason: &str,
message: Option<&str>,
at: chrono::DateTime<Utc>,
) -> TargetOutcome {
TargetOutcome::Failed {
reason: reason.to_string(),
error: AggregateLastError {
device_id: device_id.to_string(),
message: bounded(message.unwrap_or(reason)),
at: at.to_rfc3339(),
},
}
}
async fn intent_matches(
store: &Store,
key: &str,
expected: &SystemUpgradeAttempt,
) -> Result<Option<bool>> {
Ok(store
.entry(key)
.await?
.filter(|entry| entry.operation == Operation::Put)
.map(|entry| attempt_matches(&entry.value, expected)))
}
fn attempt_matches(value: &[u8], expected: &SystemUpgradeAttempt) -> bool {
serde_json::from_slice::<SystemUpgradeAttempt>(value).is_ok_and(|attempt| attempt == *expected)
}
fn status_matches(status: &SystemUpgradeStatus, attempt: &SystemUpgradeAttempt) -> bool {
status.attempt_id == attempt.attempt_id && status.run_uid == attempt.run_uid
}
fn completion_verified(status: &SystemUpgradeStatus, deadline: chrono::DateTime<Utc>) -> bool {
status
.post_completion_heartbeat_at
.is_some_and(|heartbeat| heartbeat > status.updated_at && heartbeat <= deadline)
}
async fn cleanup_intents(intents: &Store, plan: &Plan, run_uid: &str) -> Result<()> {
for target in &plan.targets {
let key = system_upgrade_intent_key(&target.device_id, run_uid);
if intents
.entry(&key)
.await?
.is_some_and(|entry| entry.operation == Operation::Put)
{
intents.delete(&key).await?;
}
}
Ok(())
}
async fn patch_failed(
runs: &Api<TaskRun>,
task_run: &TaskRun,
target_count: u32,
failed: u32,
start: Option<&Time>,
reason: &str,
error: Option<&str>,
) -> Result<()> {
let now = Time(Utc::now());
patch_status(
runs,
task_run,
&TaskRunStatus {
phase: TaskRunPhase::Failed,
target_count,
succeeded: 0,
failed,
start_time: start.cloned(),
completion_time: Some(now.clone()),
reason: Some(reason.to_string()),
last_error: error.map(|message| AggregateLastError {
device_id: String::new(),
message: bounded(message),
at: now.0.to_rfc3339(),
}),
},
)
.await
}
async fn patch_status(
runs: &Api<TaskRun>,
task_run: &TaskRun,
status: &TaskRunStatus,
) -> Result<()> {
let resource_version = task_run
.metadata
.resource_version
.as_deref()
.context("TaskRun metadata.resourceVersion missing")?;
runs.patch_status(
&task_run.name_any(),
&PatchParams::default(),
&Patch::Merge(serde_json::json!({
"metadata": { "resourceVersion": resource_version },
"status": status,
})),
)
.await?;
Ok(())
}
async fn patch_finalizer(runs: &Api<TaskRun>, task_run: &TaskRun, present: bool) -> Result<()> {
let resource_version = task_run
.metadata
.resource_version
.as_deref()
.context("TaskRun metadata.resourceVersion missing")?;
let mut finalizers = task_run.finalizers().to_vec();
if present {
finalizers.push(FINALIZER.to_string());
} else {
finalizers.retain(|finalizer| finalizer != FINALIZER);
}
runs.patch(
&task_run.name_any(),
&PatchParams::default(),
&Patch::Merge(serde_json::json!({
"metadata": {
"resourceVersion": resource_version,
"finalizers": finalizers,
},
})),
)
.await?;
Ok(())
}
fn bounded(message: &str) -> String {
message.chars().take(MAX_ERROR_CHARS).collect()
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashSet};
use chrono::DateTime;
use harmony_reconciler_contracts::UpdaterCapabilities;
use super::*;
fn attempt() -> SystemUpgradeAttempt {
SystemUpgradeAttempt {
attempt_id: "run-1".into(),
run_uid: "run-1".into(),
device_id: Id::from("device-1"),
expires_at: DateTime::parse_from_rfc3339("2026-07-24T12:00:00Z")
.unwrap()
.with_timezone(&Utc),
}
}
#[test]
fn status_must_match_attempt_and_run() {
let attempt = attempt();
let mut status = SystemUpgradeStatus {
attempt_id: attempt.attempt_id.clone(),
run_uid: attempt.run_uid.clone(),
phase: SystemUpgradePhase::Applying,
started_at: attempt.expires_at,
updated_at: attempt.expires_at,
post_completion_heartbeat_at: None,
error: None,
};
assert!(status_matches(&status, &attempt));
status.attempt_id = "other".into();
assert!(!status_matches(&status, &attempt));
status.attempt_id = attempt.attempt_id.clone();
status.run_uid = "other".into();
assert!(!status_matches(&status, &attempt));
}
#[test]
fn existing_intent_must_match_the_complete_attempt() {
let expected = attempt();
assert!(attempt_matches(
&serde_json::to_vec(&expected).unwrap(),
&expected
));
let mut changed = expected.clone();
changed.expires_at += chrono::Duration::seconds(1);
assert!(!attempt_matches(
&serde_json::to_vec(&changed).unwrap(),
&expected
));
assert!(!attempt_matches(b"not json", &expected));
}
#[test]
fn release_authorization_ignores_current_selector_labels() {
let mut device = Device::new(
"device-1",
crate::DeviceSpec {
inventory: None,
updater: Some(UpdaterCapabilities {
protocol: 1,
apt_full_upgrade_v1: true,
}),
agent_upgrade: None,
},
);
device.metadata.labels = Some(BTreeMap::from([("site".into(), "changed".into())]));
let allowed = vec!["production".to_string()];
let groups = HashSet::from(["production".to_string()]);
assert!(authorized_for_release(
&device,
&allowed,
Some(&groups),
None
));
device.spec.updater = None;
assert!(!authorized_for_release(
&device,
&allowed,
Some(&groups),
None
));
device.spec.updater = Some(UpdaterCapabilities {
protocol: 1,
apt_full_upgrade_v1: true,
});
assert!(!authorized_for_release(&device, &allowed, None, None));
}
#[test]
fn terminal_counts_cover_every_target_and_preserve_late_observed_success() {
assert_eq!(
run_result(3, 1, 1, false, false, false, false),
(TaskRunPhase::Running, 1)
);
assert_eq!(
run_result(3, 1, 1, true, true, false, false),
(TaskRunPhase::Failed, 2)
);
assert_eq!(
run_result(3, 3, 0, false, true, true, true),
(TaskRunPhase::Complete, 0)
);
assert_eq!(
run_result(3, 1, 0, false, false, false, true),
(TaskRunPhase::Failed, 2)
);
}
#[test]
fn success_requires_durable_in_deadline_heartbeat_proof() {
let terminal = DateTime::parse_from_rfc3339("2026-07-24T12:00:00Z")
.unwrap()
.with_timezone(&Utc);
let deadline = terminal + chrono::Duration::minutes(1);
let mut status = SystemUpgradeStatus {
attempt_id: "run-1".into(),
run_uid: "run-1".into(),
phase: SystemUpgradePhase::Complete,
started_at: terminal,
updated_at: terminal,
post_completion_heartbeat_at: None,
error: None,
};
assert!(!completion_verified(&status, deadline));
status.post_completion_heartbeat_at = Some(terminal);
assert!(!completion_verified(&status, deadline));
status.post_completion_heartbeat_at = Some(terminal + chrono::Duration::seconds(1));
assert!(completion_verified(&status, deadline));
status.post_completion_heartbeat_at = Some(deadline + chrono::Duration::seconds(1));
assert!(!completion_verified(&status, deadline));
}
}

24
fleet/harmony-fleet-operator/vendor/app.js vendored
View File

@@ -1,27 +1,3 @@
document.body.addEventListener('htmx:configRequest', (event) => {
event.detail.headers['x-csrf-token'] = '1';
});
// Open a modal dialog swapped into #modal-root. Lives here (not inline)
// because the production CSP forbids inline scripts/handlers.
document.body.addEventListener('htmx:afterSwap', (event) => {
if (!event.target || event.target.id !== 'modal-root') return;
const dialog = event.target.querySelector('dialog');
if (!dialog || typeof dialog.showModal !== 'function') return;
dialog.showModal();
// Backdrop click closes; closing clears the root so it can re-open.
dialog.addEventListener('click', (e) => {
if (e.target === dialog) dialog.close();
});
dialog.addEventListener('close', () => {
event.target.innerHTML = '';
});
// Keep a streaming log body scrolled to the latest line.
const body = dialog.querySelector('[sse-connect]');
if (body) {
new MutationObserver(() => {
body.scrollTop = body.scrollHeight;
}).observe(body, { childList: true });
}
});

290
fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js vendored
View File

@@ -1,290 +0,0 @@
/*
Server Sent Events Extension
============================
This extension adds support for Server Sent Events to htmx. See /www/extensions/sse.md for usage instructions.
*/
(function() {
/** @type {import("../htmx").HtmxInternalApi} */
var api
htmx.defineExtension('sse', {
/**
* Init saves the provided reference to the internal HTMX API.
*
* @param {import("../htmx").HtmxInternalApi} api
* @returns void
*/
init: function(apiRef) {
// store a reference to the internal API.
api = apiRef
// set a function in the public API for creating new EventSource objects
if (htmx.createEventSource == undefined) {
htmx.createEventSource = createEventSource
}
},
getSelectors: function() {
return ['[sse-connect]', '[data-sse-connect]', '[sse-swap]', '[data-sse-swap]']
},
/**
* onEvent handles all events passed to this extension.
*
* @param {string} name
* @param {Event} evt
* @returns void
*/
onEvent: function(name, evt) {
var parent = evt.target || evt.detail.elt
switch (name) {
case 'htmx:beforeCleanupElement':
var internalData = api.getInternalData(parent)
// Try to remove remove an EventSource when elements are removed
var source = internalData.sseEventSource
if (source) {
api.triggerEvent(parent, 'htmx:sseClose', {
source,
type: 'nodeReplaced',
})
internalData.sseEventSource.close()
}
return
// Try to create EventSources when elements are processed
case 'htmx:afterProcessNode':
ensureEventSourceOnElement(parent)
}
}
})
/// ////////////////////////////////////////////
// HELPER FUNCTIONS
/// ////////////////////////////////////////////
/**
* createEventSource is the default method for creating new EventSource objects.
* it is hoisted into htmx.config.createEventSource to be overridden by the user, if needed.
*
* @param {string} url
* @returns EventSource
*/
function createEventSource(url) {
return new EventSource(url, { withCredentials: true })
}
/**
* registerSSE looks for attributes that can contain sse events, right
* now hx-trigger and sse-swap and adds listeners based on these attributes too
* the closest event source
*
* @param {HTMLElement} elt
*/
function registerSSE(elt) {
// Add message handlers for every `sse-swap` attribute
if (api.getAttributeValue(elt, 'sse-swap')) {
// Find closest existing event source
var sourceElement = api.getClosestMatch(elt, hasEventSource)
if (sourceElement == null) {
// api.triggerErrorEvent(elt, "htmx:noSSESourceError")
return null // no eventsource in parentage, orphaned element
}
// Set internalData and source
var internalData = api.getInternalData(sourceElement)
var source = internalData.sseEventSource
var sseSwapAttr = api.getAttributeValue(elt, 'sse-swap')
var sseEventNames = sseSwapAttr.split(',')
for (var i = 0; i < sseEventNames.length; i++) {
const sseEventName = sseEventNames[i].trim()
const listener = function(event) {
// If the source is missing then close SSE
if (maybeCloseSSESource(sourceElement)) {
return
}
// If the body no longer contains the element, remove the listener
if (!api.bodyContains(elt)) {
source.removeEventListener(sseEventName, listener)
return
}
// swap the response into the DOM and trigger a notification
if (!api.triggerEvent(elt, 'htmx:sseBeforeMessage', event)) {
return
}
swap(elt, event.data)
api.triggerEvent(elt, 'htmx:sseMessage', event)
}
// Register the new listener
api.getInternalData(elt).sseEventListener = listener
source.addEventListener(sseEventName, listener)
}
}
// Add message handlers for every `hx-trigger="sse:*"` attribute
if (api.getAttributeValue(elt, 'hx-trigger')) {
// Find closest existing event source
var sourceElement = api.getClosestMatch(elt, hasEventSource)
if (sourceElement == null) {
// api.triggerErrorEvent(elt, "htmx:noSSESourceError")
return null // no eventsource in parentage, orphaned element
}
// Set internalData and source
var internalData = api.getInternalData(sourceElement)
var source = internalData.sseEventSource
var triggerSpecs = api.getTriggerSpecs(elt)
triggerSpecs.forEach(function(ts) {
if (ts.trigger.slice(0, 4) !== 'sse:') {
return
}
var listener = function (event) {
if (maybeCloseSSESource(sourceElement)) {
return
}
if (!api.bodyContains(elt)) {
source.removeEventListener(ts.trigger.slice(4), listener)
}
// Trigger events to be handled by the rest of htmx
htmx.trigger(elt, ts.trigger, event)
htmx.trigger(elt, 'htmx:sseMessage', event)
}
// Register the new listener
api.getInternalData(elt).sseEventListener = listener
source.addEventListener(ts.trigger.slice(4), listener)
})
}
}
/**
* ensureEventSourceOnElement creates a new EventSource connection on the provided element.
* If a usable EventSource already exists, then it is returned. If not, then a new EventSource
* is created and stored in the element's internalData.
* @param {HTMLElement} elt
* @param {number} retryCount
* @returns {EventSource | null}
*/
function ensureEventSourceOnElement(elt, retryCount) {
if (elt == null) {
return null
}
// handle extension source creation attribute
if (api.getAttributeValue(elt, 'sse-connect')) {
var sseURL = api.getAttributeValue(elt, 'sse-connect')
if (sseURL == null) {
return
}
ensureEventSource(elt, sseURL, retryCount)
}
registerSSE(elt)
}
function ensureEventSource(elt, url, retryCount) {
var source = htmx.createEventSource(url)
source.onerror = function(err) {
// Log an error event
api.triggerErrorEvent(elt, 'htmx:sseError', { error: err, source })
// If parent no longer exists in the document, then clean up this EventSource
if (maybeCloseSSESource(elt)) {
return
}
// Otherwise, try to reconnect the EventSource
if (source.readyState === EventSource.CLOSED) {
retryCount = retryCount || 0
retryCount = Math.max(Math.min(retryCount * 2, 128), 1)
var timeout = retryCount * 500
window.setTimeout(function() {
ensureEventSourceOnElement(elt, retryCount)
}, timeout)
}
}
source.onopen = function(evt) {
api.triggerEvent(elt, 'htmx:sseOpen', { source })
if (retryCount && retryCount > 0) {
const childrenToFix = elt.querySelectorAll("[sse-swap], [data-sse-swap], [hx-trigger], [data-hx-trigger]")
for (let i = 0; i < childrenToFix.length; i++) {
registerSSE(childrenToFix[i])
}
// We want to increase the reconnection delay for consecutive failed attempts only
retryCount = 0
}
}
api.getInternalData(elt).sseEventSource = source
var closeAttribute = api.getAttributeValue(elt, "sse-close");
if (closeAttribute) {
// close eventsource when this message is received
source.addEventListener(closeAttribute, function() {
api.triggerEvent(elt, 'htmx:sseClose', {
source,
type: 'message',
})
source.close()
});
}
}
/**
* maybeCloseSSESource confirms that the parent element still exists.
* If not, then any associated SSE source is closed and the function returns true.
*
* @param {HTMLElement} elt
* @returns boolean
*/
function maybeCloseSSESource(elt) {
if (!api.bodyContains(elt)) {
var source = api.getInternalData(elt).sseEventSource
if (source != undefined) {
api.triggerEvent(elt, 'htmx:sseClose', {
source,
type: 'nodeMissing',
})
source.close()
// source = null
return true
}
}
return false
}
/**
* @param {HTMLElement} elt
* @param {string} content
*/
function swap(elt, content) {
api.withExtensions(elt, function(extension) {
content = extension.transformResponse(content, null, elt)
})
var swapSpec = api.getSwapSpecification(elt)
var target = api.getTarget(elt)
api.swap(target, content, swapSpec)
}
function hasEventSource(node) {
return api.getInternalData(node).sseEventSource != null
}
})()

2
fleet/scripts/deploy-apache.sh
View File

@@ -1,4 +1,4 @@
cargo run -q -p example_harmony_apply_deployment -- \
cargo run -q -p harmony-fleet-deploy --bin harmony-fleet-example-deployment -- \
--namespace "fleet-demo" \
--name "testdeployment-apache" \
--target-device "paul" \

Some files were not shown because too many files have changed in this diff Show More