2026-07-28 14:21:44 +00:00
158 changed files with 18694 additions and 3880 deletions
--- a/.opencode/skills/code-derived-doc-review/SKILL.md
+++ b/.opencode/skills/code-derived-doc-review/SKILL.md
@@ -0,0 +1,70 @@
 ---
 name: code-derived-doc-review
 description: Use when writing or reviewing architecture and design documentation to derive claims from code, map claims to tests, run an external review, and improve clarity and information density with the humanizer skill.
 license: AGPL-3.0-only
 compatibility: opencode
 ---
 # Code-derived documentation review
 Keep design documentation factual, concise, and difficult to drift away from
 the codebase.
 ## Workflow
 1. Read the complete document set before editing.
 2. Identify the authoritative document for decisions, operational behavior,
   wire contracts, and implementation details. Remove duplicate specifications.
 3. Extract every architecturally significant claim: ownership, trust boundary,
   ordering, state transition, failure policy, durability guarantee, limit, and
   security constraint.
 4. For each claim, locate:
   - the implementation owner with file and symbol;
   - the test that observes the behavior;
   - the coverage level: direct, partial, or missing.
 5. Treat an unsupported claim as a finding. Correct the prose, add a behavioral
   test when the guarantee is required, or label the gap explicitly.
 6. Ask an independent read-only reviewer to compare the finished documentation
   with code and tests. Address findings, then request confirmation.
 7. Load the `humanizer` skill. Remove repetition, inflated certainty, generic
   conclusions, and AI-style transitions without removing facts or caveats.
 8. Run a density pass: each fact has one authoritative home; diagrams summarize
   and do not introduce unique guarantees; planning files point to maintained
   documentation instead of restating it.
 9. Validate links, documentation builds, diagram syntax, formatting, and stale
   terminology.
 ## Claim table
 Use this shape during review:
 | Architectural claim | Documentation | Implementation owner | Validating test | Coverage |
 |---|---|---|---|---|
 Coverage means:
 - **Direct:** the test observes the documented behavior at the owning boundary.
 - **Partial:** the test covers a component or serialization detail but not the
  full guarantee.
 - **Missing:** no automated test proves the claim.
 Do not count a compile check, helper-only test, or repeated production logic as
 behavioral proof. Do not imply integration coverage when only a unit test exists.
 ## External review prompt
 Ask the reviewer to report findings first with file and line references. Require
 checks for factual drift, hidden failure cases, security overclaims, duplicate
 authority, unsupported certainty, unclear ownership, and missing tests. The
 reviewer must not edit the files.
 ## Completion standard
 Documentation is complete when:
 - each significant claim is traceable to code;
 - each required guarantee has a direct test or an explicit coverage gap;
 - one source owns each fact;
 - external review findings are resolved;
 - the prose passes the humanizer and density checks;
 - documentation and diagram validation pass.
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1750,6 +1750,27 @@ version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
 [[package]]
 name = "const_format"
 version = "0.2.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e"
 dependencies = [
 "const_format_proc_macros",
 "konst",
 ]
 [[package]]
 name = "const_format_proc_macros"
 version = "0.2.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
 dependencies = [
 "proc-macro2",
 "quote",
 "unicode-xid",
 ]
 [[package]]
 name = "constant_time_eq"
 version = "0.4.2"
@@ -3498,12 +3519,14 @@ name = "example_harmony_apply_deployment"
 version = "0.1.0"
 dependencies = [
 "anyhow",
- "clap",
+ "async-trait",
 "harmony",
 "harmony-fleet-deploy",
 "harmony-fleet-operator",
 "harmony-reconciler-contracts",
 "harmony_app",
 "harmony_cli",
 "k8s-openapi",
 "kube",
 "serde_json",
 "tokio",
 ]
@@ -3642,6 +3665,16 @@ dependencies = [
 "serde",
 ]
 [[package]]
 name = "fs2"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
 dependencies = [
 "libc",
 "winapi",
 ]
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -4012,6 +4045,7 @@ dependencies = [
 "tokio-util",
 "url",
 "uuid",
 "vaultrs",
 "virt",
 "walkdir",
 "webbrowser",
@@ -4019,18 +4053,22 @@ dependencies = [
 [[package]]
 name = "harmony-fleet-agent"
-version = "0.1.0"
+version = "0.1.2"
 dependencies = [
 "anyhow",
 "async-nats",
 "async-trait",
 "chrono",
 "clap",
 "fs2",
 "futures-util",
 "harmony-fleet-auth",
 "harmony-reconciler-contracts",
 "harmony_secret",
 "oci-client",
 "podman-api",
 "reqwest 0.12.28",
 "sd-notify",
 "serde",
 "serde_json",
 "sha2 0.10.9",
@@ -4039,6 +4077,7 @@ dependencies = [
 "toml",
 "tracing",
 "tracing-subscriber",
 "uuid",
 ]
 [[package]]
@@ -4071,6 +4110,7 @@ dependencies = [
 "harmony",
 "harmony-fleet-auth",
 "harmony-fleet-operator",
 "harmony-reconciler-contracts",
 "harmony_app",
 "harmony_cli",
 "harmony_config",
@@ -4082,6 +4122,7 @@ dependencies = [
 "kube",
 "log",
 "non-blank-string-rs",
 "oci-client",
 "schemars 0.8.22",
 "serde",
 "serde_json",
@@ -4137,7 +4178,6 @@ dependencies = [
 "async-trait",
 "axum",
 "axum-extra",
 "base64 0.22.1",
 "chrono",
 "clap",
 "dotenvy",
@@ -4161,6 +4201,7 @@ dependencies = [
 "tracing",
 "tracing-subscriber",
 "url",
 "uuid",
 ]
 [[package]]
@@ -4229,6 +4270,7 @@ dependencies = [
 "schemars 0.8.22",
 "serde",
 "serde_json",
 "sha2 0.10.9",
 "thiserror 2.0.18",
 ]
@@ -4317,6 +4359,50 @@ dependencies = [
 "url",
 ]
 [[package]]
 name = "harmony_auth"
 version = "0.1.0"
 dependencies = [
 "async-trait",
 "chrono",
 "reqwest 0.12.28",
 "serde",
 "serde_json",
 "sha2 0.10.9",
 "thiserror 2.0.18",
 "uuid",
 ]
 [[package]]
 name = "harmony_auth_cli"
 version = "0.1.0"
 dependencies = [
 "clap",
 "harmony_auth",
 "serde_json",
 "tokio",
 "tracing-subscriber",
 ]
 [[package]]
 name = "harmony_auth_ui"
 version = "0.1.0"
 dependencies = [
 "anyhow",
 "axum",
 "axum-extra",
 "clap",
 "harmony_auth",
 "maud",
 "reqwest 0.12.28",
 "serde",
 "tokio",
 "tower-http",
 "tracing",
 "tracing-subscriber",
 "uuid",
 ]
 [[package]]
 name = "harmony_cli"
 version = "0.1.0"
@@ -4569,6 +4655,7 @@ name = "harmony_zitadel_jwt"
 version = "0.1.0"
 dependencies = [
 "anyhow",
 "base64 0.22.1",
 "chrono",
 "jsonwebtoken",
 "reqwest 0.12.28",
@@ -4748,6 +4835,15 @@ dependencies = [
 "itoa",
 ]
 [[package]]
 name = "http-auth"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822"
 dependencies = [
 "memchr",
 ]
 [[package]]
 name = "http-body"
 version = "0.4.6"
@@ -5522,6 +5618,21 @@ dependencies = [
 "simple_asn1",
 ]
 [[package]]
 name = "jwt"
 version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f"
 dependencies = [
 "base64 0.13.1",
 "crypto-common 0.1.7",
 "digest 0.10.7",
 "hmac 0.12.1",
 "serde",
 "serde_json",
 "sha2 0.10.9",
 ]
 [[package]]
 name = "k3d-rs"
 version = "0.1.0"
@@ -5554,6 +5665,21 @@ dependencies = [
 "serde_json",
 ]
 [[package]]
 name = "konst"
 version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb"
 dependencies = [
 "konst_macro_rules",
 ]
 [[package]]
 name = "konst_macro_rules"
 version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
 [[package]]
 name = "kube"
 version = "1.1.0"
@@ -6216,6 +6342,49 @@ dependencies = [
 "memchr",
 ]
 [[package]]
 name = "oci-client"
 version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b74df13319e08bc386d333d3dc289c774c88cc543cae31f5347db07b5ec2172"
 dependencies = [
 "bytes 1.11.1",
 "chrono",
 "futures-util",
 "http 1.4.0",
 "http-auth",
 "jwt",
 "lazy_static",
 "oci-spec",
 "olpc-cjson",
 "regex",
 "reqwest 0.12.28",
 "serde",
 "serde_json",
 "sha2 0.10.9",
 "thiserror 2.0.18",
 "tokio",
 "tracing",
 "unicase",
 ]
 [[package]]
 name = "oci-spec"
 version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc3da52b83ce3258fbf29f66ac784b279453c2ac3c22c5805371b921ede0d308"
 dependencies = [
 "const_format",
 "derive_builder 0.20.2",
 "getset",
 "regex",
 "serde",
 "serde_json",
 "strum 0.27.2",
 "strum_macros 0.27.2",
 "thiserror 2.0.18",
 ]
 [[package]]
 name = "octocrab"
 version = "0.44.1"
@@ -6272,6 +6441,17 @@ dependencies = [
 "tokio",
 ]
 [[package]]
 name = "olpc-cjson"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083"
 dependencies = [
 "serde",
 "serde_json",
 "unicode-normalization",
 ]
 [[package]]
 name = "once_cell"
 version = "1.21.4"
@@ -7905,6 +8085,15 @@ dependencies = [
 "untrusted",
 ]
 [[package]]
 name = "sd-notify"
 version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4"
 dependencies = [
 "libc",
 ]
 [[package]]
 name = "sec1"
 version = "0.3.0"
@@ -9428,6 +9617,12 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 [[package]]
 name = "unicase"
 version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
 [[package]]
 name = "unicode-bidi"
 version = "0.3.18"
@@ -9564,6 +9759,7 @@ dependencies = [
 "getrandom 0.4.2",
 "js-sys",
 "rand 0.10.1",
 "serde_core",
 "wasm-bindgen",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,6 +36,9 @@ members = [
  "fleet/harmony-fleet-deploy",
  "fleet/harmony-fleet-e2e",
  "harmony-reconciler-contracts",
  "harmony_auth",
  "harmony_auth_cli",
  "harmony_auth_ui",
  "examples/fleet_server_install",
  "nats/jwt",
  "nats/callout",
@@ -85,6 +88,7 @@ convert_case = "0.8"
 chrono = "0.4"
 similar = "2"
 uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] }
 fs2 = "0.4"
 pretty_assertions = "1.4.1"
 tempfile = "3.20.0"
 bollard = "0.19.1"
@@ -106,6 +110,7 @@ reqwest = { version = "0.12", features = [
  "http2",
  "json",
 ], default-features = false }
 oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] }
 assertor = "0.0.4"
 tokio-test = "0.4"
 anyhow = "1.0"
--- a/book.toml
+++ b/book.toml
@@ -2,7 +2,7 @@
 title = "Harmony"
 description = "Infrastructure orchestration that treats your platform like first-class code"
 src = "docs"
-build-dir = "book"
+# build-dir = "book"
 authors = ["NationTech"]
 [output.html]
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -6,6 +6,17 @@
 - [Typed Score References](./concepts/score-references.md)
 - [Getting Started Guide](./guides/getting-started.md)
 ## Fleet
 - [Overview and Architecture](./guides/fleet.md)
  - [Enrollment and Device Secrets](./guides/fleet-device-secrets.md)
  - [Podman Deployments and Canary Rollout](./guides/fleet-podman-deployments.md)
  - [Application Continuous Delivery](./guides/fleet-application-cd.md)
  - [Tasks and Upgrades](./guides/fleet-tasks-upgrades.md)
  - [Control-plane Operations](./guides/fleet-staging-install.md)
  - [Zitadel FAQ](./guides/fleet-zitadel-faq.md)
  - [Manual Token Mint](./guides/fleet-manual-token-mint.md)
 ## Use Cases
 - [PostgreSQL on Local K3D](./use-cases/postgresql-on-local-k3d.md)
@@ -22,13 +33,13 @@
 - [Developer Guide](./guides/developer-guide.md)
 - [Application CLI — Use Cases & Commands](./guides/application-cli.md)
 - [Harmony Auth CLI](./guides/harmony-auth-cli.md)
 - [Application Capabilities — .with(...)](./guides/application-capabilities.md)
 - [Writing a Score](./guides/writing-a-score.md)
 - [Writing a Topology](./guides/writing-a-topology.md)
 - [Adding Capabilities](./guides/adding-capabilities.md)
 - [Web Authentication and CSRF Security](./guides/web-auth-security.md)
 - [Operator Dashboard SSO (Zitadel) — setup](./guides/operator-dashboard-sso.md)
 - [Fleet Device Secrets](./guides/fleet-device-secrets.md)
 ## Configuration
@@ -37,6 +48,9 @@
 ## Reference Designs
 - [Fleet Score References](./reference/fleet-score-references.md)
 - [Fleet Agent Upgrades](./design/fleet-agent-upgrades.md)
 - [Fleet Tasks](./design/fleet-tasks.md)
  - [System-upgrade Executor](./design/fleet-system-upgrades.md)
 ## Architecture Decision Records
--- a/docs/adr/022-fleet-agent-upgrade.md
+++ b/docs/adr/022-fleet-agent-upgrade.md
@@ -4,353 +4,79 @@ Initial Author: Jean-Gabriel Gill-Couture
 Initial Date: 2026-05-06
-Last Updated Date: 2026-05-06
+Last Updated Date: 2026-07-22
 ## Status
-Accepted (design); implementation deferred — see roadmap
+Accepted. This revision replaces the dual-active design and the later signed
-`ROADMAP/fleet_platform/v0_2_plan.md`.
+switch-authorization design.
 ## Context
-The v0.1 fleet agent ships as a single static aarch64-musl binary
+Harmony Fleet currently manages IoT devices whose agents reconcile Podman
-sitting at `/usr/local/bin/fleet-agent`, started by a systemd
+workloads from NATS desired state. Agent replacement must preserve one workload
-unit dropped at install time by `FleetDeviceSetupScore`. Every
+owner across process or power loss.
 managed device runs one. Today the only "upgrade procedure" is
 `scp` + `systemctl restart` — fine for the bring-up phase, not
 fine once paying customers run real workloads on the fleet.
-Without a defined upgrade story we cannot ship a v0.1 agent into
+NationTech fleet administrators, the operator, and the NATS control plane share
-the field. The contract a customer needs is:
+one administrative trust domain. Device identity comes from a Zitadel JWT whose
 signature is validated through Zitadel JWKS and whose registered claims are
 checked before its `device_id` is interpolated into device-scoped NATS subjects.
 Separate artifact and cutover signatures are omitted because they would remain
 inside this trust domain while adding another key lifecycle.
-1. New agent versions can be rolled out without operator-side
+Future MDC fleets and their OKD control plane are outside this decision.
   manual intervention per device.
 2. Workloads currently reconciled on the device do not flap
   (start/stop/start) during the upgrade.
 3. A failed new version automatically reverts to the last
   known-good version, on its own, without page.
 4. The operator (the central one in the cluster, not the human)
   sees what version each device is running, can drive a target
   version per device, and observes upgrade progress.
 The agent itself is the only process on-device with full context
 on what's reconciling and what's healthy. Anything centralized
 (Ansible-pushed, OS-package-managed) doesn't have that signal.
 The agent must be the one driving its own swap, with the
 operator coordinating but not executing.
 ## Decision
-We adopt a **K8s rolling-update–shape upgrade**, single-host,
+The operator CAS-writes one per-device attempt. Once accepted, its UUID is bound
-agent-driven, operator-coordinated. Old version stays alive until
+to its complete content. The live agent forwards it to a root updater and keeps
-new is verified healthy from the operator's vantage point; only
+reconciling while the updater downloads, verifies, installs, and probes the
-then does the operator signal old to exit. **No version is ever
+candidate.
 erased from disk.** Symlinks select the active binary.
-### On-disk layout
+After probe success, the updater owns one local transaction:
-```
+```text
-/usr/bin/fleet-agent-v0.1.1            ← versioned binary, immutable
+stop old service
-/usr/bin/fleet-agent-v0.1.2            ← versioned binary, immutable
+select candidate atomically
-/usr/bin/fleet-agent-v0.1.3            ← versioned binary, immutable
+start candidate
-/usr/local/bin/fleet-agent             → symlink to current versioned binary
+commit on strict readiness, otherwise select and start the previous binary
 ```
- Versioned binaries are the source of truth. They live forever
+The old agent attempts to drain its current mutation only after systemd sends
-  (history-preserving, no GC). Disk use is bounded by humans
+SIGTERM. Shutdown is bounded by systemd, so an operation that exceeds the stop
-  cleaning up explicitly, not by the upgrade procedure.
+limit may be killed rather than drained.
 - The systemd unit installed by `FleetDeviceSetupScore` references
  `/usr/local/bin/fleet-agent`. Symlink swap is the cutover
  primitive — atomic on POSIX (`renameat2`).
 - Naming convention: exact crate version string, `v<MAJOR>.<MINOR>.<PATCH>`,
  no build metadata in the path. Build metadata lives in the agent's
  reported version string but not in the file path (otherwise you
  can't predict the path from a version pin).
-### State machine on the agent side
+Artifacts use HTTPS and a SHA-256 digest carried by NATS intent. The updater
 trusts its `fleet-agent` socket group, derives all managed paths, accepts no
 command, and executes candidates without root privileges. It has no NATS,
 OpenBao, or operator connection.
-```
+There is no artifact signature, second cutover authorization, device trust key,
-Running ──[operator publishes desired_version != current]──▶ Draining
+or extended probation.
   ▲                                                            │
   │                                                            │
   │                                                            ▼
   │                                                          Staging
   │                                                            │
   │                                                            ▼
   │                                                         Verifying
   │                                                            │
   │                                                            ▼
   │       ┌──────────────────────────────[smoke fails]────────┤
   │       │                                                   │
   │   [revert: symlink → previous,                            ▼
   │    stay at current]                                  Cutover-Ready
   │                                                            │
   │   [Cutover-Ready persists ≥ T_OPERATOR_OBSERVE              │
   │    until operator publishes stop_signal]                   │
   │                                                            ▼
   └────────────────────────────────────────────────────── Stopping
                                                                │
                                                                ▼
                                                              (exit)
 ```
-States in detail:
+The active service reports ready only after local dependencies, NATS, the
 complete desired-state snapshot, activation target-version check, transaction
 recovery, and required startup publications succeed. Strict readiness may reject
 a valid candidate during a coincident network outage. Retry and failure
 classification will be refined separately.
- **Running** — normal reconcile loop.
+Every pre-commit activation failure biases executable selection toward the
- **Draining** — refuses to start new podman services for new
+previous binary. `rollback-failed` blocks new attempts until direct repair.
-  desired-state writes. In-flight reconciles complete and report
+Successful readiness commits immediately. Failures after commit use systemd's
-  their final state to the operator. Existing services stay
+normal restart policy and do not trigger protocol rollback.
  running. Heartbeat continues. State is published as part of the
  agent's heartbeat (`agent_state: "draining"`).
 - **Staging** — fetch new versioned binary URL (signed,
  hash-pinned), verify, place at `/usr/bin/fleet-agent-v<new>`.
  Set chmod, ownership. No other state mutation.
 - **Verifying** — invoke the staged binary with `--self-test`. New
  binary parses its config, opens NATS connection, validates JWT,
  prints version + "ok", exits 0. **No state mutation.** Catches
  obvious breakage (missing dependency, wrong arch, corrupt
  download, broken config-schema migration) before swap.
 - **Cutover-Ready** — staged binary is healthy. Old agent updates
  the symlink atomically:
  ```
  ln -sfn /usr/bin/fleet-agent-v0.1.2 /usr/local/bin/fleet-agent.new
  mv -T /usr/local/bin/fleet-agent.new /usr/local/bin/fleet-agent
  ```
  Old agent then `systemctl start fleet-agent-v0.1.2.service` (a
  parallel transient service, not `systemctl restart` of itself).
  Both old and new are now running. New publishes its first
  heartbeat with `version=v0.1.2`. Operator sees two heartbeats
  per device for a brief window.
 - **Stopping** — operator publishes a stop signal to the old
  agent's NATS subject. Old agent receives, gracefully exits.
  systemd's `Restart=on-failure` does *not* trigger because the
  exit is `success` (rc=0, code-path-driven). New agent is now
  the only one running. systemd unit is reconfigured to point at
  the *current* symlink target on its next restart, but that's
  cosmetic — the symlink already does the job.
 ### Operator-side coordination
 The operator is the only source of truth for "what version should
 this device run". One new field per device, two new subjects.
 **New on `Device` CR / KV `device-info`:**
 - `current_version` — what the agent is running right now.
  Reported in heartbeat; reflected to the CR.
 - `desired_version` — what the operator wants the agent to run.
  Set by operator-side logic (default: latest published; eventually
  canary / %-based).
 **New NATS subjects (per-device, scoped by callout permissions):**
 - `device-cmd.<device_id>.upgrade-stop` — operator → old agent.
  Payload: `{"reason": "...", "deadline_ms": ...}`. Sent only after
  operator has observed a heartbeat from the new version with
  `current_version == desired_version` AND `agent_state == "running"`.
 - `device-state.<device_id>.upgrade` — agent → operator. Status
  events: `staging`, `verifying`, `cutover-ready`, `failed`, `done`.
  Drives `Device.status.upgrade.{phase, last_error, ...}`.
 The operator only emits `upgrade-stop` after it has independently
 verified the new agent is up. **Old agent does not stop itself
 based on its own observations.** This is the load-bearing
 property: the same operator that disagreed with the upgrade
 ("haven't seen new version's heartbeat") would never have sent
 the stop signal. Single-source-of-truth handoff.
 ### Failure modes and rollback
 - **Staging fails (download / hash mismatch):** Agent stays in
  `Running`. Reports `phase: "failed"`, `last_error`. Operator
  sees the failure; can fix the artifact + retry by re-publishing
  the same desired_version (any change to desired_version
  re-triggers the state machine).
 - **Verifying fails (smoke test rc != 0):** Agent stays in
  `Running`. Reports failure. Staged binary stays on disk for
  inspection. Operator can collect, debug, ship a fixed version.
 - **Cutover-ready, but new agent never publishes a heartbeat
  with the new version within T_HEARTBEAT_TIMEOUT (suggested
  60s):** Old agent reverts the symlink, stops the parallel
  systemd transient service, transitions back to Running with
  the old version. Reports `failed`. Same recovery path.
 - **Operator never sends stop signal (e.g., operator-side
  outage):** Old agent stays in Cutover-Ready indefinitely. Both
  agents are running; only the new one is publishing as the
  active one (the old one's writes are gated on its state). This
  is expensive (2× resource use) but safe — the operator is the
  authoritative coordinator and any other behavior would risk
  losing both agents at once.
 - **Both agents alive but new agent crashes:** systemd's
  `Restart=on-failure` on the new agent's transient unit retries.
  If it can't come back, the operator never sends the stop signal,
  the old agent stays Cutover-Ready, and a human investigates.
  The fleet keeps working on the old version — the rollback is
  implicit.
 - **Operator publishes an older `desired_version`:** Reverse
  rollout. Same mechanism, just with old/new swapped. The "new"
  binary is older, but the procedure is identical. The fact that
  no version is ever GC'd is what makes this work.
 ### What this isn't
 - **Not fleet-wide.** Per-device. Fleet-wide canary / %-based
  rollouts are operator-side orchestration **on top of** this
  primitive. The operator would publish `desired_version` to a
  rolling subset of devices and watch heartbeats. Out of scope
  for v0.2 — single-device upgrade is sufficient for a 100-Pi
  fleet which is more than the 12-month customer roadmap.
 - **Not blue/green of the entire OS.** We swap one userspace
  binary. The OS, podman, the systemd unit text, the kernel — all
  unchanged. Out of scope.
 - **Not a package manager.** Versioned binaries land at fixed
  paths because we control them. apt / dpkg / OSTree are
  orthogonal and not in the loop.
 ## Rationale
 - **No version ever erased.** Trivializes rollback (the previous
  binary is a `ln -sfn` away). Simplifies the failure tree:
  every "what if" branch resolves to "old still on disk". Disk
  cost on aarch64-musl is ~5–10 MB per version — at 12 versions
  / year, that's 100 MB after a decade of upgrades. Negligible
  compared to Pi storage.
 - **Symlink swap as cutover.** POSIX-atomic. No daemon state.
  Cheap to revert. Compatible with systemd unit references that
  point at a stable path.
 - **Old verifies new, then reports up.** This is the load-bearing
  property: it places the verification at the agent (which has
  the only complete view of its own runtime state) but the
  *commitment* at the operator (which is the only thing safe to
  trust as the cluster-wide source of truth). Either side alone
  can fail safe; only consensus advances the upgrade.
 - **Operator-driven stop, not agent self-stop.** A self-stopping
  agent could decide to exit before the operator agrees, leaving
  the cluster blind. Forcing the stop through the operator means
  any disagreement keeps the old agent alive — which is the
  desired bias.
 - **Drains in-flight work first.** Mirrors K8s pod-shutdown
  semantics. A workload reconciling at the moment of swap
  finishes its current step, reports state, then queues. New
  agent picks up the queue once it's the active version. No
  observable flap on the workload.
 - **Heartbeat-driven version reporting.** The agent already
  publishes heartbeats; adding the version field is one line.
  No new transport.
 ## Consequences
-**Pros:**
+- One process owns workload mutation; there is a bounded interval with no agent
  during cutover.
 - Preparation does not delay workload reconciliation.
 - Rollback changes only the executable. Persistent changes made before commit
  must remain readable by the previous release.
 - The first updater-capable release requires `FleetDeviceSetupScore`. The root
  updater remains outside automatic upgrades.
 - NationTech administration, the `fleet-agent` socket, the Zitadel identity
  chain, the auth callout, and NATS are trusted control-plane components.
- Bounded blast radius per upgrade (one device).
+The [fleet agent upgrade guide](../design/fleet-agent-upgrades.md) owns the
- Rollback is the same code path as upgrade — no special-case
+current sequence, recovery behavior, limits, status, and repair constraints.
-  bug class.
+Wire types remain authoritative in `harmony-reconciler-contracts`.
 - Operator's view is monotonic: heartbeats with versions are
  immutable history; there's no "did the upgrade really happen"
  state.
 - Old agent never decides to exit on its own. The most dangerous
  failure mode in self-upgrading software (premature exit) is
  designed out.
 - Compatible with eventual fleet-wide rollouts (canary, %-based)
  which become operator-side orchestration on top of this
  primitive.
 **Cons:**
 - Briefly runs two agents per device (Cutover-Ready window).
  Memory and connection-count both ~2× during that window.
  Acceptable for the upgrade duration (typically <60s).
 - Requires reliable connectivity between agent and operator to
  complete the handoff. A device whose NATS link fails mid-
  upgrade stays in Cutover-Ready until link recovers.
 - Disk grows monotonically with version count. Bounded by human
  cleanup. We do not GC.
 - New NATS subjects, new heartbeat fields, new `Device.status`
  fields. Schema bump that operators-in-the-field need to handle
  (the operator must understand "old agent reporting no version
  field" as `version: unknown`, not crash).
 ## Alternatives considered
 1. **OS-package upgrade (apt / dpkg / OSTree).** *Pros:* zero
   custom code, standard toolchain, GPG-signed.
   *Cons:* Loses the "agent verifies the new agent before swap"
   property. apt's restart hook flips the symlink and `systemctl
   restart`s; if the new binary is broken, the device is bricked
   until human intervention. Doesn't drain in-flight work. Doesn't
   know about NATS-managed pause states. Couples the upgrade
   schedule to the distro's repo, not to the cluster operator's
   intent. Rejected.
 2. **Pull-from-OCI-registry on each agent restart.** *Pros:* same
   primitive as podman / kube node-image-rotation.
   *Cons:* Coupling to a registry the device must reach — many
   customer fleets are on private subnets without registry
   access. Would mean shipping a registry mirror per fleet. Adds
   a dependency for a problem we can solve with a signed binary
   on a CDN.
 3. **Two systemd units, blue/green at the unit level.**
   `fleet-agent-v0.1.1.service` and `fleet-agent-v0.1.2.service`,
   ratchet via systemctl enable/disable. *Pros:* no symlink dance.
   *Cons:* duplicates a lot of unit-file content; harder to
   reason about what the "active" unit is (you have to ask
   systemd, not `readlink`); doesn't compose well with the
   `ExecStart=/usr/local/bin/fleet-agent` line we already ship.
   Symlink swap is the lighter primitive.
 4. **Self-stopping agent (no operator stop signal).** New agent
   tells old agent "I'm up, you can go" via NATS. *Pros:* one
   fewer subject.
   *Cons:* The new agent is also the agent we're least sure of
   — putting it in charge of the old one's lifecycle inverts the
   trust model. If the new agent has a bug that causes it to
   announce ready prematurely, the cluster goes blind. The
   operator path is the conservative choice.
 5. **Operator-pushed binary (instead of agent-pulled).** The
   operator sshes / executes a one-off command per device.
   *Pros:* operator controls timing precisely.
   *Cons:* Reintroduces SSH as a control plane (we just spent a
   month getting rid of it for the enrollment flow). Doesn't
   scale to fleets where most devices are NATted away from the
   operator.
 ## Implementation milestones
 (For a future implementer; not committed to a date here. Lives
 in the v0.2+ backlog.)
 1. **M1** — Versioned binary layout: builds produce
   `fleet-agent-v<version>` artifacts; install Score writes them
   to `/usr/bin/fleet-agent-v<version>` + creates
   `/usr/local/bin/fleet-agent` symlink. Existing tests cover the
   rest.
 2. **M2** — Version field in heartbeat + `Device.status.current_version`
   reflection on the operator side. No upgrade behavior yet.
 3. **M3** — `desired_version` field on the device-info KV +
   operator setter. No agent-side action yet.
 4. **M4** — Agent state machine, end to end, gated by a feature
   flag. Operator publishes desired_version → agent does the
   dance → operator sends stop signal → done. Includes failure-
   mode tests (download fail, smoke fail, heartbeat-timeout
   revert).
 5. **M5** — Remove the feature flag. Default-on.
 6. **M6** — Operator-side rollout strategies (canary, %-based) —
   only after M5 has been in production for 30 days against a
   real fleet.
 ## Additional Notes
 - Binary signing + signature verification is in scope for the
  `Staging` step but the *which* signing scheme (cosign / Rekor
  / minisign) is deferred until the M1 implementation. Whatever
  we pick must work on aarch64-musl Pi devices without
  additional system dependencies.
 - The N-versions-on-disk policy is "all of them, forever" per
  the constraint above. If disk pressure becomes real on some
  customer fleet, a manual GC tool can prune `/usr/bin/fleet-agent-v*`
  by date — never automatic, never as part of the upgrade
  itself.
 - See JG's *Pour l'amour des compilateurs* talk (Botpress
  Meetup, 2026-04-30) for the framing applied here:
  cardinality-matched types and operator-as-coordinator are the
  same idea, applied to one function and to one platform.
--- a/docs/coding-guide.md
+++ b/docs/coding-guide.md
@@ -215,6 +215,24 @@ pub use network::NetworkConfig;
 ---
 ## Testing
 Test infrastructure behavior against the real component whenever practical:
 use k3d for Kubernetes, Podman for containers, libvirt for VMs, and disposable
 devices for privileged or destructive paths. A mock that reimplements an
 external system can confirm the mock rather than the integration.
 Fakes remain appropriate at domain and use-case boundaries when testing core
 algorithms, failure transitions, or authorization decisions. Keep them behind
 the same capability boundary as the real adapter, and retain at least one test
 through the real adapter for behavior owned by that external system.
 E2E harnesses compose the same Scores as production. They may change the
 Topology, credentials, namespace, or target device, but must not maintain a
 parallel manifest or deployment path.
 ---
 ## Commit Style
 Follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/):
--- a/docs/design/fleet-agent-upgrades.md
+++ b/docs/design/fleet-agent-upgrades.md
@@ -0,0 +1,228 @@
 # Fleet agent upgrades
 ## Scope
 This design updates the Harmony agent on IoT devices that reconcile Podman
 workloads from NATS desired state. It covers one agent binary on one Linux
 device. Operating-system, Podman, workload, and future MDC/OKD upgrades are
 outside its scope.
 ![Fleet agent upgrade architecture](../diagrams/fleet-agent-upgrade-architecture.svg)
 ## Ownership
 | Component | Responsibility |
 |---|---|
 | Fleet operator | Select the desired release and CAS-write the current per-device attempt |
 | Active agent | Validate and forward intent, reconcile during preparation, drain on SIGTERM, publish status |
 | Root updater | Prepare and probe the candidate, stop, switch, start, commit or roll back |
 | systemd | Serialize service ownership and enforce startup and shutdown limits |
 The active process holds an exclusive lock, so only one agent can mutate
 workloads. Preparation does not pause Podman reconciliation. Running containers
 continue under their restart policies during agent cutover.
 The updater has no NATS, OpenBao, or operator client. Its Unix socket accepts
 `upgrade` and read-only `status` requests. It derives every managed path and
 accepts no shell command or caller-selected destination.
 ## Trust boundary
 The operator CAS-writes `AgentUpgradeAttempt` to the device's
 `agent-upgrade-intent` key. CAS orders competing writers; it does not make the KV
 key append-only. After local acceptance, the agent and updater bind an attempt
 UUID to the digest of its complete content.
 An attempt contains the device, target version, architecture, artifact URL, byte
 limit, SHA-256, and UUID. It has no source version or expiry: a device returning
 after days or months can upgrade directly to the current target. Any required
 data migration belongs in the binary and must handle the versions it supports.
 Artifacts are either direct HTTPS URLs or anonymous `oci://` references with an
 explicit tag or manifest digest. OCI artifacts must declare the Harmony agent
 artifact type and contain exactly one Harmony agent binary layer. HTTPS
 authenticates transport; SHA-256 binds the downloaded bytes to the NATS attempt.
 There is no artifact signature, second cutover authorization, device upgrade
 key, or OpenBao dependency. NationTech administrators, the operator, and NATS
 share one administrative trust domain, so a separate release-signing domain is
 not required today.
 The updater trusts callers admitted by the `fleet-agent` socket group to forward
 NATS intent faithfully. Compromise of that account includes control of agent
 upgrades. Installed binaries still run as `fleet-agent`, not root.
 The auth callout validates the Zitadel JWT signature through Zitadel JWKS and
 checks its registered claims. It then extracts `device_id`, rejects characters
 unsafe for NATS subjects, and interpolates the value into device-scoped data
 permissions. No unverified client-supplied device ID is used.
 ## Upgrade flow
 ![Fleet agent upgrade sequence](../diagrams/fleet-agent-upgrade-sequence.svg)
 ### Intent
 When desired and reported versions differ, the operator writes an attempt with
 a new UUID. The agent rejects an attempt for another device, the wrong
 architecture, or an invalid UUID. Reuse of a UUID with different content is
 rejected. Intent has no age or previous-version gate.
 The per-device intent key can later be replaced by CAS. An unchanged release
 whose attempt has failed is reused rather than retried automatically. A new
 attempt currently requires changed release metadata or direct control-plane
 repair.
 ### Preparation and probe
 The agent publishes `preparing` and sends one `upgrade` request. The updater
 writes `preparing` to its journal before it:
 1. downloads over HTTPS with redirects disabled;
 2. enforces the attempt limit and 100 MiB compiled ceiling;
 3. verifies SHA-256;
 4. installs or re-verifies `/usr/lib/harmony-fleet/fleet-agent-v<version>`;
 5. runs `<candidate> --self-test` as `fleet-agent` and requires its compiled
   version to match the target.
 The live agent continues reconciling during these steps.
 Probe mode parses the real configuration, reaches Podman when enabled,
 authenticates to NATS, opens required buckets, reads per-device keys, creates a
 filtered desired-state consumer, checks the updater socket, and consumes a
 complete snapshot. It starts no loops, publishes no normal status or heartbeat,
 and mutates no workload. Creating the ephemeral consumer is a JetStream
 management operation. The probe does not prove publish permissions; active
 startup exercises its startup writes before readiness.
 NATS connection attempts are limited to 15 seconds over a 3-minute retry
 window. The complete probe is limited to 4 minutes.
 ### Stop, switch, start
 After probe success, the updater writes `activating` and runs
 `systemctl stop fleet-agent.service`. On SIGTERM, the old agent stops admitting
 new runtime mutations and waits for the current mutation. It then attempts to
 publish acknowledged `stopping` status with `reason=updater`, records drain
 duration, flushes NATS, notifies systemd, and exits.
 This drain is bounded by `TimeoutStopSec=60s`. systemd may kill an agent whose
 current Podman mutation does not finish in time. The `stopping` status is
 diagnostic; the updater does not wait for it. Cutover proceeds when systemd
 reports the service inactive.
 The updater atomically replaces `/usr/local/bin/fleet-agent`, syncs its parent
 directory, and starts the service. It never deliberately runs old and new
 reconcilers together.
 ### Readiness and commit
 The new process sends `READY=1` only after it:
 - acquires the process lock and parses configuration;
 - reaches Podman when enabled;
 - reconnects to NATS and opens required buckets;
 - publishes device information;
 - reads and validates the recovered updater transaction;
 - verifies its compiled version against the activation target;
 - consumes a complete desired-state snapshot;
 - publishes transaction status when an upgrade is active.
 The local transaction need not match the current NATS intent during recovery.
 Its stored digest, target, and exact previous binary path are authoritative after
 cutover starts. Rollback restores that path and does not depend on a source
 version declared by the operator.
 Successful systemd readiness commits immediately. A network failure after a
 successful probe can therefore cause rollback. After `committed`, later crashes
 use the normal systemd restart policy; this protocol does not roll back a
 candidate that was already ready.
 ## Recovery
 ![Fleet agent durable transaction and recovery](../diagrams/fleet-agent-upgrade-recovery.svg)
 The updater journal stores attempt identity, previous and target paths, bounded
 errors, and up to 16 timed transitions.
 | Durable phase | Recovery |
 |---|---|
 | `preparing` | Restore the previous link, record `failed`, start the previous service; restore or start failure enters `rollback-failed` |
 | `activating` | Record `rolling-back`, restore the previous link, stop, then start the previous service |
 | `committed` | Keep the target |
 | `rolling-back` | Restore the previous link and repeat stop/start |
 | `failed` | Keep the previous target; no automatic retry |
 | `rollback-failed` | Refuse every new attempt until direct root repair |
 Before the updater reports systemd readiness, interrupted pre-commit recovery
 has selected the previous symlink and persisted its recovery phase. Service
 stop/start completion continues asynchronously after updater readiness so the
 agent can query local status while starting. Corrupt journal data prevents the
 updater socket from starting; state is not inferred from filesystem contents.
 Journal and binary writes use temporary files, file sync, atomic rename, and
 parent-directory sync. Symlink selection uses a temporary symlink, atomic
 rename, and parent-directory sync.
 `rollback-failed` has no remote clear operation. Repair requires root access to
 inspect the journal, active symlink, managed binaries, and both systemd units.
 The updater must not be unquarantined until the previous binary is selected and
 starts successfully.
 ## Status
 Public phases are `preparing`, `stopping`, `starting`, `complete`,
 `rolling-back`, `failed`, and `rollback-failed`. The wire contract in
 [`harmony-reconciler-contracts/src/upgrade.rs`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs)
 is authoritative for fields and serialization.
 Status includes attempt and target identity, current phase, bounded detail and
 error, attempt and transition timing, completed drain duration, typed reason,
 boot and systemd invocation IDs when available, one current journald unit
 reference, and up to 16 transitions. Log content remains in journald.
 The updater never publishes NATS status. The active agent polls the local
 transaction and reflects changes, including `complete`, into NATS and the Device
 CR.
 ## Limits
 | Limit | Value |
 |---|---:|
 | HTTPS connect / complete request | 15 seconds / 5 minutes |
 | Artifact size | 100 MiB |
 | NATS connection attempt / retry window | 15 seconds / 3 minutes |
 | Candidate probe | 4 minutes |
 | systemd stop / updater stop bound | 60 / 65 seconds |
 | systemd start / updater start bound | 4 minutes / 4 minutes 15 seconds |
 | Socket request-read / status response / upgrade response | 10 seconds / 15 seconds / 20 minutes |
 | Socket request or response | 1 MiB |
 | Concurrent socket handlers | 32 |
 | Retained transitions | 16 |
 | Detail or error | 1,024 characters |
 Only one mutating socket request runs at a time; `status` remains available
 during preparation and recovery. Versioned binaries are not garbage-collected.
 Devices currently have many GB available, so collection is deferred until disk
 pressure makes retention an operational concern.
 Rollback changes only the executable. Configuration, local databases, Podman
 labels, and any persistent format touched before commit must remain readable by
 the previous release.
 ## Bootstrap
 `FleetDeviceSetupScore` installs the bootstrap binary, updater and agent units,
 socket ownership, state directories, and active symlink. The root updater is not
 self-updated. Repairing or replacing it requires another device setup operation.
 Running device setup during an active transaction is unsupported.
 Devices installed before OCI support require one device setup operation before
 their first `oci://` upgrade. This replaces the bootstrap updater; an older
 updater cannot fetch an OCI artifact even when the active agent can accept the
 intent.
 ## Related decisions
 - [ADR-016: Harmony agent and global mesh](../adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md)
 - [ADR-022: Fleet agent upgrade procedure](../adr/022-fleet-agent-upgrade.md)
 - [ADR-023: Deploy architecture](../adr/023-deploy-architecture.md)
--- a/docs/design/fleet-system-upgrades.md
+++ b/docs/design/fleet-system-upgrades.md
@@ -0,0 +1,128 @@
 # Fleet system-upgrade executor
 ## Summary
 System upgrade is a privileged built-in Fleet task. `TaskRun` owns placement,
 the frozen target plan, deadlines, and aggregate status as described in
 [Fleet tasks](./fleet-tasks.md). This document covers the device executor.
 The unprivileged agent requests one fixed operation from the existing root
 updater over its Unix socket. The updater accepts no shell command, package
 name, repository, path, unit, or reboot argument.
 System and agent upgrades share the updater process and mutation lock. System
 upgrades have a separate protocol operation, journal, and state machine because
 package upgrades are not atomically reversible.
 The first executor runs an apt full upgrade using the device's configured,
 signed repositories and then requires a reboot. It supports Debian and
 Raspberry Pi OS. It does not configure repositories, select packages, roll back
 packages, cancel an active apt/dpkg process, or retry a terminal failure.
 ## Trust boundary
 The updater socket admits only the `fleet-agent` Unix group. The agent can ask
 for `AptFullUpgradeV1`, but cannot alter what that operation does. A compromised
 agent can invoke the upgrade and is treated as a compromised device; adding a
 second key held by the same agent would not improve that boundary.
 A system-upgrade request contains only:
 ```text
 attemptId
 runUid
 deviceId
 expiresAt
 ```
 The updater validates the identifiers and expiry before starting. Duplicate
 requests with the same content return the durable result. Reusing an attempt ID
 with different content fails. Expiry prevents an intent delayed in NATS from
 starting later; it does not interrupt apt/dpkg after mutation begins.
 The updater publishes protocol and `AptFullUpgradeV1` capability through device
 observation. Planning fails if a selected device lacks that capability. The
 privileged updater is installed out of band by `FleetDeviceSetupScore` and does
 not self-update.
 The updater has no NATS credentials. The agent relays updater status. A
 compromised agent can therefore falsify status, which is consistent with the
 device-compromise trust model.
 ## Privileged state machine
 Agent and system upgrades share one durable mutation lock. System upgrades use
 one journal per attempt under
 `/var/lib/harmony-fleet-updater/system-upgrades/`.
 Device-reported phases are:
 ```text
 blocked
 preflight
 applying
 rebooting
 verifying
 complete
 failed
 repair-required
 ```
 `blocked` is non-terminal and covers an active agent upgrade or a bounded wait
 for apt/dpkg locks. `repair-required` forbids automatic retry.
 `AptFullUpgradeV1` has one compiled transaction:
 1. Confirm package state is healthy and no configured apt source disables
   repository authentication.
 2. Run `apt-get update`.
 3. Run noninteractive `apt-get full-upgrade`, preserving locally modified
   configuration files.
 4. Audit dpkg state.
 5. Record `rebooting` and the current boot ID, then request reboot.
 6. After startup, require a changed boot ID and healthy dpkg state.
 Durable recovery is:
 | Journal state | Recovery |
 |---|---|
 | `preflight` | Repeat non-mutating checks. |
 | `applying` | Audit dpkg and enter `repair-required`; never rerun the interrupted full upgrade. |
 | `rebooting`, old boot ID | Retry the fixed reboot request up to a bound, then report `failed`. |
 | `rebooting`, changed boot ID | Persist `verifying`. |
 | `verifying` | Audit package state, then complete or require repair. |
 | `complete`, `failed`, `repair-required` | Return the same terminal result for duplicate intent. |
 The updater persists `applying` before spawning apt and `rebooting` before
 requesting reboot. Startup fails if more than one journal appears active or a
 journal is corrupt. Journal writes use a temporary
 file, fsync, and atomic rename. `repair-required` remains until an operator
 repairs the device.
 ## Task result
 Disconnect after `rebooting` is expected; request timeout is not a failure
 signal. A device succeeds only after:
 - the updater observes a boot ID different from the pre-upgrade boot ID;
 - post-boot dpkg state is healthy;
 - the agent reconnects and relays `complete`;
 - the agent records a post-completion heartbeat before the run deadline.
 The operator owns the run deadline. A device that never returns fails the run.
 Already upgraded devices are not rolled back.
 ## Invariants
 - A device runs at most one privileged mutation at a time.
 - An expired attempt cannot start.
 - Restart does not repeat a completed attempt.
 - Corrupt or incomplete package state fails closed.
 - All apt, dpkg, and reboot arguments are compiled into the updater.
 ## Remaining work
 1. Prove the path on a disposable Debian VM, then Raspberry Pi OS.
 2. Test updater and agent restart, package failure, reboot failure, expiry, and
   duplicate delivery against real systemd, apt, and dpkg.
 3. Add UTC recurring schedules after the direct run passes those tests.
--- a/docs/design/fleet-tasks.md
+++ b/docs/design/fleet-tasks.md
@@ -0,0 +1,142 @@
 # Fleet tasks
 ## Summary
 Fleet tasks are finite jobs selected with the same group and label rules as
 Fleet deployments. A task freezes its targets, runs once on each device, and
 retains a terminal result.
 The first release supports directly created, multi-device `TaskRun`s and the
 built-in system upgrade. Recurring schedules come after the direct path is
 proven on Debian and Raspberry Pi OS.
 ## `TaskRun`
 ```yaml
 apiVersion: fleet.nationtech.io/v1alpha1
 kind: TaskRun
 metadata:
  name: upgrade-device-1
 spec:
  allowedGroups: [production]
  targetSelector:
    matchLabels:
      device-id: device-1
  deadlineSeconds: 21600
  systemUpgradeV1: {}
 ```
 `systemUpgradeV1` is empty because apt, dpkg, repository, and reboot policy is
 compiled into the updater. The API cannot pass package names, repository paths,
 commands, or reboot arguments.
 Admission rejects changes to the spec. The run starts immediately and has one
 phase:
 ```text
 Planning | Running | Complete | Failed
 ```
 During `Planning`, the operator freezes every matching device that supports
 `AptFullUpgradeV1`. No match fails with `NoTargets`.
 Status records target, success, and failure counts; start and completion times;
 a reason; and the latest bounded error. The CR never contains an unbounded
 per-device map.
 ## Placement
 Task placement uses the deployment aggregator's rule:
 ```text
 allowed group membership AND targetSelector
 ```
 `allowedGroups` requires at least one group. `matchLabels` is a conjunction, and
 an empty selector matches every authorized device.
 `matchExpressions` fails closed until implemented.
 The operator checks group authorization again before creating the intent. A
 device revoked before release fails the run with `TargetRevoked`; another device
 is never substituted into the frozen plan. A group-source error retries
 planning instead of becoming `NoTargets`.
 The target plan is created atomically in JetStream before the run enters
 `Running`. Concurrent operator instances therefore load the same frozen plan.
 The run UID is also the attempt ID, so intent creation and updater execution are
 idempotent across restarts.
 ## Execution
 The operator writes the fixed system-upgrade attempt to:
 ```text
 system-upgrade-intent: <device-id>.<run-uid>
 ```
 The agent publishes updater-owned status to:
 ```text
 system-upgrade-status: <device-id>.<run-uid>
 ```
 The agent watches current and new intents, so reboot does not lose the active
 run. The operator accepts terminal status only when its attempt and run IDs
 match. For success, the agent records the first heartbeat after updater
 completion in the durable run status; that heartbeat must precede the deadline.
 A late result cannot reopen a failed run.
 Intent and status buckets are file-backed, keep one value per key, and have byte
 and age limits. The operator removes a terminal intent; terminal status remains
 for bounded diagnostics.
 The deadline is copied into the attempt as `expiresAt`. Expiry prevents a
 delayed intent from starting. It does not interrupt apt or dpkg after the
 updater has accepted the attempt.
 ## Canary release
 If any matched device has the `canary=true` label, the operator releases only
 those devices first. Every canary must complete successfully before the
 operator releases the remaining devices. A failed, missing, or timed-out canary
 stops the rollout. Completed devices are not rolled back.
 When no matched device has `canary=true`, the operator releases the full
 frozen target set immediately. There is no percentage, batch-size, or
 concurrency setting.
 The frozen target set must remain bounded outside the CR, while the CR retains
 aggregate counts and the latest error.
 ## Recurring schedules
 A later namespaced `TaskSchedule` creates immutable `TaskRun` resources. Daily,
 weekly, yearly, and five-field cron forms use UTC. Time-zone configuration is
 not supported.
 One schedule has at most one active run. Ticks during an active run are skipped,
 and restart creates at most one catch-up run for the latest eligible tick.
 Suspension prevents new runs without stopping an active run.
 ## Other workloads
 Container and agent-upgrade tasks remain deferred. Add a workload discriminator
 only when a second workload is implemented. Agent upgrades continue to use
 their existing activation, health-check, and rollback state machine.
 ## Invariants
 - A run UID identifies one immutable target plan and operation.
 - Canary and non-canary membership is frozen with the target plan.
 - Group authorization is current when an intent is released.
 - A device executes one run attempt at most once.
 - A terminal run never advances.
 - Restart does not repeat a completed device attempt.
 - No Kubernetes status map grows with fleet size.
 ## Remaining work
 1. Prove package upgrade, reboot, restart recovery, expiry, and duplicate intent
   handling on a disposable Debian VM and Raspberry Pi OS device.
 2. Add UTC schedules.
 3. Add dashboard run history and alerts.
--- a/docs/diagrams/fleet-agent-upgrade-architecture.svg
+++ b/docs/diagrams/fleet-agent-upgrade-architecture.svg
@@ -0,0 +1,63 @@
 <svg xmlns="http://www.w3.org/2000/svg" width="1200" height="540" viewBox="0 0 1200 540" role="img" aria-labelledby="title desc">
  <title id="title">IoT fleet agent upgrade ownership and trust architecture</title>
  <desc id="desc">The fleet operator writes one authenticated NATS upgrade intent. The active agent forwards it while continuing Podman reconciliation. A root updater downloads by HTTPS, verifies SHA-256, probes as fleet-agent, and performs the systemd stop, symlink switch, and start transaction.</desc>
  <defs>
    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
    <marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
    <style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
  </defs>
  <rect width="1200" height="540" fill="#ffffff"/>
  <g class="sans">
    <text x="42" y="50" fill="#0f172a" font-size="29" font-weight="750">One intent, one active workload owner</text>
    <text x="42" y="78" fill="#64748b" font-size="16">Authenticated NATS carries upgrade intent. HTTPS and SHA-256 bind the candidate bytes.</text>
    <rect x="35" y="110" width="300" height="360" rx="18" fill="#f8fafc" stroke="#cbd5e1" stroke-width="1.5"/>
    <text x="60" y="142" fill="#475569" font-size="12" font-weight="750" letter-spacing="1.4">CONTROL PLANE</text>
    <rect x="65" y="175" width="240" height="90" rx="12" fill="#172554"/>
    <text x="88" y="205" fill="#bfdbfe" font-size="12" font-weight="700">DESIRED RELEASE</text>
    <text x="88" y="235" fill="#fff" font-size="18" font-weight="700" class="mono">Device.spec</text>
    <text x="88" y="256" fill="#bfdbfe" font-size="14">version and artifact metadata</text>
    <rect x="65" y="315" width="240" height="110" rx="12" fill="#0f766e"/>
    <text x="88" y="345" fill="#ccfbf1" font-size="12" font-weight="700">FLEET OPERATOR</text>
    <text x="88" y="376" fill="#fff" font-size="17" font-weight="700">current attempt</text>
    <text x="88" y="400" fill="#ccfbf1" font-size="14">CAS and status reflection</text>
    <path d="M185 265V315" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
    <rect x="390" y="110" width="250" height="360" rx="18" fill="#f0fdfa" stroke="#5eead4" stroke-width="1.5"/>
    <text x="415" y="142" fill="#115e59" font-size="12" font-weight="750" letter-spacing="1.4">AUTHENTICATED MESH</text>
    <rect x="425" y="178" width="180" height="245" rx="14" fill="#fff" stroke="#99f6e4"/>
    <text x="515" y="216" text-anchor="middle" fill="#0f172a" font-size="21" font-weight="750">NATS KV</text>
    <text x="445" y="265" fill="#115e59" font-size="12" class="mono">agent-upgrade-intent</text>
    <text x="445" y="309" fill="#115e59" font-size="12" class="mono">agent-upgrade-status</text>
    <text x="445" y="365" fill="#64748b" font-size="12">device-scoped data subjects</text>
    <text x="445" y="390" fill="#64748b" font-size="12">identity from verified JWT</text>
    <path d="M305 370H425" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
    <path d="M425 400H305" stroke="#0f766e" stroke-width="2" marker-end="url(#arrow)"/>
    <rect x="695" y="110" width="470" height="360" rx="18" fill="#fff7ed" stroke="#fdba74" stroke-width="1.5"/>
    <text x="720" y="142" fill="#9a3412" font-size="12" font-weight="750" letter-spacing="1.4">IOT PODMAN DEVICE</text>
    <rect x="730" y="175" width="180" height="110" rx="12" fill="#172554"/>
    <text x="750" y="203" fill="#bfdbfe" font-size="12" font-weight="700">UNPRIVILEGED</text>
    <text x="750" y="234" fill="#fff" font-size="19" font-weight="750">Fleet agent</text>
    <text x="750" y="259" fill="#bfdbfe" font-size="13">forward · reconcile · report</text>
    <rect x="960" y="175" width="170" height="110" rx="12" fill="#7c2d12"/>
    <text x="980" y="203" fill="#fed7aa" font-size="12" font-weight="700">ROOT</text>
    <text x="980" y="234" fill="#fff" font-size="19" font-weight="750">Updater</text>
    <text x="980" y="259" fill="#fed7aa" font-size="13">prepare · switch · recover</text>
    <path d="M605 230H730" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
    <path d="M910 230H960" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/>
    <text x="935" y="216" text-anchor="middle" fill="#9a3412" font-size="11">upgrade</text>
    <rect x="730" y="340" width="180" height="80" rx="12" fill="#eef2ff" stroke="#a5b4fc"/>
    <text x="750" y="372" fill="#312e81" font-size="17" font-weight="700">Podman workloads</text>
    <text x="750" y="397" fill="#64748b" font-size="13">reconcile until SIGTERM</text>
    <rect x="960" y="330" width="170" height="100" rx="12" fill="#fffbeb" stroke="#fbbf24"/>
    <text x="980" y="360" fill="#78350f" font-size="17" font-weight="700">systemd + symlink</text>
    <text x="980" y="385" fill="#92400e" font-size="13">stop · inactive · switch</text>
    <text x="980" y="408" fill="#92400e" font-size="13">start · strict readiness</text>
    <path d="M820 285V340" stroke="#64748b" stroke-width="2"/>
    <path d="M1045 285V330" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/>
    <text x="42" y="510" fill="#64748b" font-size="14">NationTech administration is one trust domain; device_id comes from a JWKS-verified Zitadel JWT.</text>
  </g>
 </svg>
--- a/docs/diagrams/fleet-agent-upgrade-recovery.svg
+++ b/docs/diagrams/fleet-agent-upgrade-recovery.svg
@@ -0,0 +1,45 @@
 <svg xmlns="http://www.w3.org/2000/svg" width="1200" height="650" viewBox="0 0 1200 650" role="img" aria-labelledby="title desc">
  <title id="title">Fleet agent durable upgrade states and recovery</title>
  <desc id="desc">Preparing keeps the previous agent active; interruption selects the previous binary and fails, while recovery failure quarantines the updater. Activating covers stop, switch, strict start, and readiness; interruption enters rollback. Readiness commits immediately.</desc>
  <defs>
    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
    <marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
    <marker id="arrow-bad" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="8" markerHeight="8" orient="auto"><path d="M0 0 10 5 0 10z" fill="#be123c"/></marker>
    <style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
  </defs>
  <rect width="1200" height="650" fill="#ffffff"/>
  <g class="sans">
    <text x="40" y="48" fill="#0f172a" font-size="29" font-weight="750">The durable journal controls recovery</text>
    <text x="40" y="76" fill="#64748b" font-size="16">Every interruption before commit attempts to restore the previous binary. Rollback failure blocks new attempts.</text>
    <rect x="35" y="110" width="520" height="230" rx="18" fill="#f0fdfa" stroke="#5eead4"/>
    <text x="60" y="142" fill="#115e59" font-size="12" font-weight="750" letter-spacing="1.3">PREPARATION: PREVIOUS AGENT RECONCILES</text>
    <rect x="85" y="195" width="170" height="76" rx="12" fill="#fff" stroke="#99f6e4"/><text x="170" y="226" text-anchor="middle" fill="#0f172a" font-size="18" font-weight="700" class="mono">no journal</text><text x="170" y="251" text-anchor="middle" fill="#64748b" font-size="13">existing target stays active</text>
    <rect x="335" y="195" width="170" height="76" rx="12" fill="#0f766e"/><text x="420" y="226" text-anchor="middle" fill="#fff" font-size="18" font-weight="700" class="mono">preparing</text><text x="420" y="251" text-anchor="middle" fill="#ccfbf1" font-size="13">download, verify, probe</text>
    <path d="M255 233H335" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/>
    <text x="295" y="220" text-anchor="middle" fill="#115e59" font-size="12">upgrade request</text>
    <rect x="610" y="110" width="555" height="230" rx="18" fill="#fff7ed" stroke="#fdba74"/>
    <text x="635" y="142" fill="#9a3412" font-size="12" font-weight="750" letter-spacing="1.3">ACTIVATION: STOP, SWITCH, STRICT START</text>
    <rect x="650" y="195" width="180" height="76" rx="12" fill="#b45309"/><text x="740" y="226" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">activating</text><text x="740" y="251" text-anchor="middle" fill="#ffedd5" font-size="12">not committed</text>
    <rect x="930" y="195" width="180" height="76" rx="12" fill="#172554"/><text x="1020" y="226" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">committed</text><text x="1020" y="251" text-anchor="middle" fill="#bfdbfe" font-size="12">target retained</text>
    <path d="M555 233H650" stroke="#b45309" stroke-width="2.5" marker-end="url(#arrow-warn)"/><text x="602" y="220" text-anchor="middle" fill="#9a3412" font-size="12">probe passes</text>
    <path d="M830 233H930" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/><text x="880" y="220" text-anchor="middle" fill="#115e59" font-size="12">READY=1</text>
    <text x="880" y="292" text-anchor="middle" fill="#64748b" font-size="12">immediate commit, no wait period</text>
    <rect x="280" y="400" width="190" height="80" rx="12" fill="#be123c"/><text x="375" y="432" text-anchor="middle" fill="#fff" font-size="17" font-weight="700" class="mono">rolling-back</text><text x="375" y="458" text-anchor="middle" fill="#ffe4e6" font-size="13">restore previous target</text>
    <path d="M740 271V350H470V420" fill="none" stroke="#be123c" stroke-width="2.5" marker-end="url(#arrow-bad)"/>
    <text x="535" y="350" fill="#9f1239" font-size="13">activation interrupted before commit</text>
    <rect x="85" y="535" width="180" height="72" rx="12" fill="#f1f5f9" stroke="#94a3b8"/><text x="175" y="565" text-anchor="middle" fill="#0f172a" font-size="18" font-weight="700" class="mono">failed</text><text x="175" y="588" text-anchor="middle" fill="#64748b" font-size="12">previous service selected</text>
    <rect x="390" y="535" width="220" height="72" rx="12" fill="#fff1f2" stroke="#fb7185"/><text x="500" y="565" text-anchor="middle" fill="#9f1239" font-size="17" font-weight="700" class="mono">rollback-failed</text><text x="500" y="588" text-anchor="middle" fill="#be123c" font-size="12">hard quarantine</text>
    <path d="M330 480V510H175V535" fill="none" stroke="#0f766e" stroke-width="2.5" marker-end="url(#arrow)"/><text x="235" y="505" fill="#115e59" font-size="12">restore and start succeed</text>
    <path d="M420 271V365H175V535" fill="none" stroke="#64748b" stroke-width="2.2" marker-end="url(#arrow)"/><text x="210" y="385" fill="#475569" font-size="12">preparation interrupted: select previous, fail</text>
    <path d="M455 271V380H500V535" fill="none" stroke="#be123c" stroke-width="2" marker-end="url(#arrow-bad)"/><text x="515" y="400" fill="#9f1239" font-size="12">restore or start fails</text>
    <path d="M420 480V510H500V535" fill="none" stroke="#be123c" stroke-width="2.5" marker-end="url(#arrow-bad)"/><text x="500" y="505" fill="#9f1239" font-size="12">restore or start fails</text>
    <text x="700" y="558" fill="#475569" font-size="14" font-weight="700">Durability sequence</text>
    <text x="700" y="581" fill="#64748b" font-size="13">temporary write, file sync, atomic rename, directory sync</text>
    <text x="700" y="610" fill="#64748b" font-size="13">No automatic rollback after committed.</text>
  </g>
 </svg>
--- a/docs/diagrams/fleet-agent-upgrade-sequence.svg
+++ b/docs/diagrams/fleet-agent-upgrade-sequence.svg
@@ -0,0 +1,38 @@
 <svg xmlns="http://www.w3.org/2000/svg" width="1200" height="830" viewBox="0 0 1200 830" role="img" aria-labelledby="title desc">
  <title id="title">IoT fleet agent stop, switch, start upgrade sequence</title>
  <desc id="desc">The operator CAS-writes the current NATS attempt. The active agent forwards one upgrade request and keeps reconciling while the updater downloads, verifies, installs, and probes. The updater stops the old service, waits for inactivity, switches the symlink, starts the candidate, and commits after strict readiness. The restarted agent publishes completion.</desc>
  <defs>
    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto"><path d="M0 0 10 5 0 10z" fill="#0f766e"/></marker>
    <marker id="arrow-warn" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto"><path d="M0 0 10 5 0 10z" fill="#b45309"/></marker>
    <style>.sans{font-family:Inter,ui-sans-serif,system-ui,sans-serif}.mono{font-family:&quot;JetBrains Mono&quot;,Consolas,monospace}</style>
  </defs>
  <rect width="1200" height="830" fill="#ffffff"/>
  <g class="sans">
    <text x="40" y="44" fill="#0f172a" font-size="28" font-weight="750">Prepare live, then stop, switch, start</text>
    <text x="40" y="70" fill="#64748b" font-size="15">One authenticated intent controls the complete transaction. Strict readiness commits immediately.</text>
    <g font-size="14" font-weight="700" text-anchor="middle">
      <rect x="35" y="96" width="180" height="46" rx="10" fill="#0f766e"/><text x="125" y="125" fill="#fff">Fleet operator</text>
      <rect x="275" y="96" width="150" height="46" rx="10" fill="#f0fdfa" stroke="#5eead4"/><text x="350" y="125" fill="#115e59">NATS KV</text>
      <rect x="485" y="96" width="180" height="46" rx="10" fill="#172554"/><text x="575" y="125" fill="#fff">Active agent</text>
      <rect x="725" y="96" width="180" height="46" rx="10" fill="#7c2d12"/><text x="815" y="125" fill="#fff">Root updater</text>
      <rect x="965" y="96" width="200" height="46" rx="10" fill="#fffbeb" stroke="#fbbf24"/><text x="1065" y="125" fill="#78350f">systemd / candidate</text>
    </g>
    <g stroke="#cbd5e1" stroke-width="1.5" stroke-dasharray="5 5"><path d="M125 142V805"/><path d="M350 142V805"/><path d="M575 142V805"/><path d="M815 142V805"/><path d="M1065 142V805"/></g>
    <g font-size="13">
      <text x="45" y="178" fill="#64748b" font-weight="700">1</text><path d="M125 185H350" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="237" y="178" text-anchor="middle" fill="#115e59">CAS current attempt</text>
      <text x="45" y="228" fill="#64748b" font-weight="700">2</text><path d="M350 235H575" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="462" y="228" text-anchor="middle" fill="#115e59">authenticated intent</text>
      <path d="M575 275H815" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="695" y="268" text-anchor="middle" fill="#9a3412">one upgrade request</text>
      <rect x="500" y="302" width="150" height="62" rx="8" fill="#eef2ff"/><text x="575" y="326" text-anchor="middle" fill="#312e81" font-weight="700">keep reconciling</text><text x="575" y="347" text-anchor="middle" fill="#64748b">no early drain</text>
      <text x="45" y="395" fill="#64748b" font-weight="700">3</text><rect x="730" y="375" width="170" height="94" rx="8" fill="#fff7ed" stroke="#fdba74"/><text x="815" y="400" text-anchor="middle" fill="#9a3412" font-weight="700">preparing</text><text x="815" y="422" text-anchor="middle" fill="#9a3412">HTTPS + SHA-256</text><text x="815" y="443" text-anchor="middle" fill="#9a3412">install + fsync</text><text x="815" y="462" text-anchor="middle" fill="#9a3412">probe as fleet-agent</text>
      <text x="45" y="505" fill="#64748b" font-weight="700">4</text><path d="M815 512H1065" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="940" y="505" text-anchor="middle" fill="#9a3412">activating: systemctl stop</text>
      <path d="M1065 548H575" stroke="#b45309" stroke-width="2.2" marker-end="url(#arrow-warn)"/><text x="820" y="541" text-anchor="middle" fill="#9a3412">SIGTERM</text>
      <rect x="500" y="565" width="150" height="78" rx="8" fill="#eef2ff"/><text x="575" y="588" text-anchor="middle" fill="#312e81" font-weight="700">attempt current drain</text><text x="575" y="608" text-anchor="middle" fill="#64748b">status + flush if done</text><text x="575" y="628" text-anchor="middle" fill="#64748b">60 s systemd bound</text>
      <path d="M1065 660H815" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="940" y="653" text-anchor="middle" fill="#115e59">inactive observed</text>
      <text x="45" y="693" fill="#64748b" font-weight="700">5</text><rect x="980" y="680" width="170" height="76" rx="8" fill="#fffbeb" stroke="#fbbf24"/><text x="1065" y="704" text-anchor="middle" fill="#78350f" font-weight="700">atomic symlink switch</text><text x="1065" y="726" text-anchor="middle" fill="#92400e">start + strict READY=1</text><text x="1065" y="746" text-anchor="middle" fill="#92400e">3 min NATS bound</text>
      <text x="45" y="785" fill="#64748b" font-weight="700">6</text><path d="M1065 792H815" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="940" y="785" text-anchor="middle" fill="#115e59">READY=1: commit locally</text>
      <path d="M575 812H350" stroke="#0f766e" stroke-width="2.2" marker-end="url(#arrow)"/><text x="462" y="805" text-anchor="middle" fill="#115e59">agent reflects complete status</text>
    </g>
  </g>
 </svg>
--- a/docs/guides/fleet-application-cd.md
+++ b/docs/guides/fleet-application-cd.md
@@ -0,0 +1,147 @@
 # Fleet application continuous delivery
 An application repository owns a Rust deployment crate whose binary is named
 `harmony`. It declares image builds, runtime services, Fleet placement, and
 compiled deployment contexts. Harmony supplies the release tag at runtime and
 passes registry-returned digests into one Fleet `Deployment`.
 The reference two-service manifest is
 [`examples/harmony_apply_deployment`](../../examples/harmony_apply_deployment/src/main.rs).
 It builds frontend and backend images and advances them together as one rollout
 revision.
 ## Production command
 After checks pass, the production pipeline runs:
 ```bash
 harmony ship \
  --context production \
  --tag "$CI_COMMIT_SHA" \
  --wait
 ```
 `ship` performs these steps in order:
 1. Build every image declared by the Rust manifest.
 2. Push each image to the context's registry repository.
 3. Record the digest returned by the registry.
 4. Apply one namespaced Fleet `Deployment` containing those digests.
 5. Wait for the operator to report the exact applied rollout revision.
 6. Exit successfully when all frozen targets converge, or fail with rollout
   counts and the latest device error.
 The command prints the supplied tag and every deployed digest. The pipeline is
 roll-forward only. A failed rollout blocks that run. Shipping a repaired image
 or deployment spec changes the Fleet `Deployment` and creates the next
 generation; rerunning an unchanged commit does not. Harmony does not rewrite a
 Git manifest or roll the application back to an old tag.
 ## One CI secret
 The runner receives exactly one secret variable,
 `HARMONY_ZITADEL_KEY_JSON`.
 This is the tenant-scoped Zitadel machine key. Harmony exchanges it for the
 short-lived identity used to read the rest of the deployment configuration from
 the tenant's OpenBao instance.
 The compiled context contains non-secret coordinates such as the registry
 hostname, repository, OpenBao URL, Zitadel URL, and OpenBao role. OpenBao holds:
 - `RegistryCredentials` in the context namespace, using a push-scoped registry
  robot account;
 - Kubernetes cluster access for the tenant namespace;
 - application secrets;
 - pull-only registry credentials under
  `<device-prefix>/registry/device-pull/<reference>`.
 Each device pull secret is JSON scoped to one registry authority. The authority
 includes the port when the registry uses one:
 ```json
 {
  "registry": "hub.nationtech.io:5000",
  "username": "tenant-device-pull",
  "password": "..."
 }
 ```
 The credential schema requires all three string fields: `registry`, `username`,
 and `password`.
 The agent normalizes the host name and requires the image registry authority,
 including its port, to match before forwarding credentials to Podman.
 Do not inject registry passwords, kubeconfigs, OpenBao tokens, or device pull
 credentials into CI variables. `publish` loads `RegistryCredentials` through
 the resolved context and uses a temporary Docker configuration directory.
 ## Separate push and pull identities
 CI and devices never share registry credentials.
 | Identity | Registry access | OpenBao reader |
 |---|---|---|
 | CI publisher | Push to the application's repositories | Tenant CI machine identity |
 | Device puller | Pull only from the application's repositories | Device groups authorized for the Fleet Deployment |
 The Rust manifest places only an `image_pull_secret` reference in each private
 service. The operator adds exact referenced pull-secret paths to that
 Deployment's OpenBao policy. The agent reads the credential only when Podman
 needs to pull a missing image and sends it through Podman's registry-auth
 header. It does not run `podman login`, write an auth file, or place credentials
 in desired state, container environment, labels, or logs.
 The registry should provide separate repository-scoped robot accounts for push
 and pull. For the first hosted deployments, those repositories can live under
 `hub.nationtech.io/<tenant>`.
 ## Rust manifest
 The application crate compiles to `harmony`:
 ```toml
 [[bin]]
 name = "harmony"
 path = "src/main.rs"
 ```
 Its `HarmonyApp` implementation declares stable build and deployment policy:
 - frontend and backend build contexts and Dockerfiles;
 - one production context and tenant namespace;
 - one Fleet Deployment name;
 - allowed device groups and placement labels;
 - ports, environment, application-secret references, and pull-secret
  references.
 Those values change only when application architecture or policy changes. A
 normal release supplies a new commit tag to the same compiled manifest.
 ## Deploy an existing image
 `deploy` skips build and publication:
 ```bash
 harmony deploy \
  --context production \
  --tag "$CI_COMMIT_SHA" \
  --image frontend=hub.nationtech.io/example/frontend@sha256:... \
  --image backend=hub.nationtech.io/example/backend@sha256:... \
  --wait
 ```
 This application accepts registry digests and explicit `dev-*` tags. It rejects
 `latest` and ordinary mutable tags.
 ## Watching and debugging
 While `ship --wait` is running, the dashboard deployment page polls the current
 summary every three seconds. It shows succeeded, pending, and failed targets,
 plus the latest failing device, error, and timestamp. The CLI waits for the
 status revision matching the object UID and generation returned by its apply,
 so success from an older rollout cannot complete a newer pipeline run.
 The initial dashboard view remains aggregate. Detailed per-device rollout state
 and explicit canary-stage presentation are later work.
--- a/docs/guides/fleet-device-secrets.md
+++ b/docs/guides/fleet-device-secrets.md
@@ -1,113 +1,97 @@
-# Fleet device secrets
+# Fleet enrollment and device secrets
-How fleet devices read their deployments' secrets — image-pull
+Enrollment gives a Linux device its Fleet identity and installs the agent. It
-credentials, application config, API keys — from OpenBao, and how you
+does not, by itself, authorize group placement or make application secrets
-control who reads what. Design reasoning lives in
+available.
 [ADR-025](../adr/025-fleet-device-secret-access.md); this page is the
 operating manual.
-## The architecture in one minute
+## Enroll a device
-A device proves *who it is* with the Zitadel machine key it already
+Use the `example_fleet_device_enroll` binary as the current enrollment driver.
-uses for NATS. *What it may read* is decided by **group membership**,
+It runs locally or targets a device over SSH:
 which only admins can change. *Where it runs* is decided by labels.
 The three never mix:
 ```
 Admin                          Zitadel                       OpenBao
   │ assign device to groups ──▶ device ∈ {edge-a, …}           │
   │ write secret ──────────────────────────────────────────────▶ secret/<fleet>/<deployment>/…
   │                                                            │
 Deployment CR                                                  │
   │ allowedGroups: [edge-a] ──▶ operator attaches policy ──────▶ group edge-a may read
   │ targetSelector: labels  ──▶ operator schedules within      │   deployment-<name> subtree
   │                             authorized devices             │
   │                                                            │
 Device                                                         │
   └─ machine key ─▶ Zitadel token (groups: [edge-a]) ─▶ JWT login ─▶ reads its secrets
 ```
 Three rules to remember:
 | Concept | Who sets it | What it controls |
 |---|---|---|
 | **Group** | Admin, in Zitadel | Security boundary: which deployments' secrets a device may read |
 | **Label** | Device config (self-reported) | Placement only: which authorized devices actually run the workload — narrows, never widens |
 | **`allowedGroups`** | Deployment author | Which groups may view (run + read secrets of) the deployment |
 ## Storing a secret for a deployment
 Write it under the deployment's subtree of the fleet KV mount:
 ```bash
-bao kv put secret/<fleet-ns>/<deployment-name>/db-credentials \
+cargo run -p example_fleet_device_enroll -- \
-    username=app password=…
+  --target ssh://pi@10.0.0.42 \
  --issuer-url https://sso.example.com \
  --audience <fleet-project-id> \
  --nats-url wss://nats.example.com \
  --admin-oidc-client-id <numeric-client-id> \
  --agent-binary ./build/fleet-agent-aarch64 \
  --device-id warehouse-042 \
  --labels site=warehouse-east,arch=aarch64
 ```
-Nothing else to configure — access follows from the deployment's
+The operator must authenticate to Zitadel through the browser flow or provide
-`allowedGroups` below. Every device in an allowed group can read the
+`HARMONY_ZITADEL_ADMIN_TOKEN`. Enrollment then finds or creates the device
-whole `<deployment-name>/` subtree, whether or not the workload landed
+machine user, grants the requested project role, and mints a new JSON machine
-on it; size your groups to the audience the secret may have.
+key. Zitadel does not return an existing private key, so a repeated enrollment
 mints another key.
-## Authorizing and placing a deployment
+`FleetDeviceSetupScore` installs Podman, the unprivileged `fleet-agent` user,
 the root updater, systemd units, the agent binary, and its configuration. When a
 machine key is supplied or minted, the Score writes it to
 `/etc/fleet-agent/zitadel-key.json` with mode `0640`. The device cannot obtain
 that long-lived key from NATS or OpenBao automatically.
-```yaml
+The device ID must be a valid Kubernetes RFC 1123 subdomain. Labels are
-apiVersion: fleet.nationtech.io/v1alpha1
+self-reported placement data. Do not treat labels such as `group=production` as
-kind: Deployment
+authorization.
-metadata:
+
-  name: hello-web
+Verify the installation on the device and control plane:
-spec:
+
-  allowedGroups: [edge-a]        # security: who may view this deployment
+```bash
-  targetSelector:
+systemctl status fleet-agent harmony-fleet-updater
-    matchLabels:
+journalctl -u fleet-agent -u harmony-fleet-updater
-      hw: pi5                    # placement: which authorized devices run it
+kubectl -n <namespace> get devices.fleet.nationtech.io
  score: …
 ```
-The operator schedules onto devices that are **members of an allowed
+## Identity and tokens
 group AND match the selector**, then attaches the deployment's read
 policy to each allowed group. Existing device sessions pick the grant
 up immediately — rolling out a new deployment never touches logins,
 tokens, or per-device configuration.
-## Managing device group membership
+The device stores only its Zitadel machine key. On each NATS connection, the
 agent signs a 60-second JWT assertion and exchanges it at Zitadel's token
 endpoint. Production requires the returned `id_token`: it is a JWT that the NATS
 callout and OpenBao can verify against Zitadel's JWKS. Zitadel's default
 `access_token` is opaque and is not suitable for this path.
-Until Zitadel's first-class groups are GA, a fleet group is a Zitadel
+The `id_token` is cached in memory and renewed before expiry. No refresh token
-project role (one role per group, surfaced as a `groups` claim by the
+is stored. Rotating a machine key requires installing the replacement on the
-fleet's token Action — applied by the fleet bootstrap, not by hand).
+device and deleting the old key in Zitadel.
 Grant or remove the role on the device's machine user to move it
 between groups.
-Membership changes take effect at the device's **next login, at most
+## Group placement and OpenBao
 one token TTL later (≈ 1 h)**. This is the only eventually-consistent
 operation in the system — everything else binds immediately. If that
 window matters, use the emergency levers below.
-## Reading secrets on the device
+The intended authorization rule is:
-Nothing to do: the agent reads through `harmony_config`, which
+```text
-authenticates to OpenBao with the machine keyfile automatically
+admin-managed group membership AND self-reported target labels
-(JWT-bearer rung of `OpenbaoSecretStore`). Tokens are short-lived
+```
 batch tokens; the store re-logins transparently on expiry.
-## Revocation runbook
+`allowedGroups` names the authorized groups; `targetSelector` narrows placement
 within those groups. An empty group list authorizes no device.
-| Situation | Action | Effect |
+The operator supports a Zitadel-backed group source and OpenBao policy grant
-|---|---|---|
+synchronization, but the public production composition does not configure
-| Device leaves a group (routine) | Remove the role/group in Zitadel | Converges at next login, ≤ 1 TTL |
+either one today. Without an explicit group source, deployments match no
-| Device compromised | Disable its OpenBao identity entity, then rotate/delete its Zitadel machine key | Token dead at next request; no new logins |
+devices. Without OpenBao administrator configuration, changing
-| Secret/deployment compromised | Detach `deployment-<name>` policy from its groups (or rotate the secret) | All devices lose read immediately |
+`allowedGroups` does not create or attach OpenBao policies.
 | Whole group compromised | Detach the group's policies | Immediate, group-wide |
-## Guarantees and limits
+OpenBao use therefore requires an operator to configure all of the following:
- A device can only ever read secrets of deployments whose
+- the agent's OpenBao URL and secret prefix during device setup;
-  `allowedGroups` intersect its groups. Labels, CR edits, or a
+- Zitadel group roles and device grants;
-  compromised agent cannot widen that — membership is signed into the
+- the operator group source;
-  Zitadel token and validated against the project (`bound_audiences`),
+- OpenBao JWT auth, deployment policies, and group-policy attachments;
-  so other projects' tokens are rejected outright.
+- secret values under the configured deployment prefix.
- Visibility is **group-granular**: being scheduled is not required to
+
-  read. Finer isolation ⇒ finer groups.
+Do not assume enrollment completed these steps. The E2E tests exercise explicit
- One Zitadel project + OpenBao pair (a *cell*) comfortably serves
+group and grant setup, not the default production composition.
-  ~10⁵ devices; beyond that, shard fleets into cells
+
-  ([ADR-025](../adr/025-fleet-device-secret-access.md), Rationale).
+## Revocation
 For a compromised device, remove or deactivate its Zitadel machine keys and
 revoke any active OpenBao identity or token. Rotate secrets the device could
 read. Group removal takes effect only after the device obtains a token carrying
 the new claims; policy detachment or secret rotation is the immediate control.
 See [ADR-025](../adr/025-fleet-device-secret-access.md) for the intended group
 model. Treat its unwired portions as design until production composition adds
 and verifies the required configuration.
--- a/docs/guides/fleet-manual-token-mint.md
+++ b/docs/guides/fleet-manual-token-mint.md
@@ -1,189 +1,126 @@
-# Manual Zitadel token mint + NATS write
+# Manually mint a Fleet token
-Operator-side recipe for talking to a callout-protected NATS by
+Use this procedure to test the Zitadel-to-NATS authentication chain. It signs a
-hand: sign a JWT-bearer assertion with a Zitadel machine user's
+JWT-bearer assertion with a machine key, exchanges it for an OIDC ID token, and
-private key, exchange it for an access token, drive `nats` CLI
+passes that ID token to the NATS CLI.
 commands with the token. Useful for debugging the auth chain,
 poking the desired-state KV without the operator running, and
 validating that a deployed callout is actually accepting what
 you think it should.
-Read [fleet-zitadel-faq.md](./fleet-zitadel-faq.md) first for the
+Do not use `access_token` from the response. Zitadel returns an opaque access
-underlying mechanism (RFC 7523 JWT-bearer flow, why we sign
+token by default; production Fleet requires the verifiable `id_token`.
 locally, what each claim means).
-## Inputs you need
+## Inputs
-Five strings:
+Read the live values rather than relying on a local Zitadel cache:
-| Input | Where to find it |
+| Input | Source |
-| --- | --- |
+|---|---|
-| `OIDC_ISSUER_URL` (the Zitadel base URL) | callout Deployment env: `kubectl exec -n fleet-system deploy/fleet-callout -- printenv OIDC_ISSUER_URL` |
+| `OIDC_ISSUER_URL` | Callout environment |
-| `project_id` (becomes the access token's `aud`) | callout Deployment env: `OIDC_AUDIENCE` |
+| `PROJECT_ID` | Callout `OIDC_AUDIENCE` environment value |
-| Machine user's `userId` | the JSON keyfile's `userId` field |
+| `USER_ID` | Machine key `userId` |
-| Machine user's `keyId` | the JSON keyfile's `keyId` field |
+| `KEY_ID` | Machine key `keyId` |
-| Private RSA key (PEM) | the JSON keyfile's `key` field |
+| RSA private key | Machine key `key` |
-Get the `fleet-ops` (admin role) JSON keyfile from the cache:
+For a cached operator key:
 ```bash
 jq -r '.machine_keys["fleet-ops"]' \
-  ~/.local/share/harmony/zitadel/client-config.json \
+  ~/.local/share/harmony/zitadel/client-config.json > /tmp/fleet-ops.json
-  > /tmp/fleet-ops.json
+jq -r '.key' /tmp/fleet-ops.json > /tmp/fleet-ops.pem
 jq -r '.userId' /tmp/fleet-ops.json    # → user_id
 jq -r '.keyId'  /tmp/fleet-ops.json    # → key_id
 jq -r '.key'    /tmp/fleet-ops.json    > /tmp/fleet-ops.pem
 ```
-The cache may drift from the deployed Zitadel state if Zitadel has
+Protect and remove these temporary files when finished.
 been re-seeded; **always pull `OIDC_AUDIENCE` from the running
 callout**, not from the cache. The cache fix landed in commit
 `f4d6fb94` but older entries can still trip you up.
-## Mint script (PyJWT)
+## Mint with PyJWT
 Install `PyJWT`, not the unrelated `jwt` package.
 ```python
-# pip install PyJWT requests   ← MUST be PyJWT, not the `jwt` package.
+import json
-# The two share `import jwt`; `jwt` (the package) refuses raw PEM
+import time
 # strings and demands an AbstractJWKBase wrapper. PyJWT takes PEM
 # directly. If you ever see `TypeError: key must be an instance of
 # a class implements jwt.AbstractJWKBase`, you have the wrong one.
-import jwt, time, requests
+import jwt
 import requests
-# These come from the running callout + Zitadel. Don't reuse stale
+OIDC_ISSUER_URL = "https://sso.example.com"
-# values from a checked-in note; verify against the live cluster.
+PROJECT_ID = "..."  # OIDC_AUDIENCE from the running callout
-OIDC_ISSUER_URL = "http://sso.fleet.local:8080"
+machine_key = json.load(open("/tmp/fleet-ops.json"))
-PROJECT_ID      = "371158654839160853"   # = OIDC_AUDIENCE on callout
+private_key = open("/tmp/fleet-ops.pem").read()
 USER_ID         = "..."                  # from machine keyfile
 KEY_ID          = "..."                  # from machine keyfile
 key = open("/tmp/fleet-ops.pem").read()
 now = int(time.time())
 assertion = jwt.encode(
    {
-        "iss": USER_ID,
+        "iss": machine_key["userId"],
-        "sub": USER_ID,
+        "sub": machine_key["userId"],
-        "aud": OIDC_ISSUER_URL,   # for Zitadel itself, NOT the project_id
+        "aud": OIDC_ISSUER_URL,
        "exp": now + 60,          # Zitadel rejects exp - iat > 60s
        "iat": now,
        "exp": now + 60,
    },
-    key,
+    private_key,
    algorithm="RS256",
-    headers={"kid": KEY_ID},      # PyJWT spelling — `headers=`, not `optional_headers=`
+    headers={"kid": machine_key["keyId"]},
 )
-r = requests.post(
+response = requests.post(
-    f"{OIDC_ISSUER_URL}/oauth/v2/token",
+    f"{OIDC_ISSUER_URL.rstrip('/')}/oauth/v2/token",
    data={
        "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
-        "assertion":  assertion,
+        "assertion": assertion,
        # Three scopes:
        #   openid                                     — base OIDC
        #   urn:zitadel:iam:org:projects:roles         — PLURAL.
        #     Without this, Zitadel omits the role claim and the
        #     callout rejects with "no authorized role in token".
        #   urn:zitadel:iam:org:project:id:<id>:aud    — singular.
        #     Tells Zitadel to put <id> into the access token's
        #     `aud` claim, which the callout's audience check
        #     compares against OIDC_AUDIENCE.
        "scope": (
            "openid "
            "urn:zitadel:iam:org:projects:roles "
            f"urn:zitadel:iam:org:project:id:{PROJECT_ID}:aud"
        ),
    },
    timeout=10,
 )
-r.raise_for_status()
+response.raise_for_status()
-token = r.json()["access_token"]
+token = response.json()["id_token"]
 # Sanity check — decode without verifying signature so you can see
 # what Zitadel actually emitted. If anything below is wrong, the
 # callout will reject your token.
 print(jwt.decode(token, options={"verify_signature": False}))
 print(token)
 ```
-Expected decoded claims (the parts the callout will check):
+The decoded ID token must have the exact issuer configured on the callout, the
 Fleet project in `aud`, a `client_id`, and the Fleet project's role claim. The
 callout accepts `fleet-admin` or `device`; it strips the configured `device-`
 prefix from `client_id` for device identities.
-| Claim | What it should be | Why |
+Decoding without signature verification is diagnostic only. The callout still
-| --- | --- | --- |
+verifies the signature against Zitadel's JWKS.
 | `iss` | `OIDC_ISSUER_URL` (byte-equal) | Callout: `validation.set_issuer(&[&self.issuer_url])` |
 | `aud` | `["<PROJECT_ID>"]` | Callout: `validation.set_audience(&[&self.audience])`; the array form is Zitadel's default |
 | `exp` | ~now + 12h | Zitadel default access token TTL |
 | `client_id` | the machine user's username (`fleet-ops`, `device-vm-device-00`, …) | Callout uses this as `device_id_claim` (with optional `DEVICE_ID_PREFIX_STRIP` applied) |
 | `urn:zitadel:iam:org:project:<PROJECT_ID>:roles` | object with role names as keys (e.g. `{"fleet-admin": {"<orgId>": "<orgName>"}}`) | Callout uses this as `roles_claim` and admits the role if `fleet-admin` or `device` is present |
-If any of these is wrong, fix the script before bothering with NATS.
+## Connect to NATS
-## Drive NATS with the token
+Capture the last output line from the script:
 `nats --token=<bearer>` puts the value into the CONNECT frame's
 `auth_token`, which is what the callout expects.
 ```bash
-NATS_SERVER=192.168.122.1:30422       # libvirt host's port mapping
+TOKEN=$(python3 mint.py | tail -1)
-TOKEN=$(python3 mint.py | tail -1)    # last line is the raw token
+nats --server wss://nats.example.com --token "$TOKEN" kv ls device-info
 # Read everything (admin role allows >):
 nats --server "$NATS_SERVER" --token "$TOKEN" kv ls device-info
 nats --server "$NATS_SERVER" --token "$TOKEN" kv get device-info info.vm-device-00
 # Write a desired state — agent's KV watcher fires within 1s,
 # reconciler creates the podman container.
 nats --server "$NATS_SERVER" --token "$TOKEN" \
  kv put desired-state vm-device-00.hello-web '{
    "name": "hello-web",
    "type": "PodmanV0",
    "data": {
      "services": [{
        "name":  "testnginx",
        "image": "docker.io/nginx:latest",
        "ports": ["8080:80"]
      }]
    }
  }'
 ```
-The exact JSON shape comes from
+An administrator can inspect a device entry:
 `harmony-reconciler-contracts/src/fleet.rs` — read that crate when
 in doubt about field names, NOT this doc; this doc is a worked
 example and may drift.
 ## Common failures and what they mean
 | Symptom | Likely cause |
 | --- | --- |
 | `TypeError: key must be an instance of … AbstractJWKBase` | Wrong PyPI package. `pip uninstall jwt && pip install PyJWT`. |
 | HTTP 400 from `/oauth/v2/token`: `"invalid_grant_type"` | Forgot the percent-encoded form encoding, OR `grant_type` value mistyped. The full URN is `urn:ietf:params:oauth:grant-type:jwt-bearer`. |
 | HTTP 400: `"jwt: token is expired"` | Your assertion's `exp` is in the past. Wall-clock skew between your laptop and the cluster — sync NTP. |
 | Token mints but no `urn:zitadel:…:roles` claim | Missing the **plural** `urn:zitadel:iam:org:projects:roles` in scope. |
 | Token mints but `aud` is the issuer URL instead of the project id | Forgot the `urn:zitadel:iam:org:project:id:<id>:aud` scope. |
 | NATS CLI: `nats: Authorization Violation` | Token is good but callout rejected it — check `kubectl logs -n fleet-system -l app=fleet-callout` for the actual reason. The most common ones are "InvalidAudience" (your `aud` ≠ deployed `OIDC_AUDIENCE`) and "no authorized role in token". |
 | Callout log: `JWT validation failed: InvalidIssuer` | Trailing slash drift. `OIDC_ISSUER_URL=http://sso.fleet.local:8080/` ≠ `http://sso.fleet.local:8080`. Match exactly. |
 When the callout rejects, **its log is the source of truth**, not
 your decoded claims. The validation error includes which check
 failed; work backwards from there.
 ## Rotating the deployed `OIDC_AUDIENCE`
 If Zitadel was re-seeded and `OIDC_AUDIENCE` on the callout now
 points at a non-existent project:
 ```bash
-# 1. Confirm the live project id
+nats --server wss://nats.example.com --token "$TOKEN" \
-oc -n zitadel exec -ti deploy/zitadel -- /bin/sh -c \
+  kv get device-info info.<device-id>
  'curl -s -H "Authorization: Bearer $PAT" \
        $ZITADEL_URL/management/v1/projects/_search \
   | jq ".result[] | select(.name == \"fleet\") | .id"'
 # 2. Re-run the bring-up — the live-query fix in f4d6fb94 will
 #    refresh OIDC_AUDIENCE on the next NatsAuthCalloutScore apply.
 ```
-The shape of `mint.py` doesn't change between regular operation
+Do not use manual KV writes as the deployment workflow. Apply a namespaced
-and post-recovery — you just plug in fresh values for
+Fleet `Deployment` resource so the operator owns target planning, rollout state,
-`OIDC_AUDIENCE` and `PROJECT_ID`.
+and status.
 ## Failures
 | Symptom | Check |
 |---|---|
 | Token response has no `id_token` | Include `openid`; confirm the API application permits an ID token |
 | ID token has no project roles | Include the plural `urn:zitadel:iam:org:projects:roles` scope and verify the role grant |
 | `InvalidAudience` | Use the live callout `OIDC_AUDIENCE` and include the project audience scope |
 | `InvalidIssuer` | Match `OIDC_ISSUER_URL` exactly, including scheme, host, port, and trailing-slash behavior |
 | Missing `client_id` | Confirm the token is an ID token minted for the Fleet API application |
 | Assertion expired | Synchronize the client clock; the assertion window is 60 seconds |
 | `AbstractJWKBase` type error | Remove the `jwt` package and install `PyJWT` |
 The callout log contains the validation failure:
 ```bash
 kubectl -n <namespace> logs deploy/fleet-callout
 ```
--- a/docs/guides/fleet-podman-deployments.md
+++ b/docs/guides/fleet-podman-deployments.md
@@ -0,0 +1,82 @@
 # Fleet Podman deployments
 Fleet `Deployment` resources carry a `ReconcileScore`. `PodmanV0` supports one
 optional init container followed by one or more long-running services.
 ```yaml
 apiVersion: fleet.nationtech.io/v1alpha1
 kind: Deployment
 metadata:
  name: analysis-api
 spec:
  allowedGroups: [lab]
  targetSelector:
    matchLabels:
      role: analyzer
  rollout:
    strategy: Immediate
  score:
    type: PodmanV0
    data:
      init_container:
        name: analysis-migrate
        image: registry.example.com/analysis-migrate@sha256:<digest>
        ports: []
        env: []
        secret_env: []
        volumes:
          - host_path: /var/lib/analysis
            container_path: /data
            read_only: false
        restart_policy: "no"
      services:
        - name: analysis-api
          image: registry.example.com/analysis-api@sha256:<digest>
          ports: ["8080:8080"]
          env: []
          secret_env: []
          volumes:
            - host_path: /var/lib/analysis
              container_path: /data
              read_only: false
          restart_policy: unless-stopped
 ```
 `Immediate` starts the rollout without operator approval. When any matched
 device has the `canary=true` label, those devices must converge before the same
 revision is released to the remaining frozen targets.
 The target set and canary membership are frozen for the revision. A failed
 canary prevents release to non-canaries. Fleet does not roll back canaries that
 already converged, and it has no percentage, batch-size, concurrency, pause, or
 manual-approval setting.
 `allowedGroups` is enforced only when the operator has a group source. The
 public production composition does not currently wire one, so it fails closed
 and matches no devices until a private composition provides group membership.
 The agent preflights names, ports, ownership, and images before changing the
 deployment. It then runs `init_container` with restart policy `no` and waits for
 it to exit. Exit code zero allows service reconciliation to continue. Any other
 exit code marks the deployment failed and leaves services that have not yet
 been reconciled untouched. The next reconciliation pass retries the failed init
 container.
 The completed init container remains in Podman as the durable completion
 marker. Reconciliation skips it while the complete resolved score is unchanged.
 Changing any score field, including a regular service field, replaces and
 reruns the init container before reconciling services. Resolved secret values
 are part of the score revision, but the agent does not watch the secret source
 for changes. Init containers must be idempotent. Removing the Deployment removes
 both the init container and its services.
 Container replacement and Deployment removal ask Podman to stop each running
 container with a 30-second timeout, then force-remove it. Fleet does not wait
 indefinitely for graceful shutdown; applications must finish termination work
 within that timeout.
 `init_container` uses the same `PodmanService` fields as a service. Secret
 references are resolved before Podman receives the definition. The runtime
 always overrides its restart policy to `no`; setting another value does not
 make an init container long-running. Its `ports` list must be empty because an
 init container cannot publish host ports.
--- a/docs/guides/fleet-staging-install.md
+++ b/docs/guides/fleet-staging-install.md
@@ -1,15 +1,16 @@
-# Deploy Fleet to a remote cluster
+# Fleet control-plane operations
-Fleet uses the same lifecycle commands and Scores for every cluster. Remote
+Fleet uses the same lifecycle commands and Scores for local and remote clusters.
-contexts currently use production-equivalent replication, TLS, exposure, and
+A remote context enables the production exposure and credential paths even when
-credential behavior, including when the target cluster is used for staging.
+the cluster is used for staging. The current operator deployment has one
 replica; this guide does not imply active-active or failover support.
 ## Prerequisites
 - Docker with Buildx, Helm, and kubectl are on `PATH`.
 - The remote cluster has cert-manager and the CloudNativePG operator.
- The image registry contains credentials for `REGISTRY_USER` and
+- Registry publication credentials are stored as `RegistryCredentials` in the
-  `REGISTRY_TOKEN`.
+  remote context's OpenBao namespace.
 - The deploy identity can obtain the cluster kubeconfig through OpenBao.
 - Public DNS points `zitadel`, `openbao`, and `nats` under the context domain at
  the cluster ingress.
@@ -72,10 +73,12 @@ old releases. Disposable local and E2E clusters should be recreated instead.
 ## Provision the tenant
-Run the tenant provisioning binary with a platform-admin context. It applies
+Run the tenant provisioning binary with an explicit administrator kubeconfig.
-`TenantScore`, creates the Fleet deployer ServiceAccount and RBAC, issues its
+It applies `TenantScore`, creates the Fleet deployer ServiceAccount and RBAC,
-kubeconfig, and stores `ClusterAccess` through Harmony Config. The credential
+issues its namespace-scoped kubeconfig, and stores `ClusterAccess` through
-store role must be able to write the tenant's OpenBao namespace.
+Harmony Config. Do not store the administrator kubeconfig in OpenBao or expose
 it to the tenant CI/CD identity. The provisioning token only needs write access
 to the tenant's OpenBao namespace.
 The deployer can create namespaced Roles and RoleBindings because Helm installs
 the Fleet operator's runtime RBAC. Kubernetes prevents it from binding rights it
@@ -88,8 +91,8 @@ provisioning to create and store a replacement. This remains the rotation path
 until short-lived TokenRequest brokerage is implemented.
 ```bash
-cargo run --release --bin tenant-provision -- \
+export KUBECONFIG=/secure/path/platform-admin.kubeconfig
-  deploy --context platform-admin
+cargo run --release --bin tenant-provision
 ```
 Fleet creates the immutable NATS callout credential Secret on first deploy and
@@ -104,17 +107,15 @@ public, so the remote context uses `image_pull_secret: None`.
 references, then converges the Fleet Scores serially:
 ```bash
 export REGISTRY_USER=...
 export REGISTRY_TOKEN=...
 export HARMONY_ZITADEL_KEY_JSON=...
 cargo run -- ship --context customer-prod
 ```
-These three environment variables are required today. The target CI model
+The runner receives only `HARMONY_ZITADEL_KEY_JSON`. Context resolution uses
-keeps only `HARMONY_ZITADEL_KEY_JSON` in the runner and reads registry
+that identity to load the cluster access and secret-class registry credentials
-credentials from OpenBao through `ConfigClient`; that publication work is not
+from OpenBao. `build` remains credential-free; `publish` and `ship` resolve the
-implemented yet.
+full remote context.
 For CI, the stages can run independently. Pass the digest references printed by
 `build` into `publish` and `deploy`:
@@ -141,6 +142,48 @@ kubectl -n customer-fleet get deployments,pods,routes
 kubectl -n customer-fleet get deployments.fleet.nationtech.io,devices.fleet.nationtech.io
 ```
 For production, open `https://dashboard.<context-domain>`. Zitadel login must
 return an ID token carrying `fleet-admin`. The dashboard shows live Device and
 Deployment CR projections. It does not provide deployment creation, rollout
 approval, task history, persistent alert acknowledgement, or historical trend
 data.
 ## Troubleshooting
 Start with the component that owns the failed boundary:
 ```bash
 # Control-plane rollout and application logs
 cargo run -- status --context customer-prod
 cargo run -- logs --context customer-prod
 kubectl -n customer-fleet get pods,events
 # Operator reconciliation and dashboard
 kubectl -n customer-fleet logs deploy/harmony-fleet-operator
 # Device reconciliation and privileged updater
 systemctl status fleet-agent harmony-fleet-updater
 journalctl -u fleet-agent -u harmony-fleet-updater
 podman ps -a
 ```
 Use these symptoms to narrow the search:
 | Symptom | Check |
 |---|---|
 | Deployment remains at zero targets | Production has no group source by default; inspect operator warnings and its group-source environment |
 | Secret reference fails | Confirm device OpenBao configuration, JWT role, policy attachment, and secret path; production grant sync is not wired by default |
 | Agent cannot connect to NATS | Check device time, Zitadel reachability, ID-token mint errors, DNS, TLS, and callout logs |
 | Init container runs again | Any resolved score change changes its completion revision; keep init work idempotent |
 | Container disappears after a slow stop | Fleet gives Podman 30 seconds, then force-removes the container |
 | Dashboard alert returns after restart | Acknowledgements are in memory only |
 | System upgrade reports `repair-required` | Inspect apt and dpkg directly on the device; Fleet does not retry or roll back packages |
 The public production composition does not currently configure group placement
 or OpenBao deployment grant synchronization. Do not diagnose their absence as
 eventual convergence: supply the missing configuration in the private deploy
 composition or manage those grants separately.
 For local validation before a remote deploy:
 ```bash
--- a/docs/guides/fleet-tasks-upgrades.md
+++ b/docs/guides/fleet-tasks-upgrades.md
@@ -0,0 +1,78 @@
 # Fleet tasks and upgrades
 Fleet has two distinct upgrade paths. `TaskRun` invokes the fixed operating
 system upgrade executor. Setting `Device.spec.agentUpgrade` invokes the agent
 binary upgrade protocol.
 ## System upgrade task
 Create a namespaced `TaskRun`:
 ```yaml
 apiVersion: fleet.nationtech.io/v1alpha1
 kind: TaskRun
 metadata:
  name: upgrade-device-1
 spec:
  allowedGroups: [production]
  targetSelector:
    matchLabels:
      device-id: device-1
  deadlineSeconds: 21600
  systemUpgradeV1: {}
 ```
 Task placement uses the same group source as deployments. The public production
 composition does not configure that source, so a task has no targets until a
 private composition supplies group membership.
 The operator freezes the matching target set. Each device must advertise the
 `AptFullUpgradeV1` updater capability. The root updater then runs its compiled
 sequence: package preflight, `apt-get update`, noninteractive full upgrade, dpkg
 audit, reboot, and post-boot verification. Callers cannot supply package names,
 repositories, commands, or reboot arguments.
 If a matched target has `canary=true`, only canaries are released first. Every
 canary must complete before the remaining frozen targets are released. A failed
 or timed-out canary stops further release. Completed upgrades are not rolled
 back.
 The deadline prevents an expired intent from starting and determines when the
 operator fails a run. It does not interrupt apt or dpkg after the updater has
 accepted the attempt. An interrupted package transaction can enter
 `repair-required`; repair then requires direct device access.
 The executor exists in the agent and updater, but validation on a disposable
 Debian VM and Raspberry Pi OS device remains listed as work in the current
 design. Treat it as unproven on physical production devices until that run is
 recorded.
 Inspect progress with:
 ```bash
 kubectl -n <namespace> get taskruns
 kubectl -n <namespace> get taskrun <name> -o yaml
 journalctl -u harmony-fleet-updater
 ```
 Recurring schedules and dashboard task history are not implemented.
 ## Agent upgrade
 An agent upgrade target contains a version, architecture, artifact URL, maximum
 size, and SHA-256 digest. The operator writes a per-device attempt when desired
 and reported versions differ.
 The root updater downloads and verifies the candidate, runs its self-test and
 probe, stops the old agent, switches the active symlink, and starts the new
 agent. Readiness commits the change. Failure before commit restores the previous
 agent executable; `rollback-failed` requires root repair.
 This rollback covers only the executable. Configuration, local databases, and
 formats changed by the candidate must remain readable by the previous release.
 The root updater does not update itself. Devices installed before OCI artifact
 support need `FleetDeviceSetupScore` run once before their first `oci://`
 upgrade.
 See [Fleet agent upgrades](../design/fleet-agent-upgrades.md) for the protocol,
 timeouts, journal states, and repair conditions.
--- a/docs/guides/fleet-zitadel-faq.md
+++ b/docs/guides/fleet-zitadel-faq.md
@@ -1,185 +1,101 @@
-# Fleet × Zitadel FAQ
+# Fleet and Zitadel FAQ
-Technical reference for the Zitadel setup behind the fleet
+This page describes the authentication path used by the public Fleet production
-auth callout. Describes what exists, why it's that way, and where
+composition.
 each piece lives in the code.
-Code anchors:
+## What does Zitadel provision for Fleet?
 - `examples/fleet_e2e_demo/src/lib.rs` — bring-up flow
 - `harmony/src/modules/zitadel/setup.rs` — `ZitadelSetupScore`
 - `harmony/src/modules/zitadel/mod.rs` — Helm install
 - `nats/callout/src/handler.rs` — auth callout
 - `fleet/harmony-fleet-agent/src/credentials.rs` — JWT-bearer mint
---
+`ZitadelSetupScore` creates a Fleet project, an API application for NATS, a web
 PKCE application for the production dashboard, the `fleet-admin` and `device`
 roles, and the operator machine identity. Device enrollment creates a separate
 machine user and JSON key for each device.
-## What is an "application" in Zitadel?
+An application is an OIDC client configuration. Users belong to a Zitadel
 organization; project role grants associate a user with roles in the Fleet
 project. A project role is not a Zitadel administrator role.
-An OIDC client config: `clientId`, allowed grant types, redirect
+## How does a device authenticate?
 URIs (browser apps only), PKCE settings (browser apps only).
-Apps are not containers for users or roles — those live one
+The agent keeps the JSON machine key at
-level up at the org. An app is the entry point a service uses to
+`/etc/fleet-agent/zitadel-key.json`. On each NATS connection it:
 delegate auth to Zitadel.
-The `nats` app is **API type**: JWT-bearer / client-credentials
+1. signs a JWT assertion whose `iss` and `sub` are the machine user ID;
-only, no browser flow. Headless agents never see a login page.
+2. sends the assertion to `<issuer>/oauth/v2/token` with the RFC 7523
-The app's `clientId` is what tokens carry as `aud` and what the
+   JWT-bearer grant;
-auth callout validates against (`OIDC_AUDIENCE` env on the callout
+3. requests `openid`, project roles, and the Fleet project audience;
-Deployment).
+4. takes `id_token` from the response and presents it as the NATS bearer token.
-## Why are users and roles at org level instead of per-project?
+Production requires `id_token`, not `access_token`. Zitadel access tokens are
 opaque by default. The ID token is a JWT that the callout can verify through
 Zitadel's JWKS and that contains the audience, `client_id`, and role claims used
 by Fleet.
-Roles are defined inside a project but are essentially labels —
+The assertion lasts 60 seconds. The ID token is cached in memory and replaced
-strings + display names with no inherent permissions. Each app
+before its JWT `exp`; no refresh token is stored. The machine key remains valid
-enforces them in code (the callout maps `device` → a
+until it expires or is removed in Zitadel.
 permission template).
-Users live at org level so one identity can hold roles across
+## How is a device ID derived?
 multiple projects in the same org and SSO between them. Role
 grants are the join: "user X has roles \[A, B\] on project Y."
-The only privilege ladder Zitadel enforces directly is at the
+Enrollment conventionally creates the username `device-<device-id>`. The
-instance/org level (IAM-Owner, Org-Owner). Project roles say
+callout reads the ID token's `client_id` claim and strips `device-`. It validates
-nothing about Zitadel admin rights.
+the result before inserting it into device-scoped NATS subjects.
-## What is each service account for?
+## What happens while a device is offline?
-| User | Created by | Purpose |
+The machine key remains on disk. Once the network returns, a NATS connection
-| --- | --- | --- |
+attempt obtains a fresh ID token if the cached one is near expiry. The agent
-| `iam-admin` | Helm `FirstInstance.Org.Machine` | IAM-Owner. Its PAT (`iam-admin-pat` k8s Secret) drives the management API from `ZitadelSetupScore`. |
+cannot mint while Zitadel is unreachable, so reconnect waits until both Zitadel
-| `login-client` | Helm `FirstInstance.Org.LoginClient` | Internal — Zitadel's login UI pod uses it to call back into Zitadel. Don't touch. |
+and NATS are available.
 | `fleet-ops` | `fleet_e2e_demo` admin setup | `fleet-admin` role grant, JSON key, used by tests and admin tooling. |
 | `device-vm-device-NN` | `fleet_e2e_demo::provision_device` | One per VM. JSON key copied to `/etc/fleet-agent/zitadel-key.json`. `device` role grant. |
 | `ops-station`, `sensor-a`, `sensor-b`, `intruder` | `fleet_auth_callout` (separate example) | Leftovers from previous runs. Postgres survives cluster recreates. Harmless, deletable. |
-The `device-` prefix on per-device usernames is intentional:
+## Are machine keys equivalent to PATs?
 Zitadel emits the username verbatim in the access token's
 `client_id` claim. The callout strips `device-` to recover the
 bare device id used for NATS subject interpolation
 (`DEVICE_ID_PREFIX_STRIP=device-` env var on the callout;
 `nats/callout/src/zitadel.rs::extract_device_id`).
-## How does the agent authenticate? Are JWTs / refresh tokens cached?
+Both permit impersonation if copied. A machine key signs short-lived assertions
 sent to Zitadel, while a PAT is itself a bearer credential. Zitadel allows more
 than one key per machine user, which permits installing a replacement before
 removing the old key. Fleet does not currently use TPM-backed key storage.
-On disk the agent keeps **only the JSON machine key** (RSA
+## Does enrollment make OpenBao secrets available?
 private key) at `/etc/fleet-agent/zitadel-key.json`.
-It does NOT store:
+No. The same ID token can authenticate to an OpenBao JWT role, but enrollment
- access tokens (in memory only)
+must also configure the agent's OpenBao endpoint. Group membership, OpenBao
- refresh tokens (the JWT-bearer flow has none — RFC 7523 is
+policies, policy attachments, and secret values must exist separately.
  stateless by design)
-On every NATS (re)connect, `credentials.rs::zitadel_mint`:
+The operator has optional group-source and OpenBao grant-sync support. The
 public production composition does not wire the required group source or
 OpenBao administrator token, so those behaviors are disabled there.
-1. Builds a JWT assertion with `exp = now + 60s`, signs it with
+## How does dashboard login differ?
   the RSA key
 2. POSTs it to `<zitadel>/oauth/v2/token` with grant type
   `urn:ietf:params:oauth:grant-type:jwt-bearer`
 3. Receives an access token (~12h validity), caches it in memory
 4. Re-mints when within 5min of expiry
   (`TOKEN_REFRESH_LEEWAY_SECS`)
-## What happens to an offline agent?
+The dashboard uses an OIDC authorization-code flow with PKCE. It validates the
 returned ID token, stores it in an encrypted private cookie, and requires the
 `fleet-admin` role. Its Zitadel application is configured to include roles in
 the ID token. See [Operator dashboard SSO](./operator-dashboard-sso.md).
-| Time offline | Behavior |
+## Why is the IAM administrator PAT in a Kubernetes Secret?
 | --- | --- |
 | 0 – ~12 h | Cached access token still valid. Reconnects work transparently. |
 | > ~12 h | Token expired. Agent enters reconnect loop until network returns, then mints fresh on first successful reach. |
-The RSA key never expires until rotated server-side.
+`ZitadelSetupScore` uses the PAT for management API operations. Kubernetes
 Secrets are not encrypted at rest unless the cluster enables encryption at
 rest. Anyone who can read this Secret can act with its privileges. Moving this
 bootstrap credential to a stronger storage path remains production-hardening
 work.
-## Where are the lifetimes set?
+## Token reference
- **Access token TTL** — Zitadel UI: Org → Settings → OIDC
+| Material | Storage | Purpose |
-  Settings → "Access Token Lifetime" (default 12 h).
+|---|---|---|
- **Assertion TTL** — hardcoded 60 s in
+| Machine JSON key | Device filesystem | Signs JWT-bearer assertions |
-  `credentials.rs::ASSERTION_LIFETIME_SECS`. Zitadel rejects
+| JWT assertion | Memory and token request | Proves possession of the machine key; valid for 60 seconds |
-  assertions where `exp - iat > 60 s`; this is server-enforced,
+| OIDC ID token | Agent memory and NATS connection | Verifiable bearer used by the NATS callout and OpenBao JWT login |
-  not a knob.
+| Zitadel access token | Token response only | Opaque by default; not used for production Fleet bearer authentication |
- **Machine key TTL** — set when the key is created in
+| NATS user JWT | NATS authorization exchange | Carries the permissions issued by the auth callout |
  `harmony/src/modules/zitadel/setup.rs::create_machine_key`.
 ## Why is a JSON machine key more secure than a PAT?
 Both are "if stolen, full impersonation" — the same blast radius.
 The difference is in leak surface:
 - **PAT**: a 60-char bearer string sent on every authenticated
  request. Every log line, every env dump, every misrouted
  request is a leak opportunity.
 - **JSON key**: an RSA private key. Only ever signs short-lived
  (60 s) assertions sent to one endpoint
  (`<zitadel>/oauth/v2/token`). The bearer token NATS sees is
  the access token — short-lived (12 h max), scoped, distinct
  from the long-term secret. A full network capture of the
  agent ↔ NATS traffic yields only access tokens that expire
  within 12 h.
 Plus: Zitadel allows multiple keys per machine user, so rotation
 is zero-downtime (mint new → push to device → delete old). PATs
 rotate one-at-a-time and are disruptive.
 What this does not defend against: a fully compromised device
 where the attacker reads the keyfile. That requires hardware
 (TPM / secure element) and is out of scope.
 ## The machine keys expire in year 9999. Isn't that effectively forever?
 Yes. Currently set in `ZitadelSetupScore::create_machine_key` as
 a known-bad default chosen for demo convenience (re-running tests
 shouldn't produce expired keys mid-run). Tracked as a known issue.
 ## Why is the IAM-Owner PAT stored as a plain k8s Secret?
 K8s Secrets are base64-encoded, **not** encrypted at rest unless
 etcd encryption-at-rest is explicitly enabled with a KMS provider.
 Anyone with `get secrets` in the `zitadel` namespace effectively
 has Zitadel admin.
 The PAT exists because `ZitadelSetupScore` calls Zitadel's
 management API (create project, role, machine user, mint key),
 which requires IAM-Owner privileges. A PAT is the simplest
 credential that survives across applies.
 This is a known production-hardening gap. Harmony has the
 `harmony_secret` crate (ADR-020) with OpenBao and local-encrypted-file
 backends; the Score is currently wired against a k8s Secret only.
 ## What lifetime is set for the human admin password — why does the ConfigMap show one that doesn't work?
 `ZitadelScore` regenerates a random admin password on every apply
 and writes it to the rendered ConfigMap. Helm's `FirstInstance`
 block only seeds Postgres on the **first** install against an
 empty DB, so re-applies render a new ConfigMap password but leave
 the original Postgres hash untouched. The displayed password is
 stale on every apply after the first.
 To recover access: use the `iam-admin-pat` to call Zitadel's
 management API and reset the human admin's password directly.
 Tracked as a known bug.
 ## Quick reference — tokens on the wire
 | Token | Lives where | Lifetime | Signed by | Purpose |
 | --- | --- | --- | --- | --- |
 | **Assertion** | Agent memory, in-flight | 60 s | Agent (RSA key) | "I'm machine user X — give me an access token" |
 | **Access token** | Agent memory + on-the-wire to NATS | ~12 h | Zitadel | "Zitadel says I'm device X with role `device`" |
 | **NATS user JWT** | NATS server connection state | callout-defined (~30 s) | Auth callout (NKey) | "I have these permissions on these subjects" |
 The agent only holds the RSA key on disk and the access token
 in memory. The NATS user JWT is server-internal — agents don't
 see it.
 ## Code map
 | Topic | File |
-| --- | --- |
+|---|---|
-| Helm install, masterkey, admin password | `harmony/src/modules/zitadel/mod.rs` |
+| Fleet identity composition | `fleet/harmony-fleet-deploy/src/app.rs` |
-| Project/role/machine user provisioning | `harmony/src/modules/zitadel/setup.rs` |
+| Machine-key token exchange | `harmony_zitadel_jwt/src/lib.rs` |
-| Per-device machine user + key handoff | `examples/fleet_e2e_demo/src/lib.rs::provision_device` |
+| Shared agent/operator credentials | `fleet/harmony-fleet-auth/src/credentials.rs` |
-| JWT-bearer mint | `fleet/harmony-fleet-agent/src/credentials.rs::zitadel_mint` |
+| Callout validation | `nats/callout/src/zitadel.rs` |
-| Auth callout decision tree | `nats/callout/src/handler.rs::decide` |
+| Device enrollment | `fleet/harmony-fleet-deploy/src/device_setup.rs` |
-| Per-device permission template | `nats/callout/src/permissions.rs::device_default` |
+| Manual diagnosis | [Manual token mint](./fleet-manual-token-mint.md) |
 | End-to-end rehearsal runbook | `examples/fleet_e2e_demo/RUNBOOK.md` |
 | Manual JWT-bearer mint + NATS write recipe | [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) |
--- a/docs/guides/fleet.md
+++ b/docs/guides/fleet.md
@@ -0,0 +1,104 @@
 # Harmony Fleet
 Harmony Fleet deploys Podman workloads and runs bounded maintenance operations
 on Linux devices outside a Kubernetes cluster. A Kubernetes operator holds the
 desired state, while an agent on each device reconciles its local Podman runtime.
 This fits small edge installations where devices may disconnect and reconnect.
 Fleet is not a general Kubernetes distribution. The current workload contract is
 `PodmanV0`, and the current maintenance task is an apt full upgrade for Debian and
 Raspberry Pi OS.
 ## Data flow
 ```text
 Deployment and TaskRun CRs
          |
          v
 Kubernetes operator -----> NATS JetStream KV -----> device agent -----> Podman
          ^                         ^                       |
          |                         |                       v
          +------ CR status <-------+------------- device status
 ```
 The operator watches namespaced `Deployment`, `Device`, and `TaskRun` resources.
 It resolves targets, writes per-device intent to NATS, and aggregates reported
 state. Agents do not receive Kubernetes credentials and do not call the
 Kubernetes API.
 Zitadel authenticates operators and devices. A device keeps a Zitadel machine
 key and exchanges a signed assertion for an OIDC `id_token`. The NATS auth
 callout validates that token and grants device-scoped subjects. OpenBao can use
 the same verifiable `id_token` for its JWT login. Zitadel access tokens are
 opaque by default and are not used as the production NATS or OpenBao bearer.
 ## Trust boundaries
 - Kubernetes and the Fleet operator are the control plane. An operator
  compromise can change workload and maintenance intent.
 - NATS carries desired and reported state. The auth callout restricts a device
  identity to its subjects, but NATS remains in the control-plane trust domain.
 - The unprivileged agent controls workloads owned by the `fleet-agent` account.
 - A root updater exposes fixed agent-upgrade and system-upgrade operations over a
  Unix socket. It does not accept arbitrary commands, package names, or paths.
 - Device labels are self-reported placement data, not authorization data.
 - A copied machine key permits impersonation of that device until the key is
  revoked. Hardware-backed key storage is not implemented.
 ## Current production composition
 The production deploy composition installs PostgreSQL, Zitadel, NATS, the auth
 callout, OpenBao, the operator, and the dashboard. It does **not** currently pass
 a Zitadel group source or OpenBao administrator token to the operator. As a
 result, `allowedGroups` placement fails closed and deployment-to-group OpenBao
 grant synchronization is disabled unless a private deploy composition supplies
 the corresponding operator configuration.
 Enrollment installs a machine key on the device only when the enrollment
 operator provides or mints one. It does not automatically make application
 secrets available. OpenBao endpoint configuration, group placement, policies,
 and secret data are separate setup steps.
 ## Dashboard
 Production exposes the operator's web dashboard through Zitadel SSO and requires
 the `fleet-admin` role. The live service reads Device and Deployment CRs and can:
 - show current device liveness and deployment aggregate state;
 - list device inventory and labels;
 - derive alerts for stale devices and failing deployments;
 - quarantine a device by adding the blacklist label;
 - send an agent command from a device page.
 Alert acknowledgements are held in operator memory and disappear after restart.
 The displayed 24-hour trend is currently static presentation, not historical
 telemetry. The dashboard does not create deployments, show `TaskRun` history, or
 provide rollout controls.
 For local UI work, `serve-web --mock` uses generated data and proves only the web
 surface, not Fleet connectivity.
 ## Guide
 1. [Enroll a device](./fleet-device-secrets.md).
 2. [Deploy Podman workloads](./fleet-podman-deployments.md).
 3. [Run tasks and upgrades](./fleet-tasks-upgrades.md).
 4. [Deploy and operate the control plane](./fleet-staging-install.md).
 5. Use the [Zitadel FAQ](./fleet-zitadel-faq.md) and [manual token
   mint](./fleet-manual-token-mint.md) when diagnosing authentication.
 ## Limits
 - The production operator is a single replica. Active-active reconciliation and
  tested control-plane failover are not provided.
 - There is no published or validated fleet-size target. Some controllers still
  poll complete device sets.
 - Deployment rollout supports one mode: immediate release, with an optional
  `canary=true` gate. There are no percentages, batches, approval steps, or
  workload rollback.
 - System upgrades are direct `TaskRun` resources. Recurring schedules are not
  implemented.
 - Agent upgrades can restore the previous agent executable if activation fails.
  Workload rollback and operating-system package rollback are not implemented.
 - A device is trusted to report its own labels, inventory, workload status, and
  task status.
--- a/docs/guides/harmony-auth-cli.md
+++ b/docs/guides/harmony-auth-cli.md
@@ -0,0 +1,376 @@
 # Harmony Auth CLI
 > **Status: read-only commands implemented.** Mutation commands follow the
 > ADR-027 group migration.
 `harmony-auth` inspects and manages the relationship between Zitadel
 identities and OpenBao access. It presents tenants, identities, and Harmony
 permissions first. JWT roles, policy names, subject claims, and HCL remain
 available through advanced output.
 The CLI is the preferred interface while the web UI matures. Both interfaces
 use the same `harmony_auth` operations and return the same effective access.
 ## Mental model
 The CLI has two main views:
 - `identity`: who an identity is and what it can access
 - `tenant`: who can access a tenant or project
 Harmony permissions are the primary authorization vocabulary:
 | Permission | Intended identity | Effect |
 |---|---|---|
 | `tenant-admin` | Human | Read, create, change, and delete secrets in a tenant or project |
 | `cd-deployer` | Service account | Read deployment secrets in one project |
 | `read-only` | Human or service account | Read secrets in a tenant or project |
 Existing OpenBao policies that do not correspond to a Harmony assignment are
 shown as imported access. The CLI does not rename, rewrite, or hide them.
 ## Command tree
 ```text
 harmony-auth
 ├── connection check
 ├── identity list
 ├── identity show <subject-id>
 ├── tenant list
 └── tenant show <tenant>
 ```
 There are no flat aliases. `harmony-auth list` is not valid.
 ## Connection and credentials
 Every command except `--help` and `--version` requires:
 | Flag | Environment | Meaning |
 |---|---|---|
 | `--zitadel-url` | `ZITADEL_URL` | Zitadel base URL |
 | `--openbao-url` | `OPENBAO_URL` | OpenBao base URL |
 | none | `ZITADEL_PAT` | Zitadel service-account PAT |
 | none | `OPENBAO_TOKEN` | Temporary OpenBao administrator token |
 Secrets are environment-only because command-line arguments remain in shell
 history and may be visible in the process list. Secret values never appear in
 help output, normal output, JSON, or logs.
 Example:
 ```sh
 export ZITADEL_URL=https://sso.example.com
 export ZITADEL_PAT=...
 export OPENBAO_URL=https://secrets.example.com
 export OPENBAO_TOKEN=...
 harmony-auth connection check
 ```
 The CLI does not persist profiles or credentials. Browser profile storage and
 session credential refresh remain web UI concerns.
 `connection check` attempts both backends even when one fails. It reports each
 status without printing provider response bodies:
 ```text
 Zitadel  connected
 OpenBao  connected
 ```
 ## Identity commands
 ### List identities
 ```sh
 harmony-auth identity list
 harmony-auth identity list --search folk
 harmony-auth identity list --kind human
 harmony-auth identity list --tenant devsights
 harmony-auth identity list --tenant devsights --kind service
 ```
 Filters combine with AND semantics. `--search` matches display name, login, or
 email. `--kind` accepts `human` or `service`. Tenant matching uses parsed
 assignment and imported-policy scopes, not string-prefix matching.
 Imported scopes are recognized only from wildcard roots:
 - `<mount>/data/<tenant>/*` is tenant-wide.
 - `<mount>/data/<tenant>/<project>/*` is project-specific.
 - Exact secret paths and wildcard paths below a project are custom access. They
  remain visible but do not affect tenant filters or summaries.
 Tenant and project components must pass the same slug validation as managed
 Harmony scopes. `harmony_auth` returns both the raw paths and parsed scopes;
 frontends never infer scopes themselves.
 Human output keeps the tenant visible:
 ```text
 ACTIVE  HUMAN    Alice Example  alice@example.com
        subject  241696899342475267
        devsights                 Tenant Admin
 ACTIVE  SERVICE  Folk CD        folk-cd
        subject  241697058442100739
        devsights/folk-timesheet CD Deployer
 ```
 An identity without recognized access is still listed with `No access`.
 ### Show an identity
 ```sh
 harmony-auth identity show 241696899342475267
 ```
 Default output includes identity metadata, Harmony assignments, and imported
 access:
 ```text
 Alice Example
  Subject: 241696899342475267
  Login:   alice@example.com
  Kind:    Human
  Status:  Active
 Harmony permissions
  019b...  Tenant Admin  devsights
 Imported OpenBao access
  legacy-folk-reader  devsights/folk-timesheet  Read secrets
 ```
 Use `--advanced` to inspect implementation details:
 ```sh
 harmony-auth identity show 241696899342475267 --advanced
 ```
 Advanced output adds matching JWT roles, bound subject, audiences, attached
 policy names, and the exact HCL returned by OpenBao. A built-in or inaccessible
 policy remains listed with `Policy body unavailable`.
 ## Planned group-based mutations
 The first release does not grant or revoke access. Existing per-subject JWT
 roles are discovery input, not a writable authorization model.
 ADR-027 makes Zitadel roles named `<tenant>:owner`, `<tenant>:deployer`, and
 `<tenant>:viewer` authoritative. One shared OpenBao JWT role reads the `groups`
 claim, and OpenBao external groups attach policies. Mutation commands ship only
 after `harmony_auth` implements that model end to end.
 ### Grant a permission (planned)
 ```sh
 harmony-auth identity grant 241696899342475267 \
  --permission tenant-admin \
  --tenant devsights
 ```
 Grant is review-only by default:
 ```text
 Plan
  Identity:   Alice Example (241696899342475267)
  Permission: Tenant Admin
  Scope:      devsights
  Effect:     Read, create, change, and delete secrets
 No changes applied. Re-run with --apply to continue.
 ```
 Apply the reviewed request explicitly:
 ```sh
 harmony-auth identity grant 241696899342475267 \
  --permission tenant-admin \
  --tenant devsights \
  --apply
 ```
 `cd-deployer` requires `--project`. Other permissions accept an optional
 project. Permission applicability and scope validation come from
 `harmony_auth`; the CLI does not duplicate those rules.
 Applying a grant changes the Zitadel role assignment. It does not create or
 edit a per-subject OpenBao JWT role. Applying the same grant twice must report
 `changed: false`.
 ### Revoke a permission (planned)
 The assignment ID comes from `identity show`:
 ```sh
 harmony-auth identity revoke 241696899342475267 019b...
 ```
 Revoke is also review-only by default. It shows identity, permission, scope,
 and the warning that already-issued OpenBao tokens remain valid until expiry or
 revocation. `--apply` performs the removal:
 ```sh
 harmony-auth identity revoke 241696899342475267 019b... --apply
 ```
 Imported per-subject access cannot be revoked through this command because it
 is outside the ADR-027 group model. Its OpenBao policy name remains visible for
 manual migration.
 ## Tenant commands
 ### List tenants
 ```sh
 harmony-auth tenant list
 ```
 Tenants and projects are discovered from managed assignments and recognized
 OpenBao policy paths:
 ```text
 TENANT     PROJECT          HUMANS  SERVICES
 devsights  All projects     2       0
 devsights  folk-timesheet   1       1
 detexion   harmony-fleet    0       2
 ```
 ### Show a tenant
 ```sh
 harmony-auth tenant show devsights
 harmony-auth tenant show devsights --project folk-timesheet
 ```
 Output lists matching scopes and identities with their access. With
 `--project`, tenant-wide access and access to that exact project are included;
 other projects are excluded:
 ```text
 devsights/folk-timesheet
  Alice Example  Human    Read-only
  Folk CD        Service  CD Deployer
 ```
 This is an authorization view, not a secret-value browser. Secret listing,
 creation, update, and reveal are outside the first release.
 ## JSON output
 Every implemented command accepts `--json`. JSON is written to stdout;
 diagnostics and logs are written to stderr. The envelope is versioned:
 ```json
 {
  "schema_version": 1,
  "command": "identity.list",
  "result": {}
 }
 ```
 The first release uses these result shapes:
 | Command | `result` fields |
 |---|---|
 | `connection check` | `zitadel: { connected }`, `openbao: { connected }` |
 | `identity list` | `identities: [{ identity, access }]` |
 | `identity show` | `identity`, `access` |
 | `tenant list` | `tenants: [{ scope, humans, services }]` |
 | `tenant show` | `tenant`, `project`, `identities: [{ identity, access }]` |
 `identity` contains `subject_id`, `kind`, `display_name`, `login_name`,
 `email`, and `active`. Identity kinds are `human` and `service`; `email` is a
 string or `null`.
 `scope` contains `tenant` and `project`, where `project` is a string or `null`.
 An assignment contains `id`, `subject_id`, `permission`, `scope`,
 `policy_name`, and `created_at`. IDs are UUID strings, timestamps are RFC 3339,
 and permissions are `tenant_admin`, `cd_deployer`, or `read_only`.
 `access` contains:
 - `assignments`: assignment objects as defined above
 - `imported`: objects with `role_name`, `policy_name`, `secret_paths`,
  `scopes`, and `effect`
 - `roles`: objects with `name`, `subject_id`, `bound_audiences`, and `policies`
 Each role policy contains `name` and `body`. Policy bodies are strings only for
 `identity show --advanced`; otherwise `body` is `null`. Imported
 `secret_paths`, audiences, and scopes are arrays. `effect` is the
 plain-language interpretation returned by `harmony_auth`.
 `connection check` always returns both status objects and exits `1` when either
 `connected` value is false. A top-level error is used only when the command
 cannot produce those statuses.
 Domain and backend errors use stdout when `--json` is active:
 ```json
 {
  "schema_version": 1,
  "command": "identity.show",
  "error": {
    "kind": "not_found",
    "message": "identity not found"
  }
 }
 ```
 Error kinds are `not_found`, `invalid`, and `backend`. Clap usage errors remain
 on stderr because command parsing fails before JSON dispatch.
 Planned grant and revoke results will add `applied` and `changed` when those
 commands are implemented. Their JSON schema is not frozen by this release.
 ## Exit and error behavior
 | Exit | Meaning |
 |---|---|
 | `0` | Query completed, plan produced, or mutation applied/no-op |
 | `2` | Invalid command, missing connection value, not found, or invalid input |
 | `1` | Zitadel or OpenBao request failed |
 Errors name the failed operation and backend but do not print credentials,
 provider response bodies, or stack traces. `RUST_LOG=info` enables operation
 logs on stderr.
 ## Architecture boundary
 ```text
 harmony_auth_ui  ─┐
                  ├──> harmony_auth ──> Zitadel + OpenBao
 harmony_auth_cli ─┘
 ```
 `harmony_auth` owns:
 - identities, scopes, permissions, assignments, roles, policies, and plans
 - Zitadel identity discovery
 - OpenBao role and policy discovery
 - tenant and identity access queries
 - permission applicability and scope validation
 - assignment discovery and current legacy assignment operations used by the UI
 - provider request construction and response interpretation
 `harmony_auth_cli` owns:
 - Clap arguments and environment mapping
 - terminal and JSON rendering
 - binary exit codes and logging setup
 `harmony_auth_ui` owns HTTP routes, browser sessions, cookies, forms, HTML, CSS,
 and browser JavaScript. Neither frontend may parse OpenBao policies, infer
 tenants, reconcile JWT roles, or implement permission rules.
 ## First-release limits
 - Authorization discovers existing per-subject JWT roles as imported access.
 - Grant and revoke wait for Zitadel role and OpenBao external-group operations.
 - The CLI does not create Zitadel identities.
 - The CLI does not provide a generic OpenBao policy editor.
 - Tenant administrators are not yet authenticated as constrained actors; the
  supplied OpenBao token determines backend authority.
 - The CLI does not store profiles or credentials.
--- a/docs/guides/operator-dashboard-sso.md
+++ b/docs/guides/operator-dashboard-sso.md
@@ -5,21 +5,20 @@ public client). Distinct from the agent/callout machine auth
 ([fleet-zitadel-faq](./fleet-zitadel-faq.md)); the security rationale is in
 [web-auth-security](./web-auth-security.md). Code: `harmony_zitadel_auth/`.
-## Quickstart (staging)
+## Deployment
-1. **Zitadel app** — create a **Web** application, auth method **PKCE** (no client
+`FleetApp` declares a dedicated dashboard Web PKCE application with the
-   secret), redirect URI `https://fleet-stg.<base>/auth/callback`, post-logout URI
+dashboard callback and logout URLs. The operator's Device Code application is
-   `https://fleet-stg.<base>/`. Copy its **Client ID**.
+separate. `ZitadelSetupScore` reconciles both and publishes the dashboard client
-2. **Seed config** in OpenBao (namespace `fleet-staging`) — the deploy derives every
+ID. `FleetOperatorScore` builds `ZitadelAuthConfig` from that output and the
-   host from `base_domain`, so you set only:
+dashboard Ingress, then generates and retains the session cookie key in the
-   - `FleetDeployConfig.operator_oidc_client_id` = the Client ID
+operator Secret. No client ID or cookie key is entered by hand.
   - `FleetDeployConfig.operator_trusted_audiences` = `["<Client ID>"]`
   - `FleetDeploySecrets.operator_cookie_key_b64` = `openssl rand -base64 64`
 3. **Deploy**: `./fleet/scripts/dev-deploy-operator.sh`
 4. Open `https://fleet-stg.<base>/` → Zitadel login → back to the dashboard.
-`fleet_staging_install` already generates the cookie key, so a fresh install needs
+The dashboard requires the exact Zitadel project role `fleet-admin`. Create the
-only the Client ID + audiences.
+role on the fleet project and grant it to each operator. Login automatically
 requests `urn:zitadel:iam:org:project:roles`, so the project-level **Assert Roles
 on Authentication** setting is not required. Users without the role receive a
 403 response that retains the session cookie and includes a sign-out link.
 ## Local dev (`serve-web`)
@@ -39,17 +38,15 @@ on the app's **Development Mode** (Zitadel rejects non-HTTPS redirects otherwise
 - **Cookie key** — `cookie_key_b64` must decode to ≥64 bytes, else the dashboard
  refuses to start (`cookie_key_b64 must decode to at least 64 bytes`; reconcile
  keeps running).
 - **403 after login** — confirm the user has the exact `fleet-admin` project role
  and that the aggregate roles claim is present in the ID token.
 ## Config reference
-The operator reads `ZitadelAuthConfig` + `OperatorCookieKey` via ConfigClient. The
+The operator reads `ZitadelAuthConfig` and `OperatorCookieKey` through
-deploy derives `zitadel_base` / `base_url` / `logout_redirect_uri` from `base_domain`
+ConfigClient. The deploy derives `zitadel_base`, `base_url`, client ID, trusted
-(`https://sso-stg.<base>`, `https://fleet-stg.<base>`, `…/`) and fixes
+audience, logout URI, and `scope = openid profile email`. The login flow adds the
-`scope = openid profile email`; you supply `client_id`, `trusted_audiences`,
+aggregate roles scope once if it is absent. All endpoints derive from
-`cookie_key_b64`. All endpoints derive from `zitadel_base`:
+`zitadel_base`:
 `/.well-known/openid-configuration`, `/oauth/v2/authorize`, `/oauth/v2/token`,
 `/oidc/v1/end_session`.
 > The dashboard only checks that the user authenticated — no role gate yet
 > ([web-auth-security](./web-auth-security.md) §3,
 > [ROADMAP/09](../../ROADMAP/09-sso-config-hardening.md)).
--- a/examples/fleet_device_enroll/src/main.rs
+++ b/examples/fleet_device_enroll/src/main.rs
@@ -28,7 +28,7 @@ use harmony::inventory::Inventory;
 use harmony::modules::fleet::ensure_fleet_ssh_keypair;
 use harmony::modules::linux::{LinuxHostTopology, LinuxLocalhostTopology, SshCredentials};
 use harmony_fleet_deploy::{
-    AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
+    AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
 };
 use harmony_types::id::Id;
@@ -52,6 +52,10 @@ use harmony::topology::{VirtualMachineSpec, VmArchitecture, VmFirstBootConfig};
             credentials inline (browser SSO or pre-acquired token)"
 )]
 struct Cli {
    /// Apply a changed existing device configuration without prompting.
    #[arg(long)]
    yes: bool,
    // ---- target ----------------------------------------------------------
    /// Where to apply the score.
    ///
@@ -112,6 +116,14 @@ struct Cli {
    #[arg(long)]
    nats_url: Option<String>,
    /// OpenBao endpoint used for deployment secrets and private image pulls.
    #[arg(long, requires = "openbao_secret_prefix")]
    openbao_url: Option<String>,
    /// KV prefix containing this fleet's deployment and pull secrets.
    #[arg(long, requires = "openbao_url")]
    openbao_secret_prefix: Option<String>,
    // ---- device identity -------------------------------------------------
    /// Device id baked into the agent's TOML, the Zitadel machine
    /// username (`device-<device_id>`), and the Kubernetes Device CR
@@ -302,9 +314,13 @@ async fn main() -> Result<()> {
        auth,
        agent_binary_path: agent_binary,
        hosts_entries: vec![],
-        openbao: None,
+        openbao: cli
            .openbao_url
            .clone()
            .zip(cli.openbao_secret_prefix.clone())
            .map(|(url, secret_prefix)| DeviceOpenbao { url, secret_prefix }),
    };
-    let setup_score = FleetDeviceSetupScore::new(setup_config);
+    let setup_score = FleetDeviceSetupScore::new(setup_config).overwrite_existing_config(cli.yes);
    #[cfg(feature = "vm-rehearsal")]
    if cli.vm_rehearsal {
--- a/examples/fleet_load_test/src/main.rs
+++ b/examples/fleet_load_test/src/main.rs
@@ -46,6 +46,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 use tokio::task::JoinSet;
 const LOAD_TEST_GROUP: &str = "fleet-load-test";
 #[derive(Parser, Debug, Clone)]
 #[command(
    name = "fleet_load_test",
@@ -406,7 +408,7 @@ async fn apply_one_cr(
    let cr = Deployment::new(
        &group.cr_name,
        DeploymentSpec {
-            allowed_groups: None,
+            allowed_groups: vec![LOAD_TEST_GROUP.to_string()],
            target_selector: LabelSelector {
                match_labels: Some(match_labels),
                match_expressions: None,
@@ -416,9 +418,11 @@ async fn apply_one_cr(
            // for each matched device; that's wire noise we accept
            // as part of the realism.
            score: ReconcileScore::PodmanV0(PodmanV0Score {
                init_container: None,
                services: vec![PodmanService {
                    name: group.cr_name.clone(),
                    image: "docker.io/library/nginx:alpine".to_string(),
                    image_pull_secret: None,
                    ports: vec!["8080:80".to_string()],
                    env: vec![],
                    secret_env: vec![],
@@ -468,6 +472,7 @@ async fn publish_one_info(bucket: kv::Store, device: DevicePlan) -> Result<()> {
        device_id: Id::from(device.device_id.clone()),
        labels: BTreeMap::from([("group".to_string(), device.cr_name.clone())]),
        inventory: None,
        updater: None,
        updated_at: Utc::now(),
    };
    let key = device_info_key(&device.device_id);
@@ -495,6 +500,7 @@ async fn simulate_state_loop(
            device_id: Id::from(device.device_id.clone()),
            deployment: deployment.clone(),
            phase,
            rollout_revision: None,
            last_event_at: Utc::now(),
            last_error: matches!(phase, Phase::Failed)
                .then(|| format!("synthetic failure @{}", device.device_id)),
@@ -529,6 +535,7 @@ async fn simulate_heartbeat_loop(
        let hb = HeartbeatPayload {
            device_id: Id::from(device.device_id.clone()),
            at: Utc::now(),
            agent_version: None,
        };
        if let Ok(payload) = serde_json::to_vec(&hb) {
            if bucket.put(&hb_key, payload.into()).await.is_ok() {
--- a/examples/fleet_server_install/src/main.rs
+++ b/examples/fleet_server_install/src/main.rs
@@ -95,6 +95,9 @@ struct Cli {
    /// `RUST_LOG` value injected into the operator pod's env.
    #[arg(long, default_value = "info,kube_runtime=warn")]
    log_level: String,
    /// Static `device=group|group;...` membership for local testing.
    #[arg(long)]
    device_groups: Option<String>,
    /// Hostname Zitadel should answer on. When set, Zitadel + its
    /// PostgreSQL cluster are installed alongside the operator.
@@ -138,12 +141,15 @@ async fn main() -> Result<()> {
    // NatsScore install creates. ClusterIP and LoadBalancer both
    // expose the same `<release>.<namespace>:4222` for in-cluster
    // callers.
-    let operator = FleetOperatorScore::new(&cli.operator_image)
+    let mut operator = FleetOperatorScore::new(&cli.operator_image)
        .namespace(&cli.operator_namespace)
        .release_name(&cli.operator_release)
        .image_pull_policy(&cli.operator_image_pull_policy)
        .messaging(&nats.client_ref())
        .log_level(&cli.log_level);
    if let Some(groups) = cli.device_groups {
        operator = operator.device_groups(groups);
    }
    // FleetServerScore now takes NatsK8sScore (auth-callout-aware,
    // OKD-Route-aware) — see `fleet_staging_install` for the
--- a/examples/fleet_typed_deploy/src/bin/tenant-provision.rs
+++ b/examples/fleet_typed_deploy/src/bin/tenant-provision.rs
@@ -1,8 +1,20 @@
-use example_fleet_typed_deploy::{credential_store, platform_context, tenant_config};
+use std::{env, path::PathBuf};
-use harmony_fleet_deploy::provision_fleet_tenant_with_context;
+
 use anyhow::Context;
 use example_fleet_typed_deploy::{credential_store, fleet_context, tenant_config};
 use harmony_fleet_deploy::provision_fleet_tenant_with_kubeconfig;
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    provision_fleet_tenant_with_context(platform_context()?, tenant_config(), credential_store()?)
+    let kubeconfig = env::var_os("KUBECONFIG")
-        .await
+        .map(PathBuf::from)
        .context("set KUBECONFIG to your cluster-admin kubeconfig")?;
    provision_fleet_tenant_with_kubeconfig(
        fleet_context()?,
        kubeconfig,
        tenant_config(),
        credential_store()?,
        false,
    )
    .await
 }
--- a/examples/fleet_typed_deploy/src/lib.rs
+++ b/examples/fleet_typed_deploy/src/lib.rs
@@ -18,26 +18,6 @@ pub fn fleet_context() -> anyhow::Result<Context> {
    })
 }
 pub fn platform_context() -> anyhow::Result<Context> {
    Ok(Context {
        name: context_name!("platform-admin"),
        namespace: "platform-system".parse()?,
        spec: ContextSpec::Remote(RemoteContext {
            registry: oci_registry!("registry.example.com"),
            repository: oci_repository!("customer/fleet"),
            domain: domain!("fleet.example.com"),
            image_pull_secret: None,
            access: OpenBaoClusterAccess {
                namespace: openbao_namespace!("platform/admin"),
                url: http_url!("https://secrets.example.com"),
                role: "platform-admin".parse()?,
                zitadel_url: http_url!("https://identity.example.com"),
                zitadel_audience: "openbao".parse()?,
            },
        }),
    })
 }
 pub fn tenant_config() -> TenantConfig {
    TenantConfig {
        id: "customer-fleet".into(),
--- a/examples/harmony_apply_deployment/Cargo.toml
+++ b/examples/harmony_apply_deployment/Cargo.toml
@@ -5,15 +5,17 @@ edition = "2024"
 license.workspace = true
 [[bin]]
-name = "harmony_apply_deployment"
+name = "harmony"
 path = "src/main.rs"
 [dependencies]
 harmony = { path = "../../harmony" }
 harmony_app = { path = "../../harmony_app" }
 harmony_cli = { path = "../../harmony_cli" }
 harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
 harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
 harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
 kube = { workspace = true, features = ["runtime", "derive"] }
 k8s-openapi = { workspace = true }
-serde_json.workspace = true
+async-trait.workspace = true
 tokio.workspace = true
 anyhow.workspace = true
 clap.workspace = true
--- a/examples/harmony_apply_deployment/backend.Dockerfile
+++ b/examples/harmony_apply_deployment/backend.Dockerfile
@@ -0,0 +1,2 @@
 FROM busybox:1.37
 CMD ["sleep", "infinity"]
--- a/examples/harmony_apply_deployment/frontend.Dockerfile
+++ b/examples/harmony_apply_deployment/frontend.Dockerfile
@@ -0,0 +1,2 @@
 FROM busybox:1.37
 CMD ["httpd", "-f", "-p", "8080"]
--- a/examples/harmony_apply_deployment/src/main.rs
+++ b/examples/harmony_apply_deployment/src/main.rs
@@ -1,240 +1,168 @@
-//! Typed-Rust applier for the harmony fleet `Deployment` CR.
+use async_trait::async_trait;
-//!
+use harmony::score::Score;
-//! Builds a `Deployment` CR via the typed `DeploymentSpec` +
+use harmony::topology::K8sAnywhereTopology;
-//! `PodmanV0Score` + `kube::Api`, then either applies it directly
+use harmony_app::{
-//! through the kube client or prints it to stdout so the user can
+    AppContext, AppError, AppIdentity, Context, ContextCatalog, ContextSpec, HarmonyApp, ImageRefs,
-//! pipe into `kubectl apply -f -`.
+    ImageSpec, LocalContext, OpenBaoClusterAccess, Profile, RemoteContext,
 //!
 //! The CRD is domain-agnostic — it's "declarative reconcile intent
 //! for a set of devices matched by label selector," which is the
 //! same shape whether the fleet is Pi podman, OKD clusters, or
 //! KVM VMs. The name `harmony_apply_deployment` reflects that
 //! (not `iot_`-anything), in line with the review call to position
 //! the operator as a generic fleet/reconcile tool.
 //!
 //! The CRD types live in `harmony_fleet_operator`; the score types
 //! live in `harmony_reconciler_contracts` (PodmanV0 being the first
 //! reconciler variant — future variants drop in alongside).
 //!
 //! Typical demo-driver usage:
 //!
 //!     # apply an nginx deployment
 //!     cargo run -q -p example_harmony_apply_deployment -- \
 //!         --target-device fleet-smoke-vm-arm \
 //!         --image nginx:latest
 //!
 //!     # print the CR JSON (lets the user kubectl-apply it manually)
 //!     cargo run -q -p example_harmony_apply_deployment -- \
 //!         --target-device fleet-smoke-vm-arm \
 //!         --image nginx:latest --print | kubectl apply -f -
 //!
 //!     # upgrade the same deployment to a newer image
 //!     cargo run -q -p example_harmony_apply_deployment -- \
 //!         --target-device fleet-smoke-vm-arm \
 //!         --image nginx:1.26
 //!
 //!     # delete the deployment
 //!     cargo run -q -p example_harmony_apply_deployment -- --delete
 use anyhow::{Context, Result};
 use clap::Parser;
 use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
 use harmony_reconciler_contracts::{
    EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, VolumeMount,
 };
 use harmony_fleet_deploy::FleetDeploymentScore;
 use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
 use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
 use kube::Client;
 use kube::api::{Api, DeleteParams, Patch, PatchParams};
 use std::collections::BTreeMap;
-#[derive(Parser, Debug)]
+struct ExampleApp;
-#[command(
+
-    name = "harmony_apply_deployment",
+#[async_trait]
-    about = "Build + apply a harmony fleet Deployment CR from typed Rust (no yaml)"
+impl HarmonyApp<K8sAnywhereTopology> for ExampleApp {
-)]
+    fn identity(&self, ctx: &AppContext) -> AppIdentity {
-struct Cli {
+        AppIdentity {
-    /// Kubernetes namespace for the Deployment CR.
+            name: "example-app".into(),
-    #[arg(long, default_value = "fleet-demo")]
+            namespace: ctx.namespace().into(),
-    namespace: String,
+        }
-    /// Deployment CR name. Also used as the KV key suffix and
+    }
-    /// podman container name on the device.
+
-    #[arg(long, default_value = "hello-world")]
+    fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
-    name: String,
+        Ok(["frontend", "backend"]
-    /// Shortcut: if set, picks a single device by id. Shorthand for
+            .map(|name| ImageSpec {
-    /// `--selector device-id=<target_device>` — the agent publishes
+                name: name.into(),
-    /// a `device-id=<id>` label on its DeviceInfo by default so this
+                image: ctx.image(name),
-    /// works without any cluster-side label pre-wiring.
+                context: env!("CARGO_MANIFEST_DIR").into(),
-    #[arg(long, default_value = "fleet-smoke-vm")]
+                dockerfile: format!("{}/{name}.Dockerfile", env!("CARGO_MANIFEST_DIR")).into(),
-    target_device: String,
+                build_args: Vec::new(),
-    /// Repeatable `key=value` label selector. Takes precedence over
+            })
-    /// `--target-device` when provided. All pairs AND together.
+            .into())
-    #[arg(long = "selector", value_name = "KEY=VALUE")]
+    }
-    selectors: Vec<String>,
+
-    /// Container image to run.
+    fn validate_deploy_images(&self, images: &ImageRefs) -> Result<(), AppError> {
-    #[arg(long, default_value = "docker.io/library/nginx:latest")]
+        if images
-    image: String,
+            .iter()
-    /// `host:container` port mapping exposed on the device.
+            .any(|(name, _)| !matches!(name, "frontend" | "backend"))
-    #[arg(long, default_value = "8080:80")]
+        {
-    port: String,
+            return Err(AppError::InvalidComposition(
-    /// Repeatable `KEY=VALUE` env var injected into the container.
+                "only frontend and backend images are accepted".into(),
-    #[arg(long = "env", value_name = "KEY=VALUE")]
+            ));
-    envs: Vec<String>,
+        }
-    /// Repeatable bind-mount in `host_path:container_path[:ro]` form.
+        for name in ["frontend", "backend"] {
-    /// Append `:ro` for read-only.
+            let image = images.require(name)?;
-    #[arg(long = "volume", value_name = "HOST:CONTAINER[:ro]")]
+            let digest_pinned = harmony_app::is_digest_pinned(image);
-    volumes: Vec<String>,
+            let dev_tagged = !image.contains('@')
-    /// Container restart policy.
+                && image.rsplit_once(':').is_some_and(|(repository, tag)| {
-    #[arg(long, value_enum, default_value_t = CliRestart::UnlessStopped)]
+                    !repository.is_empty() && tag.starts_with("dev-")
-    restart: CliRestart,
+                });
-    /// Delete the Deployment CR instead of applying it.
+            if !digest_pinned && !dev_tagged {
-    #[arg(long)]
+                return Err(AppError::InvalidComposition(format!(
-    delete: bool,
+                    "image '{name}' must be digest-pinned or use a dev-* tag"
-    /// Print the CR as JSON to stdout instead of applying it.
+                )));
-    /// Useful for piping into `kubectl apply -f -`.
+            }
-    #[arg(long)]
+        }
-    print: bool,
+        Ok(())
    }
    async fn scores(
        &self,
        ctx: &AppContext,
        images: &ImageRefs,
    ) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
        let services = ["frontend", "backend"]
            .map(|name| {
                Ok(PodmanService {
                    name: name.into(),
                    image: images.require(name)?.into(),
                    image_pull_secret: (ctx.profile() == Profile::Prod)
                        .then(|| "application-images".into()),
                    ports: (name == "frontend")
                        .then(|| "8080:8080".into())
                        .into_iter()
                        .collect(),
                    env: Vec::new(),
                    secret_env: Vec::new(),
                    volumes: Vec::new(),
                    restart_policy: Default::default(),
                })
            })
            .into_iter()
            .collect::<Result<Vec<_>, AppError>>()?;
        let deployment = Deployment::new(
            "example-app",
            DeploymentSpec {
                allowed_groups: vec!["application".into()],
                target_selector: LabelSelector {
                    match_labels: Some([("application".into(), "example".into())].into()),
                    match_expressions: None,
                },
                score: ReconcileScore::PodmanV0(PodmanV0Score {
                    init_container: None,
                    services,
                }),
                rollout: Rollout {
                    strategy: RolloutStrategy::Immediate,
                },
            },
        );
        Ok(vec![Box::new(FleetDeploymentScore::new(
            deployment,
            ctx.namespace(),
        ))])
    }
 }
 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> anyhow::Result<()> {
-    let cli = Cli::parse();
+    harmony_cli::app::app_main(
-    let cr = build_cr(&cli);
+        ExampleApp,
-
+        ContextCatalog::new([
-    if cli.print {
+            Context {
-        println!("{}", serde_json::to_string_pretty(&cr)?);
+                name: "local".parse()?,
-        return Ok(());
+                namespace: "example-app".parse()?,
-    }
+                spec: ContextSpec::Local(LocalContext::ManagedK3d),
    let client = Client::try_default()
        .await
        .context("building kube client (is KUBECONFIG set?)")?;
    let api: Api<Deployment> = Api::namespaced(client, &cli.namespace);
    if cli.delete {
        match api.delete(&cli.name, &DeleteParams::default()).await {
            Ok(_) => println!("deleted deployment '{}/{}'", cli.namespace, cli.name),
            Err(kube::Error::Api(ae)) if ae.code == 404 => {
                println!(
                    "deployment '{}/{}' not found (already gone)",
                    cli.namespace, cli.name
                )
            }
            Err(e) => anyhow::bail!("delete failed: {e}"),
        }
        return Ok(());
    }
    // Server-side apply so repeated invocations (upgrades) patch
    // the existing CR instead of erroring with "already exists."
    let params = PatchParams::apply("harmony-apply-deployment").force();
    let applied = api
        .patch(&cli.name, &params, &Patch::Apply(&cr))
        .await
        .context("applying Deployment CR")?;
    let meta = applied.metadata;
    println!(
        "applied deployment '{}/{}' (resourceVersion={}, image={})",
        cli.namespace,
        meta.name.as_deref().unwrap_or("?"),
        meta.resource_version.as_deref().unwrap_or("?"),
        cli.image,
    );
    Ok(())
 }
 /// Mirrors the contract `RestartPolicy` so we can keep the CLI
 /// schema stable even if the underlying enum gains variants.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 enum CliRestart {
    No,
    UnlessStopped,
    OnFailure,
    Always,
 }
 impl From<CliRestart> for RestartPolicy {
    fn from(c: CliRestart) -> Self {
        match c {
            CliRestart::No => RestartPolicy::No,
            CliRestart::UnlessStopped => RestartPolicy::UnlessStopped,
            CliRestart::OnFailure => RestartPolicy::OnFailure,
            CliRestart::Always => RestartPolicy::Always,
        }
    }
 }
 fn parse_env(s: &str) -> Result<(String, String)> {
    let (k, v) = s
        .split_once('=')
        .ok_or_else(|| anyhow::anyhow!("--env expects KEY=VALUE, got {s:?}"))?;
    Ok((k.to_string(), v.to_string()))
 }
 fn parse_volume(s: &str) -> Result<VolumeMount> {
    let parts: Vec<&str> = s.split(':').collect();
    let (host, cont, ro) = match parts.as_slice() {
        [host, cont] => (host, cont, false),
        [host, cont, mode] if *mode == "ro" => (host, cont, true),
        [host, cont, mode] if *mode == "rw" => (host, cont, false),
        _ => anyhow::bail!("--volume expects HOST:CONTAINER[:ro|rw], got {s:?}"),
    };
    Ok(VolumeMount {
        host_path: host.to_string(),
        container_path: cont.to_string(),
        read_only: ro,
    })
 }
 fn build_cr(cli: &Cli) -> Deployment {
    let env: Vec<EnvVar> = cli
        .envs
        .iter()
        .map(|s| EnvVar::from(parse_env(s).expect("--env validated")))
        .collect();
    let volumes: Vec<VolumeMount> = cli
        .volumes
        .iter()
        .map(|s| parse_volume(s).expect("--volume validated"))
        .collect();
    let score = PodmanV0Score {
        services: vec![PodmanService {
            name: cli.name.clone(),
            image: cli.image.clone(),
            ports: vec![cli.port.clone()],
            env,
            secret_env: vec![],
            volumes,
            restart_policy: cli.restart.into(),
        }],
    };
    let payload = ReconcileScore::PodmanV0(score);
    let mut match_labels = BTreeMap::new();
    if cli.selectors.is_empty() {
        match_labels.insert("device-id".to_string(), cli.target_device.clone());
    } else {
        for kv in &cli.selectors {
            let (k, v) = kv
                .split_once('=')
                .unwrap_or_else(|| panic!("--selector expects KEY=VALUE, got '{kv}'"));
            match_labels.insert(k.to_string(), v.to_string());
        }
    }
    Deployment::new(
        &cli.name,
        DeploymentSpec {
            allowed_groups: None,
            target_selector: LabelSelector {
                match_labels: Some(match_labels),
                match_expressions: None,
            },
-            score: payload,
+            Context {
-            rollout: Rollout {
+                name: "production".parse()?,
-                strategy: RolloutStrategy::Immediate,
+                namespace: "example-app".parse()?,
                spec: ContextSpec::Remote(RemoteContext {
                    registry: "registry.example.com".parse()?,
                    repository: "apps".parse()?,
                    domain: "example.com".parse()?,
                    image_pull_secret: None,
                    access: OpenBaoClusterAccess {
                        namespace: "platform/example-app".parse()?,
                        url: "https://secrets.example.com".parse()?,
                        role: "deployer".parse()?,
                        zitadel_url: "https://identity.example.com".parse()?,
                        zitadel_audience: "openbao".parse()?,
                    },
                }),
            },
-        },
+        ])?,
    )
    .await
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn standalone_deploy_allows_digests_and_dev_tags_only() {
        let digest = "a".repeat(64);
        assert!(
            ExampleApp
                .validate_deploy_images(&ImageRefs::new([
                    (
                        "frontend".into(),
                        format!("registry/frontend@sha256:{digest}")
                    ),
                    ("backend".into(), "registry/backend:dev-123".into()),
                ]))
                .is_ok()
        );
        assert!(
            ExampleApp
                .validate_deploy_images(&ImageRefs::new([
                    ("frontend".into(), "registry/frontend:latest".into()),
                    ("backend".into(), "registry/backend:dev-123".into()),
                ]))
                .is_err()
        );
    }
 }
--- a/examples/harmony_sso/src/main.rs
+++ b/examples/harmony_sso/src/main.rs
@@ -250,6 +250,7 @@ async fn main() -> anyhow::Result<()> {
        openshift: false,
        tls_issuer: None,
        node_port: None,
        create_namespace: true,
    }
    .interpret(&Inventory::autoload(), &topology)
    .await
--- a/examples/monitoring_with_tenant/src/main.rs
+++ b/examples/monitoring_with_tenant/src/main.rs
@@ -37,6 +37,7 @@ async fn main() {
                memory_request_gb: 4.0,
                memory_limit_gb: 4.0,
                storage_total_gb: 10.0,
                service_limit: 10,
            },
            network_policy: TenantNetworkPolicy::default(),
        },
--- a/examples/openbao/src/main.rs
+++ b/examples/openbao/src/main.rs
@@ -95,6 +95,7 @@ async fn main() -> Result<()> {
        openshift: cfg.openshift,
        tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()),
        node_port: None,
        create_namespace: true,
    };
    // JWT auth composes in only when both issuer and audience are set; it
@@ -130,6 +131,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
        users: vec![],
        jwt_auth,
        oidc_application: None,
        endpoint: None,
    };
    let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![Box::new(deploy), Box::new(setup)];
--- a/fleet/README.md
+++ b/fleet/README.md
@@ -142,6 +142,38 @@ Run this from the private deploy repository whose binary calls
 See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap.
 ### Releases
 `scripts/release.sh` publishes immutable releases. Control-plane and agent
 versions are independent; the operator and NATS callout share one control-plane
 version.
 ```bash
 # Publish either release independently.
 fleet/scripts/release.sh --control-version 0.4.0 --publish-only
 fleet/scripts/release.sh --agent-version 0.2.0 --publish-only
 # Publish both, deploy through a private deploy crate, and upgrade one Device.
 fleet/scripts/release.sh \
  --control-version 0.4.0 \
  --agent-version 0.2.0 \
  --deploy-manifest /path/to/Cargo.toml \
  --deploy-bin private-deploy \
  --context production \
  --namespace fleet \
  --device device-1
 ```
 Control-plane images publish to `hub.nationtech.io/harmony`. The agent is a
 raw, single-layer OCI artifact and Devices receive a manifest-digest reference.
 Registry pulls are anonymous, while publication uses `REGISTRY_USER` and
 `REGISTRY_TOKEN`. Run the script with `--help` for its tool and deployment
 inputs.
 Before the first OCI-based upgrade, rerun `FleetDeviceSetupScore` for each
 device to install an updater that understands `oci://` references. The updater
 does not update itself. Direct `https://` agent artifacts remain supported.
 ### Connecting to the operator
 The operator runs as a single-replica Deployment in the context namespace.
--- a/fleet/agent-reconciliation-and-upgrade-plan.md
+++ b/fleet/agent-reconciliation-and-upgrade-plan.md
@@ -0,0 +1,168 @@
 # Fleet agent reconciliation and upgrade plan
 ## Decisions
 - One agent instance owns workload reconciliation for a device at a time.
 - NATS desired state is authoritative. Podman labels are the durable observed
  state. The agent rebuilds its in-memory state from both after every restart.
 - Desired entries carry their JetStream revision. Older watch or snapshot data
  never replaces a newer revision.
 - KV watches accelerate convergence. A periodic full snapshot repairs missed
  events and drives orphan cleanup.
 - One worker serializes runtime mutations. New events replace older intent
  before the next reconciliation pass.
 - A deployment upgrade validates and pulls first, then sends the old container
  its normal stop signal and waits without a deadline. The replacement does not
  start until the old container exits. A stuck shutdown requires a separate,
  explicit operator force-kill action; reconciliation never escalates to kill
  on its own. The current agent still has a fixed 30-second stop timeout; this
  decision describes the required replacement behavior.
 - An existing container owned by another deployment or by a human is a
  conflict. The agent reports it and does not delete the container.
 - Agent upgrade never runs two workload reconcilers. The candidate runs only a
  non-workload-mutating probe. A root-owned updater then switches the permanent
  systemd service and rolls back if the new agent does not become ready.
 Deployment rollout freezes the authorized devices matched for one revision. If
 any frozen device has `canary=true`, only those devices receive the revision
 first. Every canary must report successful deployment convergence for that
 revision before the operator releases the remaining devices. A failed or missing
 canary blocks the rollout. Without a canary, the operator releases the full
 frozen set immediately.
 Agents that predate rollout acknowledgements still accept and reconcile the
 desired score, but cannot prove which revision reached `Running`. Upgrade agents
 before enabling this operator rollout behavior; otherwise a legacy canary
 remains pending and safely blocks the remainder.
 ## Deployment reconciliation
 ### Inputs and reconstruction
 The reconciler owns one map from `DeploymentName` to the latest desired value.
 A value is a valid unresolved score or an invalid payload with its error. At
 startup and every periodic resync it replaces that map from the current
 `desired-state.<device>.>` KV snapshot. Live puts and deletes update the same
 map and wake the worker. A failed or incomplete snapshot cannot authorize
 deletion. Because KV key enumeration and value reads are not atomic, a managed
 deployment must be absent from two consecutive complete snapshots before it is
 removed as an orphan. Any newer watch revision resets that absence count.
 An explicit revisioned delete schedules removal immediately; the two-snapshot
 rule applies only when reconstructing an absent key.
 No local deployment database is needed. After a reboot or power loss, current
 KV values reconstruct intent and Harmony-managed Podman labels reconstruct
 observed state.
 ### Reconciliation pass
 For one complete desired snapshot:
 1. Validate every deployment. A duplicate service name or explicit host port
   fails the conflicting deployments but does not block unrelated deployments.
 2. Resolve secrets for the current score revision.
 3. Ask the runtime to converge each valid deployment.
 4. List Harmony-managed runtime deployments and remove deployments absent from
   two complete snapshots. Cleanup runs before retrying a desired deployment
   that needs the same service name.
 5. Publish coarse `Phase` plus bounded `lastError`. Only an acknowledged phase
   and error pair is deduplicated; failed status writes remain dirty.
 The runtime convergence operation preflights the full score before stopping
 anything. It validates ports and service names, pulls missing images, and checks
 container ownership. If the score has an init container, the runtime runs it
 with no restart policy and requires exit code zero before reconciling services.
 Its exited container records completion for the resolved score. A score change
 reruns initialization, so init containers must be idempotent. Preflight prevents
 known destructive failures; it cannot make Podman operations transactional.
 For each changed service the runtime performs stop, remove, create, start, and
 inspect in that order. Successfully changed services remain changed if a later
 service fails, and the next pass resumes from observed labels. Unchanged running
 services are untouched. Missing or stopped desired services are recreated or
 restarted. Services no longer in the score are removed with the same
 graceful-stop behavior.
 A malformed current payload remains present for orphan accounting, reports
 `Failed`, and causes no workload mutation. The last valid workload is preserved
 but no longer reported healthy until the payload is corrected or deleted.
 If a newer revision arrives during a runtime call, that call may finish but its
 result is not published. The worker immediately reconciles the latest revision;
 it never interrupts one Podman operation halfway.
 ### Edge cases
 The first implementation and its tests cover:
 - new deployment;
 - unchanged deployment as a no-op;
 - upgrade with stop-before-replace ordering;
 - preflight failure without stopping the old version;
 - exited desired container reported as failed with its exit details; Podman
  restart count is included for diagnosis but no time-window crashloop
  classifier is introduced;
 - online deletion;
 - offline deletion followed by orphan cleanup when the device returns;
 - agent restart and host reboot;
 - power loss before stop, after stop, after remove, after create, and after
  start, each repaired by the next full pass;
 - service-name conflict between deployments;
 - conflict with an unmanaged or differently owned container;
 - desired container manually stopped or deleted;
 - malformed desired payload replacing, rather than preserving, old intent;
 - partial failure in a multi-service deployment;
 - Podman unavailability and retry;
 - failed state publication and retry;
 - watch interruption or missed events repaired by full resync;
 - failed and incomplete full snapshots;
 - puts and deletes before, during, and after snapshot enumeration;
 - invalid desired keys and failed state deletion;
 - a newer desired revision arriving during reconciliation.
 Secret values remain cached for one desired revision. Secret-only rotation is
 out of scope until the secret source exposes a revision or watch contract.
 ## Agent upgrade
 The signed authorization and probation design was replaced by stop-switch-start.
 The maintained protocol is
 [`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md);
 the original implementation plan remains as a link-preserving pointer in
 [`agent-upgrade-stop-switch-start-plan.md`](agent-upgrade-stop-switch-start-plan.md).
 Deferred follow-up:
 - retry and classify activation readiness failures before rollback or quarantine;
 - add an isolated VM agent-upgrade E2E covering success and failure recovery;
 - rerun `FleetDeviceSetupScore` to refresh the non-self-updating bootstrap updater,
  then verify an anonymous pull from a digest-pinned OCI artifact and a
  target/binary version mismatch. The mismatch must fail during candidate
  self-test without stopping the active agent; updater child processes must not
  emit stray systemd notifications or expected broken-pipe warnings;
 - replace the operator's two-second full-device scan before fleet scale;
 - add binary garbage collection when disk pressure makes retention relevant.
 ## Device commands
 The dashboard sends bounded one-shot exec requests over Core NATS request/reply.
 Exec is an administrative capability, not a sandbox: it runs as the
 `fleet-agent` Unix account and can access that account's credentials and rootless
 Podman runtime. Only a verified dashboard session with the exact `fleet-admin`
 project role may use it.
 Manual QA must cover stdout, stderr and non-zero exit status, timeout cleanup,
 output truncation, an offline device, and denial for a dashboard user without the
 `fleet-admin` role.
 ## Delivery checkpoints
 1. Rework deployment reconciliation around a complete snapshot and a fakeable
   runtime boundary. Add the full unit matrix and retain the existing VM E2E.
 2. Update Podman convergence for ownership checks, preflight, graceful
   stop-before-replace, inspection, and orphan inventory.
 3. Amend ADR-022 and add attempt-scoped upgrade contracts.
 4. Add the updater, probe mode, exclusive agent lock, durable transaction, and
   operator coordination.
 5. Run unit tests, compile environment-gated E2E, run focused local VM tests if
   available without production credentials, and complete independent reviews.
 6. Production QA is manual and stepwise: deployment recovery, bounded device
   commands, bootstrap-updater diagnostics, then one agent upgrade attempt with
   rollback rehearsed before any canary rollout.
--- a/fleet/agent-upgrade-stop-switch-start-plan.md
+++ b/fleet/agent-upgrade-stop-switch-start-plan.md
@@ -0,0 +1,18 @@
 # Fleet agent upgrade: stop, switch, start
 ## Status
 Implemented. This plan replaced the signed two-step authorization protocol for
 the IoT Podman fleet agent.
 The maintained documents are:
 - [`docs/adr/022-fleet-agent-upgrade.md`](../docs/adr/022-fleet-agent-upgrade.md)
  for the decision and invariants;
 - [`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md)
  for the current protocol, recovery behavior, limits, and operational caveats;
 - `harmony-reconciler-contracts/src/upgrade.rs` for wire fields and phases;
 - `fleet/harmony-fleet-agent/src/updater.rs` for the local transaction.
 This file is retained so links from the implementation work remain valid. It is
 not an independent protocol specification.
--- a/fleet/harmony-fleet-agent/Cargo.toml
+++ b/fleet/harmony-fleet-agent/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "harmony-fleet-agent"
-version = "0.1.0"
+version = "0.1.2"
 edition = "2024"
 rust-version = "1.85"
@@ -15,7 +15,7 @@ futures-util = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha2.workspace = true
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["process"] }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 anyhow = { workspace = true }
@@ -23,3 +23,8 @@ clap = { workspace = true }
 toml = { workspace = true }
 thiserror = { workspace = true }
 podman-api = "0.9"
 sd-notify = "0.4"
 reqwest.workspace = true
 oci-client.workspace = true
 uuid.workspace = true
 fs2.workspace = true
--- a/fleet/harmony-fleet-agent/Dockerfile
+++ b/fleet/harmony-fleet-agent/Dockerfile
@@ -41,6 +41,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=builder /app/target/release/harmony-fleet-agent /usr/local/bin/harmony-fleet-agent
 RUN install -d -o 65532 -g 65532 -m 0700 /run/harmony-fleet-agent
 # Non-root runtime. 65532 is the `nonroot` UID convention from
 # distroless. Pairs with `securityContext.runAsNonRoot: true` in
 # whatever Pod spec the harness or production helm chart applies.
--- a/fleet/harmony-fleet-agent/src/command_server.rs
+++ b/fleet/harmony-fleet-agent/src/command_server.rs
@@ -1,10 +1,7 @@
 //! Agent-side request/reply command server.
 //!
-//! Subscribes to `device-commands.<device_id>.>` and dispatches one
+//! Subscribes to `device-commands.<device_id>.>` and returns one reply
-//! handler per verb. Single-shot replies for v1; streaming verbs
+//! for each ping or bounded exec request.
 //! (logs, exec follow-up) will reuse this loop and write multiple
 //! frames to the inbox, terminating with the `X-Harmony-Final`
 //! header.
 //!
 //! Runs alongside the KV reconciler in the agent's top-level
 //! `tokio::select!`. Independent of the podman runtime: when
@@ -12,23 +9,42 @@
 //! the command server still runs (ping is useful for "is this device
 //! online" health-checks regardless).
 use std::io;
 use std::os::unix::process::CommandExt;
 use std::process::Stdio;
 use std::sync::Arc;
-use std::time::Instant;
+use std::time::{Duration, Instant};
 use async_nats::Client;
 use async_nats::Subject;
 use futures_util::StreamExt;
 use harmony_reconciler_contracts::{
-    HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subscription,
+    CommandRequest, ExecReply, HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subject,
    device_command_subscription,
 };
 use serde::Serialize;
 use thiserror::Error;
 use tokio::io::{AsyncRead, AsyncReadExt};
 use tokio::sync::Semaphore;
 const EXEC_MAX_COMMAND: usize = 16 * 1024;
 const EXEC_MAX_OUTPUT: usize = 128 * 1024;
 const EXEC_DEADLINE: Duration = Duration::from_secs(25);
 const EXEC_CONCURRENCY: usize = 4;
 const COMMAND_CONCURRENCY: usize = 32;
 const SIGKILL: i32 = 9;
 const ESRCH: i32 = 3;
 unsafe extern "C" {
    fn kill(pid: i32, signal: i32) -> i32;
 }
 pub struct CommandServer {
    device_id: Id,
    client: Client,
    agent_version: &'static str,
    started_at: Instant,
    exec_permits: Semaphore,
 }
 impl CommandServer {
@@ -36,46 +52,39 @@ impl CommandServer {
        Self {
            device_id,
            client,
-            agent_version: env!("CARGO_PKG_VERSION"),
+            agent_version: crate::VERSION,
            started_at: Instant::now(),
            exec_permits: Semaphore::new(EXEC_CONCURRENCY),
        }
    }
    pub async fn run(self: Arc<Self>) -> Result<(), CommandServerError> {
        let subject = device_command_subscription(&self.device_id.to_string());
        tracing::info!(subject = %subject, "command server subscribing");
-        let mut sub = self.client.subscribe(subject.clone()).await.map_err(|e| {
+        let sub = self.client.subscribe(subject.clone()).await.map_err(|e| {
            CommandServerError::Subscribe {
                subject: subject.clone(),
                source: e,
            }
        })?;
-        while let Some(msg) = sub.next().await {
+        sub.for_each_concurrent(COMMAND_CONCURRENCY, |msg| {
            let me = self.clone();
-            tokio::spawn(async move {
+            async move {
                match me.dispatch(msg).await {
                    Ok(()) => tracing::debug!("command handled"),
                    Err(e) => {
                        tracing::error!(command_error = %e, "failed to handle command")
                    }
                };
-            });
+            }
-        }
+        })
        .await;
        tracing::warn!("command server subscription ended");
        Ok(())
    }
    async fn dispatch(&self, msg: async_nats::Message) -> Result<(), CommandError> {
-        // Subject token after the device id is the verb. Pattern is
+        let verb_token = msg.subject.rsplit('.').next().unwrap_or_default();
        // `device-commands.<id>.<verb>` — we own both ends so this
        // unwrap shape is safe under normal routing.
        // FIXME do not unwrap here, we cannot affoard to crash an entire fleet because a verb is
        // added or removed or format changed. Log an error and move on maybe we could list supported verbs.
        let verb_token = if let Some(verb) = msg.subject.rsplit('.').next() {
            verb
        } else {
            return Err(CommandError::InvalidFormat(msg.subject.to_string()));
        };
        let request_id = msg
            .headers
            .as_ref()
@@ -96,9 +105,19 @@ impl CommandServer {
            }
        };
-        if verb_token == Verb::Ping.as_subject_token() {
+        let device_id = self.device_id.to_string();
        if msg.subject.as_str() == device_command_subject(&device_id, Verb::Ping) {
            self.reply_ping(reply_to).await?;
            Ok(())
        } else if msg.subject.as_str() == device_command_subject(&device_id, Verb::Exec) {
            let reply = match serde_json::from_slice(&msg.payload) {
                Ok(CommandRequest::Exec { command }) => {
                    run_shell(&command, EXEC_DEADLINE, &self.exec_permits).await
                }
                Ok(_) => exec_error("expected exec request body"),
                Err(error) => exec_error(format!("invalid exec request: {error}")),
            };
            self.publish_reply(reply_to, &reply).await
        } else {
            tracing::warn!(verb = %verb_token, "unknown command verb");
            Err(CommandError::UnknownVerb(verb_token.to_string()))
@@ -111,7 +130,15 @@ impl CommandServer {
            agent_version: self.agent_version.to_string(),
            uptime_s: self.started_at.elapsed().as_secs(),
        };
-        let payload = serde_json::to_vec(&reply).map_err(CommandError::SerializeReply)?;
+        self.publish_reply(reply_to, &reply).await
    }
    async fn publish_reply(
        &self,
        reply_to: Subject,
        reply: &impl Serialize,
    ) -> Result<(), CommandError> {
        let payload = serde_json::to_vec(reply).map_err(CommandError::SerializeReply)?;
        self.client
            .publish(reply_to, payload.into())
            .await
@@ -119,13 +146,164 @@ impl CommandServer {
    }
 }
 async fn run_shell(command: &str, deadline: Duration, permits: &Semaphore) -> ExecReply {
    if command.trim().is_empty() {
        return exec_error("command must not be empty");
    }
    if command.len() > EXEC_MAX_COMMAND {
        return exec_error(format!("command exceeds {EXEC_MAX_COMMAND} byte limit"));
    }
    let _permit = match permits.try_acquire() {
        Ok(permit) => permit,
        Err(_) => return exec_error("too many commands are already running"),
    };
    let mut command_process = std::process::Command::new("sh");
    command_process
        .arg("-c")
        // The outer shell terminates same-group background jobs after the command.
        .arg("sh -c \"$1\"; status=$?; trap '' TERM; kill -TERM 0; exit $status")
        .arg("harmony-exec")
        .arg(command)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        // A dedicated process group lets timeout cleanup reach shell descendants.
        .process_group(0);
    let mut child = match tokio::process::Command::from(command_process)
        .kill_on_drop(true)
        .spawn()
    {
        Ok(child) => child,
        Err(error) => return exec_error(format!("failed to spawn shell: {error}")),
    };
    let Some(process_group) = child.id().and_then(|id| i32::try_from(id).ok()) else {
        return exec_error("spawned shell has no process id");
    };
    let mut stdout = child.stdout.take().expect("piped stdout");
    let mut stderr = child.stderr.take().expect("piped stderr");
    let completed = tokio::time::timeout(
        deadline,
        collect_output(&mut child, &mut stdout, &mut stderr),
    )
    .await;
    let (status, stdout, stderr, truncated) = match completed {
        Ok(Ok(result)) => result,
        outcome => {
            // SAFETY: process_group is the positive PID returned for the child whose
            // process group was set to that PID before exec.
            if unsafe { kill(-process_group, SIGKILL) } == -1 {
                tracing::warn!(error = %io::Error::last_os_error(), "failed to kill exec process group");
                let _ = child.start_kill();
            }
            let _ = tokio::time::timeout(Duration::from_secs(1), child.wait()).await;
            return match outcome {
                Ok(Err(error)) => exec_error(format!("failed waiting for command output: {error}")),
                Err(_) => exec_error(format!(
                    "command exceeded {}s deadline and was killed",
                    deadline.as_secs_f64()
                )),
                Ok(Ok(_)) => unreachable!(),
            };
        }
    };
    // The wrapper sends SIGTERM before exiting. Kill any same-group process
    // that ignored it; ESRCH means the group is already gone.
    let cleanup = unsafe { kill(-process_group, SIGKILL) };
    if cleanup == -1 {
        let error = io::Error::last_os_error();
        if error.raw_os_error() != Some(ESRCH) {
            tracing::warn!(%error, "failed to clean up exec process group");
        }
    }
    let (stdout, stderr, utf8_truncated) = bounded_strings(stdout, stderr);
    ExecReply {
        exit_code: status.code().unwrap_or(-1),
        stdout,
        stderr,
        truncated: truncated || utf8_truncated,
    }
 }
 async fn collect_output(
    child: &mut tokio::process::Child,
    stdout: &mut (impl AsyncRead + Unpin),
    stderr: &mut (impl AsyncRead + Unpin),
 ) -> io::Result<(std::process::ExitStatus, Vec<u8>, Vec<u8>, bool)> {
    let (mut stdout_open, mut stderr_open) = (true, true);
    let (mut stdout_output, mut stderr_output) = (Vec::new(), Vec::new());
    let (mut stdout_chunk, mut stderr_chunk) = ([0; 8192], [0; 8192]);
    let (mut remaining, mut truncated) = (EXEC_MAX_OUTPUT, false);
    let mut status = None;
    while status.is_none() || stdout_open || stderr_open {
        tokio::select! {
            result = child.wait(), if status.is_none() => status = Some(result?),
            result = stdout.read(&mut stdout_chunk), if stdout_open => {
                let read = result?;
                stdout_open = read != 0;
                retain_output(&mut stdout_output, &stdout_chunk[..read], &mut remaining, &mut truncated);
            }
            result = stderr.read(&mut stderr_chunk), if stderr_open => {
                let read = result?;
                stderr_open = read != 0;
                retain_output(&mut stderr_output, &stderr_chunk[..read], &mut remaining, &mut truncated);
            }
        }
    }
    Ok((
        status.expect("loop waits for status"),
        stdout_output,
        stderr_output,
        truncated,
    ))
 }
 fn retain_output(output: &mut Vec<u8>, chunk: &[u8], remaining: &mut usize, truncated: &mut bool) {
    let retained = chunk.len().min(*remaining);
    output.extend_from_slice(&chunk[..retained]);
    *remaining -= retained;
    *truncated |= retained < chunk.len();
 }
 fn bounded_strings(stdout: Vec<u8>, stderr: Vec<u8>) -> (String, String, bool) {
    let mut left = EXEC_MAX_OUTPUT;
    let mut truncated = false;
    let mut convert = |bytes: Vec<u8>| {
        let text = String::from_utf8_lossy(&bytes);
        let end = text
            .char_indices()
            .map(|(index, _)| index)
            .take_while(|index| *index <= left)
            .last()
            .unwrap_or(0);
        let end = if text.len() <= left { text.len() } else { end };
        truncated |= end < text.len();
        left -= end;
        text[..end].to_string()
    };
    let stdout = convert(stdout);
    let stderr = convert(stderr);
    (stdout, stderr, truncated)
 }
 fn exec_error(message: impl Into<String>) -> ExecReply {
    ExecReply {
        exit_code: -1,
        stdout: String::new(),
        stderr: message.into(),
        truncated: false,
    }
 }
 /// Failure modes the per-message dispatcher can report. Stays
 /// `pub(crate)` for now — the run loop logs and continues on each
 /// variant rather than surfacing them to a caller.
 #[derive(Debug, Error, Serialize)]
 pub(crate) enum CommandError {
    #[error("invalid command subject: {0}")]
    InvalidFormat(String),
    #[error("unknown verb: {0}")]
    UnknownVerb(String),
    #[error("command message had no reply inbox")]
@@ -151,3 +329,103 @@ pub enum CommandServerError {
        source: async_nats::SubscribeError,
    },
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[tokio::test]
    async fn exec_captures_stdout_and_nonzero_stderr() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        let reply = run_shell(
            "printf output; printf error >&2; exit 7",
            EXEC_DEADLINE,
            &permits,
        )
        .await;
        assert_eq!(reply.exit_code, 7);
        assert_eq!(reply.stdout, "output");
        assert_eq!(reply.stderr, "error");
        assert!(!reply.truncated);
    }
    #[tokio::test]
    async fn exec_rejects_empty_and_oversized_commands() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        for command in ["", " \t\n"] {
            let reply = run_shell(command, EXEC_DEADLINE, &permits).await;
            assert_eq!(reply.exit_code, -1);
            assert!(reply.stderr.contains("empty"));
        }
        let oversized = "x".repeat(EXEC_MAX_COMMAND + 1);
        let reply = run_shell(&oversized, EXEC_DEADLINE, &permits).await;
        assert_eq!(reply.exit_code, -1);
        assert!(reply.stderr.contains("limit"));
    }
    #[tokio::test]
    async fn exec_caps_combined_output() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        let reply = run_shell(
            "yes o | head -c 100000; yes e | head -c 100000 >&2",
            EXEC_DEADLINE,
            &permits,
        )
        .await;
        assert_eq!(reply.exit_code, 0);
        assert!(reply.truncated);
        assert!(reply.stdout.len() + reply.stderr.len() <= EXEC_MAX_OUTPUT);
    }
    #[tokio::test]
    async fn exec_timeout_kills_process_group() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        let marker = std::env::temp_dir().join(format!(
            "harmony-exec-timeout-{}-{}",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        let command = format!("(sleep 0.3; touch '{}') & wait", marker.to_string_lossy());
        let reply = run_shell(&command, Duration::from_millis(100), &permits).await;
        assert_eq!(reply.exit_code, -1);
        assert!(reply.stderr.contains("deadline"));
        tokio::time::sleep(Duration::from_millis(300)).await;
        assert!(!marker.exists(), "background process survived timeout");
    }
    #[tokio::test]
    async fn exec_success_terminates_detached_same_group_processes() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        let marker = std::env::temp_dir().join(format!(
            "harmony-exec-success-{}-{}",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        let command = format!(
            "(trap '' TERM; sleep 0.2; touch '{}') >/dev/null 2>&1 &",
            marker.to_string_lossy()
        );
        let reply = run_shell(&command, EXEC_DEADLINE, &permits).await;
        assert_eq!(reply.exit_code, 0);
        tokio::time::sleep(Duration::from_millis(250)).await;
        assert!(
            !marker.exists(),
            "background process survived successful exec"
        );
    }
    #[tokio::test]
    async fn exec_rejects_when_concurrency_is_exhausted() {
        let permits = Semaphore::new(EXEC_CONCURRENCY);
        let _held = permits.acquire_many(EXEC_CONCURRENCY as u32).await.unwrap();
        let reply = run_shell("printf started", Duration::from_secs(2), &permits).await;
        assert_eq!(reply.exit_code, -1);
        assert!(reply.stderr.contains("already running"));
    }
 }
--- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs
+++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs
@@ -3,15 +3,11 @@
 //! Thin wrapper around three KV buckets: [`BUCKET_DEVICE_INFO`],
 //! [`BUCKET_DEVICE_STATE`], [`BUCKET_DEVICE_HEARTBEAT`].
 //!
 //! Failure mode: log and swallow. The KV is the source of truth —
 //! a dropped put gets corrected on the next reconcile transition
 //! or operator watch reconnection.
 use async_nats::jetstream::{self, kv};
 use harmony_reconciler_contracts::{
    BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
-    DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, device_heartbeat_key,
+    DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, UpdaterCapabilities,
-    device_info_key, device_state_key,
+    device_heartbeat_key, device_info_key, device_state_key,
 };
 use std::collections::BTreeMap;
@@ -27,37 +23,20 @@ pub struct FleetPublisher {
    heartbeat_bucket: kv::Store,
 }
 #[async_trait::async_trait]
 pub trait DeploymentStatePublisher: Send + Sync {
    async fn write(&self, state: &DeploymentState) -> anyhow::Result<()>;
    async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()>;
 }
 impl FleetPublisher {
-    /// Open every bucket the agent needs, creating those that don't
+    /// Open the operator-owned buckets used by the agent.
    /// exist yet. Idempotent with operator-side creation.
    pub async fn connect(client: async_nats::Client, device_id: Id) -> anyhow::Result<Self> {
        let jetstream = jetstream::new(client.clone());
-        let info_bucket = jetstream
+        let info_bucket = jetstream.get_key_value(BUCKET_DEVICE_INFO).await?;
-            .create_key_value(kv::Config {
+        let state_bucket = jetstream.get_key_value(BUCKET_DEVICE_STATE).await?;
-                bucket: BUCKET_DEVICE_INFO.to_string(),
+        let heartbeat_bucket = jetstream.get_key_value(BUCKET_DEVICE_HEARTBEAT).await?;
                // If this is as I think, it would be useful to keep a history of the last 10 device
                // info, with a timestamp
                history: 1,
                ..Default::default()
            })
            .await?;
        let state_bucket = jetstream
            .create_key_value(kv::Config {
                bucket: BUCKET_DEVICE_STATE.to_string(),
                // If this is as I think, it would be useful to keep a history of the last 10 states
                // a device had, with a timestamp
                history: 1,
                ..Default::default()
            })
            .await?;
        let heartbeat_bucket = jetstream
            .create_key_value(kv::Config {
                bucket: BUCKET_DEVICE_HEARTBEAT.to_string(),
                history: 1,
                ..Default::default()
            })
            .await?;
        Ok(Self {
            device_id,
@@ -74,22 +53,19 @@ impl FleetPublisher {
        &self,
        labels: BTreeMap<String, String>,
        inventory: Option<InventorySnapshot>,
-    ) {
+        updater: Option<UpdaterCapabilities>,
    ) -> anyhow::Result<()> {
        let info = DeviceInfo {
            device_id: self.device_id.clone(),
            labels,
            inventory,
            updater,
            updated_at: chrono::Utc::now(),
        };
        let key = device_info_key(&self.device_id.to_string());
-        match serde_json::to_vec(&info) {
+        let payload = serde_json::to_vec(&info)?;
-            Ok(payload) => {
+        self.info_bucket.put(&key, payload.into()).await?;
-                if let Err(e) = self.info_bucket.put(&key, payload.into()).await {
+        Ok(())
                    tracing::warn!(%key, error = %e, "publish_device_info: kv put failed");
                }
            }
            Err(e) => tracing::warn!(error = %e, "publish_device_info: serialize failed"),
        }
    }
    /// Tiny liveness ping. Called every 30s.
@@ -97,6 +73,7 @@ impl FleetPublisher {
        let hb = HeartbeatPayload {
            device_id: self.device_id.clone(),
            at: chrono::Utc::now(),
            agent_version: Some(crate::VERSION.to_string()),
        };
        let key = device_heartbeat_key(&self.device_id.to_string());
        match serde_json::to_vec(&hb) {
@@ -114,17 +91,12 @@ impl FleetPublisher {
    /// bucket picks up this put and updates CR status counters.
    /// Also fans out the same payload on `device-state.<device_id>`
    /// for live observers that don't want to consume the KV stream.
-    pub async fn write_deployment_state(&self, state: &DeploymentState) {
+    pub async fn write_deployment_state(&self, state: &DeploymentState) -> anyhow::Result<()> {
        let key = device_state_key(&self.device_id.to_string(), &state.deployment);
-        match serde_json::to_vec(state) {
+        let payload = serde_json::to_vec(state)?;
-            Ok(payload) => {
+        self.state_bucket.put(&key, payload.clone().into()).await?;
-                if let Err(e) = self.state_bucket.put(&key, payload.clone().into()).await {
+        self.publish_direct_state(payload).await;
-                    tracing::warn!(%key, error = %e, "write_deployment_state: kv put failed");
+        Ok(())
                }
                self.publish_direct_state(payload).await;
            }
            Err(e) => tracing::warn!(error = %e, "write_deployment_state: serialize failed"),
        }
    }
    /// Emit a tiny presence pulse on `device-state.<device_id>` so live
@@ -154,10 +126,20 @@ impl FleetPublisher {
    /// Delete the authoritative current-phase entry, e.g. when the
    /// Deployment CR is removed and the agent has torn down the
    /// container.
-    pub async fn delete_deployment_state(&self, deployment: &DeploymentName) {
+    pub async fn delete_deployment_state(&self, deployment: &DeploymentName) -> anyhow::Result<()> {
        let key = device_state_key(&self.device_id.to_string(), deployment);
-        if let Err(e) = self.state_bucket.delete(&key).await {
+        self.state_bucket.delete(&key).await?;
-            tracing::debug!(%key, error = %e, "delete_deployment_state: kv delete failed");
+        Ok(())
-        }
+    }
 }
 #[async_trait::async_trait]
 impl DeploymentStatePublisher for FleetPublisher {
    async fn write(&self, state: &DeploymentState) -> anyhow::Result<()> {
        self.write_deployment_state(state).await
    }
    async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()> {
        self.delete_deployment_state(deployment).await
    }
 }
--- a/fleet/harmony-fleet-agent/src/main.rs
+++ b/fleet/harmony-fleet-agent/src/main.rs
@@ -3,11 +3,15 @@ mod config;
 mod fleet_publisher;
 mod podman;
 mod reconciler;
 mod system_upgrade;
 mod system_upgrade_service;
 mod updater;
 mod upgrade;
 use std::sync::Arc;
 use std::time::Duration;
-use anyhow::{Context, Error, Result};
+use anyhow::{Context, Result};
 use clap::Parser;
 use config::AgentConfig;
 use harmony_fleet_auth::{
@@ -18,20 +22,33 @@ use harmony_fleet_auth::{
 type Creds = Arc<CredentialSource>;
 use futures_util::StreamExt;
 use harmony_reconciler_contracts::{
-    BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter,
+    BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE,
    BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, BUCKET_SYSTEM_UPGRADE_INTENT,
    BUCKET_SYSTEM_UPGRADE_STATUS, Id, InventorySnapshot, agent_upgrade_intent_key,
    agent_upgrade_status_key, desired_state_watch_filter, device_heartbeat_key, device_info_key,
 };
 use crate::command_server::CommandServer;
 use crate::fleet_publisher::FleetPublisher;
-use crate::podman::PodmanRuntime;
+use crate::podman::{PodmanRuntime, SecretSource};
-use crate::reconciler::Reconciler;
+use crate::reconciler::{Reconciler, SnapshotEntry};
 /// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
 /// events are accelerators.
 const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
 const NATS_CONNECT_WINDOW: Duration = Duration::from_secs(3 * 60);
 const NATS_CONNECT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15);
 pub(crate) const VERSION: &str = match option_env!("HARMONY_FLEET_AGENT_VERSION") {
    Some(version) => version,
    None => env!("CARGO_PKG_VERSION"),
 };
 #[derive(Parser)]
-#[command(name = "fleet-agent-v0", about = "IoT agent for Raspberry Pi devices")]
+#[command(
    name = "fleet-agent-v0",
    version = VERSION,
    about = "IoT agent for Raspberry Pi devices"
 )]
 struct Cli {
    #[arg(
        long,
@@ -41,85 +58,243 @@ struct Cli {
        default_value = "/etc/fleet-agent/config.toml"
    )]
    config: std::path::PathBuf,
    #[arg(long)]
    self_test: bool,
    #[arg(long, requires = "self_test", hide = true)]
    expected_version: Option<String>,
    #[arg(long)]
    updater: bool,
    #[arg(long, default_value = updater::DEFAULT_SOCKET)]
    updater_socket: std::path::PathBuf,
 }
 fn acquire_process_lock() -> Result<std::fs::File> {
    use fs2::FileExt;
    let file = std::fs::OpenOptions::new()
        .create(true)
        .truncate(false)
        .read(true)
        .write(true)
        .open("/run/harmony-fleet-agent/agent.lock")?;
    file.try_lock_exclusive()
        .context("another fleet agent instance is already active")?;
    Ok(file)
 }
 async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result<async_nats::Client> {
    let urls = &cfg.nats.urls;
    tracing::info!(device_id = %cfg.agent.device_id, "connecting to NATS {urls:?}");
-    // The auth callback is invoked on every (re)connect, so a fresh
+    let started = tokio::time::Instant::now();
-    // Zitadel access token is minted automatically when the cached one
+    let mut last_error = None;
-    // is near-expiry — that's how we hold the "never lose connectivity"
+    let mut attempt = 0;
-    // guarantee even across token rollovers and NATS pod restarts.
+    loop {
-    let client = connect_options_with_credentials(creds)
+        let remaining = NATS_CONNECT_WINDOW.saturating_sub(started.elapsed());
-        .ping_interval(Duration::from_secs(10))
+        if remaining.is_zero() {
-        // Surface async-nats's connection lifecycle in our logs. This
+            break;
-        // is load-bearing for ops: a device that quietly disconnects
+        }
-        // is exactly the failure mode we promise won't happen, and
+        attempt += 1;
-        // operators need to see the reconnect attempts to debug.
+        // The callback mints a fresh token on every connect and reconnect.
-        .event_callback(|event| async move {
+        let connect = connect_options_with_credentials(creds.clone())
-            use async_nats::Event;
+            .ping_interval(Duration::from_secs(10))
-            match event {
+            .connection_timeout(NATS_CONNECT_ATTEMPT_TIMEOUT)
-                Event::Connected => tracing::info!("NATS connected"),
+            .event_callback(|event| async move {
-                Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"),
+                use async_nats::Event;
-                Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"),
+                match event {
-                Event::SlowConsumer(sid) => {
+                    Event::Connected => tracing::info!("NATS connected"),
-                    tracing::warn!(sid = %sid, "NATS slow consumer")
+                    Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"),
                    Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"),
                    Event::SlowConsumer(sid) => {
                        tracing::warn!(sid = %sid, "NATS slow consumer")
                    }
                    Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"),
                    Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"),
                    Event::Closed => tracing::debug!("NATS connection closed"),
                    other => tracing::debug!(?other, "NATS event"),
                }
-                Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"),
+            })
-                Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"),
+            .connect(cfg.nats.urls.as_slice());
-                Event::Closed => tracing::error!("NATS connection closed"),
+        match tokio::time::timeout(NATS_CONNECT_ATTEMPT_TIMEOUT.min(remaining), connect).await {
-                other => tracing::debug!(?other, "NATS event"),
+            Ok(Ok(client)) => {
                tracing::info!(urls = ?cfg.nats.urls, "connected to NATS");
                return Ok(client);
            }
-        })
+            Ok(Err(error)) => last_error = Some(error.to_string()),
-        .connect(cfg.nats.urls.as_slice())
+            Err(_) => last_error = Some("connection attempt timed out".to_string()),
-        .await?;
+        }
-    tracing::info!(urls = ?cfg.nats.urls, "connected to NATS");
+        tracing::warn!(attempt, error = %last_error.as_deref().unwrap(), "NATS connection failed; retrying");
-    Ok(client)
+        tokio::time::sleep(
            Duration::from_secs(5).min(NATS_CONNECT_WINDOW.saturating_sub(started.elapsed())),
        )
        .await;
    }
    anyhow::bail!(
        "NATS connection failed after bounded retries: {}",
        last_error.as_deref().unwrap_or("connection window expired")
    )
 }
-async fn watch_desired_state(
+async fn desired_state_store(
    client: async_nats::Client,
-    device_id: Id,
+) -> Result<async_nats::jetstream::kv::Store> {
-    reconciler: Arc<Reconciler>,
+    Ok(async_nats::jetstream::new(client)
-) -> Result<()> {
+        .get_key_value(BUCKET_DESIRED_STATE)
-    let jetstream = async_nats::jetstream::new(client);
+        .await?)
-    let bucket = jetstream
+}
-        .create_key_value(async_nats::jetstream::kv::Config {
+
-            bucket: BUCKET_DESIRED_STATE.to_string(),
+async fn probe_services(
    client: &async_nats::Client,
    device_id: &Id,
    updater_socket: &std::path::Path,
 ) -> Result<Vec<SnapshotEntry>> {
    let jetstream = async_nats::jetstream::new(client.clone());
    let desired = jetstream.get_key_value(BUCKET_DESIRED_STATE).await?;
    let id = device_id.to_string();
    for (bucket, key) in [
        (BUCKET_AGENT_UPGRADE_INTENT, agent_upgrade_intent_key(&id)),
        (BUCKET_AGENT_UPGRADE_STATUS, agent_upgrade_status_key(&id)),
        (BUCKET_DEVICE_INFO, device_info_key(&id)),
        (BUCKET_DEVICE_HEARTBEAT, device_heartbeat_key(&id)),
    ] {
        jetstream.get_key_value(bucket).await?.get(key).await?;
    }
    jetstream
        .get_key_value(BUCKET_DEVICE_STATE)
        .await?
        .status()
        .await?;
    for bucket in [BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS] {
        jetstream.get_key_value(bucket).await?.status().await?;
    }
    let updater = updater::UpdaterClient::new(updater_socket);
    tokio::time::timeout(Duration::from_secs(15), updater.status())
        .await
        .context("updater status probe timed out")??;
    let capabilities = tokio::time::timeout(Duration::from_secs(15), updater.capabilities())
        .await
        .context("updater capabilities probe timed out")??;
    if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 {
        anyhow::bail!("updater does not support AptFullUpgradeV1");
    }
    load_desired_snapshot(&desired, device_id).await
 }
 async fn load_desired_snapshot(
    bucket: &async_nats::jetstream::kv::Store,
    device_id: &Id,
 ) -> Result<Vec<SnapshotEntry>> {
    let prefix = format!("{device_id}.");
    let filter = format!(
        "{}{}",
        bucket.prefix,
        desired_state_watch_filter(&device_id.to_string())
    );
    let consumer = bucket
        .stream
        .create_consumer(async_nats::jetstream::consumer::pull::OrderedConfig {
            filter_subject: filter,
            deliver_policy: async_nats::jetstream::consumer::DeliverPolicy::LastPerSubject,
            ..Default::default()
        })
        .await?;
    let pending = consumer.cached_info().num_pending;
    if pending == 0 {
        return Ok(Vec::new());
    }
    let mut entries = Vec::new();
    let mut snapshot = consumer.messages().await?;
    for _ in 0..pending {
        let message = tokio::time::timeout(Duration::from_secs(10), snapshot.next())
            .await
            .context("timed out before desired-state snapshot completed")?
            .context("desired-state snapshot ended before completion")??;
        let info = message
            .info()
            .map_err(|error| anyhow::anyhow!(error.to_string()))?;
        let key = message
            .message
            .subject
            .strip_prefix(&bucket.prefix)
            .context("desired-state subject has the wrong bucket prefix")?
            .to_string();
        let deleted = message.message.headers.as_ref().is_some_and(|headers| {
            headers
                .get("KV-Operation")
                .is_some_and(|operation| matches!(operation.as_str(), "DEL" | "PURGE"))
                || headers.get("Nats-Marker-Reason").is_some()
        });
        if !deleted && key.starts_with(&prefix) {
            entries.push(SnapshotEntry {
                key,
                revision: info.stream_sequence,
                value: message.message.payload.to_vec(),
            });
        }
    }
    Ok(entries)
 }
 async fn watch_desired_state(
    bucket: async_nats::jetstream::kv::Store,
    device_id: Id,
    reconciler: Arc<Reconciler>,
 ) -> Result<()> {
    let key_filter = desired_state_watch_filter(&device_id.to_string());
    tracing::info!(filter = %key_filter, "watching KV keys");
-
+    loop {
-    let mut watch = bucket.watch(&key_filter).await?;
+        let mut watch = match bucket.watch(&key_filter).await {
-    while let Some(result) = watch.next().await {
+            Ok(watch) => watch,
-        let entry = match result {
+            Err(error) => {
-            Ok(e) => e,
+                tracing::warn!(%error, "desired-state watch start failed");
-            Err(e) => {
+                tokio::time::sleep(Duration::from_secs(1)).await;
                tracing::warn!(error = %e, "watch error");
                continue;
            }
        };
-
+        while let Some(result) = watch.next().await {
-        tracing::debug!(key = %entry.key, "bucket watch new value {entry:?}");
+            let entry = match result {
-
+                Ok(entry) => entry,
-        match entry.operation {
+                Err(error) => {
-            async_nats::jetstream::kv::Operation::Put => {
+                    tracing::warn!(%error, "desired-state watch failed; restarting");
-                if let Err(e) = reconciler.apply(&entry.key, &entry.value).await {
+                    break;
                    tracing::warn!(key = %entry.key, error = %e, "apply failed");
                }
-            }
+            };
-            async_nats::jetstream::kv::Operation::Delete
+            let result = match entry.operation {
-            | async_nats::jetstream::kv::Operation::Purge => {
+                async_nats::jetstream::kv::Operation::Put => {
-                if let Err(e) = reconciler.remove(&entry.key).await {
+                    reconciler
-                    tracing::warn!(key = %entry.key, error = %e, "remove failed");
+                        .put(&entry.key, entry.revision, &entry.value)
                        .await
                }
                async_nats::jetstream::kv::Operation::Delete
                | async_nats::jetstream::kv::Operation::Purge => {
                    reconciler.delete(&entry.key, entry.revision).await
                }
            };
            if let Err(error) = result {
                tracing::warn!(key = %entry.key, %error, "desired-state event rejected");
            }
        }
        tokio::time::sleep(Duration::from_secs(1)).await;
    }
 }
 async fn snapshot_loop(
    bucket: async_nats::jetstream::kv::Store,
    device_id: Id,
    reconciler: Arc<Reconciler>,
 ) {
    let mut interval = tokio::time::interval(RECONCILE_INTERVAL);
    interval.tick().await;
    loop {
        interval.tick().await;
        let generation = reconciler.generation().await;
        match load_desired_snapshot(&bucket, &device_id).await {
            Ok(snapshot) => {
                if let Err(error) = reconciler.replace_snapshot(snapshot, generation).await {
                    tracing::warn!(%error, "desired-state snapshot rejected");
                }
            }
            Err(error) => tracing::warn!(%error, "desired-state snapshot failed"),
        }
    }
    Ok(())
 }
 /// Tiny liveness-only loop: push a `HeartbeatPayload` into the
@@ -158,7 +333,7 @@ fn local_inventory() -> InventorySnapshot {
            .map(|n| n.get() as u32)
            .unwrap_or(0),
        memory_mb: sys_memory_total_mb().unwrap_or(0),
-        agent_version: env!("CARGO_PKG_VERSION").to_string(),
+        agent_version: VERSION.to_string(),
    }
 }
@@ -189,6 +364,18 @@ async fn main() -> Result<()> {
    tracing_subscriber::fmt().with_env_filter(filter).init();
    let cli = Cli::parse();
    if cli.updater {
        return updater::run_server(&cli.updater_socket).await;
    }
    if let Some(expected) = cli.expected_version.as_deref()
        && expected != VERSION
    {
        anyhow::bail!(
            "candidate version mismatch: expected {expected}, binary reports {}",
            VERSION
        );
    }
    let _process_lock = (!cli.self_test).then(acquire_process_lock).transpose()?;
    let cfg = config::load_config(&cli.config)?;
    tracing::info!(
        device_id = %cfg.agent.device_id,
@@ -242,7 +429,7 @@ async fn main() -> Result<()> {
            )
            .map_err(|e| anyhow::anyhow!("building OpenBao secret store: {e}"))?;
            tracing::info!(url = %ob.url, prefix = %ob.secret_prefix, "OpenBao secret store ready");
-            Some(reconciler::SecretSource {
+            Some(SecretSource {
                store: Arc::new(store),
                prefix: ob.secret_prefix.clone(),
            })
@@ -253,12 +440,22 @@ async fn main() -> Result<()> {
        (None, _) => None,
    };
-    let client = connect_nats(&cfg, creds).await.map_err(|e| {
+    let client = connect_nats(&cfg, creds).await.map_err(|error| {
-        let msg = format!("Nats connection FAILED : {e}");
+        tracing::error!(%error, "NATS connection failed");
-        tracing::error!(msg);
+        error
        Error::msg(msg)
    })?;
    if cli.self_test {
        let snapshot = probe_services(&client, &device_id, &cli.updater_socket).await?;
        if let Some(topology) = topology {
            let reconciler = Reconciler::new(device_id.clone(), topology, None, secrets);
            let generation = reconciler.generation().await;
            reconciler.replace_snapshot(snapshot, generation).await?;
        }
        tracing::info!(version = VERSION, "self-test ok");
        return Ok(());
    }
    // Publish surface. Opens the three KV buckets (idempotent
    // creates). Must be live before the reconciler starts so
    // writes on the first desired-state KV watch land on the wire.
@@ -278,9 +475,29 @@ async fn main() -> Result<()> {
    startup_labels
        .entry("device-id".to_string())
        .or_insert_with(|| device_id.to_string());
    let updater_capabilities = if cli.updater_socket.exists() {
        match updater::UpdaterClient::new(&cli.updater_socket)
            .capabilities()
            .await
        {
            Ok(capabilities) => Some(capabilities),
            Err(error) if !cfg.agent.runtime_enabled => {
                tracing::warn!(%error, "updater unavailable; capabilities omitted");
                None
            }
            Err(error) => return Err(error.context("reading required updater capabilities")),
        }
    } else {
        None
    };
    fleet
-        .publish_device_info(startup_labels, Some(inventory_snapshot.clone()))
+        .publish_device_info(
-        .await;
+            startup_labels,
            Some(inventory_snapshot.clone()),
            updater_capabilities,
        )
        .await
        .context("publishing device registration")?;
    // Reconciler exists only when a podman topology is available.
    // Without it, the desired-state watch + periodic reconcile arms
@@ -295,6 +512,70 @@ async fn main() -> Result<()> {
        ))
    });
    let updater_socket = cli
        .updater_socket
        .to_str()
        .context("non-UTF-8 updater socket")?;
    let upgrade_service = if cli.updater_socket.exists() {
        match upgrade::UpgradeService::connect(client.clone(), device_id.clone(), updater_socket)
            .await
        {
            Ok(service) => Some(service),
            Err(error) if !cfg.agent.runtime_enabled => {
                tracing::warn!(%error, "updater unavailable; automatic agent upgrades disabled");
                None
            }
            Err(error) => return Err(error.context("connecting required fleet updater")),
        }
    } else if cfg.agent.runtime_enabled {
        anyhow::bail!(
            "required fleet updater socket '{}' is unavailable",
            cli.updater_socket.display()
        );
    } else {
        tracing::warn!(
            socket = %cli.updater_socket.display(),
            "updater unavailable; automatic agent upgrades disabled"
        );
        None
    };
    let system_upgrade_service = if cli.updater_socket.exists() {
        match system_upgrade_service::SystemUpgradeService::connect(
            client.clone(),
            device_id.clone(),
            &cli.updater_socket,
        )
        .await
        {
            Ok(service) => Some(service),
            Err(error) if !cfg.agent.runtime_enabled => {
                tracing::warn!(%error, "updater unavailable; system upgrades disabled");
                None
            }
            Err(error) => return Err(error.context("connecting required system upgrade relay")),
        }
    } else if cfg.agent.runtime_enabled {
        anyhow::bail!(
            "required fleet updater socket '{}' is unavailable",
            cli.updater_socket.display()
        );
    } else {
        None
    };
    let desired_bucket = desired_state_store(client.clone()).await?;
    let snapshot = load_desired_snapshot(&desired_bucket, &device_id).await?;
    if let Some(reconciler) = &reconciler {
        let generation = reconciler.generation().await;
        reconciler.replace_snapshot(snapshot, generation).await?;
    }
    if let Some(upgrade_service) = &upgrade_service {
        upgrade_service.ensure_active_startup().await?;
    }
    sd_notify::notify(false, &[sd_notify::NotifyState::Ready])
        .context("notifying systemd that initialization completed")?;
    let command_server = Arc::new(CommandServer::new(device_id.clone(), client.clone()));
    let ctrlc = async {
@@ -313,34 +594,96 @@ async fn main() -> Result<()> {
    let watch: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
        match reconciler.as_ref() {
-            Some(r) => Box::pin(watch_desired_state(
+            Some(reconciler) => Box::pin(watch_desired_state(
-                client.clone(),
+                desired_bucket.clone(),
                device_id.clone(),
-                r.clone(),
+                reconciler.clone(),
            )),
            None => Box::pin(async {
                std::future::pending::<()>().await;
                Ok(())
            }),
        };
-    let reconcile: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> =
+    let snapshots: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> =
        match reconciler.as_ref() {
-            Some(r) => Box::pin(r.clone().run_periodic(RECONCILE_INTERVAL)),
+            Some(reconciler) => Box::pin(snapshot_loop(
-            None => Box::pin(std::future::pending::<()>()),
+                desired_bucket,
                device_id.clone(),
                reconciler.clone(),
            )),
            _ => Box::pin(std::future::pending()),
        };
    let heartbeat = publish_heartbeat_loop(fleet);
    let commands = command_server.run();
    let mut worker = reconciler
        .as_ref()
        .map(|reconciler| tokio::spawn(reconciler.clone().run()));
    let worker_finished = async {
        match worker.as_mut() {
            Some(worker) => worker.await.context("reconciler worker task"),
            None => std::future::pending().await,
        }
    };
    let upgrades: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
        match upgrade_service.as_ref() {
            Some(service) => Box::pin(service.clone().run()),
            None => Box::pin(async {
                std::future::pending::<()>().await;
                Ok(())
            }),
        };
    let system_upgrades: std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send>> =
        match system_upgrade_service {
            Some(service) => Box::pin(service.run()),
            None => Box::pin(std::future::pending()),
        };
-    tokio::select! {
+    enum Shutdown {
-        // Waiting on ctrlc in a select will automatically terminate other branches when
+        Interrupt,
-        // ctrlc happens.
+        Terminate,
        _ = ctrlc => {},
        r = sigterm => { r?; }
        r = watch => { r?; }
        _ = reconcile => {}
        _ = heartbeat => {}
        r = commands => { r?; }
    }
    let signal = tokio::select! {
        _ = ctrlc => Shutdown::Interrupt,
        r = sigterm => { r?; Shutdown::Terminate }
        r = watch => { r?; anyhow::bail!("desired-state watch exited unexpectedly") }
        _ = snapshots => anyhow::bail!("desired-state snapshot loop exited unexpectedly"),
        r = worker_finished => { r?; anyhow::bail!("reconciler worker exited unexpectedly") }
        r = upgrades => { r?; anyhow::bail!("agent upgrade loop exited unexpectedly") }
        r = system_upgrades => { r?; anyhow::bail!("system upgrade relay exited unexpectedly") }
        _ = heartbeat => anyhow::bail!("heartbeat loop exited unexpectedly"),
        r = commands => { r?; anyhow::bail!("command server exited unexpectedly") }
    };
    let drain_started = tokio::time::Instant::now();
    if let Some(reconciler) = &reconciler {
        reconciler.drain().await;
    }
    let drain_duration_ms = drain_started
        .elapsed()
        .as_millis()
        .min(u128::from(u64::MAX)) as u64;
    if let Some(worker) = &worker {
        worker.abort();
    }
    let acknowledgement = if matches!(signal, Shutdown::Terminate)
        && let Some(upgrade_service) = &upgrade_service
    {
        upgrade_service
            .acknowledge_shutdown(drain_duration_ms)
            .await
            .map(|_| ())
    } else {
        Ok(())
    };
    let flush = client
        .flush()
        .await
        .context("flushing NATS during shutdown");
    let stopping = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping])
        .context("notifying systemd that shutdown started");
    acknowledgement?;
    flush?;
    stopping?;
    Ok(())
 }
--- a/fleet/harmony-fleet-agent/src/podman.rs
+++ b/fleet/harmony-fleet-agent/src/podman.rs
@@ -4,20 +4,63 @@ use std::time::Duration;
 use anyhow::{Result, anyhow, bail};
 use futures_util::StreamExt;
-use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, RestartPolicy, VolumeMount};
+use harmony_reconciler_contracts::{
    DEVICE_PULL_SECRET_PATH, PodmanService, PodmanV0Score, RestartPolicy, VolumeMount,
 };
 use harmony_secret::SecretStore;
 use oci_client::Reference;
 use podman_api::Podman;
-use podman_api::models::{ContainerMount, PortMapping};
+use podman_api::models::{ContainerMount, ContainerStatus, PortMapping};
 use podman_api::opts::{
    ContainerCreateOpts, ContainerDeleteOpts, ContainerListFilter, ContainerListOpts,
-    ContainerRestartPolicy, ContainerStopOpts, PullOpts,
+    ContainerRestartPolicy, ContainerStopOpts, ContainerWaitOpts, PullOpts, RegistryAuth,
 };
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
 use std::fmt;
 use std::sync::Arc;
 const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment";
 const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256";
 const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by";
 const MANAGED_BY_VALUE: &str = "harmony";
-const STOP_TIMEOUT: Duration = Duration::from_secs(300);
+const STOP_TIMEOUT: Duration = Duration::from_secs(30);
 #[async_trait::async_trait]
 pub trait WorkloadRuntime: Send + Sync {
    async fn reconcile(
        &self,
        deployment: &str,
        score: &PodmanV0Score,
        secrets: Option<&SecretSource>,
    ) -> Result<()>;
    async fn remove_deployment(&self, deployment: &str) -> Result<()>;
    async fn managed_deployments(&self) -> Result<HashSet<String>>;
 }
 #[derive(Clone)]
 pub struct SecretSource {
    pub store: Arc<dyn SecretStore>,
    pub prefix: String,
 }
 #[derive(Deserialize)]
 struct RegistryCredential {
    registry: String,
    username: String,
    password: String,
 }
 impl fmt::Debug for RegistryCredential {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("RegistryCredential")
            .field("registry", &self.registry)
            .field("username", &"[REDACTED]")
            .field("password", &"[REDACTED]")
            .finish()
    }
 }
 pub struct PodmanRuntime {
    podman: Podman,
@@ -42,93 +85,276 @@ impl PodmanRuntime {
        Ok(())
    }
-    pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> {
+    pub async fn reconcile(
-        for service in &score.services {
+        &self,
-            self.ensure_service_running(service, deployment).await?;
+        deployment: &str,
-        }
+        score: &PodmanV0Score,
        secrets: Option<&SecretSource>,
    ) -> Result<()> {
        score
            .validate_init_container()
            .map_err(anyhow::Error::msg)?;
        score.image_pull_secrets().map_err(anyhow::Error::msg)?;
        let desired = score
-            .services
+            .init_container
            .iter()
            .chain(&score.services)
            .map(|service| service.name.as_str())
            .collect::<HashSet<_>>();
-        for container in self.deployment_containers(deployment).await? {
+        let stale = self.preflight(deployment, score, &desired, secrets).await?;
-            let name = container_name(&container);
+        if let Some(init) = &score.init_container {
-            if !desired.contains(name.as_str()) {
+            self.ensure_init_completed(init, deployment, &score.revision())
-                self.remove_service(&name).await?;
+                .await?;
            }
        }
        for name in stale {
            self.remove_service(&name, deployment).await?;
            log_action(deployment, &name, "container", "remove");
        }
        for service in &score.services {
            self.ensure_service_running(service, deployment, secrets)
                .await?;
        }
        for service in &score.services {
            self.ensure_service_healthy(service).await?;
        }
        Ok(())
    }
    async fn preflight(
        &self,
        deployment: &str,
        score: &PodmanV0Score,
        desired: &HashSet<&str>,
        secrets: Option<&SecretSource>,
    ) -> Result<Vec<String>> {
        let mut names = HashSet::new();
        let mut host_ports = HashSet::new();
        let containers = self.all_containers().await?;
        for service in score.init_container.iter().chain(&score.services) {
            validate_service_name(&service.name)?;
            if !names.insert(&service.name) {
                bail!("duplicate service name '{}'", service.name);
            }
            for port in &service.ports {
                let requested = parse_port_mapping(port)?;
                let host_port = requested.host_port.unwrap_or_default();
                if !host_ports.insert(host_port) {
                    bail!("host port {host_port} is requested more than once");
                }
                if containers.iter().any(|container| {
                    let name = container_name(container);
                    let removable_stale =
                        is_owned_by(container, deployment) && !desired.contains(name.as_str());
                    name != service.name
                        && !removable_stale
                        && container.ports.as_ref().is_some_and(|ports| {
                            ports
                                .iter()
                                .any(|port| port.host_port == requested.host_port)
                        })
                }) {
                    bail!(
                        "host port {} is already in use",
                        requested.host_port.unwrap_or_default()
                    );
                }
            }
            if let Some(existing) = self.get_by_name(&service.name).await? {
                ensure_owned_by(&existing, deployment)?;
            }
            self.ensure_image_present(service, deployment, secrets)
                .await?;
        }
        Ok(containers
            .iter()
            .filter(|container| {
                is_owned_by(container, deployment)
                    && !desired.contains(container_name(container).as_str())
            })
            .map(container_name)
            .collect())
    }
    async fn ensure_init_completed(
        &self,
        init: &PodmanService,
        deployment: &str,
        score_hash: &str,
    ) -> Result<()> {
        let existing = self.get_by_name(&init.name).await?;
        let action = init_action(existing.as_ref(), score_hash);
        if action == InitAction::Complete {
            return Ok(());
        }
        if action == InitAction::Replace {
            self.remove_service(&init.name, deployment).await?;
        }
        let container = match action {
            InitAction::Create | InitAction::Replace => {
                let created = self
                    .podman
                    .containers()
                    .create(
                        &container_create_opts(init, deployment, score_hash, RestartPolicy::No)?
                            .build(),
                    )
                    .await?;
                self.podman.containers().get(created.id)
            }
            _ => self.podman.containers().get(&init.name),
        };
        if action != InitAction::Wait {
            container.start(None).await?;
            log_action(deployment, &init.name, "init", action.as_str());
        } else {
            tracing::info!(
                %deployment,
                service = %init.name,
                role = "init",
                "waiting for workload init"
            );
        }
        container
            .wait(
                &ContainerWaitOpts::builder()
                    .conditions([ContainerStatus::Exited])
                    .build(),
            )
            .await?;
        let exit_code = container
            .inspect()
            .await?
            .state
            .and_then(|state| state.exit_code)
            .ok_or_else(|| anyhow!("init container '{}' has no exit code", init.name))?;
        if exit_code != 0 {
            bail!(
                "init container '{}' failed with exit code {exit_code}",
                init.name
            );
        }
        tracing::info!(
            %deployment,
            service = %init.name,
            role = "init",
            "workload init completed"
        );
        Ok(())
    }
    pub async fn remove_deployment(&self, deployment: &str) -> Result<()> {
        for container in self.deployment_containers(deployment).await? {
-            self.remove_service(&container_name(&container)).await?;
+            self.remove_service(&container_name(&container), deployment)
                .await?;
        }
        Ok(())
    }
-    pub async fn remove_service(&self, name: &str) -> Result<()> {
+    pub async fn managed_deployments(&self) -> Result<HashSet<String>> {
        let opts = ContainerListOpts::builder()
            .all(true)
            .filter([ContainerListFilter::LabelKeyVal(
                MANAGED_BY_LABEL.to_string(),
                MANAGED_BY_VALUE.to_string(),
            )])
            .build();
        Ok(self
            .podman
            .containers()
            .list(&opts)
            .await?
            .into_iter()
            .filter_map(|container| {
                container
                    .labels
                    .and_then(|labels| labels.get(DEPLOYMENT_LABEL).cloned())
            })
            .collect())
    }
    async fn remove_service(&self, name: &str, deployment: &str) -> Result<()> {
        let Some(existing) = self.get_by_name(name).await? else {
            return Ok(());
        };
        ensure_owned_by(&existing, deployment)?;
        let id = existing
            .id
            .clone()
            .ok_or_else(|| anyhow!("container '{name}' has no id"))?;
        let opts = ContainerStopOpts::builder()
            .timeout(STOP_TIMEOUT.as_secs() as usize)
            .build();
-        let container = self.podman.containers().get(name);
+        let container = self.podman.containers().get(&id);
-        if container.exists().await.unwrap_or(false) {
+        if existing.state.as_deref() == Some("running") {
-            let _ = container.stop(&opts).await;
+            container.stop(&opts).await?;
        }
-        self.remove_container(name).await
+        self.remove_container(&id).await
    }
    async fn ensure_service_running(
        &self,
        service: &PodmanService,
        deployment: &str,
        secrets: Option<&SecretSource>,
    ) -> Result<()> {
        let existing = self.get_by_name(&service.name).await?;
-        if let Some(existing) = existing.as_ref() {
+        let action = service_action(existing.as_ref(), service);
-            if matches_spec(existing, service) {
+        match action {
-                if existing.state.as_deref() == Some("running") {
+            ServiceAction::Keep => return Ok(()),
-                    return Ok(());
+            ServiceAction::Start => {
-                }
+                let existing = existing
                    .as_ref()
                    .expect("start requires an existing container");
                let id = existing.id.clone().unwrap_or_else(|| service.name.clone());
                self.podman.containers().get(id).start(None).await?;
                log_action(deployment, &service.name, "service", action.as_str());
                return Ok(());
            }
-            self.remove_container(&service.name).await?;
+            ServiceAction::Replace => {
                self.remove_service(&service.name, deployment).await?;
            }
            ServiceAction::Create => {}
        }
-        self.ensure_image_present(&service.image).await?;
+        self.ensure_image_present(service, deployment, secrets)
-        let labels = HashMap::from([
+            .await?;
-            (MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
+        let created = self
-            (DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
+            .podman
-            (SPEC_LABEL.to_string(), spec_hash(service)?),
+            .containers()
-        ]);
+            .create(
-        let ports = service
+                &container_create_opts(
-            .ports
+                    service,
-            .iter()
+                    deployment,
-            .map(|port| parse_port_mapping(port))
+                    &spec_hash(service)?,
-            .collect::<Result<Vec<_>>>()?;
+                    service.restart_policy,
-        let env: HashMap<String, String> = service
+                )?
-            .env
+                .build(),
-            .iter()
+            )
-            .map(|env| (env.name.clone(), env.value.clone()))
+            .await?;
            .collect();
        let mounts = service
            .volumes
            .iter()
            .map(volume_to_mount)
            .collect::<Vec<_>>();
        let mut builder = ContainerCreateOpts::builder()
            .name(&service.name)
            .image(&service.image)
            .labels(labels)
            .portmappings(ports)
            .env(env)
            .restart_policy(map_restart_policy(service.restart_policy));
        if !mounts.is_empty() {
            builder = builder.mounts(mounts);
        }
        let created = self.podman.containers().create(&builder.build()).await?;
        self.podman.containers().get(created.id).start(None).await?;
        log_action(deployment, &service.name, "service", action.as_str());
        Ok(())
    }
    async fn ensure_service_healthy(&self, service: &PodmanService) -> Result<()> {
        let inspected = self
            .podman
            .containers()
            .get(&service.name)
            .inspect()
            .await?;
        let state = inspected
            .state
            .ok_or_else(|| anyhow!("service '{}' has no runtime state", service.name))?;
        let restarts = inspected.restart_count.unwrap_or_default();
        if state.running != Some(true) {
            bail!(
                "service '{}' is not healthy: status={}, exit_code={}, restarts={restarts}",
                service.name,
                state.status.as_deref().unwrap_or("unknown"),
                state.exit_code.unwrap_or_default(),
            );
        }
        Ok(())
    }
@@ -143,7 +369,12 @@ impl PodmanRuntime {
            .list(&opts)
            .await?
            .into_iter()
-            .next())
+            .find(|container| container_name(container) == name))
    }
    async fn all_containers(&self) -> Result<Vec<podman_api::models::ListContainer>> {
        let opts = ContainerListOpts::builder().all(true).build();
        Ok(self.podman.containers().list(&opts).await?)
    }
    async fn deployment_containers(
@@ -152,25 +383,64 @@ impl PodmanRuntime {
    ) -> Result<Vec<podman_api::models::ListContainer>> {
        let opts = ContainerListOpts::builder()
            .all(true)
-            .filter([ContainerListFilter::LabelKeyVal(
+            .filter([
-                DEPLOYMENT_LABEL.to_string(),
+                ContainerListFilter::LabelKeyVal(
-                deployment.to_string(),
+                    MANAGED_BY_LABEL.to_string(),
-            )])
+                    MANAGED_BY_VALUE.to_string(),
                ),
                ContainerListFilter::LabelKeyVal(
                    DEPLOYMENT_LABEL.to_string(),
                    deployment.to_string(),
                ),
            ])
            .build();
        Ok(self.podman.containers().list(&opts).await?)
    }
-    async fn ensure_image_present(&self, image: &str) -> Result<()> {
+    async fn ensure_image_present(
        &self,
        service: &PodmanService,
        deployment: &str,
        secrets: Option<&SecretSource>,
    ) -> Result<()> {
        let image = &service.image;
        let images = self.podman.images();
        if images.get(image).exists().await? {
            return Ok(());
        }
-        let mut pull = images.pull(&PullOpts::builder().reference(image).build());
+        let credential = match &service.image_pull_secret {
            None => None,
            Some(reference) => {
                let source = secrets.ok_or_else(|| {
                    anyhow!(
                        "service '{}' references an image pull secret but this device has no secret store configured",
                        service.name
                    )
                })?;
                let namespace = format!("{}/{}", source.prefix, DEVICE_PULL_SECRET_PATH);
                let bytes = source
                    .store
                    .get_raw(&namespace, reference)
                    .await
                    .map_err(|error| {
                        anyhow!("fetching image pull secret '{reference}': {error}")
                    })?;
                Some(
                    serde_json::from_slice::<RegistryCredential>(&bytes).map_err(|error| {
                        anyhow!("parsing image pull secret '{reference}': {error}")
                    })?,
                )
            }
        };
        let opts = pull_opts(image, credential)?;
        tracing::debug!(%deployment, service = %service.name, %image, "pulling workload image");
        let mut pull = images.pull(&opts);
        while let Some(event) = pull.next().await {
            if let Some(error) = event?.error {
                bail!("podman pull {image} failed: {error}");
            }
        }
        tracing::info!(%deployment, service = %service.name, %image, "workload image pulled");
        Ok(())
    }
@@ -184,6 +454,66 @@ impl PodmanRuntime {
    }
 }
 #[async_trait::async_trait]
 impl WorkloadRuntime for PodmanRuntime {
    async fn reconcile(
        &self,
        deployment: &str,
        score: &PodmanV0Score,
        secrets: Option<&SecretSource>,
    ) -> Result<()> {
        self.reconcile(deployment, score, secrets).await
    }
    async fn remove_deployment(&self, deployment: &str) -> Result<()> {
        self.remove_deployment(deployment).await
    }
    async fn managed_deployments(&self) -> Result<HashSet<String>> {
        self.managed_deployments().await
    }
 }
 fn registry_auth(image: &str, credential: RegistryCredential) -> Result<RegistryAuth> {
    let reference: Reference = image
        .parse()
        .map_err(|error| anyhow!("invalid image reference '{image}': {error}"))?;
    let expected = normalize_registry_host(&credential.registry)?;
    let actual = reference.registry().to_ascii_lowercase();
    if expected != actual {
        bail!(
            "image pull credential is scoped to registry '{}' but image '{}' uses registry '{}'",
            credential.registry,
            image,
            reference.registry()
        );
    }
    Ok(RegistryAuth::builder()
        .username(credential.username)
        .password(credential.password)
        .server_address(reference.registry())
        .build())
 }
 fn normalize_registry_host(host: &str) -> Result<String> {
    if host.is_empty()
        || host != host.trim()
        || host.contains(['/', '@', '?', '#'])
        || host.ends_with(':')
    {
        bail!("invalid registry host '{host}'");
    }
    Ok(host.to_ascii_lowercase())
 }
 fn pull_opts(image: &str, credential: Option<RegistryCredential>) -> Result<PullOpts> {
    let mut builder = PullOpts::builder().reference(image);
    if let Some(credential) = credential {
        builder.auth(registry_auth(image, credential)?);
    }
    Ok(builder.build())
 }
 fn default_user_socket() -> PathBuf {
    if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") {
        return PathBuf::from(format!("{dir}/podman/podman.sock"));
@@ -229,11 +559,185 @@ fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanSe
        .is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected))
 }
-fn spec_hash(service: &PodmanService) -> Result<String> {
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    Ok(format!(
+enum ServiceAction {
-        "{:x}",
+    Keep,
-        Sha256::digest(serde_json::to_vec(service)?)
+    Start,
-    ))
+    Replace,
    Create,
 }
 impl ServiceAction {
    fn as_str(self) -> &'static str {
        match self {
            Self::Keep => "keep",
            Self::Start => "start",
            Self::Replace => "replace",
            Self::Create => "create",
        }
    }
 }
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 enum InitAction {
    Complete,
    Wait,
    Start,
    Replace,
    Create,
 }
 impl InitAction {
    fn as_str(self) -> &'static str {
        match self {
            Self::Complete => "complete",
            Self::Wait => "wait",
            Self::Start => "start",
            Self::Replace => "replace",
            Self::Create => "create",
        }
    }
 }
 fn log_action(deployment: &str, service: &str, role: &str, action: &str) {
    tracing::info!(
        %deployment,
        %service,
        %role,
        %action,
        "workload runtime action completed"
    );
 }
 fn init_action(
    observed: Option<&podman_api::models::ListContainer>,
    score_hash: &str,
 ) -> InitAction {
    match observed {
        None => InitAction::Create,
        Some(observed)
            if observed
                .labels
                .as_ref()
                .and_then(|labels| labels.get(SPEC_LABEL))
                .is_none_or(|hash| hash != score_hash) =>
        {
            InitAction::Replace
        }
        Some(observed)
            if observed.state.as_deref() == Some("exited") && observed.exit_code == Some(0) =>
        {
            InitAction::Complete
        }
        Some(observed)
            if observed.state.as_deref() == Some("exited")
                && observed.exit_code.is_some_and(|code| code != 0) =>
        {
            // A fresh container prevents Podman's previous exit event from
            // satisfying the retry's wait before the new process exits.
            InitAction::Replace
        }
        Some(observed) if observed.state.as_deref() == Some("running") => InitAction::Wait,
        Some(_) => InitAction::Start,
    }
 }
 fn service_action(
    observed: Option<&podman_api::models::ListContainer>,
    service: &PodmanService,
 ) -> ServiceAction {
    match observed {
        None => ServiceAction::Create,
        Some(observed) if !matches_spec(observed, service) => ServiceAction::Replace,
        Some(observed) if observed.state.as_deref() == Some("running") => ServiceAction::Keep,
        Some(_) => ServiceAction::Start,
    }
 }
 fn ensure_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> Result<()> {
    let labels = observed.labels.as_ref();
    let managed = labels.and_then(|labels| labels.get(MANAGED_BY_LABEL));
    let owner = labels.and_then(|labels| labels.get(DEPLOYMENT_LABEL));
    if managed.map(String::as_str) != Some(MANAGED_BY_VALUE)
        || owner.map(String::as_str) != Some(deployment)
    {
        bail!(
            "container '{}' conflicts with deployment '{deployment}' (owner: {})",
            container_name(observed),
            owner.map(String::as_str).unwrap_or("unmanaged")
        );
    }
    Ok(())
 }
 fn is_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> bool {
    let labels = observed.labels.as_ref();
    labels
        .and_then(|labels| labels.get(MANAGED_BY_LABEL))
        .map(String::as_str)
        == Some(MANAGED_BY_VALUE)
        && labels
            .and_then(|labels| labels.get(DEPLOYMENT_LABEL))
            .map(String::as_str)
            == Some(deployment)
 }
 fn validate_service_name(name: &str) -> Result<()> {
    if name.is_empty()
        || !name
            .chars()
            .all(|character| character.is_ascii_alphanumeric() || "_.-".contains(character))
        || !name
            .chars()
            .next()
            .is_some_and(|character| character.is_ascii_alphanumeric())
    {
        bail!("invalid service name '{name}'");
    }
    Ok(())
 }
 fn spec_hash(value: &impl serde::Serialize) -> Result<String> {
    Ok(format!("{:x}", Sha256::digest(serde_json::to_vec(value)?)))
 }
 fn container_create_opts(
    service: &PodmanService,
    deployment: &str,
    spec_hash: &str,
    restart_policy: RestartPolicy,
 ) -> Result<podman_api::opts::ContainerCreateOptsBuilder> {
    let labels = HashMap::from([
        (MANAGED_BY_LABEL.to_string(), MANAGED_BY_VALUE.to_string()),
        (DEPLOYMENT_LABEL.to_string(), deployment.to_string()),
        (SPEC_LABEL.to_string(), spec_hash.to_string()),
    ]);
    let ports = service
        .ports
        .iter()
        .map(|port| parse_port_mapping(port))
        .collect::<Result<Vec<_>>>()?;
    let env = service
        .env
        .iter()
        .map(|env| (env.name.clone(), env.value.clone()))
        .collect::<HashMap<_, _>>();
    let mounts = service
        .volumes
        .iter()
        .map(volume_to_mount)
        .collect::<Vec<_>>();
    let mut builder = ContainerCreateOpts::builder()
        .name(&service.name)
        .image(&service.image)
        .labels(labels)
        .portmappings(ports)
        .env(env)
        .restart_policy(map_restart_policy(restart_policy));
    if !mounts.is_empty() {
        builder = builder.mounts(mounts);
    }
    Ok(builder)
 }
 fn container_name(container: &podman_api::models::ListContainer) -> String {
@@ -282,6 +786,7 @@ mod tests {
        PodmanService {
            name: "web".into(),
            image: "nginx:latest".into(),
            image_pull_secret: None,
            ports: vec!["8080:80".into()],
            env: vec![EnvVar::new("MODE", "prod")],
            secret_env: vec![],
@@ -290,6 +795,19 @@ mod tests {
        }
    }
    fn observed(
        name: &str,
        state: &str,
        labels: HashMap<String, String>,
    ) -> podman_api::models::ListContainer {
        serde_json::from_value(serde_json::json!({
            "Names": [name],
            "State": state,
            "Labels": labels,
        }))
        .unwrap()
    }
    #[test]
    fn spec_hash_is_stable_and_covers_desired_state() {
        let original = service();
@@ -299,4 +817,178 @@ mod tests {
        changed.env[0].value = "dev".into();
        assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap());
    }
    #[test]
    fn replacement_is_selected_before_create_for_changed_spec() {
        let original = service();
        let observed = observed(
            &original.name,
            "running",
            HashMap::from([
                (MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()),
                (DEPLOYMENT_LABEL.into(), "deployment-a".into()),
                (SPEC_LABEL.into(), spec_hash(&original).unwrap()),
            ]),
        );
        let mut changed = original;
        changed.image = "nginx:new".into();
        assert_eq!(
            service_action(Some(&observed), &changed),
            ServiceAction::Replace
        );
        assert_eq!(STOP_TIMEOUT, Duration::from_secs(30));
    }
    #[test]
    fn ownership_requires_both_harmony_and_deployment_labels() {
        let mut observed = observed("web", "running", HashMap::new());
        assert!(ensure_owned_by(&observed, "deployment-a").is_err());
        observed.labels = Some(HashMap::from([
            (MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()),
            (DEPLOYMENT_LABEL.into(), "deployment-a".into()),
        ]));
        assert!(ensure_owned_by(&observed, "deployment-a").is_ok());
        assert!(ensure_owned_by(&observed, "deployment-b").is_err());
    }
    #[test]
    fn non_running_service_is_restarted_regardless_of_exit_code() {
        let service = service();
        let observed: podman_api::models::ListContainer =
            serde_json::from_value(serde_json::json!({
                "Names": [service.name],
                "State": "exited",
                "Exited": true,
                "ExitCode": 42,
                "Labels": { SPEC_LABEL: spec_hash(&service).unwrap() },
            }))
            .unwrap();
        assert_eq!(
            service_action(Some(&observed), &service),
            ServiceAction::Start
        );
        let stopped: podman_api::models::ListContainer =
            serde_json::from_value(serde_json::json!({
                "Names": [service.name],
                "State": "exited",
                "Exited": true,
                "ExitCode": 0,
                "Labels": { SPEC_LABEL: spec_hash(&service).unwrap() },
            }))
            .unwrap();
        assert_eq!(
            service_action(Some(&stopped), &service),
            ServiceAction::Start
        );
    }
    #[test]
    fn init_completion_is_reused_only_for_the_same_score() {
        let completed = observed(
            "init",
            "exited",
            HashMap::from([(SPEC_LABEL.into(), "score-a".into())]),
        );
        let mut completed: podman_api::models::ListContainer = completed;
        completed.exit_code = Some(0);
        assert_eq!(
            init_action(Some(&completed), "score-a"),
            InitAction::Complete
        );
        assert_eq!(
            init_action(Some(&completed), "score-b"),
            InitAction::Replace
        );
        completed.exit_code = Some(1);
        assert_eq!(
            init_action(Some(&completed), "score-a"),
            InitAction::Replace
        );
    }
    #[test]
    fn invalid_service_names_are_rejected() {
        assert!(validate_service_name("web-1").is_ok());
        assert!(validate_service_name("").is_err());
        assert!(validate_service_name("/web").is_err());
    }
    #[test]
    fn pull_options_are_anonymous_without_credentials_and_scoped_to_image_registry() {
        let anonymous = pull_opts("docker.io/library/nginx:latest", None).unwrap();
        assert_eq!(
            anonymous.serialize().unwrap(),
            "reference=docker.io%2Flibrary%2Fnginx%3Alatest"
        );
        let authenticated = pull_opts(
            "registry.example:5000/team/app:v1",
            Some(RegistryCredential {
                registry: "registry.example:5000".into(),
                username: "device".into(),
                password: "pull-token".into(),
            }),
        )
        .unwrap();
        assert_eq!(
            authenticated.serialize().unwrap(),
            "reference=registry.example%3A5000%2Fteam%2Fapp%3Av1"
        );
        let auth = registry_auth(
            "registry.example:5000/team/app:v1",
            RegistryCredential {
                registry: "REGISTRY.EXAMPLE:5000".into(),
                username: "device".into(),
                password: "pull-token".into(),
            },
        )
        .unwrap();
        let RegistryAuth::Password { server_address, .. } = auth else {
            panic!("expected password registry auth")
        };
        assert_eq!(server_address.as_deref(), Some("registry.example:5000"));
        let error = registry_auth(
            "other.example:5000/team/app:v1",
            RegistryCredential {
                registry: "registry.example:5000".into(),
                username: "device".into(),
                password: "pull-token".into(),
            },
        )
        .unwrap_err();
        assert!(error.to_string().contains("scoped to registry"));
        let error = registry_auth(
            "registry.example/team/app:v1",
            RegistryCredential {
                registry: "registry.example:5000".into(),
                username: "device".into(),
                password: "pull-token".into(),
            },
        )
        .unwrap_err();
        assert!(error.to_string().contains("scoped to registry"));
    }
    #[test]
    fn registry_credentials_are_redacted() {
        let credential = RegistryCredential {
            registry: "registry.example".into(),
            username: "device-user".into(),
            password: "super-secret".into(),
        };
        let debug = format!("{credential:?}");
        assert!(!debug.contains("device-user"));
        assert!(!debug.contains("super-secret"));
        assert_eq!(
            debug,
            "RegistryCredential { registry: \"registry.example\", username: \"[REDACTED]\", password: \"[REDACTED]\" }"
        );
    }
 }
--- a/fleet/harmony-fleet-agent/src/reconciler.rs
+++ b/fleet/harmony-fleet-agent/src/reconciler.rs
--- a/fleet/harmony-fleet-agent/src/system_upgrade.rs
+++ b/fleet/harmony-fleet-agent/src/system_upgrade.rs
@@ -0,0 +1,661 @@
 use std::path::{Path, PathBuf};
 use anyhow::{Context, Result, bail};
 use chrono::{DateTime, Utc};
 use harmony_reconciler_contracts::{SystemUpgradeAttempt, SystemUpgradePhase, SystemUpgradeStatus};
 use serde::{Deserialize, Serialize};
 use crate::updater::{bounded_error, safe_token, sync_directory};
 const JOURNAL_DIR: &str = "/var/lib/harmony-fleet-updater/system-upgrades";
 const BOOT_ID: &str = "/proc/sys/kernel/random/boot_id";
 const MAX_REBOOT_REQUESTS: u8 = 3;
 const TERMINAL_REPLAY_DAYS: i64 = 2;
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub(super) struct CommandSpec {
    pub program: &'static str,
    pub args: &'static [&'static str],
    pub env: &'static [(&'static str, &'static str)],
 }
 const AUDIT: CommandSpec = CommandSpec {
    program: "/usr/bin/dpkg",
    args: &["--audit"],
    env: &[],
 };
 const CHECK: CommandSpec = CommandSpec {
    program: "/usr/bin/apt-get",
    args: &["-o", "DPkg::Lock::Timeout=300", "check"],
    env: &[],
 };
 const UPDATE: CommandSpec = CommandSpec {
    program: "/usr/bin/apt-get",
    args: &[
        "-o",
        "DPkg::Lock::Timeout=300",
        "-o",
        "APT::Update::Error-Mode=any",
        "-o",
        "Acquire::AllowInsecureRepositories=false",
        "-o",
        "Acquire::AllowDowngradeToInsecureRepositories=false",
        "-o",
        "Acquire::AllowWeakRepositories=false",
        "update",
    ],
    env: &[],
 };
 const FULL_UPGRADE: CommandSpec = CommandSpec {
    program: "/usr/bin/apt-get",
    args: &[
        "-o",
        "DPkg::Lock::Timeout=300",
        "-o",
        "Dpkg::Use-Pty=0",
        "-o",
        "Dpkg::Options::=--force-confold",
        "-o",
        "APT::Get::AllowUnauthenticated=false",
        "-y",
        "full-upgrade",
    ],
    env: &[
        ("DEBIAN_FRONTEND", "noninteractive"),
        ("APT_LISTCHANGES_FRONTEND", "none"),
        ("NEEDRESTART_MODE", "a"),
    ],
 };
 const REBOOT: CommandSpec = CommandSpec {
    program: "/bin/systemctl",
    args: &["reboot", "--no-wall"],
    env: &[],
 };
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(super) struct Journal {
    attempt_id: String,
    run_uid: String,
    device_id: harmony_reconciler_contracts::Id,
    attempt_digest: String,
    phase: SystemUpgradePhase,
    started_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,
    error: Option<String>,
    boot_id: Option<String>,
    reboot_requests: u8,
 }
 impl Journal {
    pub(super) fn status(&self) -> SystemUpgradeStatus {
        SystemUpgradeStatus {
            attempt_id: self.attempt_id.clone(),
            run_uid: self.run_uid.clone(),
            phase: self.phase,
            started_at: self.started_at,
            updated_at: self.updated_at,
            post_completion_heartbeat_at: None,
            error: self.error.clone(),
        }
    }
    fn transition(&mut self, phase: SystemUpgradePhase, error: Option<String>) {
        self.phase = phase;
        self.error = error;
        self.updated_at = Utc::now();
    }
    fn validate(&self, expected_attempt_id: &str) -> Result<()> {
        if self.attempt_id != expected_attempt_id
            || uuid::Uuid::parse_str(&self.attempt_id).is_err()
            || !safe_token(&self.run_uid)
            || !safe_token(&self.device_id.to_string())
            || self.attempt_digest.len() != 64
            || !self.attempt_digest.chars().all(|c| c.is_ascii_hexdigit())
            || self.reboot_requests > MAX_REBOOT_REQUESTS
            || self.phase == SystemUpgradePhase::Blocked
            || matches!(
                self.phase,
                SystemUpgradePhase::Rebooting | SystemUpgradePhase::Verifying
            ) && self.boot_id.as_deref().is_none_or(str::is_empty)
        {
            bail!("invalid system upgrade journal");
        }
        Ok(())
    }
 }
 pub(super) enum Acceptance {
    Existing(SystemUpgradeStatus),
    New(Journal),
 }
 pub(super) fn accept(
    attempt: &SystemUpgradeAttempt,
    existing: Option<&Journal>,
    now: DateTime<Utc>,
 ) -> Result<Acceptance> {
    validate(attempt)?;
    if let Some(journal) = existing {
        if journal.attempt_digest != attempt.digest() {
            bail!("system upgrade attempt id was reused with different content");
        }
        return Ok(Acceptance::Existing(journal.status()));
    }
    if attempt.expires_at <= now {
        bail!("system upgrade attempt has expired");
    }
    Ok(Acceptance::New(Journal {
        attempt_id: attempt.attempt_id.clone(),
        run_uid: attempt.run_uid.clone(),
        device_id: attempt.device_id.clone(),
        attempt_digest: attempt.digest(),
        phase: SystemUpgradePhase::Preflight,
        started_at: now,
        updated_at: now,
        error: None,
        boot_id: None,
        reboot_requests: 0,
    }))
 }
 fn validate(attempt: &SystemUpgradeAttempt) -> Result<()> {
    uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid attempt id")?;
    if !safe_token(&attempt.run_uid) {
        bail!("invalid run uid");
    }
    if !safe_token(&attempt.device_id.to_string()) {
        bail!("invalid device id");
    }
    Ok(())
 }
 pub(super) async fn read(attempt_id: &str) -> Result<Option<Journal>> {
    uuid::Uuid::parse_str(attempt_id).context("invalid attempt id")?;
    let path = journal_path(attempt_id);
    match tokio::fs::read(&path).await {
        Ok(bytes) => {
            let journal: Journal = serde_json::from_slice(&bytes)
                .with_context(|| format!("reading system upgrade journal {}", path.display()))?;
            journal.validate(attempt_id)?;
            Ok(Some(journal))
        }
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
        Err(error) => Err(error.into()),
    }
 }
 pub(super) async fn write(journal: &Journal) -> Result<()> {
    let path = journal_path(&journal.attempt_id);
    let parent = path
        .parent()
        .context("system upgrade journal has no parent")?;
    tokio::fs::create_dir_all(parent).await?;
    let temporary = path.with_extension("tmp");
    let mut file = tokio::fs::File::create(&temporary).await?;
    use tokio::io::AsyncWriteExt;
    file.write_all(&serde_json::to_vec(journal)?).await?;
    file.sync_all().await?;
    tokio::fs::rename(&temporary, &path).await?;
    sync_directory(parent)?;
    if journal.phase != SystemUpgradePhase::Preflight {
        tracing::info!(
            attempt_id = %journal.attempt_id,
            run_uid = %journal.run_uid,
            phase = ?journal.phase,
            reboot_requests = journal.reboot_requests,
            "system upgrade state persisted"
        );
    }
    Ok(())
 }
 pub(super) async fn recover_active() -> Result<Option<Journal>> {
    let mut entries = match tokio::fs::read_dir(JOURNAL_DIR).await {
        Ok(entries) => entries,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
        Err(error) => return Err(error.into()),
    };
    let mut active = None;
    while let Some(entry) = entries.next_entry().await? {
        let path = entry.path();
        if path.extension().and_then(|value| value.to_str()) != Some("json") {
            continue;
        }
        let journal: Journal = serde_json::from_slice(&tokio::fs::read(&path).await?)
            .with_context(|| format!("corrupt system upgrade journal {}", path.display()))?;
        let attempt_id = path
            .file_stem()
            .and_then(|value| value.to_str())
            .context("invalid system upgrade journal filename")?;
        journal.validate(attempt_id)?;
        if !retain_journal(&journal, Utc::now()) {
            tokio::fs::remove_file(path).await?;
            continue;
        }
        if !journal.phase.is_terminal() && active.replace(journal).is_some() {
            bail!("multiple active system upgrade journals");
        }
    }
    Ok(active)
 }
 fn retain_journal(journal: &Journal, now: DateTime<Utc>) -> bool {
    journal.phase == SystemUpgradePhase::RepairRequired
        || !journal.phase.is_terminal()
        || journal.updated_at + chrono::Duration::days(TERMINAL_REPLAY_DAYS) >= now
 }
 pub(super) async fn run(mut journal: Journal) -> Result<SystemUpgradeStatus> {
    let action = recovery_action(&journal, &current_boot_id()?);
    if journal.phase == SystemUpgradePhase::Rebooting {
        tracing::info!(
            attempt_id = %journal.attempt_id,
            run_uid = %journal.run_uid,
            ?action,
            reboot_requests = journal.reboot_requests,
            "recovering system upgrade after reboot request"
        );
    }
    let result = match action {
        RecoveryAction::Apply => apply(&mut journal).await,
        RecoveryAction::Repair => {
            let audit = run_command(&AUDIT, true).await;
            let message = match audit {
                Ok(()) => "system upgrade was interrupted while applying packages".into(),
                Err(error) => format!("system upgrade was interrupted; {error}"),
            };
            journal.transition(SystemUpgradePhase::RepairRequired, Some(message));
            write(&journal).await
        }
        RecoveryAction::Reboot => request_reboot(&mut journal).await,
        RecoveryAction::Verify => verify(&mut journal).await,
        RecoveryAction::None => Ok(()),
    };
    if let Err(error) = result {
        if journal.phase == SystemUpgradePhase::Rebooting {
            journal.error = Some(bounded_error(&error.to_string()));
            journal.updated_at = Utc::now();
            write(&journal).await?;
        } else if !journal.phase.is_terminal() {
            let phase = if journal.phase == SystemUpgradePhase::Applying {
                SystemUpgradePhase::RepairRequired
            } else {
                SystemUpgradePhase::Failed
            };
            journal.transition(phase, Some(bounded_error(&error.to_string())));
            write(&journal).await?;
        }
        return Err(error);
    }
    Ok(journal.status())
 }
 async fn apply(journal: &mut Journal) -> Result<()> {
    run_command(&AUDIT, true).await?;
    run_command(&CHECK, false).await?;
    validate_sources().await?;
    run_command(&UPDATE, false).await?;
    journal.transition(SystemUpgradePhase::Applying, None);
    write(journal).await?;
    let upgrade = run_command(&FULL_UPGRADE, false).await;
    let health = match run_command(&AUDIT, true).await {
        Ok(()) => run_command(&CHECK, false).await,
        error => error,
    };
    if let Err(error) = health {
        journal.transition(
            SystemUpgradePhase::RepairRequired,
            Some(bounded_error(&error.to_string())),
        );
        write(journal).await?;
        return Ok(());
    }
    if let Err(error) = upgrade {
        journal.transition(
            SystemUpgradePhase::Failed,
            Some(bounded_error(&error.to_string())),
        );
        write(journal).await?;
        return Ok(());
    }
    journal.boot_id = Some(current_boot_id()?);
    journal.transition(SystemUpgradePhase::Rebooting, None);
    request_reboot(journal).await
 }
 async fn validate_sources() -> Result<()> {
    let mut paths = vec![PathBuf::from("/etc/apt/sources.list")];
    match tokio::fs::read_dir("/etc/apt/sources.list.d").await {
        Ok(mut entries) => {
            while let Some(entry) = entries.next_entry().await? {
                let path = entry.path();
                if matches!(
                    path.extension().and_then(|extension| extension.to_str()),
                    Some("list" | "sources")
                ) {
                    paths.push(path);
                }
            }
        }
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
        Err(error) => return Err(error.into()),
    }
    for path in paths {
        let contents = match tokio::fs::read_to_string(&path).await {
            Ok(contents) => contents,
            Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
            Err(error) => return Err(error.into()),
        };
        if source_disables_authentication(&contents) {
            bail!(
                "apt source {} disables repository authentication",
                path.display()
            );
        }
    }
    Ok(())
 }
 fn source_disables_authentication(contents: &str) -> bool {
    contents.lines().any(|line| {
        let line = line.trim();
        if line.is_empty() || line.starts_with('#') {
            return false;
        }
        let lower = line.to_ascii_lowercase();
        let options = lower
            .strip_prefix("deb ")
            .or_else(|| lower.strip_prefix("deb-src "))
            .and_then(|line| line.strip_prefix('['))
            .and_then(|line| line.split_once(']').map(|(options, _)| options));
        if let Some(options) = options {
            let options = options.replace(" =", "=").replace("= ", "=");
            return options.split_ascii_whitespace().any(|option| {
                matches!(
                    option.split_once('='),
                    Some((
                        "trusted" | "allow-insecure" | "allow-weak",
                        "yes" | "true" | "1"
                    ))
                )
            });
        }
        matches!(
            lower
                .split_once(':')
                .map(|(key, value)| (key.trim(), value.trim())),
            Some((
                "trusted" | "allow-insecure" | "allow-weak",
                "yes" | "true" | "1"
            ))
        )
    })
 }
 async fn request_reboot(journal: &mut Journal) -> Result<()> {
    while journal.reboot_requests < MAX_REBOOT_REQUESTS {
        journal.reboot_requests += 1;
        write(journal).await?;
        match run_command(&REBOOT, false).await {
            Ok(()) => tokio::time::sleep(std::time::Duration::from_secs(30)).await,
            Err(error) => {
                journal.error = Some(bounded_error(&error.to_string()));
                journal.updated_at = Utc::now();
                write(journal).await?;
                tokio::time::sleep(std::time::Duration::from_secs(2)).await;
            }
        }
    }
    journal.transition(
        SystemUpgradePhase::Failed,
        Some("reboot did not change the boot id".into()),
    );
    write(journal).await?;
    Ok(())
 }
 async fn verify(journal: &mut Journal) -> Result<()> {
    journal.transition(SystemUpgradePhase::Verifying, None);
    write(journal).await?;
    let result = run_command(&AUDIT, true).await;
    let result = match result {
        Ok(()) => run_command(&CHECK, false).await,
        error => error,
    };
    if let Err(error) = result {
        journal.transition(
            SystemUpgradePhase::RepairRequired,
            Some(bounded_error(&error.to_string())),
        );
        return write(journal).await;
    }
    journal.transition(SystemUpgradePhase::Complete, None);
    write(journal).await
 }
 async fn run_command(spec: &CommandSpec, require_empty_stdout: bool) -> Result<()> {
    let mut command = tokio::process::Command::new(spec.program);
    command
        .env_remove("NOTIFY_SOCKET")
        .envs(spec.env.iter().copied())
        .args(spec.args)
        .kill_on_drop(true);
    let output = command.output().await?;
    if !output.status.success() {
        bail!("{} failed: {}", spec.program, output.status);
    }
    if require_empty_stdout && !output.stdout.iter().all(u8::is_ascii_whitespace) {
        bail!("dpkg audit reported incomplete package state");
    }
    Ok(())
 }
 fn current_boot_id() -> Result<String> {
    Ok(std::fs::read_to_string(BOOT_ID)?.trim().to_string())
 }
 #[derive(Debug, PartialEq, Eq)]
 enum RecoveryAction {
    Apply,
    Repair,
    Reboot,
    Verify,
    None,
 }
 fn recovery_action(journal: &Journal, boot_id: &str) -> RecoveryAction {
    match journal.phase {
        SystemUpgradePhase::Preflight => RecoveryAction::Apply,
        SystemUpgradePhase::Applying => RecoveryAction::Repair,
        SystemUpgradePhase::Rebooting if journal.boot_id.as_deref() == Some(boot_id) => {
            RecoveryAction::Reboot
        }
        SystemUpgradePhase::Rebooting => RecoveryAction::Verify,
        SystemUpgradePhase::Verifying => RecoveryAction::Verify,
        _ => RecoveryAction::None,
    }
 }
 fn journal_path(attempt_id: &str) -> PathBuf {
    Path::new(JOURNAL_DIR).join(format!("{attempt_id}.json"))
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use chrono::TimeDelta;
    use harmony_reconciler_contracts::Id;
    fn attempt() -> SystemUpgradeAttempt {
        SystemUpgradeAttempt {
            attempt_id: uuid::Uuid::new_v4().to_string(),
            run_uid: "run-1".into(),
            device_id: Id::from("device-1"),
            expires_at: Utc::now() + TimeDelta::minutes(5),
        }
    }
    fn journal(phase: SystemUpgradePhase) -> Journal {
        let attempt = attempt();
        let Acceptance::New(mut journal) = accept(&attempt, None, Utc::now()).unwrap() else {
            unreachable!()
        };
        journal.phase = phase;
        journal.boot_id = Some("old-boot".into());
        journal
    }
    #[test]
    fn validates_expiry_and_duplicate_digest() {
        let now = Utc::now();
        let original = attempt();
        let Acceptance::New(journal) = accept(&original, None, now).unwrap() else {
            unreachable!()
        };
        assert!(matches!(
            accept(&original, Some(&journal), now + TimeDelta::hours(1)).unwrap(),
            Acceptance::Existing(_)
        ));
        let mut changed = original.clone();
        changed.run_uid = "changed".into();
        assert!(accept(&changed, Some(&journal), now).is_err());
        let mut expired = attempt();
        expired.expires_at = now;
        assert!(accept(&expired, None, now).is_err());
        expired.expires_at = now + TimeDelta::minutes(1);
        expired.attempt_id = "invalid".into();
        assert!(accept(&expired, None, now).is_err());
        expired.attempt_id = uuid::Uuid::new_v4().to_string();
        expired.run_uid = "unsafe/run".into();
        assert!(accept(&expired, None, now).is_err());
    }
    #[test]
    fn command_policy_is_exact() {
        assert_eq!(
            AUDIT,
            CommandSpec {
                program: "/usr/bin/dpkg",
                args: &["--audit"],
                env: &[]
            }
        );
        assert_eq!(CHECK.args, &["-o", "DPkg::Lock::Timeout=300", "check"]);
        assert_eq!(
            UPDATE.args,
            &[
                "-o",
                "DPkg::Lock::Timeout=300",
                "-o",
                "APT::Update::Error-Mode=any",
                "-o",
                "Acquire::AllowInsecureRepositories=false",
                "-o",
                "Acquire::AllowDowngradeToInsecureRepositories=false",
                "-o",
                "Acquire::AllowWeakRepositories=false",
                "update"
            ]
        );
        assert_eq!(
            FULL_UPGRADE.args,
            &[
                "-o",
                "DPkg::Lock::Timeout=300",
                "-o",
                "Dpkg::Use-Pty=0",
                "-o",
                "Dpkg::Options::=--force-confold",
                "-o",
                "APT::Get::AllowUnauthenticated=false",
                "-y",
                "full-upgrade"
            ]
        );
        assert_eq!(
            FULL_UPGRADE.env,
            &[
                ("DEBIAN_FRONTEND", "noninteractive"),
                ("APT_LISTCHANGES_FRONTEND", "none"),
                ("NEEDRESTART_MODE", "a")
            ]
        );
        assert_eq!(
            REBOOT,
            CommandSpec {
                program: "/bin/systemctl",
                args: &["reboot", "--no-wall"],
                env: &[]
            }
        );
    }
    #[test]
    fn apt_sources_cannot_disable_authentication() {
        for source in [
            "deb [trusted=yes] https://example.invalid stable main",
            "deb [allow-insecure = yes] https://example.invalid stable main",
            "deb [allow-weak=yes] https://example.invalid stable main",
            "Types: deb\nTrusted: yes\nURIs: https://example.invalid",
            "Types: deb\nAllow-Insecure: true\nURIs: https://example.invalid",
            "Types: deb\nAllow-Weak: 1\nURIs: https://example.invalid",
        ] {
            assert!(
                source_disables_authentication(source),
                "accepted insecure source: {source}"
            );
        }
        assert!(!source_disables_authentication(
            "deb [signed-by=/etc/apt/keyrings/vendor.gpg] https://example.invalid stable main"
        ));
        assert!(!source_disables_authentication(
            "deb https://trusted=yes.example.invalid stable main"
        ));
        assert!(!source_disables_authentication(
            "# deb [trusted=yes] disabled"
        ));
    }
    #[test]
    fn recovery_never_reapplies_an_interrupted_upgrade() {
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Preflight), "old-boot"),
            RecoveryAction::Apply
        );
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Applying), "old-boot"),
            RecoveryAction::Repair
        );
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Rebooting), "old-boot"),
            RecoveryAction::Reboot
        );
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Rebooting), "new-boot"),
            RecoveryAction::Verify
        );
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Verifying), "new-boot"),
            RecoveryAction::Verify
        );
        assert_eq!(
            recovery_action(&journal(SystemUpgradePhase::Complete), "new-boot"),
            RecoveryAction::None
        );
    }
    #[test]
    fn terminal_journals_expire_but_repair_state_remains() {
        let now = Utc::now();
        let mut complete = journal(SystemUpgradePhase::Complete);
        complete.updated_at = now - chrono::Duration::days(TERMINAL_REPLAY_DAYS + 1);
        assert!(!retain_journal(&complete, now));
        let mut repair = complete;
        repair.phase = SystemUpgradePhase::RepairRequired;
        assert!(retain_journal(&repair, now));
    }
 }
--- a/fleet/harmony-fleet-agent/src/system_upgrade_service.rs
+++ b/fleet/harmony-fleet-agent/src/system_upgrade_service.rs
@@ -0,0 +1,325 @@
 use std::time::Duration;
 use anyhow::{Context, Result, bail};
 use async_nats::jetstream::kv::{Operation, Store};
 use chrono::Utc;
 use futures_util::StreamExt;
 use harmony_reconciler_contracts::{
    BUCKET_DEVICE_HEARTBEAT, BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS,
    HeartbeatPayload, Id, SystemUpgradeAttempt, SystemUpgradePhase, SystemUpgradeStatus,
    device_heartbeat_key, system_upgrade_intent_key, system_upgrade_intent_watch_filter,
    system_upgrade_status_key,
 };
 use crate::updater::{UpdaterClient, safe_token};
 const RETRY_INTERVAL: Duration = Duration::from_secs(1);
 #[derive(Clone)]
 pub struct SystemUpgradeService {
    device_id: Id,
    intent: Store,
    status: Store,
    heartbeat: Store,
    updater: UpdaterClient,
 }
 impl SystemUpgradeService {
    pub async fn connect(
        client: async_nats::Client,
        device_id: Id,
        updater_socket: &std::path::Path,
    ) -> Result<Self> {
        let jetstream = async_nats::jetstream::new(client);
        let intent = jetstream
            .get_key_value(BUCKET_SYSTEM_UPGRADE_INTENT)
            .await?;
        let status = jetstream
            .get_key_value(BUCKET_SYSTEM_UPGRADE_STATUS)
            .await?;
        let heartbeat = jetstream.get_key_value(BUCKET_DEVICE_HEARTBEAT).await?;
        let updater = UpdaterClient::new(updater_socket);
        let capabilities = updater.capabilities().await?;
        if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 {
            bail!("updater does not support AptFullUpgradeV1");
        }
        Ok(Self {
            device_id,
            intent,
            status,
            heartbeat,
            updater,
        })
    }
    pub async fn run(self) -> Result<()> {
        let filter = system_upgrade_intent_watch_filter(&self.device_id.to_string());
        loop {
            let mut intents = match self.intent.watch_with_history(&filter).await {
                Ok(intents) => intents,
                Err(error) => {
                    tracing::warn!(%error, "system upgrade intent watch start failed");
                    tokio::time::sleep(RETRY_INTERVAL).await;
                    continue;
                }
            };
            while let Some(entry) = intents.next().await {
                match entry {
                    Ok(entry) if entry.operation == Operation::Put => {
                        let result = serde_json::from_slice(&entry.value)
                            .context("decoding system upgrade attempt")
                            .and_then(|attempt| {
                                validate_intent(&self.device_id, &entry.key, &attempt)
                                    .map(|()| attempt)
                            });
                        match result {
                            Ok(attempt) => self.relay(attempt).await,
                            Err(error) => {
                                tracing::warn!(key = %entry.key, %error, "system upgrade attempt rejected")
                            }
                        }
                    }
                    Ok(_) => {}
                    Err(error) => {
                        tracing::warn!(%error, "system upgrade intent watch failed; restarting");
                        break;
                    }
                }
            }
            tokio::time::sleep(RETRY_INTERVAL).await;
        }
    }
    async fn relay(&self, attempt: SystemUpgradeAttempt) {
        let mut blocked_since = None;
        let mut blocked_published = false;
        let recovered = match self
            .updater
            .system_upgrade_status(&attempt.attempt_id)
            .await
        {
            Ok(Some(status))
                if status.attempt_id == attempt.attempt_id && status.run_uid == attempt.run_uid =>
            {
                Some(status)
            }
            Ok(Some(_)) => {
                tracing::warn!(attempt_id = %attempt.attempt_id, "updater status does not match system upgrade intent");
                return;
            }
            Ok(None) | Err(_) => None,
        };
        let accepted = if let Some(status) = recovered {
            status
        } else {
            loop {
                if blocked_since.is_some() && attempt.expires_at <= Utc::now() {
                    return;
                }
                match self.updater.start_system_upgrade(&attempt).await {
                    Ok(status) => break status,
                    Err(error)
                        if error
                            .to_string()
                            .contains("another upgrade is already in progress") =>
                    {
                        let now = Utc::now();
                        let started_at = *blocked_since.get_or_insert(now);
                        let status = SystemUpgradeStatus {
                            attempt_id: attempt.attempt_id.clone(),
                            run_uid: attempt.run_uid.clone(),
                            phase: SystemUpgradePhase::Blocked,
                            started_at,
                            updated_at: now,
                            post_completion_heartbeat_at: None,
                            error: Some(error.to_string()),
                        };
                        match self.publish(&attempt, status).await {
                            Ok(()) if !blocked_published => {
                                tracing::info!(
                                    attempt_id = %attempt.attempt_id,
                                    run_uid = %attempt.run_uid,
                                    "system upgrade blocked status published"
                                );
                                blocked_published = true;
                            }
                            Ok(()) => {}
                            Err(error) => {
                                tracing::warn!(%error, "publishing blocked system upgrade status failed")
                            }
                        }
                        if attempt.expires_at <= Utc::now() {
                            return;
                        }
                        tokio::time::sleep(RETRY_INTERVAL).await;
                    }
                    Err(error) if temporary_updater_error(&error) => {
                        tracing::warn!(%error, "updater unavailable during system upgrade submit; retrying");
                        if attempt.expires_at <= Utc::now() {
                            return;
                        }
                        tokio::time::sleep(RETRY_INTERVAL).await;
                    }
                    Err(error) => {
                        tracing::warn!(%error, "system upgrade submit failed");
                        return;
                    }
                }
            }
        };
        let accepted = self.attach_heartbeat(accepted).await;
        let accepted_published = match self.publish(&attempt, accepted.clone()).await {
            Ok(()) => true,
            Err(error) => {
                tracing::warn!(%error, "publishing accepted system upgrade status failed");
                false
            }
        };
        if terminal_relay_complete(&accepted) && accepted_published {
            return;
        }
        let mut last = accepted_published.then_some(accepted);
        loop {
            match self
                .updater
                .system_upgrade_status(&attempt.attempt_id)
                .await
            {
                Ok(Some(status)) => {
                    let status = self.attach_heartbeat(status).await;
                    if last.as_ref() == Some(&status) {
                        tokio::time::sleep(RETRY_INTERVAL).await;
                        continue;
                    }
                    let terminal = terminal_relay_complete(&status);
                    let published = match self.publish(&attempt, status.clone()).await {
                        Ok(()) => {
                            last = Some(status);
                            true
                        }
                        Err(error) => {
                            tracing::warn!(%error, "publishing system upgrade status failed");
                            false
                        }
                    };
                    if terminal && published {
                        return;
                    }
                }
                Ok(_) => {}
                Err(error) => {
                    tracing::warn!(%error, "updater system upgrade status unavailable; retrying")
                }
            }
            tokio::time::sleep(RETRY_INTERVAL).await;
        }
    }
    async fn attach_heartbeat(&self, mut status: SystemUpgradeStatus) -> SystemUpgradeStatus {
        if status.phase != SystemUpgradePhase::Complete
            || status.post_completion_heartbeat_at.is_some()
        {
            return status;
        }
        let key = device_heartbeat_key(&self.device_id.to_string());
        let heartbeat = match self.heartbeat.entry(&key).await {
            Ok(Some(entry)) if entry.operation == Operation::Put => {
                serde_json::from_slice::<HeartbeatPayload>(&entry.value).ok()
            }
            _ => None,
        };
        if let Some(heartbeat) = heartbeat
            && heartbeat.device_id == self.device_id
            && heartbeat.at > status.updated_at
        {
            status.post_completion_heartbeat_at = Some(heartbeat.at);
        }
        status
    }
    async fn publish(
        &self,
        attempt: &SystemUpgradeAttempt,
        status: SystemUpgradeStatus,
    ) -> Result<()> {
        if status.attempt_id != attempt.attempt_id || status.run_uid != attempt.run_uid {
            bail!("updater system upgrade status does not match current attempt");
        }
        let key = system_upgrade_status_key(&self.device_id.to_string(), &status.run_uid);
        self.status
            .put(&key, serde_json::to_vec(&status)?.into())
            .await?;
        if status.phase != SystemUpgradePhase::Blocked {
            tracing::info!(
                attempt_id = %status.attempt_id,
                run_uid = %status.run_uid,
                phase = ?status.phase,
                heartbeat_confirmed = status.post_completion_heartbeat_at.is_some(),
                "system upgrade status published"
            );
        }
        Ok(())
    }
 }
 fn terminal_relay_complete(status: &SystemUpgradeStatus) -> bool {
    status.phase.is_terminal()
        && (status.phase != SystemUpgradePhase::Complete
            || status.post_completion_heartbeat_at.is_some())
 }
 fn temporary_updater_error(error: &anyhow::Error) -> bool {
    error.downcast_ref::<std::io::Error>().is_some()
        || error.to_string().starts_with("updater response timed out")
 }
 fn validate_intent(device_id: &Id, key: &str, attempt: &SystemUpgradeAttempt) -> Result<()> {
    uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid system upgrade attempt id")?;
    if &attempt.device_id != device_id {
        bail!("system upgrade attempt targets another device");
    }
    if !safe_token(&attempt.run_uid) {
        bail!("invalid system upgrade run uid");
    }
    if key != system_upgrade_intent_key(&device_id.to_string(), &attempt.run_uid) {
        bail!("system upgrade intent key does not match its device and run");
    }
    Ok(())
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use chrono::TimeDelta;
    fn attempt(device_id: &str) -> SystemUpgradeAttempt {
        SystemUpgradeAttempt {
            attempt_id: uuid::Uuid::new_v4().to_string(),
            run_uid: "run-1".into(),
            device_id: Id::from(device_id),
            expires_at: Utc::now() + TimeDelta::minutes(5),
        }
    }
    #[test]
    fn rejects_wrong_device_and_bad_keys() {
        let device_id = Id::from("device-1");
        let wrong = attempt("device-2");
        assert!(validate_intent(&device_id, "device-1.run-1", &wrong).is_err());
        let invalid = attempt("device-1");
        assert!(validate_intent(&device_id, "device-1.wrong", &invalid).is_err());
    }
    #[test]
    fn intent_and_status_keys_map_to_the_attempt_run() {
        let device_id = Id::from("device-1");
        let attempt = attempt("device-1");
        assert!(validate_intent(&device_id, "device-1.run-1", &attempt).is_ok());
        assert!(validate_intent(&device_id, "device-1.other", &attempt).is_err());
        assert_eq!(
            system_upgrade_status_key(&device_id.to_string(), &attempt.run_uid),
            "device-1.run-1"
        );
    }
 }
--- a/fleet/harmony-fleet-agent/src/updater.rs
+++ b/fleet/harmony-fleet-agent/src/updater.rs
--- a/fleet/harmony-fleet-agent/src/upgrade.rs
+++ b/fleet/harmony-fleet-agent/src/upgrade.rs
--- a/fleet/harmony-fleet-deploy/Cargo.toml
+++ b/fleet/harmony-fleet-deploy/Cargo.toml
@@ -20,6 +20,10 @@ path = "src/main.rs"
 name = "harmony-fleet-crds-deploy"
 path = "src/bin/harmony-fleet-crds-deploy.rs"
 [[bin]]
 name = "harmony-fleet-release"
 path = "src/bin/harmony-fleet-release.rs"
 [dependencies]
 harmony = { path = "../../harmony", features = ["podman"] }
 harmony_cli = { path = "../../harmony_cli" }
@@ -29,6 +33,7 @@ harmony_types = { path = "../../harmony_types" }
 harmony_macros = { path = "../../harmony_macros" }
 harmony-fleet-auth = { path = "../harmony-fleet-auth" }
 harmony-fleet-operator = { path = "../harmony-fleet-operator" }
 harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
 harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
 anyhow = { workspace = true }
@@ -40,6 +45,7 @@ kube = { workspace = true, features = ["runtime", "derive"] }
 log = { workspace = true }
 env_logger = { workspace = true }
 non-blank-string-rs = "1"
 oci-client.workspace = true
 inquire.workspace = true
 schemars = "0.8"
 serde = { workspace = true }
--- a/fleet/harmony-fleet-deploy/src/app.rs
+++ b/fleet/harmony-fleet-deploy/src/app.rs
@@ -22,11 +22,18 @@ const PROJECT: &str = "fleet";
 const ADMIN_ROLE: &str = "fleet-admin";
 const DEVICE_ROLE: &str = "device";
 const OPERATOR_APP: &str = "harmony-fleet-operator";
 const DASHBOARD_APP: &str = "harmony-fleet-dashboard";
 const OPERATOR_USER: &str = "fleet-operator";
 const NATS_ACCOUNT: &str = "FLEET";
 pub struct FleetApp;
 impl FleetApp {
    pub fn official_images(tag: &str) -> Vec<ImageSpec> {
        fleet_images(|name| format!("hub.nationtech.io/harmony/{name}:{tag}"))
    }
 }
 #[async_trait]
 impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
    fn identity(&self, ctx: &AppContext) -> AppIdentity {
@@ -37,22 +44,7 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
    }
    fn images(&self, ctx: &AppContext) -> Result<Vec<ImageSpec>, AppError> {
-        Ok(vec![
+        Ok(fleet_images(|name| ctx.image(name)))
            ImageSpec {
                name: "operator".to_string(),
                image: ctx.image("harmony-fleet-operator"),
                context: ".".into(),
                dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
                build_args: Vec::new(),
            },
            ImageSpec {
                name: "callout".to_string(),
                image: ctx.image("harmony-nats-callout"),
                context: ".".into(),
                dockerfile: "nats/callout/Dockerfile".into(),
                build_args: Vec::new(),
            },
        ])
    }
    async fn scores(
@@ -60,6 +52,12 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
        ctx: &AppContext,
        images: &ImageRefs,
    ) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
        let operator_image = images
            .get("operator")
            .map_or_else(|| ctx.image("harmony-fleet-operator"), str::to_owned);
        let callout_image = images
            .get("callout")
            .map_or_else(|| ctx.image("harmony-nats-callout"), str::to_owned);
        let namespace = ctx.namespace();
        let image_pull_secret = ctx.image_pull_secret();
@@ -73,16 +71,38 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
            zitadel = zitadel.http(Some(8080));
        }
        let provider = zitadel.provider_ref();
        let dashboard_host =
            (ctx.profile() == Profile::Prod).then(|| ctx.service_host("dashboard"));
        let identity = ZitadelSetupScore::for_provider(&provider, namespace, namespace)
            .application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode)
            .api_application(PROJECT, "nats")
            .role(PROJECT, ADMIN_ROLE, "Fleet Admin")
            .role(PROJECT, DEVICE_ROLE, "Device")
-            .machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE])
+            .machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE]);
-            .port_forward("zitadel")
+        let identity = if let Some(host) = &dashboard_host {
-            .groups_claim();
+            identity.application(
                PROJECT,
                DASHBOARD_APP,
                ZitadelAppType::WebPkce {
                    redirect_uris: vec![format!("https://{host}/auth/callback")],
                    post_logout_redirect_uris: vec![format!("https://{host}/")],
                    id_token_role_assertion: true,
                },
            )
        } else {
            identity
        };
        let identity = if ctx.profile() == Profile::Local {
            identity.port_forward("zitadel")
        } else {
            identity
        }
        .groups_claim();
        let application = identity.application_ref(OPERATOR_APP);
        let dashboard_application = dashboard_host
            .as_ref()
            .map(|_| identity.application_ref(DASHBOARD_APP));
        let operator_identity = identity.machine_identity_ref(OPERATOR_USER);
        let credentials =
@@ -97,26 +117,30 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
        // I feel like this should not be a standalone nats score but rather be configuration passed
        // to the main nats score that is installing the nats cluster
        let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT)
-            .with_jetstream_size("2Gi");
+            .with_jetstream_size("2Gi")
            .create_namespace(false);
        let nats = if ctx.profile() == Profile::Prod {
            nats.websocket(ctx.service_host("nats"), "letsencrypt-prod")
        } else {
            nats
        };
        let account = nats.account_ref();
-        let callout =
+        let mut callout =
            NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth")
                .credentials(&credentials.credentials_ref())
                .with_oidc(&provider, &application)
-                .image(images.require("callout")?)
+                .image(callout_image)
                .image_pull_secret(image_pull_secret.clone())
                .admin_role(ADMIN_ROLE)
                .device_role(DEVICE_ROLE)
                .device_id_claim("client_id");
        callout.device_id_prefix_strip = "device-".to_string();
        let nats = nats.with_auth_callout(&callout.auth_callout_ref());
-        let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao"));
+        let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao"))
            .create_namespace(false);
        if ctx.profile() == Profile::Prod {
            openbao.openshift = true;
            openbao = openbao.tls("letsencrypt-prod");
        }
        let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application(
@@ -124,12 +148,26 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
            &application,
            OpenbaoJwtAuth::oidc("fleet-device"),
        );
        let openbao_setup = if ctx.profile() == Profile::Prod {
            openbao_setup.endpoint(format!("https://{}", ctx.service_host("openbao")))
        } else {
            openbao_setup
        };
-        let operator = FleetOperatorScore::new(images.require("operator")?)
+        let operator = FleetOperatorScore::new(operator_image)
            .namespace(namespace)
            .image_pull_secret(image_pull_secret)
            .messaging(&nats.client_ref())
            .identity(&provider, &application, &operator_identity);
        let operator = if let Some((host, dashboard_application)) =
            dashboard_host.zip(dashboard_application)
        {
            operator
                .ingress(host, Some("letsencrypt-prod".to_string()))
                .web_auth(&dashboard_application)
        } else {
            operator
        };
        Ok(vec![
            Box::new(postgres),
@@ -145,6 +183,25 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetApp {
    }
 }
 fn fleet_images(image: impl Fn(&str) -> String) -> Vec<ImageSpec> {
    vec![
        ImageSpec {
            name: "operator".to_string(),
            image: image("harmony-fleet-operator"),
            context: ".".into(),
            dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(),
            build_args: Vec::new(),
        },
        ImageSpec {
            name: "callout".to_string(),
            image: image("harmony-nats-callout"),
            context: ".".into(),
            dockerfile: "nats/callout/Dockerfile".into(),
            build_args: Vec::new(),
        },
    ]
 }
 pub struct FleetCrdsApp;
 #[async_trait]
@@ -167,31 +224,35 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetCrdsApp {
 pub struct FleetTenantProvisionApp {
    tenant: TenantConfig,
-    credential_store: Arc<ConfigClient>,
+    credential_store: TenantCredentialStore,
    allow_insecure_source: bool,
 }
 enum TenantCredentialStore {
    Client(Arc<ConfigClient>),
    OpenBao(OpenBaoClusterAccess),
 }
 impl FleetTenantProvisionApp {
    pub fn new(tenant: TenantConfig, credential_store: Arc<ConfigClient>) -> Self {
        Self {
            tenant,
-            credential_store,
+            credential_store: TenantCredentialStore::Client(credential_store),
            allow_insecure_source: false,
        }
    }
-    pub async fn from_openbao(
+    pub fn from_openbao(tenant: TenantConfig, credential_store: OpenBaoClusterAccess) -> Self {
-        tenant: TenantConfig,
+        Self {
-        credential_store: &OpenBaoClusterAccess,
+            tenant,
-    ) -> Result<Self> {
+            credential_store: TenantCredentialStore::OpenBao(credential_store),
-        let source = harmony_config::openbao_source(
+            allow_insecure_source: false,
-            credential_store.namespace.as_ref(),
+        }
-            Some(credential_store.url.to_string()),
+    }
-            Some(credential_store.zitadel_url.to_string()),
+
-            Some(credential_store.zitadel_audience.to_string()),
+    pub fn allow_insecure_source(mut self) -> Self {
-            Some(credential_store.role.to_string()),
+        self.allow_insecure_source = true;
-        )
+        self
        .await
        .ok_or_else(|| anyhow::anyhow!("tenant credential store is unavailable"))?;
        Ok(Self::new(tenant, Arc::new(ConfigClient::new(vec![source]))))
    }
 }
@@ -209,6 +270,21 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetTenantProvisionApp {
        _ctx: &AppContext,
        _images: &ImageRefs,
    ) -> Result<Vec<Box<dyn Score<K8sAnywhereTopology>>>, AppError> {
        let credential_store = match &self.credential_store {
            TenantCredentialStore::Client(client) => client.clone(),
            TenantCredentialStore::OpenBao(store) => {
                let source = harmony_config::openbao_source(
                    store.namespace.as_ref(),
                    Some(store.url.to_string()),
                    Some(store.zitadel_url.to_string()),
                    Some(store.zitadel_audience.to_string()),
                    Some(store.role.to_string()),
                )
                .await
                .ok_or_else(|| AppError::Deploy("tenant credential store is unavailable".into()))?;
                Arc::new(ConfigClient::new(vec![source]))
            }
        };
        let namespace = self
            .tenant
            .name
@@ -224,7 +300,8 @@ impl HarmonyApp<K8sAnywhereTopology> for FleetTenantProvisionApp {
                    .parse()
                    .expect("static Kubernetes name is valid"),
                fleet_deployer_rules(),
-                self.credential_store.clone(),
+                credential_store,
                self.allow_insecure_source,
            )),
        ])
    }
@@ -301,6 +378,12 @@ fn fleet_deployer_rules() -> Vec<PolicyRule> {
            verbs: verbs(),
            ..Default::default()
        },
        PolicyRule {
            api_groups: Some(vec!["route.openshift.io".to_string()]),
            resources: Some(vec!["routes/custom-host".to_string()]),
            verbs: vec!["create".to_string()],
            ..Default::default()
        },
        PolicyRule {
            api_groups: Some(vec!["policy".to_string()]),
            resources: Some(vec!["poddisruptionbudgets".to_string()]),
@@ -335,15 +418,122 @@ fn fleet_deployer_rules() -> Vec<PolicyRule> {
            verbs: verbs(),
            ..Default::default()
        },
        PolicyRule {
            api_groups: Some(vec!["fleet.nationtech.io".to_string()]),
            resources: Some(vec!["taskruns".to_string()]),
            verbs: ["get", "list", "watch"].map(String::from).to_vec(),
            ..Default::default()
        },
        PolicyRule {
            api_groups: Some(vec!["fleet.nationtech.io".to_string()]),
            resources: Some(vec!["taskruns/status".to_string()]),
            verbs: ["get", "update", "patch"].map(String::from).to_vec(),
            ..Default::default()
        },
    ]
 }
 #[cfg(test)]
-mod tenant_tests {
+mod tests {
    use super::*;
    use harmony::topology::tenant::TenantNetworkPolicy;
    use harmony_types::id::Id;
    fn prod_context() -> harmony_app::Context {
        harmony_app::Context {
            name: "prod".parse().unwrap(),
            namespace: "fleet".parse().unwrap(),
            spec: harmony_app::ContextSpec::Remote(harmony_app::RemoteContext {
                registry: "hub.nationtech.io".parse().unwrap(),
                repository: "harmony".parse().unwrap(),
                domain: "fleet.example.com".parse().unwrap(),
                image_pull_secret: None,
                access: OpenBaoClusterAccess {
                    namespace: "customer/fleet".parse().unwrap(),
                    url: "https://secrets.example.com".parse().unwrap(),
                    role: "fleet-deployer".parse().unwrap(),
                    zitadel_url: "https://identity.example.com".parse().unwrap(),
                    zitadel_audience: "openbao".parse().unwrap(),
                },
            }),
        }
    }
    async fn serialized_fleet(context: harmony_app::Context, images: ImageRefs) -> String {
        let context = AppContext::load_metadata(&context, "test", None);
        FleetApp
            .scores(&context, &images)
            .await
            .unwrap()
            .iter()
            .map(|score| serde_json::to_string(&score.serialize()).unwrap())
            .collect()
    }
    #[test]
    fn official_images_share_the_release_tag() {
        let images = FleetApp::official_images("0.0.6");
        assert_eq!(images.len(), 2);
        assert!(images.iter().all(
            |image| image.image.starts_with("hub.nationtech.io/harmony/")
                && image.image.ends_with(":0.0.6")
        ));
    }
    #[tokio::test]
    async fn fleet_uses_supplied_images() {
        let serialized = serialized_fleet(
            prod_context(),
            ImageRefs::new([
                (
                    "operator".to_string(),
                    "registry.example/operator@sha256:operator".to_string(),
                ),
                (
                    "callout".to_string(),
                    "registry.example/callout@sha256:callout".to_string(),
                ),
            ]),
        )
        .await;
        assert!(serialized.contains("registry.example/operator@sha256:operator"));
        assert!(serialized.contains("registry.example/callout@sha256:callout"));
        assert!(!serialized.contains("\"port_forward_service\":\"zitadel\""));
        assert!(serialized.contains("https://openbao.fleet.example.com"));
        assert!(serialized.contains("dashboard.fleet.example.com"));
        assert!(serialized.contains("https://dashboard.fleet.example.com/auth/callback"));
        assert!(
            serialized
                .contains("\"app_name\":\"harmony-fleet-operator\",\"app_type\":\"DeviceCode\"")
        );
        assert!(serialized.contains("\"app_name\":\"harmony-fleet-dashboard\""));
        assert!(serialized.contains("zitadel-harmony-fleet-dashboard-oidc"));
    }
    #[tokio::test]
    async fn fleet_deploy_uses_the_context_release_tag() {
        let serialized = serialized_fleet(prod_context(), ImageRefs::default()).await;
        assert!(serialized.contains("hub.nationtech.io/harmony/harmony-fleet-operator:test"));
        assert!(serialized.contains("hub.nationtech.io/harmony/harmony-nats-callout:test"));
    }
    #[tokio::test]
    async fn local_fleet_uses_zitadel_port_forward() {
        let serialized = serialized_fleet(
            harmony_app::Context {
                name: "local".parse().unwrap(),
                namespace: "fleet".parse().unwrap(),
                spec: harmony_app::ContextSpec::Local(harmony_app::LocalContext::ManagedK3d),
            },
            ImageRefs::default(),
        )
        .await;
        assert!(serialized.contains("\"port_forward_service\":\"zitadel\""));
    }
    #[test]
    fn deployer_permissions_exclude_cluster_resources() {
        let rules = fleet_deployer_rules();
@@ -355,6 +545,17 @@ mod tenant_tests {
            rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()])
                && rule.resources.as_deref() == Some(&["routes".to_string()])
        }));
        assert!(rules.iter().any(|rule| {
            rule.api_groups.as_deref() == Some(&[String::new()])
                && rule.resources.as_deref()
                    == Some(&["pods/exec".to_string(), "pods/portforward".to_string()])
                && rule.verbs == ["create".to_string()]
        }));
        assert!(rules.iter().any(|rule| {
            rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()])
                && rule.resources.as_deref() == Some(&["routes/custom-host".to_string()])
                && rule.verbs == ["create".to_string()]
        }));
        assert!(rules.iter().all(|rule| {
            !rule.resources.as_ref().is_some_and(|resources| {
                resources
@@ -362,6 +563,16 @@ mod tenant_tests {
                    .any(|resource| resource == "customresourcedefinitions")
            })
        }));
        assert!(rules.iter().any(|rule| {
            rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()])
                && rule.resources.as_deref() == Some(&["taskruns".to_string()])
                && rule.verbs == ["get", "list", "watch"].map(String::from)
        }));
        assert!(rules.iter().any(|rule| {
            rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()])
                && rule.resources.as_deref() == Some(&["taskruns/status".to_string()])
                && rule.verbs == ["get", "update", "patch"].map(String::from)
        }));
    }
    #[tokio::test]
--- a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-example-deployment.rs
+++ b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-example-deployment.rs
@@ -0,0 +1,83 @@
 use clap::Parser;
 use harmony::{inventory::Inventory, topology::K8sAnywhereTopology};
 use harmony_fleet_deploy::FleetDeploymentScore;
 use harmony_fleet_operator::{Deployment, DeploymentSpec, Rollout, RolloutStrategy};
 use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileScore};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
 use kube::{Api, Client, api::DeleteParams};
 #[derive(Parser)]
 struct Args {
    #[arg(long, default_value = "fleet-demo")]
    namespace: String,
    #[arg(long, default_value = "hello-world")]
    name: String,
    #[arg(long)]
    target_device: String,
    #[arg(long = "allowed-group", default_value = "application")]
    allowed_groups: Vec<String>,
    #[arg(long, default_value = "docker.io/library/nginx:latest")]
    image: String,
    #[arg(long, default_value = "8080:80")]
    port: String,
    #[arg(long)]
    delete: bool,
 }
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
    let args = Args::parse();
    if args.delete {
        let deployments: Api<Deployment> =
            Api::namespaced(Client::try_default().await?, &args.namespace);
        deployments
            .delete(&args.name, &DeleteParams::default())
            .await?;
        return Ok(());
    }
    let deployment = Deployment::new(
        &args.name,
        DeploymentSpec {
            allowed_groups: args.allowed_groups,
            target_selector: LabelSelector {
                match_labels: Some([("device-id".into(), args.target_device)].into()),
                match_expressions: None,
            },
            score: ReconcileScore::PodmanV0(PodmanV0Score {
                init_container: None,
                services: vec![PodmanService {
                    name: args.name.clone(),
                    image: args.image,
                    image_pull_secret: None,
                    ports: vec![args.port],
                    env: Vec::new(),
                    secret_env: Vec::new(),
                    volumes: Vec::new(),
                    restart_policy: Default::default(),
                }],
            }),
            rollout: Rollout {
                strategy: RolloutStrategy::Immediate,
            },
        },
    );
    harmony_cli::run(
        Inventory::autoload(),
        K8sAnywhereTopology::from_env(),
        vec![Box::new(FleetDeploymentScore::new(
            deployment,
            args.namespace,
        ))],
        Some(harmony_cli::Args {
            yes: true,
            filter: None,
            interactive: false,
            all: true,
            number: 0,
            list: false,
        }),
    )
    .await
    .map_err(|error| anyhow::anyhow!(error.to_string()))
 }
--- a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs
+++ b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs
@@ -0,0 +1,194 @@
 use std::collections::BTreeMap;
 use std::path::PathBuf;
 use anyhow::{Context, bail};
 use clap::{Parser, Subcommand};
 use harmony_app::{PublicationTopology, publish::build_images};
 use harmony_fleet_deploy::FleetApp;
 use harmony_reconciler_contracts::upgrade::{AGENT_OCI_ARTIFACT_TYPE, AGENT_OCI_LAYER_MEDIA_TYPE};
 use oci_client::annotations::ORG_OPENCONTAINERS_IMAGE_VERSION;
 use oci_client::client::{Config, ImageLayer};
 use oci_client::errors::{OciDistributionError, OciErrorCode};
 use oci_client::manifest::OciImageManifest;
 use oci_client::secrets::RegistryAuth;
 use oci_client::{Client, Reference};
 #[derive(Parser)]
 struct Args {
    #[command(subcommand)]
    release: Release,
 }
 #[derive(Subcommand)]
 enum Release {
    Control {
        #[arg(long)]
        version: String,
        #[arg(long)]
        push: bool,
    },
    Agent {
        #[arg(long)]
        binary: PathBuf,
        #[arg(long)]
        reference: String,
        #[arg(long)]
        version: String,
    },
 }
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
    harmony_cli::cli_logger::init();
    match Args::parse().release {
        Release::Control { version, push } => publish_control(&version, push).await,
        Release::Agent {
            binary,
            reference,
            version,
        } => publish_agent(binary, &reference, &version).await,
    }
 }
 async fn publish_control(version: &str, push: bool) -> anyhow::Result<()> {
    let images = FleetApp::official_images(version);
    let registry = PublicationTopology::Registry {
        registry: "hub.nationtech.io".to_string(),
    };
    if push {
        let client = Client::default();
        let auth = registry_auth()?;
        let mut existing = Vec::new();
        for image in &images {
            let reference = Reference::try_from(image.image.as_str())?;
            match client.pull_manifest(&reference, &auth).await {
                Ok((_, digest)) => existing.push((
                    image.name.clone(),
                    format!(
                        "{}/{}@{digest}",
                        reference.registry(),
                        reference.repository()
                    ),
                )),
                Err(error) if manifest_is_missing(&error) => {}
                Err(error) => return Err(error).context("checking control-plane release"),
            }
        }
        if existing.len() == images.len() {
            for (name, image) in existing {
                println!("{name}={image}");
            }
            return Ok(());
        }
        if !existing.is_empty() {
            bail!("control-plane release is incomplete; refusing to overwrite existing tags");
        }
    }
    let refs = build_images(&images, &registry)?;
    let refs = if push {
        let credentials = registry_credentials()?;
        harmony_app::publish::publish_images(&images, &refs, &registry, Some(&credentials))?
    } else {
        refs
    };
    for (name, image) in refs.iter() {
        println!("{name}={image}");
    }
    Ok(())
 }
 async fn publish_agent(binary: PathBuf, reference: &str, version: &str) -> anyhow::Result<()> {
    let reference = Reference::try_from(reference).context("invalid agent OCI reference")?;
    if reference.tag().is_none() || reference.digest().is_some() {
        bail!("agent publication requires a tag reference");
    }
    let auth = registry_auth()?;
    let client = Client::default();
    let layer = ImageLayer::new(
        std::fs::read(&binary)
            .with_context(|| format!("reading agent binary {}", binary.display()))?,
        AGENT_OCI_LAYER_MEDIA_TYPE.to_string(),
        None,
    );
    match client.pull_manifest(&reference, &auth).await {
        Ok((oci_client::manifest::OciManifest::Image(manifest), digest))
            if manifest.artifact_type.as_deref() == Some(AGENT_OCI_ARTIFACT_TYPE)
                && manifest
                    .annotations
                    .as_ref()
                    .and_then(|annotations| annotations.get(ORG_OPENCONTAINERS_IMAGE_VERSION))
                    .map(String::as_str)
                    == Some(version)
                && matches!(manifest.layers.as_slice(), [existing]
                    if existing.media_type == AGENT_OCI_LAYER_MEDIA_TYPE
                        && existing.digest == layer.sha256_digest()) =>
        {
            println!(
                "agent=oci://{}/{}@{digest}",
                reference.registry(),
                reference.repository()
            );
            return Ok(());
        }
        Ok(_) => bail!(
            "agent artifact tag exists with different content: {}",
            reference.whole()
        ),
        Err(error) if manifest_is_missing(&error) => {}
        Err(error) => return Err(error).context("checking agent artifact tag"),
    }
    let config = Config::new(
        b"{}".to_vec(),
        "application/vnd.oci.empty.v1+json".to_string(),
        None,
    );
    let mut annotations = BTreeMap::new();
    annotations.insert(
        ORG_OPENCONTAINERS_IMAGE_VERSION.to_string(),
        version.to_string(),
    );
    let mut manifest =
        OciImageManifest::build(std::slice::from_ref(&layer), &config, Some(annotations));
    manifest.artifact_type = Some(AGENT_OCI_ARTIFACT_TYPE.to_string());
    client
        .push(&reference, &[layer], config, &auth, Some(manifest))
        .await
        .context("publishing agent OCI artifact")?;
    let (_, digest) = client
        .pull_manifest(&reference, &auth)
        .await
        .context("resolving published agent manifest")?;
    println!(
        "agent=oci://{}/{}@{digest}",
        reference.registry(),
        reference.repository()
    );
    Ok(())
 }
 fn registry_auth() -> anyhow::Result<RegistryAuth> {
    let credentials = registry_credentials()?;
    Ok(RegistryAuth::Basic(credentials.username, credentials.token))
 }
 fn registry_credentials() -> anyhow::Result<harmony_app::RegistryCredentials> {
    Ok(harmony_app::RegistryCredentials {
        username: std::env::var("REGISTRY_USER").context("REGISTRY_USER is required")?,
        token: std::env::var("REGISTRY_TOKEN").context("REGISTRY_TOKEN is required")?,
    })
 }
 fn manifest_is_missing(error: &OciDistributionError) -> bool {
    matches!(error, OciDistributionError::ImageManifestNotFoundError(_))
        || matches!(
            error,
            OciDistributionError::RegistryError { envelope, .. }
                if envelope.errors.iter().any(|error| matches!(
                    &error.code,
                    OciErrorCode::ManifestUnknown
                        | OciErrorCode::NameUnknown
                        | OciErrorCode::NotFound
                ))
        )
 }
--- a/fleet/harmony-fleet-deploy/src/deployment.rs
+++ b/fleet/harmony-fleet-deploy/src/deployment.rs
@@ -0,0 +1,210 @@
 use std::time::Duration;
 use async_trait::async_trait;
 use harmony::data::Version;
 use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
 use harmony::inventory::Inventory;
 use harmony::score::Score;
 use harmony::topology::{K8sclient, Topology};
 use harmony_fleet_operator::{Deployment, DeploymentStatus};
 use harmony_types::id::Id;
 use kube::ResourceExt;
 use serde::Serialize;
 const ROLLOUT_TIMEOUT: Duration = Duration::from_secs(900);
 #[derive(Debug, Clone, Serialize)]
 pub struct FleetDeploymentScore {
    pub deployment: Deployment,
    pub namespace: String,
 }
 impl FleetDeploymentScore {
    pub fn new(deployment: Deployment, namespace: impl Into<String>) -> Self {
        Self {
            deployment,
            namespace: namespace.into(),
        }
    }
 }
 impl<T: Topology + K8sclient> Score<T> for FleetDeploymentScore {
    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
        Box::new(FleetDeploymentInterpret(self.clone()))
    }
    fn name(&self) -> String {
        format!("{} [FleetDeploymentScore]", self.deployment.name_any())
    }
 }
 #[derive(Debug)]
 struct FleetDeploymentInterpret(FleetDeploymentScore);
 #[async_trait]
 impl<T: Topology + K8sclient> Interpret<T> for FleetDeploymentInterpret {
    async fn execute(
        &self,
        _inventory: &Inventory,
        topology: &T,
    ) -> Result<Outcome, InterpretError> {
        let client = topology.k8s_client().await.map_err(InterpretError::new)?;
        let applied = client
            .apply(&self.0.deployment, Some(&self.0.namespace))
            .await
            .map_err(|error| InterpretError::new(format!("apply Fleet Deployment: {error}")))?;
        let revision = applied.rollout_revision().ok_or_else(|| {
            InterpretError::new("applied Fleet Deployment has no UID or generation".to_string())
        })?;
        let name = applied.name_any();
        let mut last_observation = "operator has not reported rollout status".to_string();
        let completed = tokio::time::timeout(ROLLOUT_TIMEOUT, async {
            loop {
                let current = client
                    .get_resource::<Deployment>(&name, Some(&self.0.namespace))
                    .await
                    .map_err(|error| {
                        InterpretError::new(format!("read Fleet Deployment status: {error}"))
                    })?
                    .ok_or_else(|| {
                        InterpretError::new("Fleet Deployment disappeared".to_string())
                    })?;
                if let Some(aggregate) = current
                    .status
                    .as_ref()
                    .filter(|status| status.rollout_revision.as_deref() == Some(&revision))
                    .and_then(|status| status.aggregate.as_ref())
                {
                    last_observation = rollout_observation(aggregate);
                }
                match rollout_result(current.status.as_ref(), &revision) {
                    Ok(Some(message)) => return Ok(message),
                    Ok(None) => {}
                    Err(error) => return Err(InterpretError::new(error)),
                }
                tokio::time::sleep(Duration::from_secs(1)).await;
            }
        })
        .await
        .map_err(|_| {
            InterpretError::new(format!(
                "Fleet Deployment {}/{} did not complete within 15 minutes: {last_observation}",
                self.0.namespace, name
            ))
        })??;
        Ok(Outcome::success(completed))
    }
    fn get_name(&self) -> InterpretName {
        InterpretName::Custom("FleetDeploymentInterpret")
    }
    fn get_version(&self) -> Version {
        Version::from(env!("CARGO_PKG_VERSION")).expect("package version")
    }
    fn get_status(&self) -> InterpretStatus {
        InterpretStatus::QUEUED
    }
    fn get_children(&self) -> Vec<Id> {
        Vec::new()
    }
 }
 fn rollout_result(
    status: Option<&DeploymentStatus>,
    revision: &str,
 ) -> Result<Option<String>, String> {
    let Some(status) = status.filter(|status| status.rollout_revision.as_deref() == Some(revision))
    else {
        return Ok(None);
    };
    let Some(aggregate) = &status.aggregate else {
        return Ok(None);
    };
    if aggregate.matched_device_count == 0 {
        return Err("rollout matched no devices".to_string());
    }
    if aggregate.failed > 0 {
        let mut message = format!("rollout failed: {}", rollout_observation(aggregate));
        if let Some(error) = &aggregate.last_error {
            message.push_str(&format!(
                "; last error from {} at {}: {}",
                error.device_id, error.at, error.message
            ));
        }
        return Err(message);
    }
    if aggregate.pending > 0 {
        return Ok(None);
    }
    Ok(Some(format!(
        "Fleet rollout complete: {} target(s), {} succeeded",
        aggregate.matched_device_count, aggregate.succeeded
    )))
 }
 fn rollout_observation(aggregate: &harmony_fleet_operator::DeploymentAggregate) -> String {
    format!(
        "{} succeeded, {} pending, {} failed across {} target(s)",
        aggregate.succeeded, aggregate.pending, aggregate.failed, aggregate.matched_device_count,
    )
 }
 #[cfg(test)]
 mod tests {
    use harmony_fleet_operator::{AggregateLastError, DeploymentAggregate};
    use super::*;
    fn status(revision: &str, matched: u32, failed: u32, pending: u32) -> DeploymentStatus {
        DeploymentStatus {
            rollout_revision: Some(revision.to_string()),
            aggregate: Some(DeploymentAggregate {
                matched_device_count: matched,
                succeeded: matched - failed - pending,
                failed,
                pending,
                last_error: (failed > 0).then(|| AggregateLastError {
                    device_id: "device-1".into(),
                    message: "container failed".into(),
                    at: "2026-01-01T00:00:00Z".into(),
                }),
            }),
        }
    }
    #[test]
    fn rollout_requires_the_exact_revision() {
        assert_eq!(
            rollout_result(Some(&status("uid:1", 1, 0, 0)), "uid:2"),
            Ok(None)
        );
    }
    #[test]
    fn zero_targets_fail_the_waited_deployment() {
        assert_eq!(
            rollout_result(Some(&status("uid:1", 0, 0, 0)), "uid:1"),
            Err("rollout matched no devices".into())
        );
    }
    #[test]
    fn rollout_failure_reports_last_error() {
        assert_eq!(
            rollout_result(Some(&status("uid:1", 1, 1, 0)), "uid:1"),
            Err("rollout failed: 0 succeeded, 0 pending, 1 failed across 1 target(s); last error from device-1 at 2026-01-01T00:00:00Z: container failed".into())
        );
    }
    #[test]
    fn rollout_observation_preserves_current_counts() {
        let status = status("uid:1", 5, 0, 2);
        assert_eq!(
            rollout_observation(status.aggregate.as_ref().unwrap()),
            "3 succeeded, 2 pending, 0 failed across 5 target(s)"
        );
    }
 }
--- a/fleet/harmony-fleet-deploy/src/device_setup.rs
+++ b/fleet/harmony-fleet-deploy/src/device_setup.rs
@@ -290,18 +290,48 @@ impl FleetDeviceSetupConfig {
 Description=IoT Agent (Harmony)
 After=network-online.target
 Wants=network-online.target
 Requires=harmony-fleet-updater.service
 After=harmony-fleet-updater.service
 [Service]
-Type=simple
+Type=notify
 NotifyAccess=main
 User=fleet-agent
 RuntimeDirectory=harmony-fleet-agent
 RuntimeDirectoryMode=0700
 Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml
 Environment=RUST_LOG=info
 ExecStart=/usr/local/bin/fleet-agent
 TimeoutStartSec=4min
 TimeoutStopSec=60s
 Restart=on-failure
 RestartSec=5
 StandardOutput=journal
 StandardError=journal
 [Install]
 WantedBy=multi-user.target
 "#
    }
    pub fn render_updater_systemd_unit(&self) -> &'static str {
        r#"[Unit]
 Description=Harmony Fleet Agent Updater
 Before=fleet-agent.service
 [Service]
 Type=notify
 NotifyAccess=main
 User=root
 Group=fleet-agent
 RuntimeDirectory=harmony-fleet-updater
 RuntimeDirectoryMode=0750
 StateDirectory=harmony-fleet-updater
 StateDirectoryMode=0700
 ExecStart=/usr/lib/harmony-fleet/fleet-agent-bootstrap --updater
 Restart=on-failure
 RestartSec=5
 [Install]
 WantedBy=multi-user.target
 "#
@@ -428,11 +458,21 @@ async fn resolve_zitadel_enroll(
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct FleetDeviceSetupScore {
    pub config: FleetDeviceSetupConfig,
    #[serde(skip)]
    overwrite_existing_config: bool,
 }
 impl FleetDeviceSetupScore {
    pub fn new(config: FleetDeviceSetupConfig) -> Self {
-        Self { config }
+        Self {
            config,
            overwrite_existing_config: false,
        }
    }
    pub fn overwrite_existing_config(mut self, overwrite: bool) -> Self {
        self.overwrite_existing_config = overwrite;
        self
    }
 }
@@ -444,6 +484,7 @@ impl<T: Topology + LinuxHostConfiguration> Score<T> for FleetDeviceSetupScore {
    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
        Box::new(FleetDeviceSetupInterpret {
            config: self.config.clone(),
            overwrite_existing_config: self.overwrite_existing_config,
            version: Version::from("0.1.0").expect("static version"),
            status: InterpretStatus::QUEUED,
        })
@@ -453,6 +494,7 @@ impl<T: Topology + LinuxHostConfiguration> Score<T> for FleetDeviceSetupScore {
 #[derive(Debug)]
 struct FleetDeviceSetupInterpret {
    config: FleetDeviceSetupConfig,
    overwrite_existing_config: bool,
    version: Version,
    status: InterpretStatus,
 }
@@ -543,13 +585,14 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
                        }
                    }
                }
-                let confirmed = inquire::Confirm::new(
+                let confirmed = self.overwrite_existing_config
-                    "Device already has /etc/fleet-agent/config.toml with different content.\n  \
+                    || inquire::Confirm::new(
-                     Overwrite it and apply the new config?",
+                        "Device already has /etc/fleet-agent/config.toml with different content.\n  \
-                )
+                         Overwrite it and apply the new config?",
-                .with_default(false)
+                    )
-                .prompt()
+                    .with_default(false)
-                .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
+                    .prompt()
                    .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
                if !confirmed {
                    return Err(InterpretError::new(
                        "User aborted: refused to overwrite existing config".to_string(),
@@ -596,8 +639,10 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
        }
        // 1. Dependencies.
-        info!("[{tag}] Step 2/7 — ensuring system packages: podman, systemd-container");
+        info!(
-        for pkg in ["podman", "systemd-container"] {
+            "[{tag}] Step 2/7 — ensuring system packages: podman, systemd-container, systemd-timesyncd"
        );
        for pkg in ["podman", "systemd-container", "systemd-timesyncd"] {
            let r = PackageInstaller::ensure_package(topology, pkg)
                .await
                .map_err(wrap)?;
@@ -605,6 +650,12 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
                change_count += 1;
            }
        }
        let r = SystemdManager::ensure_system_unit_active(topology, "systemd-timesyncd")
            .await
            .map_err(wrap)?;
        if r.changed {
            change_count += 1;
        }
        // 2. fleet-agent user. Not `--system`: Ubuntu's useradd skips
        // subuid/subgid auto-allocation for system users on the
@@ -651,19 +702,20 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
            change_count += 1;
        }
-        // 4. Binary. Ship via ansible's native copy-from-local-file
+        // 4. Bootstrap binary. The root updater owns the active symlink;
        // normal agent upgrades never replace this privileged helper.
        // path (`FileSource::LocalPath`). Ansible handles binary
        // content over SFTP and reports `changed: true` only when the
        // remote file actually differs from the local one — so
        // re-running this Score without a new binary is a true NOOP.
        info!(
-            "[{tag}] Step 5/7 — uploading agent binary {} -> /usr/local/bin/fleet-agent",
+            "[{tag}] Step 5/8 — uploading agent bootstrap {}",
            cfg.agent_binary_path.display()
        );
        let binary_r = FileDelivery::ensure_file(
            topology,
            &FileSpec {
-                path: "/usr/local/bin/fleet-agent".to_string(),
+                path: "/usr/lib/harmony-fleet/fleet-agent-bootstrap".to_string(),
                source: FileSource::LocalPath(cfg.agent_binary_path.clone()),
                owner: Some("root".to_string()),
                group: Some("root".to_string()),
@@ -729,8 +781,33 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
            change_count += 1;
        }
-        // 6. systemd unit for the agent itself.
+        // 6. Root updater must be active before the agent. On first
-        info!("[{tag}] Step 7/7 — installing fleet-agent.service");
+        // install it creates /usr/local/bin/fleet-agent as an atomic
        // symlink to the bootstrap binary.
        info!("[{tag}] Step 7/8 — installing harmony-fleet-updater.service");
        let updater_unit = SystemdUnitSpec {
            name: "harmony-fleet-updater".to_string(),
            unit_content: cfg.render_updater_systemd_unit().to_string(),
            scope: SystemdScope::System,
            start_immediately: true,
        };
        let updater_unit_r = SystemdManager::ensure_systemd_unit(topology, &updater_unit)
            .await
            .map_err(wrap)?;
        if updater_unit_r.changed {
            change_count += 1;
        }
        if binary_r.changed || updater_unit_r.changed {
            SystemdManager::restart_service(
                topology,
                "harmony-fleet-updater",
                SystemdScope::System,
            )
            .await
            .map_err(wrap)?;
        }
        info!("[{tag}] Step 8/8 — installing fleet-agent.service");
        let unit = SystemdUnitSpec {
            name: "fleet-agent".to_string(),
            unit_content: cfg.render_systemd_unit().to_string(),
@@ -745,7 +822,8 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInte
        }
        // 7. Restart the agent iff anything that affects it changed.
-        let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed || key_r;
+        let needs_restart =
            toml_r.changed || unit_r.changed || updater_unit_r.changed || binary_r.changed || key_r;
        let service_state = if needs_restart {
            info!("[{tag}] 🔄 Restarting fleet-agent (config/binary/unit changed)");
            SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System)
@@ -1109,4 +1187,23 @@ mod tests {
        let toml = cfg.render_toml();
        assert!(toml.contains("danger_accept_invalid_certs = true"));
    }
    #[test]
    fn systemd_service_is_ready_only_after_agent_notification() {
        let config = base_config(BTreeMap::new());
        let unit = config.render_systemd_unit();
        assert!(unit.contains("Type=notify\n"));
        assert!(unit.contains("NotifyAccess=main\n"));
        assert!(unit.contains("TimeoutStartSec=4min\n"));
        assert!(unit.contains("TimeoutStopSec=60s\n"));
        assert!(unit.contains("Requires=harmony-fleet-updater.service\n"));
        assert!(unit.contains("RuntimeDirectory=harmony-fleet-agent\n"));
        let updater = config.render_updater_systemd_unit();
        assert!(updater.contains("User=root\n"));
        assert!(updater.contains("Type=notify\n"));
        assert!(updater.contains("Group=fleet-agent\n"));
        assert!(updater.contains("RuntimeDirectoryMode=0750\n"));
        assert!(updater.contains("fleet-agent-bootstrap --updater\n"));
    }
 }
--- a/fleet/harmony-fleet-deploy/src/lib.rs
+++ b/fleet/harmony-fleet-deploy/src/lib.rs
@@ -12,11 +12,13 @@
 pub mod agent;
 mod app;
 mod deployment;
 mod device_setup;
 pub mod operator;
 pub use agent::{FleetAgentScore, PodTarget};
 pub use app::{FleetApp, FleetCrdsApp, FleetTenantProvisionApp};
 pub use deployment::FleetDeploymentScore;
 pub use device_setup::{
    AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
    HostsEntry, merge_hosts_file,
@@ -33,12 +35,44 @@ pub async fn deploy_fleet_crds_with_context(context: harmony_app::Context) -> an
    harmony_cli::app::app_main(FleetCrdsApp, contexts).await
 }
 pub async fn deploy_fleet_crds_with_kubeconfig(
    context: harmony_app::Context,
    kubeconfig: std::path::PathBuf,
 ) -> anyhow::Result<()> {
    deploy_with_kubeconfig(&FleetCrdsApp, context, kubeconfig).await
 }
 pub async fn provision_fleet_tenant_with_context(
    context: harmony_app::Context,
    tenant: harmony::topology::tenant::TenantConfig,
    credential_store: harmony_app::OpenBaoClusterAccess,
 ) -> anyhow::Result<()> {
-    let app = FleetTenantProvisionApp::from_openbao(tenant, &credential_store).await?;
+    let app = FleetTenantProvisionApp::from_openbao(tenant, credential_store);
    let contexts = harmony_app::ContextCatalog::new([context])?;
    harmony_cli::app::app_main(app, contexts).await
 }
 pub async fn provision_fleet_tenant_with_kubeconfig(
    context: harmony_app::Context,
    kubeconfig: std::path::PathBuf,
    tenant: harmony::topology::tenant::TenantConfig,
    credential_store: harmony_app::OpenBaoClusterAccess,
    allow_insecure_source: bool,
 ) -> anyhow::Result<()> {
    let mut app = FleetTenantProvisionApp::from_openbao(tenant, credential_store);
    if allow_insecure_source {
        app = app.allow_insecure_source();
    }
    deploy_with_kubeconfig(&app, context, kubeconfig).await
 }
 async fn deploy_with_kubeconfig(
    app: &dyn harmony_app::HarmonyApp<harmony::topology::K8sAnywhereTopology>,
    context: harmony_app::Context,
    kubeconfig: std::path::PathBuf,
 ) -> anyhow::Result<()> {
    harmony_cli::cli_logger::init();
    let ctx = harmony_app::AppContext::from_kubeconfig(&context, "bootstrap", kubeconfig)?;
    harmony_app::deploy(app, ctx.topology(), &ctx).await?;
    Ok(())
 }
--- a/fleet/harmony-fleet-deploy/src/operator/chart.rs
+++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs
@@ -67,8 +67,7 @@ pub struct ChartOptions {
    /// at `…/harmony-fleet-operator-chart:<tag>` matching the image tag.
    pub chart_version: Option<String>,
    /// JSON of the dashboard's `ZitadelAuthConfig`, stored in the
-    /// operator Secret under [`ENV_WEB_AUTH_CONFIG`]. `None` leaves the
+    /// operator Secret under [`ENV_WEB_AUTH_CONFIG`].
    /// dashboard unauthenticated (dev/e2e).
    pub web_auth_config_json: Option<String>,
    /// JSON of the dashboard's `OperatorCookieKey`, stored under
    /// [`ENV_WEB_COOKIE_KEY`].
@@ -76,6 +75,7 @@ pub struct ChartOptions {
    pub identity: Option<OperatorIdentityRefs>,
    pub identity_version: Option<String>,
    pub image_pull_secret: Option<K8sName>,
    pub device_groups: Option<String>,
 }
 #[derive(Debug, Clone, Serialize)]
@@ -110,6 +110,7 @@ impl Default for ChartOptions {
            identity: None,
            identity_version: None,
            image_pull_secret: None,
            device_groups: None,
        }
    }
 }
@@ -183,12 +184,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
 /// (with the JSON keyfile inlined under `key_json`). Returns `None`
 /// when no credentials are configured (no-auth dev mode).
 pub fn operator_secret(opts: &ChartOptions) -> Option<Secret> {
-    let creds = opts.credentials.as_ref()?;
+    if opts.credentials.is_none()
        && opts.web_auth_config_json.is_none()
        && opts.web_cookie_key_json.is_none()
    {
        return None;
    }
    let mut data: BTreeMap<String, ByteString> = BTreeMap::new();
-    data.insert(
+    if let Some(creds) = &opts.credentials {
-        SECRET_KEY_CREDENTIALS_TOML.to_string(),
+        data.insert(
-        ByteString(creds.credentials_toml.as_bytes().to_vec()),
+            SECRET_KEY_CREDENTIALS_TOML.to_string(),
-    );
+            ByteString(creds.credentials_toml.as_bytes().to_vec()),
        );
    }
    // Dashboard auth config + cookie key (when configured) ride in the
    // same Secret; the Deployment sources them as HARMONY_CONFIG_* env
    // for the operator's ConfigClient.
@@ -282,7 +290,7 @@ fn role() -> Role {
            // Device liveness: the device-status reconciler patches the
            // status subresource — a distinct RBAC resource from `devices`.
            PolicyRule {
-                api_groups: Some(vec![group]),
+                api_groups: Some(vec![group.clone()]),
                resources: Some(vec!["devices/status".to_string()]),
                verbs: vec!["get", "update", "patch"]
                    .into_iter()
@@ -290,6 +298,24 @@ fn role() -> Role {
                    .collect(),
                ..Default::default()
            },
            PolicyRule {
                api_groups: Some(vec![group.clone()]),
                resources: Some(vec!["taskruns".to_string()]),
                verbs: vec!["get", "list", "watch", "patch"]
                    .into_iter()
                    .map(String::from)
                    .collect(),
                ..Default::default()
            },
            PolicyRule {
                api_groups: Some(vec![group]),
                resources: Some(vec!["taskruns/status".to_string()]),
                verbs: vec!["get", "update", "patch"]
                    .into_iter()
                    .map(String::from)
                    .collect(),
                ..Default::default()
            },
        ]),
    }
 }
@@ -456,8 +482,7 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
    env.push(secret_env(ENV_WEB_AUTH_CONFIG));
    env.push(secret_env(ENV_WEB_COOKIE_KEY));
    // Secret-grant sync (OpenBao) + the device-group scheduling gate
-    // (Zitadel role grants) — ADR-025. All optional: absent, the
+    // (Zitadel role grants) — ADR-025. Missing group data fails closed.
    // operator logs and runs ungated/without grant sync.
    for name in [
        "OPENBAO_URL",
        "OPENBAO_TOKEN",
@@ -467,6 +492,13 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
    ] {
        env.push(secret_env(name));
    }
    if let Some(groups) = &opts.device_groups {
        env.push(EnvVar {
            name: "FLEET_DEVICE_GROUPS".to_string(),
            value: Some(groups.clone()),
            ..Default::default()
        });
    }
    // Namespace deliberately omitted — same rationale as the
    // ServiceAccount: helm fills in the release namespace at install
@@ -641,6 +673,27 @@ mod tests {
        };
        assert!(grants_patch("deployments/status"));
        assert!(grants_patch("devices/status"));
        assert!(grants_patch("taskruns/status"));
    }
    #[test]
    fn role_grants_only_required_taskrun_access() {
        let rules = role().rules.unwrap();
        assert!(rules.iter().any(|rule| {
            rule.resources.as_deref() == Some(&["taskruns".to_string()])
                && rule.verbs == ["get", "list", "watch", "patch"].map(String::from)
        }));
        assert!(rules.iter().any(|rule| {
            rule.resources.as_deref() == Some(&["taskruns/status".to_string()])
                && rule.verbs == ["get", "update", "patch"].map(String::from)
        }));
        assert!(rules.iter().all(|rule| {
            !rule.resources.as_ref().is_some_and(|resources| {
                resources.iter().any(|resource| {
                    resource.contains("schedule") || resource == "taskruns/finalizers"
                })
            })
        }));
    }
    #[test]
@@ -699,6 +752,32 @@ mod tests {
        );
    }
    #[test]
    fn deployment_injects_static_device_groups() {
        let deployment = operator_deployment(&ChartOptions {
            device_groups: Some("pi-01=edge-a".to_string()),
            ..Default::default()
        });
        let env = deployment
            .spec
            .unwrap()
            .template
            .spec
            .unwrap()
            .containers
            .into_iter()
            .next()
            .unwrap()
            .env
            .unwrap();
        assert_eq!(
            env.iter()
                .find(|env| env.name == "FLEET_DEVICE_GROUPS")
                .and_then(|env| env.value.as_deref()),
            Some("pi-01=edge-a")
        );
    }
    // The chart bakes these env names at publish time; the operator's
    // ConfigClient derives them from the struct names at runtime. Lock
    // them together so a rename can't silently desync the two.
@@ -715,4 +794,19 @@ mod tests {
            format!("HARMONY_CONFIG_{}", OperatorCookieKey::KEY)
        );
    }
    #[test]
    fn web_auth_does_not_require_static_nats_credentials() {
        let secret = operator_secret(&ChartOptions {
            web_auth_config_json: Some("auth".to_string()),
            web_cookie_key_json: Some("cookie".to_string()),
            ..Default::default()
        })
        .expect("web auth requires an operator Secret");
        let data = secret.data.unwrap();
        assert_eq!(data[ENV_WEB_AUTH_CONFIG].0, b"auth");
        assert_eq!(data[ENV_WEB_COOKIE_KEY].0, b"cookie");
        assert!(!data.contains_key(SECRET_KEY_CREDENTIALS_TOML));
    }
 }
--- a/fleet/harmony-fleet-deploy/src/operator/score.rs
+++ b/fleet/harmony-fleet-deploy/src/operator/score.rs
@@ -41,7 +41,7 @@ use harmony::modules::nats::NatsClientRef;
 use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef};
 use harmony::score::Score;
 use harmony::topology::{HelmCommand, K8sclient, Topology};
-use harmony_fleet_operator::{Deployment, Device};
+use harmony_fleet_operator::{Deployment, Device, TaskRun};
 use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret};
 use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use kube::CustomResourceExt;
@@ -56,7 +56,7 @@ use crate::operator::chart::{
 pub struct FleetCrdsScore;
 fn fleet_crds() -> Vec<CustomResourceDefinition> {
-    vec![Deployment::crd(), Device::crd()]
+    vec![Deployment::crd(), Device::crd(), TaskRun::crd()]
 }
 impl<T: Topology + K8sclient> Score<T> for FleetCrdsScore {
@@ -163,19 +163,12 @@ pub struct FleetOperatorScore {
    /// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no
    /// host) serves plain HTTP — the right default on issuer-less k3d.
    pub cluster_issuer: Option<String>,
-    /// Dashboard SSO config + cookie key, baked into the operator Secret
+    /// Dashboard Web PKCE application used to derive SSO configuration.
-    /// for the pod's `ConfigClient` to read. `None` leaves the dashboard
+    pub web_auth: Option<OidcApplicationRef>,
    /// unauthenticated (dev/e2e).
    pub web_auth: Option<WebAuth>,
    pub identity: Option<OperatorIdentityRefs>,
    pub image_pull_secret: Option<K8sName>,
-}
+    /// Static `device=group|group;...` membership for dev and tests.
-
+    pub device_groups: Option<String>,
 /// The dashboard's auth inputs the operator reads via `ConfigClient`.
 #[derive(Debug, Clone, Serialize)]
 pub struct WebAuth {
    pub config: harmony_zitadel_auth::ZitadelAuthConfig,
    pub cookie: harmony_zitadel_auth::OperatorCookieKey,
 }
 impl FleetOperatorScore {
@@ -198,6 +191,7 @@ impl FleetOperatorScore {
            web_auth: None,
            identity: None,
            image_pull_secret: None,
            device_groups: None,
        }
    }
@@ -210,14 +204,9 @@ impl FleetOperatorScore {
        self
    }
-    /// Configure dashboard SSO: the `ZitadelAuthConfig` + cookie key are
+    /// Configure dashboard SSO from the declared identity and Ingress.
-    /// baked into the operator Secret for the pod's `ConfigClient`.
+    pub fn web_auth(mut self, application: &OidcApplicationRef) -> Self {
-    pub fn web_auth(
+        self.web_auth = Some(application.clone());
        mut self,
        config: harmony_zitadel_auth::ZitadelAuthConfig,
        cookie: harmony_zitadel_auth::OperatorCookieKey,
    ) -> Self {
        self.web_auth = Some(WebAuth { config, cookie });
        self
    }
@@ -260,6 +249,11 @@ impl FleetOperatorScore {
        self
    }
    pub fn device_groups(mut self, groups: impl Into<String>) -> Self {
        self.device_groups = Some(groups.into());
        self
    }
    /// Set the operator's NATS auth-callout credentials (zitadel-jwt
    /// `[credentials]` TOML). Applied as the operator Secret before the
    /// helm install — including on the published-chart (CD) path.
@@ -296,9 +290,11 @@ pub struct FleetOperatorInterpret {
 async fn smoke_test_operator<T: K8sclient>(
    namespace: &str,
    expected_config_hash: &str,
    require_dashboard: bool,
    topology: &T,
 ) -> Result<(), InterpretError> {
    let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
    let mut last_observation = "no operator pod observed".to_string();
    tokio::time::timeout(Duration::from_secs(180), async {
        loop {
            let pods = k8s
@@ -310,32 +306,62 @@ async fn smoke_test_operator<T: K8sclient>(
                    ),
                )
                .await;
-            if let Ok(pods) = pods {
+            match pods {
-                for pod in pods.items {
+                Ok(pods) if pods.items.is_empty() => {
-                    let has_expected_config = pod
+                    last_observation = "no operator pod matched the release label".to_string();
-                        .metadata
+                }
-                        .annotations
+                Ok(pods) => {
-                        .as_ref()
+                    for pod in pods.items {
-                        .and_then(|annotations| {
+                        let name = pod.metadata.name.unwrap_or_default();
-                            annotations.get("harmony.nationtech.io/config-hash")
+                        let has_expected_config = pod
-                        })
+                            .metadata
-                        .is_some_and(|hash| hash == expected_config_hash);
+                            .annotations
-                    if let Some(name) = pod.metadata.name
+                            .as_ref()
-                        && has_expected_config
+                            .and_then(|annotations| {
-                        && k8s
+                                annotations.get("harmony.nationtech.io/config-hash")
-                            .pod_logs(namespace, &name, Some(100))
+                            })
-                            .await
+                            .is_some_and(|hash| hash == expected_config_hash);
-                            .is_ok_and(|logs| logs.contains("KV bucket ready"))
+                        if !has_expected_config {
-                    {
+                            last_observation = format!("pod {name} has stale configuration");
-                        return;
+                            continue;
                        }
                        match k8s.pod_logs(namespace, &name, Some(100)).await {
                            Ok(logs)
                                if logs.contains("KV bucket ready")
                                    && (!require_dashboard
                                        || logs.contains(
                                            "fleet operator web frontend listening",
                                        )) =>
                            {
                                return;
                            }
                            Ok(logs) => {
                                last_observation = if !logs.contains("KV bucket ready") {
                                    format!("pod {name} logs lack `KV bucket ready`")
                                } else {
                                    format!(
                                        "pod {name} logs lack `fleet operator web frontend listening`"
                                    )
                                };
                            }
                            Err(error) => {
                                last_observation =
                                    format!("reading pod {name} logs failed: {error}");
                            }
                        }
                    }
                }
                Err(error) => last_observation = format!("listing operator pods failed: {error}"),
            }
            tokio::time::sleep(Duration::from_secs(2)).await;
        }
    })
    .await
-    .map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string()))
+    .map_err(|_| {
        InterpretError::new(format!(
            "operator did not become ready: {last_observation}"
        ))
    })
 }
 #[async_trait]
@@ -360,6 +386,14 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
                self.score.namespace
            )));
        }
        if let Some(web_auth) = &self.score.web_auth
            && web_auth.namespace() != self.score.namespace
        {
            return Err(InterpretError::new(format!(
                "dashboard application output must be in namespace '{}'",
                self.score.namespace
            )));
        }
        let k8s = topology.k8s_client().await.map_err(InterpretError::new)?;
        k8s.ensure_namespace(&self.score.namespace)
            .await
@@ -369,99 +403,119 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
                    self.score.namespace
                ))
            })?;
-        let identity_version = if let Some(identity) = &self.score.identity {
+        let (identity_version, web_client_id) = if let Some(identity) = &self.score.identity {
-            Some(
+            let (version, client_id) = tokio::time::timeout(Duration::from_secs(180), async {
-                tokio::time::timeout(Duration::from_secs(180), async {
+                loop {
-                    loop {
+                    let application = k8s
-                        let application = k8s
+                        .get_resource::<ConfigMap>(
-                            .get_resource::<ConfigMap>(
+                            identity.application.config_map_name(),
-                                identity.application.config_map_name(),
+                            Some(identity.application.namespace()),
-                                Some(identity.application.namespace()),
+                        )
-                            )
+                        .await
-                            .await
+                        .ok()
-                            .ok()
+                        .flatten();
-                            .flatten();
+                    let machine = k8s
-                        let machine = k8s
+                        .get_resource::<Secret>(
-                            .get_resource::<Secret>(
+                            identity.machine.secret_name(),
-                                identity.machine.secret_name(),
+                            Some(identity.machine.namespace()),
-                                Some(identity.machine.namespace()),
+                        )
-                            )
+                        .await
-                            .await
+                        .ok()
-                            .ok()
+                        .flatten();
-                            .flatten();
+                    let web_client_id = if let Some(web_auth) = &self.score.web_auth {
-                        if let (Some(application), Some(machine)) = (application, machine)
+                        k8s.get_resource::<ConfigMap>(
-                            && let Some(project_id) = application
+                            web_auth.config_map_name(),
-                                .data
+                            Some(web_auth.namespace()),
-                                .as_ref()
+                        )
-                                .and_then(|data| data.get(identity.application.project_id_key()))
+                        .await
-                        {
+                        .ok()
-                            return format!(
+                        .flatten()
                        .and_then(|config_map| config_map.data)
                        .and_then(|data| data.get(web_auth.client_id_key()).cloned())
                        .map(Some)
                    } else {
                        Some(None)
                    };
                    if let (Some(application), Some(machine)) = (application, machine)
                        && let Some(data) = application.data.as_ref()
                        && let Some(project_id) = data.get(identity.application.project_id_key())
                        && let Some(web_client_id) = web_client_id
                    {
                        return (
                            format!(
                                "{}:{}",
                                project_id,
                                machine.metadata.resource_version.unwrap_or_default()
-                            );
+                            ),
-                        }
+                            web_client_id,
-                        tokio::time::sleep(Duration::from_secs(2)).await;
+                        );
                    }
-                })
+                    tokio::time::sleep(Duration::from_secs(2)).await;
-                .await
+                }
-                .map_err(|_| {
+            })
-                    InterpretError::new("timed out waiting for operator identity refs".to_string())
+            .await
-                })?,
+            .map_err(|_| {
-            )
+                InterpretError::new("timed out waiting for operator identity refs".to_string())
            })?;
            (Some(version), client_id)
        } else {
-            None
+            (None, None)
        };
        let credentials = self.score.credentials.clone();
-        // Apply the credentials Secret BEFORE the helm install (the
+        // Apply environment-specific credentials before Helm creates the pod.
-        // chart's Deployment references it via secretKeyRef). Applied
+        // Keeping the Secret outside the chart avoids competing field owners.
-        // directly, not via the chart — it's environment-specific. The
+        let (web_auth_config_json, web_cookie_key_json) = if self.score.web_auth.is_some() {
-        // published-chart CD path runs without credentials today, so
+            let identity = self.score.identity.as_ref().ok_or_else(|| {
-        // this is a no-op there.
+                InterpretError::new("dashboard web auth requires an operator identity".to_string())
-        let (web_auth_config_json, web_cookie_key_json) = match &self.score.web_auth {
+            })?;
-            Some(w) => (
+            let host = self.score.operator_ui_host.as_ref().ok_or_else(|| {
-                Some(serde_json::to_string(&w.config).map_err(|e| {
+                InterpretError::new("dashboard web auth requires an Ingress".to_string())
            })?;
            let client_id = web_client_id.as_ref().expect("identity resolved above");
            let scheme = if self.score.cluster_issuer.is_some() {
                "https"
            } else {
                "http"
            };
            let base_url = format!("{scheme}://{host}");
            let config = harmony_zitadel_auth::ZitadelAuthConfig {
                zitadel_base: identity.provider.issuer(),
                base_url: base_url.clone(),
                client_id: client_id.clone(),
                scope: "openid profile email".to_string(),
                trusted_audiences: vec![client_id.clone()],
                logout_redirect_uri: format!("{base_url}/"),
            };
            let existing_secret = k8s
                .get_resource::<Secret>(chart::SECRET_NAME, Some(&self.score.namespace))
                .await
                .map_err(|e| InterpretError::new(format!("read operator Secret: {e}")))?;
            let cookie = existing_secret
                .as_ref()
                .and_then(|secret| secret.data.as_ref())
                .and_then(|data| data.get(chart::ENV_WEB_COOKIE_KEY))
                .map(|value| {
                    serde_json::from_slice::<harmony_zitadel_auth::OperatorCookieKey>(&value.0)
                        .map_err(|e| {
                            InterpretError::new(format!("parse existing OperatorCookieKey: {e}"))
                        })
                })
                .transpose()?
                .unwrap_or_else(harmony_zitadel_auth::OperatorCookieKey::generate);
            (
                Some(serde_json::to_string(&config).map_err(|e| {
                    InterpretError::new(format!("serialize ZitadelAuthConfig: {e}"))
                })?),
-                Some(serde_json::to_string(&w.cookie).map_err(|e| {
+                Some(serde_json::to_string(&cookie).map_err(|e| {
                    InterpretError::new(format!("serialize OperatorCookieKey: {e}"))
                })?),
-            ),
+            )
-            None => (None, None),
+        } else {
            (None, None)
        };
        let expected_config_hash = chart::config_hash(&ChartOptions {
            credentials: credentials.clone(),
            web_auth_config_json: web_auth_config_json.clone(),
            web_cookie_key_json: web_cookie_key_json.clone(),
            identity: self.score.identity.clone(),
            identity_version: identity_version.clone(),
            image_pull_secret: self.score.image_pull_secret.clone(),
            ..ChartOptions::default()
        });
        if let Some(creds) = &credentials
            && let Some(secret) = operator_secret(&ChartOptions {
                credentials: Some(creds.clone()),
                web_auth_config_json: web_auth_config_json.clone(),
                web_cookie_key_json: web_cookie_key_json.clone(),
                identity: self.score.identity.clone(),
                identity_version: identity_version.clone(),
                image_pull_secret: self.score.image_pull_secret.clone(),
                ..ChartOptions::default()
            })
        {
            info!(
                "Applying operator credentials Secret '{}' in {}",
                chart::SECRET_NAME,
                self.score.namespace
            );
            K8sResourceScore::single(secret, Some(self.score.namespace.clone()))
                .interpret(inventory, topology)
                .await?;
        }
        let tmp = tempfile::tempdir()
            .map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?;
-        let chart_path = build_chart(&ChartOptions {
+        let chart_options = ChartOptions {
            output_dir: tmp.path().to_path_buf(),
            image: self.score.image.clone(),
            image_pull_policy: self.score.image_pull_policy.clone(),
@@ -474,8 +528,22 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
            identity: self.score.identity.clone(),
            identity_version,
            image_pull_secret: self.score.image_pull_secret.clone(),
-        })
+            device_groups: self.score.device_groups.clone(),
-        .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
+        };
        let expected_config_hash = chart::config_hash(&chart_options);
        if let Some(secret) = operator_secret(&chart_options) {
            info!(
                "Applying operator credentials Secret '{}' in {}",
                chart::SECRET_NAME,
                self.score.namespace
            );
            K8sResourceScore::single(secret, Some(self.score.namespace.clone()))
                .interpret(inventory, topology)
                .await?;
        }
        let chart_path = build_chart(&chart_options)
            .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?;
        let chart_path_str = chart_path
            .to_str()
            .ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?;
@@ -507,10 +575,6 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
            ))
        })?;
        if credentials.is_some() || self.score.identity.is_some() {
            smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?;
        }
        // Expose the UI. Applied after the chart so the backing Service
        // (shipped in the chart) exists. Skipped when no host is set —
        // dev/e2e harnesses keep the operator cluster-internal.
@@ -545,6 +609,31 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
            details.push(format!("operator UI:        {scheme}://{host}"));
        }
        if credentials.is_some() || self.score.identity.is_some() {
            smoke_test_operator(
                &self.score.namespace,
                &expected_config_hash,
                self.score.web_auth.is_some(),
                topology,
            )
            .await?;
        }
        info!(
            r#"
 ===== FLEET VERIFICATION =====
 kubectl -n {namespace} get \
  pods,statefulsets,deployments,pvc,services,routes,certificates
 kubectl -n {namespace} logs \
  deploy/{release} --tail=200 | grep 'KV bucket ready'
 kubectl -n {namespace} get resourcequota
 =============================="#,
            namespace = self.score.namespace,
            release = self.score.release_name,
        );
        Ok(Outcome::success_with_details(helm_outcome.message, details))
    }
@@ -577,7 +666,8 @@ mod tests {
    #[test]
    fn fleet_crds_are_namespaced_resources() {
        let crds = fleet_crds();
-        assert_eq!(crds.len(), 2);
+        assert_eq!(crds.len(), 3);
        assert!(crds.iter().any(|crd| crd.spec.names.kind == "TaskRun"));
        for crd in crds {
            assert_eq!(
                crd.spec.scope,
--- a/fleet/harmony-fleet-e2e/Cargo.toml
+++ b/fleet/harmony-fleet-e2e/Cargo.toml
@@ -14,6 +14,10 @@ path = "src/lib.rs"
 name = "ping"
 path = "tests/ping.rs"
 [[test]]
 name = "exec"
 path = "tests/exec.rs"
 [[test]]
 name = "operator"
 path = "tests/operator.rs"
--- a/fleet/harmony-fleet-e2e/src/images.rs
+++ b/fleet/harmony-fleet-e2e/src/images.rs
@@ -106,6 +106,7 @@ async fn build_and_load_binary_image(
        format!(
            r#"FROM docker.io/library/archlinux:base
 COPY target/release/{crate_name} /usr/local/bin/{crate_name}
 RUN if [ "{crate_name}" = "harmony-fleet-agent" ]; then install -d -o 65532 -g 65532 -m 0700 /run/harmony-fleet-agent; fi
 USER 65532:65532
 ENTRYPOINT ["/usr/local/bin/{crate_name}"]
 "#
--- a/fleet/harmony-fleet-e2e/src/stack.rs
+++ b/fleet/harmony-fleet-e2e/src/stack.rs
@@ -4,7 +4,7 @@
 //! same `*Score` types production uses. The only thing this module
 //! owns is the *plumbing* around them: ensure a k3d cluster exists,
 //! sideload the agent image into it, mint a per-bring-up namespace,
-//! compose [`NatsScore`] + [`FleetAgentScore`] against the test
+//! compose [`NatsScore`] + [`FleetOperatorScore`] + [`FleetAgentScore`] against the test
 //! Topology, wait for pods to be Ready, and hand the test an admin
 //! NATS client.
 //!
@@ -85,6 +85,7 @@ pub struct StackOptions {
    pub log_level: String,
    pub deploy_agent: bool,
    pub deploy_operator: bool,
    pub device_groups: Option<String>,
 }
 impl Default for StackOptions {
@@ -96,7 +97,8 @@ impl Default for StackOptions {
            operator_image: OPERATOR_IMAGE_TAG.to_string(),
            log_level: "info".to_string(),
            deploy_agent: true,
-            deploy_operator: false,
+            deploy_operator: true,
            device_groups: None,
        }
    }
 }
@@ -107,6 +109,7 @@ impl StackOptions {
    pub fn infra_only() -> Self {
        Self {
            num_devices: 0,
            deploy_operator: false,
            ..Self::default()
        }
    }
@@ -313,6 +316,9 @@ impl Stack {
                .namespace(namespace.clone())
                .messaging(&nats_ref)
                .log_level(opts.log_level.clone());
            if let Some(groups) = &opts.device_groups {
                operator = operator.device_groups(groups);
            }
            if let Some(handles) = callout_handles.as_ref() {
                operator = operator.identity(
--- a/fleet/harmony-fleet-e2e/tests/exec.rs
+++ b/fleet/harmony-fleet-e2e/tests/exec.rs
@@ -0,0 +1,39 @@
 //! Operator-to-agent exec over Core NATS request/reply.
 use harmony_fleet_e2e::{StackOptions, shared_stack};
 use harmony_fleet_operator::commands::{CommandError, FleetCommandsClient};
 const E2E_ENV: &str = "HARMONY_FLEET_E2E";
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn operator_can_execute_a_bounded_agent_command() -> anyhow::Result<()> {
    if !matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true")) {
        eprintln!("skipping {E2E_ENV}-gated exec e2e test");
        return Ok(());
    }
    let stack = shared_stack(StackOptions::default()).await?;
    let client = FleetCommandsClient::new(stack.nats_client.clone());
    let reply = client
        .exec(
            &stack.device_ids[0],
            "printf stdout; printf stderr >&2; exit 7",
        )
        .await?;
    assert_eq!(reply.exit_code, 7);
    assert_eq!(reply.stdout, "stdout");
    assert_eq!(reply.stderr, "stderr");
    assert!(!reply.truncated);
    let delayed = client
        .exec(&stack.device_ids[0], "sleep 11; printf delayed")
        .await?;
    assert_eq!(delayed.stdout, "delayed");
    assert!(matches!(
        client.exec("missing-device", "true").await,
        Err(CommandError::DeviceOffline)
    ));
    Ok(())
 }
--- a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs
+++ b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs
@@ -109,7 +109,7 @@ async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> {
        FLEET_NS.to_string(),
    );
    grants
-        .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
+        .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()], vec![])])
        .await?;
    // Device in edge-a: login binds the group, secret readable, batch token.
@@ -126,12 +126,14 @@ async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> {
    // Detach binds at request time: the in-hand token loses access with
    // no re-login involved.
-    grants.set_deployment_groups(&[(dn("web"), vec![])]).await?;
+    grants
        .set_deployment_groups(&[(dn("web"), vec![], vec![])])
        .await?;
    bao.assert_denied(&login_a.token, "web/config").await?;
    // Re-attach binds at request time too.
    grants
-        .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
+        .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()], vec![])])
        .await?;
    bao.assert_can_read(&login_a.token, "web/config").await?;
@@ -169,6 +171,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
            openshift: false,
            tls_issuer: None,
            node_port: None,
            create_namespace: true,
        }),
        Box::new(OpenbaoSetupScore {
            instance: instance.clone(),
@@ -188,6 +191,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
                max_ttl: "1h".to_string(),
            }),
            oidc_application: None,
            endpoint: None,
        }),
    ];
    for score in scores {
--- a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs
+++ b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs
@@ -90,6 +90,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
            openshift: false,
            tls_issuer: None,
            node_port: None,
            create_namespace: true,
        }),
        Box::new(OpenbaoSetupScore {
            instance: instance.clone(),
@@ -105,6 +106,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
            }],
            jwt_auth: None,
            oidc_application: None,
            endpoint: None,
        }),
    ];
--- a/fleet/harmony-fleet-e2e/tests/operator.rs
+++ b/fleet/harmony-fleet-e2e/tests/operator.rs
@@ -11,7 +11,7 @@ use k8s_openapi::api::authorization::v1::{
 use k8s_openapi::api::core::v1::Namespace;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
 use kube::Client;
-use kube::api::{Api, DeleteParams, ObjectMeta, PostParams};
+use kube::api::{Api, DeleteParams, ObjectMeta, Patch, PatchParams, PostParams};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -51,9 +51,13 @@ async fn operator_writes_desired_state_for_matching_device() -> anyhow::Result<(
    let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
    create_device(&devices, "desired-state-device").await?;
    create_device(&devices, "unauthorized-device").await?;
    create_fleet_deployment(&deployments, "desired-state-test").await?;
    wait_for_desired_state_entry(&stack, "desired-state-device", "desired-state-test", true)
        .await?;
    tokio::time::sleep(Duration::from_secs(2)).await;
    wait_for_desired_state_entry(&stack, "unauthorized-device", "desired-state-test", false)
        .await?;
    Ok(())
 }
@@ -85,6 +89,40 @@ async fn operator_deletes_desired_state_when_deployment_is_deleted() -> anyhow::
    Ok(())
 }
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn operator_deletes_desired_state_when_selector_stops_matching() -> anyhow::Result<()> {
    if !e2e_enabled() {
        skip_e2e();
        return Ok(());
    }
    let stack = operator_stack().await?;
    let client = Client::try_default().await?;
    let devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
    let deployments: Api<Deployment> = Api::namespaced(client, &stack.namespace);
    create_device(&devices, "retarget-device").await?;
    create_fleet_deployment(&deployments, "retarget-test").await?;
    wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", true).await?;
    deployments
        .patch(
            "retarget-test",
            &PatchParams::default(),
            &Patch::Merge(serde_json::json!({
                "spec": {
                    "targetSelector": {
                        "matchLabels": { "device-id": "missing" }
                    }
                }
            })),
        )
        .await?;
    wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", false).await?;
    Ok(())
 }
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
    if !e2e_enabled() {
@@ -121,15 +159,12 @@ async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
    create_device(&devices, "other-tenant-device").await?;
    create_fleet_deployment(&deployments, "other-tenant-deployment").await?;
-    let sentinel = format!(
+    let sentinel = "isolation-sentinel";
        "isolation-sentinel-{}",
        &uuid::Uuid::new_v4().simple().to_string()[..8]
    );
    let tenant_devices: Api<Device> = Api::namespaced(client.clone(), &stack.namespace);
    let tenant_deployments: Api<Deployment> = Api::namespaced(client.clone(), &stack.namespace);
-    create_device(&tenant_devices, &sentinel).await?;
+    create_device(&tenant_devices, sentinel).await?;
-    create_fleet_deployment(&tenant_deployments, &sentinel).await?;
+    create_fleet_deployment(&tenant_deployments, sentinel).await?;
-    wait_for_desired_state_entry(&stack, &sentinel, &sentinel, true).await?;
+    wait_for_desired_state_entry(&stack, sentinel, sentinel, true).await?;
    let deployment = deployments.get("other-tenant-deployment").await?;
    assert!(
@@ -173,10 +208,10 @@ async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> {
        .await?;
    assert!(!review.status.is_some_and(|status| status.allowed));
    tenant_deployments
-        .delete(&sentinel, &DeleteParams::default())
+        .delete(sentinel, &DeleteParams::default())
        .await?;
    tenant_devices
-        .delete(&sentinel, &DeleteParams::default())
+        .delete(sentinel, &DeleteParams::default())
        .await?;
    namespaces
        .delete(&namespace, &DeleteParams::default())
@@ -196,6 +231,17 @@ async fn operator_stack() -> anyhow::Result<Arc<harmony_fleet_e2e::Stack>> {
    let stack = shared_stack(StackOptions {
        deploy_agent: false,
        deploy_operator: true,
        device_groups: Some(
            [
                "desired-state-device",
                "cleanup-device",
                "retarget-device",
                "other-tenant-device",
                "isolation-sentinel",
            ]
            .map(|device| format!("{device}=e2e"))
            .join(";"),
        ),
        ..StackOptions::default()
    })
    .await?;
@@ -219,7 +265,14 @@ async fn fleet_deployments(namespace: &str) -> anyhow::Result<Api<Deployment>> {
 }
 async fn create_device(devices: &Api<Device>, name: &str) -> anyhow::Result<()> {
-    let device = Device::new(name, DeviceSpec { inventory: None });
+    let device = Device::new(
        name,
        DeviceSpec {
            inventory: None,
            updater: None,
            agent_upgrade: None,
        },
    );
    devices.create(&PostParams::default(), &device).await?;
    Ok(())
 }
@@ -231,12 +284,14 @@ async fn create_fleet_deployment(deployments: &Api<Deployment>, name: &str) -> a
            ..Default::default()
        },
        spec: DeploymentSpec {
-            allowed_groups: None,
+            allowed_groups: vec!["e2e".to_string()],
            target_selector: LabelSelector::default(),
            score: ReconcileScore::PodmanV0(PodmanV0Score {
                init_container: None,
                services: vec![PodmanService {
                    name: "hello".to_string(),
                    image: "docker.io/library/hello-world:latest".to_string(),
                    image_pull_secret: None,
                    ports: vec![],
                    env: vec![],
                    secret_env: vec![],
--- a/fleet/harmony-fleet-e2e/tests/ping.rs
+++ b/fleet/harmony-fleet-e2e/tests/ping.rs
@@ -64,7 +64,7 @@ async fn operator_can_ping_agent() -> anyhow::Result<()> {
    );
    assert!(
        !reply.agent_version.is_empty(),
-        "agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)"
+        "agent_version must be non-empty"
    );
    Ok(())
 }
--- a/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs
+++ b/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs
@@ -91,6 +91,18 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
        "sudo -iu fleet-agent podman ps must show our service, got:\n{}",
        ps.stdout,
    );
    let init = device
        .ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.State.Status}} {{.State.ExitCode}}'")
        .await?
        .into_successful()
        .map_err(|e| anyhow::anyhow!("init container inspect failed: {e}"))?;
    assert_eq!(init.stdout.trim(), "exited 0");
    let init_id = device
        .ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.Id}}'")
        .await?
        .into_successful()
        .map_err(|e| anyhow::anyhow!("init container ID inspect failed: {e}"))?
        .stdout;
    // ---- phase 2: upgrade ----
    tracing::info!(
@@ -132,6 +144,13 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
        }
        tokio::time::sleep(Duration::from_secs(2)).await;
    }
    let upgraded_init_id = device
        .ssh("sudo -iu fleet-agent podman inspect hello-web-init --format '{{.Id}}'")
        .await?
        .into_successful()
        .map_err(|e| anyhow::anyhow!("upgraded init container ID inspect failed: {e}"))?
        .stdout;
    assert_ne!(init_id.trim(), upgraded_init_id.trim());
    // ---- phase 3: delete ----
    tracing::info!(
@@ -162,8 +181,8 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
        .into_successful()
        .map_err(|e| anyhow::anyhow!("final podman ps failed: {e}"))?;
    assert!(
-        !ps_final.stdout.contains("hello-web-svc"),
+        !ps_final.stdout.contains("hello-web-svc") && !ps_final.stdout.contains("hello-web-init"),
-        "container hello-web-svc still present after delete:\n{}",
+        "deployment containers still present after delete:\n{}",
        ps_final.stdout,
    );
@@ -172,6 +191,16 @@ async fn vm_agent_drives_full_deploy_lifecycle() -> anyhow::Result<()> {
 fn podman_score(image_tag: &str) -> PodmanV0Score {
    PodmanV0Score {
        init_container: Some(PodmanService {
            name: "hello-web-init".to_string(),
            image: "docker.io/library/hello-world:latest".to_string(),
            image_pull_secret: None,
            ports: vec![],
            env: vec![],
            secret_env: vec![],
            volumes: vec![],
            restart_policy: RestartPolicy::No,
        }),
        services: vec![PodmanService {
            name: "hello-web-svc".to_string(),
            // Pin upstream to docker.io/library so the VM doesn't
@@ -179,6 +208,7 @@ fn podman_score(image_tag: &str) -> PodmanV0Score {
            // image ships with. nginx:alpine is multi-arch and the
            // smallest battle-tested long-running ARM image.
            image: format!("docker.io/library/{image_tag}"),
            image_pull_secret: None,
            ports: vec![],
            env: vec![],
            secret_env: vec![],
--- a/fleet/harmony-fleet-e2e/tests/vm_isolation.rs
+++ b/fleet/harmony-fleet-e2e/tests/vm_isolation.rs
@@ -83,9 +83,11 @@ async fn agent_ignores_other_devices_keys() -> anyhow::Result<()> {
    let admin = AdminKv::connect(&stack.infra.nats_client).await?;
    let score = PodmanV0Score {
        init_container: None,
        services: vec![PodmanService {
            name: "intruder-svc".to_string(),
            image: "docker.io/library/nginx:alpine".to_string(),
            image_pull_secret: None,
            ports: vec![],
            env: vec![],
            secret_env: vec![],
--- a/fleet/harmony-fleet-e2e/tests/vm_ping.rs
+++ b/fleet/harmony-fleet-e2e/tests/vm_ping.rs
@@ -61,7 +61,7 @@ async fn agent_on_vm_replies_to_ping() -> anyhow::Result<()> {
    );
    assert!(
        !reply.agent_version.is_empty(),
-        "agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)",
+        "agent_version must be non-empty",
    );
    Ok(())
 }
--- a/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs
+++ b/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs
@@ -59,9 +59,11 @@ fn callout_stack_options() -> VmStackOptions {
 fn secret_score(service: &str, spec_marker: &str) -> PodmanV0Score {
    PodmanV0Score {
        init_container: None,
        services: vec![PodmanService {
            name: service.to_string(),
            image: "docker.io/library/nginx:alpine".to_string(),
            image_pull_secret: None,
            ports: vec![],
            // Changing the marker changes the serialized spec, which
            // is how the tests force a re-apply (and with it a fresh
@@ -186,7 +188,7 @@ async fn device_reads_deployment_secret_via_sso_groups() -> anyhow::Result<()> {
    bao.put_secret(&deployment, "db_password", &secret_value)
        .await?;
    bao.grants()
-        .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
+        .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()], vec![])])
        .await?;
    admin
@@ -256,7 +258,7 @@ async fn removed_policy_denies_secret_fetch() -> anyhow::Result<()> {
    bao.put_secret(&deployment, "db_password", "soon-revoked")
        .await?;
    bao.grants()
-        .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
+        .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()], vec![])])
        .await?;
    // Sanity: with the grant in place the deployment converges.
--- a/fleet/harmony-fleet-operator/Cargo.toml
+++ b/fleet/harmony-fleet-operator/Cargo.toml
@@ -36,8 +36,8 @@ futures-util = { workspace = true }
 thiserror.workspace = true
 async-trait.workspace = true
 url.workspace = true
 base64.workspace = true
 reqwest.workspace = true
 uuid.workspace = true
 axum = { version = "0.8", optional = true }
 axum-extra = { version = "0.10", features = ["cookie", "cookie-private"], optional = true }
--- a/fleet/harmony-fleet-operator/README.md
+++ b/fleet/harmony-fleet-operator/README.md
@@ -33,17 +33,11 @@ SSR + HTMX), so the runtime/macro footprint was dead weight. Maud is a
 compile-time HTML macro that produces a `Markup` value — smaller dep tree,
 faster compiles, same Rust-flavored ergonomics.
 **Why HTMX + xterm.js for interactivity?** A real terminal needs xterm.js in
 the browser regardless; once that JS exists, HTMX (~14 KB) is a rounding
 error and lets every other interaction stay declarative in markup
 (`hx-post`, `hx-target`, `hx-swap`).
 **Why everything bundled?** The operator already ships as a single
-container. Tailwind CSS, HTMX, and the HTMX SSE extension are all embedded
+container. Tailwind CSS, HTMX, and the small CSRF helper are embedded so
-via `include_bytes!` so air-gapped clusters get the dashboard with nothing
+air-gapped clusters need nothing extra to mount. The only build-time external
-extra to mount. The only build-time external is the standalone `tailwindcss`
+is the standalone `tailwindcss` v4 CLI. A missing CLI produces a warning and
-v4 CLI — missing-CLI degrades gracefully (warning + empty embedded CSS); the
+empty embedded CSS; local development uses `--css-from` instead.
 dev workflow uses `--css-from` instead anyway.
 ### Running it locally (mock data, no NATS, no kube)
@@ -76,9 +70,8 @@ Open <http://localhost:18080>.
 `--mock` uses [`MockFleetService`](src/service/mock.rs), an in-memory
 seeded dataset (10 fake devices in mixed states, 4 deployments). You can
-click "Blacklist" on a row and the row will swap in place to reflect the
+blacklist a device and see its updated detail page. This exercises the same
-new status — this exercises the same `FleetService` API the real impl
+`FleetService` API as production without NATS or a Kubernetes cluster.
 will satisfy. No NATS, no Kubernetes cluster needed.
 #### Iteration cost
@@ -120,21 +113,21 @@ fleet/harmony-fleet-operator/
 │       ├── assets.rs        ← embedded Tailwind/HTMX bytes
 │       └── views/
 │           ├── dashboard.rs
-│           ├── devices.rs   ← also exposes `row()` for HTMX swaps
+│           ├── devices.rs
 │           └── deployments.rs
 ├── style/
 │   └── input.css            ← Tailwind v4 entry point
 └── vendor/
-    ├── htmx.min.js          ← HTMX v2.0.9
+    ├── app.js               ← CSRF header helper
-    └── htmx-ext-sse.js      ← SSE extension (used by future log-tail views)
+    └── htmx.min.js          ← HTMX v2.0.9
 ```
 ### What's deferred
- **Real `FleetService` impl** (wraps the kube client + NATS KV the
+- **`fleet-admin` authorization.** Zitadel login is implemented, but any
-  reconcilers already use). `serve-web` without `--mock` currently errors
+  authenticated tenant user can currently reach privileged dashboard routes.
-  out.
+- **Live log tail.** Add a typed, authorized agent transport before exposing it.
- **Zitadel SSO + admin-role check.** v1 assumes an oauth2-proxy fronts the
+- **Device commands.** Define a bounded typed protocol and authorization before
-  dashboard at the cluster edge.
+  adding command controls.
- **Live log tail** (SSE-based, HTMX `sse-swap`) — the wiring is in place.
+- **Persistent alert state and receivers.** Per-alert acknowledgement currently
- **Interactive shell** (xterm.js + axum WS + portable-pty) — separate design.
+  lives in operator memory; receiver configuration remains deployment-time.
--- a/fleet/harmony-fleet-operator/src/access.rs
+++ b/fleet/harmony-fleet-operator/src/access.rs
@@ -12,6 +12,8 @@ pub struct StaticDeviceGroups {
 }
 impl StaticDeviceGroups {
    /// Parse `device=group|group;...`. The reserved `*` device supplies
    /// membership to every device in local test and load-test setups.
    pub fn parse(spec: &str) -> Self {
        let map = spec
            .split(';')
@@ -44,7 +46,7 @@ impl DeviceGroupSource for StaticDeviceGroups {
 #[derive(Debug, Default)]
 pub struct InMemoryDeploymentSecretGrants {
-    grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
+    grants: Mutex<BTreeMap<DeploymentName, (BTreeSet<String>, BTreeSet<String>)>>,
 }
 impl InMemoryDeploymentSecretGrants {
@@ -57,7 +59,16 @@ impl InMemoryDeploymentSecretGrants {
            .lock()
            .unwrap()
            .get(deployment)
-            .map(|groups| groups.iter().cloned().collect())
+            .map(|(groups, _)| groups.iter().cloned().collect())
            .unwrap_or_default()
    }
    pub fn pull_secrets_for(&self, deployment: &DeploymentName) -> Vec<String> {
        self.grants
            .lock()
            .unwrap()
            .get(deployment)
            .map(|(_, references)| references.iter().cloned().collect())
            .unwrap_or_default()
    }
 }
@@ -66,14 +77,20 @@ impl InMemoryDeploymentSecretGrants {
 impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
    async fn set_deployment_groups(
        &self,
-        grants: &[(DeploymentName, Vec<String>)],
+        grants: &[(DeploymentName, Vec<String>, Vec<String>)],
    ) -> Result<(), SecretAccessError> {
        let mut current = self.grants.lock().unwrap();
-        for (deployment, groups) in grants {
+        for (deployment, groups, image_pull_secrets) in grants {
            if groups.is_empty() {
                current.remove(deployment);
            } else {
-                current.insert(deployment.clone(), groups.iter().cloned().collect());
+                current.insert(
                    deployment.clone(),
                    (
                        groups.iter().cloned().collect(),
                        image_pull_secrets.iter().cloned().collect(),
                    ),
                );
            }
        }
        Ok(())
@@ -86,10 +103,11 @@ mod tests {
    #[tokio::test]
    async fn static_groups_parse_membership() {
-        let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
+        let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b; *=load ;;junk");
        let groups = source.device_groups().await.unwrap();
-        assert_eq!(groups.len(), 2);
+        assert_eq!(groups.len(), 3);
        assert_eq!(groups["pi-01"].len(), 2);
        assert!(groups["*"].contains("load"));
    }
    #[tokio::test]
@@ -97,12 +115,12 @@ mod tests {
        let grants = InMemoryDeploymentSecretGrants::new();
        let deployment = DeploymentName::try_new("web").unwrap();
        grants
-            .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()])])
+            .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".into()], vec![])])
            .await
            .unwrap();
        assert_eq!(grants.groups_for(&deployment), vec!["edge-a"]);
        grants
-            .set_deployment_groups(&[(deployment.clone(), vec![])])
+            .set_deployment_groups(&[(deployment.clone(), vec![], vec![])])
            .await
            .unwrap();
        assert!(grants.groups_for(&deployment).is_empty());
--- a/fleet/harmony-fleet-operator/src/agent_upgrade.rs
+++ b/fleet/harmony-fleet-operator/src/agent_upgrade.rs
@@ -0,0 +1,218 @@
 use std::time::Duration;
 use anyhow::{Context, Result, bail};
 use harmony_reconciler_contracts::{
    AgentUpgradeAttempt, AgentUpgradeStatus, BUCKET_AGENT_UPGRADE_INTENT,
    BUCKET_AGENT_UPGRADE_STATUS, Id, agent_upgrade_intent_key, agent_upgrade_status_key,
 };
 use kube::api::{Api, ListParams, Patch, PatchParams};
 use kube::{Client, ResourceExt};
 use serde_json::json;
 use crate::crd::{
    AgentUpgradeTarget, Device, DeviceUpgradeJournalRef, DeviceUpgradeStatus,
    DeviceUpgradeTransition,
 };
 pub async fn run(
    client: Client,
    namespace: &str,
    jetstream: async_nats::jetstream::Context,
 ) -> Result<()> {
    let intents = jetstream
        .create_key_value(async_nats::jetstream::kv::Config {
            bucket: BUCKET_AGENT_UPGRADE_INTENT.into(),
            ..Default::default()
        })
        .await?;
    let statuses = jetstream
        .create_key_value(async_nats::jetstream::kv::Config {
            bucket: BUCKET_AGENT_UPGRADE_STATUS.into(),
            ..Default::default()
        })
        .await?;
    let devices: Api<Device> = Api::namespaced(client, namespace);
    // TODO: Replace full-device polling with watch-driven work before fleet scale.
    let mut ticker = tokio::time::interval(Duration::from_secs(2));
    loop {
        ticker.tick().await;
        for device in devices.list(&ListParams::default()).await?.items {
            if let Err(error) = reconcile_device(&devices, &intents, &statuses, device).await {
                tracing::warn!(device = %error.0, error = %error.1, "agent upgrade reconcile failed");
            }
        }
    }
 }
 async fn reconcile_device(
    devices: &Api<Device>,
    intents: &async_nats::jetstream::kv::Store,
    statuses: &async_nats::jetstream::kv::Store,
    device: Device,
 ) -> std::result::Result<(), (String, anyhow::Error)> {
    let id = device.name_any();
    reconcile_device_inner(devices, intents, statuses, device)
        .await
        .map_err(|error| (id, error))
 }
 async fn reconcile_device_inner(
    devices: &Api<Device>,
    intents: &async_nats::jetstream::kv::Store,
    statuses: &async_nats::jetstream::kv::Store,
    device: Device,
 ) -> Result<()> {
    let id = device.name_any();
    let status = statuses
        .get(agent_upgrade_status_key(&id))
        .await?
        .map(|bytes| serde_json::from_slice::<AgentUpgradeStatus>(&bytes))
        .transpose()?;
    if let Some(status) = status.as_ref() {
        let phase = serde_json::to_value(status.phase)?
            .as_str()
            .context("upgrade phase did not serialize as a string")?
            .to_string();
        let reflected = DeviceUpgradeStatus {
            attempt_id: status.attempt_id.clone(),
            target_version: status.target_version.clone(),
            phase,
            started_at: status.started_at.to_rfc3339(),
            updated_at: status.updated_at.to_rfc3339(),
            reason: status.reason.map(|reason| {
                serde_json::to_value(reason)
                    .expect("upgrade reason is serializable")
                    .as_str()
                    .expect("upgrade reason serializes as a string")
                    .to_string()
            }),
            detail: status.detail.clone(),
            error: status.error.clone(),
            drain_duration_ms: status.drain_duration_ms,
            boot_id: status.boot_id.clone(),
            invocation_id: status.invocation_id.clone(),
            journal: status
                .journal
                .as_ref()
                .map(|journal| DeviceUpgradeJournalRef {
                    unit: journal.unit.clone(),
                    since: journal.since.to_rfc3339(),
                }),
            transitions: status
                .transitions
                .iter()
                .map(|transition| DeviceUpgradeTransition {
                    phase: serde_json::to_value(transition.phase)
                        .expect("upgrade phase is serializable")
                        .as_str()
                        .expect("upgrade phase serializes as a string")
                        .to_string(),
                    entered_at: transition.entered_at.to_rfc3339(),
                    exited_at: transition.exited_at.map(|at| at.to_rfc3339()),
                    duration_ms: transition.duration_ms,
                })
                .collect(),
        };
        if device
            .status
            .as_ref()
            .and_then(|value| value.agent_upgrade.as_ref())
            != Some(&reflected)
        {
            devices
                .patch_status(
                    &id,
                    &PatchParams::default(),
                    &Patch::Merge(&json!({ "status": { "agentUpgrade": reflected } })),
                )
                .await?;
        }
    }
    let Some(target) = device.spec.agent_upgrade.as_ref() else {
        return Ok(());
    };
    let current_version = device
        .status
        .as_ref()
        .and_then(|status| status.current_version.as_deref())
        .context("device has not reported an agent version")?;
    if current_version == target.version {
        return Ok(());
    }
    let intent_key = agent_upgrade_intent_key(&id);
    let existing_entry = intents.entry(&intent_key).await?;
    let existing = existing_entry
        .as_ref()
        .filter(|entry| entry.operation == async_nats::jetstream::kv::Operation::Put)
        .map(|entry| serde_json::from_slice::<AgentUpgradeAttempt>(&entry.value))
        .transpose()?;
    match existing {
        Some(attempt) if attempt_matches(&attempt, target) => {}
        Some(attempt)
            if status.as_ref().is_some_and(|status| {
                status.attempt_id == attempt.attempt_id && !status.phase.is_terminal()
            }) =>
        {
            bail!("attempt '{}' is still active", attempt.attempt_id)
        }
        _ => {
            let attempt = AgentUpgradeAttempt {
                attempt_id: uuid::Uuid::new_v4().to_string(),
                device_id: Id::from(id.clone()),
                target_version: target.version.clone(),
                architecture: target.architecture.clone(),
                artifact_url: target.artifact_url.clone(),
                max_bytes: target.max_bytes,
                sha256: target.sha256.clone(),
            };
            let value = serde_json::to_vec(&attempt)?.into();
            if let Some(entry) = existing_entry {
                intents.update(&intent_key, value, entry.revision).await?;
            } else {
                intents.create(&intent_key, value).await?;
            }
        }
    }
    Ok(())
 }
 fn attempt_matches(attempt: &AgentUpgradeAttempt, target: &AgentUpgradeTarget) -> bool {
    attempt.target_version == target.version
        && attempt.architecture == target.architecture
        && attempt.artifact_url == target.artifact_url
        && attempt.max_bytes == target.max_bytes
        && attempt.sha256 == target.sha256
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn release_metadata_must_match_the_existing_attempt() {
        let target = AgentUpgradeTarget {
            version: "0.2.0".into(),
            architecture: "aarch64".into(),
            artifact_url: "https://example.invalid/agent".into(),
            max_bytes: 10,
            sha256: "digest".into(),
        };
        let attempt = AgentUpgradeAttempt {
            attempt_id: uuid::Uuid::new_v4().to_string(),
            device_id: Id::from("device"),
            target_version: target.version.clone(),
            architecture: target.architecture.clone(),
            artifact_url: target.artifact_url.clone(),
            max_bytes: target.max_bytes,
            sha256: target.sha256.clone(),
        };
        assert!(attempt_matches(&attempt, &target));
        let mut changed = target;
        changed.artifact_url = "https://example.invalid/repacked-agent".into();
        assert!(!attempt_matches(&attempt, &changed));
    }
 }
--- a/fleet/harmony-fleet-operator/src/commands.rs
+++ b/fleet/harmony-fleet-operator/src/commands.rs
@@ -15,12 +15,15 @@ use std::time::Duration;
 use async_nats::Client;
 use async_nats::error::Error as NatsError;
-use harmony_reconciler_contracts::{PingReply, Verb, device_command_subject};
+use harmony_reconciler_contracts::{
    CommandRequest, ExecReply, HDR_REQUEST_ID, PingReply, Verb, device_command_subject,
 };
 /// Default reply timeout for a single-shot command. 5 s is plenty for
 /// a healthy device on a LAN; offline devices get short-circuited
 /// earlier by NATS's `no_responders` reply.
 pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
 pub const EXEC_TIMEOUT: Duration = Duration::from_secs(30);
 #[derive(Debug, thiserror::Error)]
 pub enum CommandError {
@@ -76,6 +79,28 @@ impl FleetCommandsClient {
        let reply: PingReply = serde_json::from_slice(&resp.payload)?;
        Ok(reply)
    }
    pub async fn exec(&self, device_id: &str, command: &str) -> Result<ExecReply, CommandError> {
        let subject = device_command_subject(device_id, Verb::Exec);
        let payload = serde_json::to_vec(&CommandRequest::Exec {
            command: command.to_string(),
        })
        .expect("command request is serializable");
        let mut headers = async_nats::HeaderMap::new();
        headers.insert(HDR_REQUEST_ID, uuid::Uuid::new_v4().to_string());
        let response = self
            .nc
            .send_request(
                subject,
                async_nats::Request::new()
                    .headers(headers)
                    .payload(payload.into())
                    .timeout(Some(EXEC_TIMEOUT)),
            )
            .await
            .map_err(|error| map_request_error(error, EXEC_TIMEOUT))?;
        Ok(serde_json::from_slice(&response.payload)?)
    }
 }
 /// Map an async-nats `RequestError` to our typed surface. The `kind`
--- a/fleet/harmony-fleet-operator/src/controller.rs
+++ b/fleet/harmony-fleet-operator/src/controller.rs
@@ -34,6 +34,7 @@ use kube::runtime::watcher::Config as WatcherConfig;
 use kube::{Api, Client, ResourceExt};
 use harmony_fleet_operator::Deployment;
 use harmony_fleet_operator::rollout;
 const FINALIZER: &str = "fleet.nationtech.io/finalizer";
@@ -52,11 +53,21 @@ pub enum Error {
 pub struct Context {
    pub client: Client,
    pub kv: Store,
    pub rollout_plans: Store,
 }
-pub async fn run(client: Client, namespace: &str, kv: Store) -> anyhow::Result<()> {
+pub async fn run(
    client: Client,
    namespace: &str,
    kv: Store,
    rollout_plans: Store,
 ) -> anyhow::Result<()> {
    let api: Api<Deployment> = Api::namespaced(client.clone(), namespace);
-    let ctx = Arc::new(Context { client, kv });
+    let ctx = Arc::new(Context {
        client,
        kv,
        rollout_plans,
    });
    tracing::info!("starting Deployment controller");
    kube::runtime::controller::Controller::new(api, WatcherConfig::default())
@@ -87,7 +98,7 @@ async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Er
            // its own kube watch and writes KV entries for matching
            // devices. Long requeue so we're not pointlessly polling.
            FinalizerEvent::Apply(_) => Ok(Action::requeue(Duration::from_secs(300))),
-            FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv).await,
+            FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv, &ctx.rollout_plans).await,
        }
    })
    .await
@@ -101,11 +112,17 @@ async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Er
    })
 }
-async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
+async fn cleanup(obj: Arc<Deployment>, kv: &Store, plans: &Store) -> Result<Action, Error> {
    let name = obj.name_any();
    let deployment_name =
        DeploymentName::try_new(&name).map_err(|e| Error::InvalidName(name, e.to_string()))?;
    let suffix = format!(".{}", deployment_name.as_str());
    let plan_key = obj.metadata.uid.as_deref().map(rollout::deployment_key);
    if let Some(key) = plan_key.as_deref() {
        rollout::close(plans, key)
            .await
            .map_err(|error| Error::Kv(format!("closing rollout plan: {error}")))?;
    }
    let mut removed = 0u64;
    let mut keys = kv
@@ -121,6 +138,11 @@ async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
            removed += 1;
        }
    }
    if let Some(key) = plan_key.as_deref() {
        rollout::delete(plans, key)
            .await
            .map_err(|error| Error::Kv(format!("deleting rollout plan: {error}")))?;
    }
    tracing::info!(
        deployment = %deployment_name,
        removed,
--- a/fleet/harmony-fleet-operator/src/crd.rs
+++ b/fleet/harmony-fleet-operator/src/crd.rs
@@ -1,4 +1,4 @@
-use harmony_reconciler_contracts::InventorySnapshot;
+use harmony_reconciler_contracts::{InventorySnapshot, UpdaterCapabilities};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
 use kube::CustomResource;
 use schemars::JsonSchema;
@@ -10,7 +10,7 @@ pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileSc
 /// to the pattern K8s itself uses for DaemonSet nodeSelector, Service
 /// pod selector, etc. The operator resolves the selector against
 /// `Device` CRs at reconcile time; no list of device ids on spec.
-#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
+#[derive(CustomResource, Serialize, Clone, Debug, JsonSchema)]
 #[kube(
    group = "fleet.nationtech.io",
    version = "v1alpha1",
@@ -22,21 +22,41 @@ pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileSc
 )]
 #[serde(rename_all = "camelCase")]
 pub struct DeploymentSpec {
-    /// Device groups allowed to *view* this deployment — run it and
+    /// Device groups allowed to run this deployment and read its secrets
-    /// read its secrets (ADR-025). Membership is admin-managed identity
+    /// (ADR-025). Membership is admin-managed identity, never
-    /// (Zitadel role grants), never device-reported labels: labels can
+    /// device-reported labels. Empty means no device is authorized.
-    /// narrow placement below, but only groups grant. Absent = no group
+    pub allowed_groups: Vec<String>,
    /// gating and no secret access — the selector alone places a
    /// secretless workload.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub allowed_groups: Option<Vec<String>>,
    /// Which devices this deployment targets, *within* the allowed
    /// groups. Matches against `Device.metadata.labels`.
    pub target_selector: LabelSelector,
    pub score: ReconcileScore,
    /// Retained for existing manifests. Rollouts begin immediately, with
    /// `canary=true` targets gated before the remaining frozen targets.
    pub rollout: Rollout,
 }
 impl<'de> Deserialize<'de> for DeploymentSpec {
    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        #[derive(Deserialize)]
        #[serde(rename_all = "camelCase")]
        struct StoredSpec {
            #[serde(default)]
            allowed_groups: Vec<String>,
            target_selector: LabelSelector,
            score: ReconcileScore,
            rollout: Rollout,
        }
        let stored = StoredSpec::deserialize(deserializer)?;
        Ok(Self {
            allowed_groups: stored.allowed_groups,
            target_selector: stored.target_selector,
            score: stored.score,
            rollout: stored.rollout,
        })
    }
 }
 #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
 pub struct Rollout {
    pub strategy: RolloutStrategy,
@@ -50,6 +70,9 @@ pub enum RolloutStrategy {
 #[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeploymentStatus {
    /// Exact `<metadata.uid>:<metadata.generation>` represented by `aggregate`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub rollout_revision: Option<String>,
    /// Per-deployment rollup. Present once the aggregator has
    /// evaluated the selector at least once.
    #[serde(skip_serializing_if = "Option::is_none")]
@@ -60,21 +83,18 @@ pub struct DeploymentStatus {
 #[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeploymentAggregate {
-    /// How many Device CRs currently match `spec.targetSelector`.
+    /// Number of devices in the frozen rollout plan. The three phase counters
-    /// The three phase counters below sum to this; targeted-but-
+    /// below sum to this; unreleased devices and devices without state are
-    /// unreported devices are folded into `pending`.
+    /// folded into `pending`.
    pub matched_device_count: u32,
    pub succeeded: u32,
    pub failed: u32,
    pub pending: u32,
-    /// Device id of the most recent device reporting a failure, with
+    /// Most recent reported failure details, when available.
    /// its short error message. Cleared when that device transitions
    /// back to Running.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub last_error: Option<AggregateLastError>,
 }
-#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct AggregateLastError {
    pub device_id: String,
@@ -82,6 +102,16 @@ pub struct AggregateLastError {
    pub at: String,
 }
 impl Deployment {
    pub fn rollout_revision(&self) -> Option<String> {
        self.metadata
            .uid
            .as_deref()
            .zip(self.metadata.generation)
            .map(|(uid, generation)| format!("{uid}:{generation}"))
    }
 }
 /// A physical/virtual device registered with a tenant's fleet.
 ///
 /// Created by the operator from `DeviceInfo` entries in the NATS
@@ -90,11 +120,8 @@ pub struct AggregateLastError {
 /// reflects it here.
 ///
 /// `metadata.labels` carries the device's routing labels. `spec.
-/// inventory` holds the hardware/OS snapshot. No status subresource
+/// inventory` holds the hardware/OS snapshot. The status subresource
-/// today — liveness is queried from the NATS `device-heartbeat`
+/// reflects heartbeat liveness and the running agent version.
 /// bucket directly; when a CR-side reflection (Reachable / Stale
 /// conditions) becomes useful, it'll land with its own reconciler
 /// rather than sitting here as speculative surface.
 #[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
 #[kube(
    group = "fleet.nationtech.io",
@@ -111,20 +138,70 @@ pub struct DeviceSpec {
    /// Rarely changes after first publish.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub inventory: Option<InventorySnapshot>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub updater: Option<UpdaterCapabilities>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub agent_upgrade: Option<AgentUpgradeTarget>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct AgentUpgradeTarget {
    pub version: String,
    pub architecture: String,
    pub artifact_url: String,
    pub max_bytes: u64,
    pub sha256: String,
 }
 /// Operator-maintained liveness reflection of the NATS
 /// `device-heartbeat` bucket onto the CR, so `kubectl get devices` and
 /// the dashboard see reachability without reading NATS. Written by the
 /// device-status reconciler.
-#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeviceStatus {
    /// RFC 3339 timestamp of the last heartbeat seen. `None` until the
    /// device has pinged at least once.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub last_heartbeat: Option<String>,
    pub reachability: Reachability,
    pub current_version: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub agent_upgrade: Option<DeviceUpgradeStatus>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeviceUpgradeStatus {
    pub attempt_id: String,
    pub target_version: String,
    pub phase: String,
    pub started_at: String,
    pub updated_at: String,
    pub reason: Option<String>,
    pub detail: Option<String>,
    pub error: Option<String>,
    pub drain_duration_ms: Option<u64>,
    pub boot_id: Option<String>,
    pub invocation_id: Option<String>,
    pub journal: Option<DeviceUpgradeJournalRef>,
    pub transitions: Vec<DeviceUpgradeTransition>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeviceUpgradeTransition {
    pub phase: String,
    pub entered_at: String,
    pub exited_at: Option<String>,
    pub duration_ms: Option<u64>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DeviceUpgradeJournalRef {
    pub unit: String,
    pub since: String,
 }
 /// Coarse liveness derived from heartbeat freshness. Failing/Pending
@@ -145,10 +222,44 @@ pub enum Reachability {
 mod tests {
    use kube::CustomResourceExt;
-    use super::Device;
+    use super::{Deployment, DeploymentAggregate, Device};
    #[test]
    fn device_is_namespaced() {
        assert_eq!(Device::crd().spec.scope, "Namespaced");
    }
    #[test]
    fn deployment_requires_allowed_groups() {
        let crd = serde_json::to_value(Deployment::crd()).unwrap();
        let required = crd
            .pointer("/spec/versions/0/schema/openAPIV3Schema/properties/spec/required")
            .and_then(serde_json::Value::as_array)
            .unwrap();
        assert!(required.iter().any(|field| field == "allowedGroups"));
    }
    #[test]
    fn legacy_deployment_without_allowed_groups_fails_closed() {
        let deployment: Deployment = serde_json::from_value(serde_json::json!({
            "apiVersion": "fleet.nationtech.io/v1alpha1",
            "kind": "Deployment",
            "metadata": { "name": "legacy" },
            "spec": {
                "targetSelector": {},
                "score": { "type": "PodmanV0", "data": { "services": [] } },
                "rollout": { "strategy": "Immediate" }
            }
        }))
        .unwrap();
        assert!(deployment.spec.allowed_groups.is_empty());
    }
    #[test]
    fn aggregate_serializes_cleared_error() {
        let value = serde_json::to_value(DeploymentAggregate::default()).unwrap();
        assert_eq!(value["lastError"], serde_json::Value::Null);
    }
 }
--- a/fleet/harmony-fleet-operator/src/device_reconciler.rs
+++ b/fleet/harmony-fleet-operator/src/device_reconciler.rs
@@ -13,7 +13,7 @@
 use anyhow::Result;
 use async_nats::jetstream::kv::{Operation, Store};
 use futures_util::StreamExt;
-use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo};
+use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo, device_info_key};
 use kube::Client;
 use kube::api::{Api, DeleteParams, Patch, PatchParams};
 use std::collections::BTreeMap;
@@ -62,6 +62,10 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()>
                        continue;
                    }
                };
                if !info_key_matches(&entry.key, &info) {
                    tracing::warn!(key = %entry.key, device = %info.device_id, "device-reconciler: key does not match payload device_id");
                    continue;
                }
                if let Err(e) = upsert_device(&devices, namespace, &info).await {
                    tracing::warn!(
                        device = %info.device_id,
@@ -85,12 +89,7 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()>
 async fn upsert_device(api: &Api<Device>, namespace: &str, info: &DeviceInfo) -> Result<()> {
    let name = info.device_id.to_string();
-    let mut device = Device::new(
+    let mut device = device_from_info(info);
        &name,
        DeviceSpec {
            inventory: info.inventory.clone(),
        },
    );
    device.metadata.namespace = Some(namespace.to_string());
    device.metadata.labels = Some(clean_labels(&info.labels));
@@ -104,6 +103,21 @@ async fn upsert_device(api: &Api<Device>, namespace: &str, info: &DeviceInfo) ->
    Ok(())
 }
 fn device_from_info(info: &DeviceInfo) -> Device {
    Device::new(
        &info.device_id.to_string(),
        DeviceSpec {
            inventory: info.inventory.clone(),
            updater: info.updater.clone(),
            agent_upgrade: None,
        },
    )
 }
 fn info_key_matches(key: &str, info: &DeviceInfo) -> bool {
    key == device_info_key(&info.device_id.to_string())
 }
 async fn delete_device(api: &Api<Device>, name: &str) -> Result<()> {
    match api.delete(name, &DeleteParams::default()).await {
        Ok(_) => {
@@ -149,6 +163,9 @@ fn is_label_value(s: &str) -> bool {
 #[cfg(test)]
 mod tests {
    use chrono::Utc;
    use harmony_reconciler_contracts::{Id, UpdaterCapabilities};
    use super::*;
    #[test]
@@ -167,4 +184,22 @@ mod tests {
        assert!(!is_label_value("has space"));
        assert!(!is_label_value(&"x".repeat(64)));
    }
    #[test]
    fn device_info_identity_and_capability_are_reflected() {
        let info = DeviceInfo {
            device_id: Id::from("device-1"),
            labels: BTreeMap::new(),
            inventory: None,
            updater: Some(UpdaterCapabilities {
                protocol: 1,
                apt_full_upgrade_v1: true,
            }),
            updated_at: Utc::now(),
        };
        assert!(info_key_matches("info.device-1", &info));
        assert!(!info_key_matches("info.device-2", &info));
        assert_eq!(device_from_info(&info).spec.updater, info.updater);
    }
 }
--- a/fleet/harmony-fleet-operator/src/device_status.rs
+++ b/fleet/harmony-fleet-operator/src/device_status.rs
@@ -14,7 +14,7 @@
 use std::collections::HashMap;
 use std::time::Duration;
-use anyhow::Result;
+use anyhow::{Context, Result, bail};
 use async_nats::jetstream::kv::{Operation, Store};
 use chrono::{DateTime, Utc};
 use futures_util::StreamExt;
@@ -24,7 +24,54 @@ use kube::{Client, ResourceExt};
 use serde_json::json;
 use tokio::sync::Mutex;
-use crate::crd::{Device, Reachability};
+use crate::crd::{Device, DeviceStatus, Reachability};
 #[derive(Clone, Debug, PartialEq, Eq)]
 struct ObservedHeartbeat {
    received_at: DateTime<Utc>,
    agent_version: Option<String>,
 }
 fn heartbeat_status(heartbeat: Option<ObservedHeartbeat>, now: DateTime<Utc>) -> DeviceStatus {
    match heartbeat {
        Some(heartbeat) => DeviceStatus {
            last_heartbeat: Some(heartbeat.received_at.to_rfc3339()),
            reachability: reachability(heartbeat.received_at, now),
            current_version: heartbeat.agent_version,
            agent_upgrade: None,
        },
        None => DeviceStatus {
            last_heartbeat: None,
            reachability: Reachability::Unknown,
            current_version: None,
            agent_upgrade: None,
        },
    }
 }
 fn observe_heartbeat(
    key: &str,
    payload: &[u8],
    received_at: DateTime<Utc>,
 ) -> Result<(String, ObservedHeartbeat)> {
    let device_id = key
        .strip_prefix("heartbeat.")
        .context("heartbeat key has no heartbeat. prefix")?;
    let heartbeat: HeartbeatPayload = serde_json::from_slice(payload)?;
    if heartbeat.device_id.to_string() != device_id {
        bail!(
            "heartbeat payload device {} does not match authorized key {device_id}",
            heartbeat.device_id
        );
    }
    Ok((
        device_id.to_string(),
        ObservedHeartbeat {
            received_at,
            agent_version: heartbeat.agent_version,
        },
    ))
 }
 /// A device with no heartbeat within this window is `Stale`. Agents
 /// ping every 30 s, so this tolerates ~2 missed pings.
@@ -44,7 +91,7 @@ pub async fn run(
        })
        .await?;
-    let heartbeats: Mutex<HashMap<String, DateTime<Utc>>> = Mutex::new(HashMap::new());
+    let heartbeats: Mutex<HashMap<String, Option<ObservedHeartbeat>>> = Mutex::new(HashMap::new());
    let devices: Api<Device> = Api::namespaced(client, namespace);
    tokio::try_join!(
@@ -56,7 +103,7 @@ pub async fn run(
 async fn watch_heartbeats(
    bucket: &Store,
-    heartbeats: &Mutex<HashMap<String, DateTime<Utc>>>,
+    heartbeats: &Mutex<HashMap<String, Option<ObservedHeartbeat>>>,
 ) -> Result<()> {
    let mut watch = bucket.watch_with_history(">").await?;
    tracing::info!("device-status: watching device-heartbeat KV");
@@ -70,46 +117,63 @@ async fn watch_heartbeats(
        };
        match entry.operation {
            Operation::Put => {
-                if let Ok(hb) = serde_json::from_slice::<HeartbeatPayload>(&entry.value) {
+                let Some(received_at) = DateTime::from_timestamp(
-                    heartbeats
+                    entry.created.unix_timestamp(),
-                        .lock()
+                    entry.created.nanosecond(),
-                        .await
+                ) else {
-                        .insert(hb.device_id.to_string(), hb.at);
+                    tracing::warn!(key = %entry.key, "device-status: invalid NATS timestamp");
                    continue;
                };
                match observe_heartbeat(&entry.key, &entry.value, received_at) {
                    Ok((device_id, heartbeat)) => {
                        heartbeats.lock().await.insert(device_id, Some(heartbeat));
                    }
                    Err(error) => {
                        tracing::warn!(key = %entry.key, %error, "device-status: invalid heartbeat")
                    }
                }
            }
            Operation::Delete | Operation::Purge => {
                if let Some(id) = entry.key.strip_prefix("heartbeat.") {
-                    heartbeats.lock().await.remove(id);
+                    heartbeats.lock().await.insert(id.to_string(), None);
                }
            }
        }
    }
-    Ok(())
+    bail!("device-status: heartbeat watch ended")
 }
 async fn patch_loop(
    devices: &Api<Device>,
-    heartbeats: &Mutex<HashMap<String, DateTime<Utc>>>,
+    heartbeats: &Mutex<HashMap<String, Option<ObservedHeartbeat>>>,
 ) -> Result<()> {
    // Last status written per device, to skip no-op patches.
-    let mut applied: HashMap<String, (Reachability, DateTime<Utc>)> = HashMap::new();
+    let mut applied: HashMap<String, DeviceStatus> = HashMap::new();
    let mut ticker = tokio::time::interval(TICK);
    loop {
        ticker.tick().await;
-        let snapshot: Vec<(String, DateTime<Utc>)> = heartbeats
+        let snapshot: Vec<(String, Option<ObservedHeartbeat>)> = heartbeats
            .lock()
            .await
            .iter()
-            .map(|(k, v)| (k.clone(), *v))
+            .map(|(k, v)| (k.clone(), v.clone()))
            .collect();
        let now = Utc::now();
-        for (id, at) in snapshot {
+        for (id, heartbeat) in snapshot {
-            let reachability = reachability(at, now);
+            let status = heartbeat_status(heartbeat, now);
-            if applied.get(&id) == Some(&(reachability, at)) {
+            if applied.get(&id) == Some(&status) {
                continue;
            }
-            if patch_status(devices, &id, reachability, at).await {
+            if patch_status(devices, &id, &status).await {
-                applied.insert(id, (reachability, at));
+                if status.reachability == Reachability::Unknown {
                    let mut heartbeats = heartbeats.lock().await;
                    if matches!(heartbeats.get(&id), Some(None)) {
                        heartbeats.remove(&id);
                        applied.remove(&id);
                        continue;
                    }
                }
                applied.insert(id, status);
            }
        }
    }
@@ -128,29 +192,19 @@ fn reachability(last_heartbeat: DateTime<Utc>, now: DateTime<Utc>) -> Reachabili
 /// Returns whether the patch succeeded (so we only cache applied state
 /// on success and retry next tick otherwise).
-async fn patch_status(
+async fn patch_status(devices: &Api<Device>, id: &str, status: &DeviceStatus) -> bool {
-    devices: &Api<Device>,
+    let patch = json!({ "status": status });
    id: &str,
    reachability: Reachability,
    last_heartbeat: DateTime<Utc>,
 ) -> bool {
    let status = json!({
        "status": {
            "lastHeartbeat": last_heartbeat.to_rfc3339(),
            "reachability": reachability,
        }
    });
    match devices
-        .patch_status(id, &PatchParams::default(), &Patch::Merge(&status))
+        .patch_status(id, &PatchParams::default(), &Patch::Merge(&patch))
        .await
    {
        Ok(d) => {
-            tracing::debug!(device = %d.name_any(), ?reachability, "device-status: patched");
+            tracing::debug!(device = %d.name_any(), reachability = ?status.reachability, "device-status: patched");
            true
        }
-        // A heartbeat can outrace the Device CR's creation by the
+        // Retry a heartbeat that outraced CR creation, but a tombstone
-        // device-reconciler; skip this tick and retry on the next.
+        // for an absent Device is already converged.
-        Err(kube::Error::Api(ae)) if ae.code == 404 => false,
+        Err(kube::Error::Api(ae)) if ae.code == 404 => status.reachability == Reachability::Unknown,
        Err(e) => {
            tracing::warn!(%id, error = %e, "device-status: patch failed");
            false
@@ -161,6 +215,7 @@ async fn patch_status(
 #[cfg(test)]
 mod tests {
    use super::*;
    use harmony_reconciler_contracts::Id;
    #[test]
    fn reachable_within_window_stale_after() {
@@ -174,4 +229,50 @@ mod tests {
            Reachability::Stale
        );
    }
    #[test]
    fn observation_uses_nats_time_and_authorized_key_identity() {
        let device_time = Utc::now() - chrono::Duration::hours(2);
        let server_time = Utc::now();
        let payload = serde_json::to_vec(&HeartbeatPayload {
            device_id: Id::from("device-1".to_string()),
            at: device_time,
            agent_version: Some("1.2.3".to_string()),
        })
        .unwrap();
        let (device_id, observed) =
            observe_heartbeat("heartbeat.device-1", &payload, server_time).unwrap();
        assert_eq!(device_id, "device-1");
        assert_eq!(observed.received_at, server_time);
        assert_eq!(observed.agent_version.as_deref(), Some("1.2.3"));
        assert_eq!(
            reachability(observed.received_at, server_time),
            Reachability::Reachable
        );
        assert!(observe_heartbeat("heartbeat.other-device", &payload, server_time).is_err());
    }
    #[test]
    fn unknown_version_clears_previous_status_value() {
        let status = DeviceStatus {
            last_heartbeat: Some(Utc::now().to_rfc3339()),
            reachability: Reachability::Reachable,
            current_version: None,
            agent_upgrade: None,
        };
        assert!(serde_json::to_value(status).unwrap()["currentVersion"].is_null());
    }
    #[test]
    fn missing_heartbeat_clears_status() {
        let status = heartbeat_status(None, Utc::now());
        assert_eq!(status.reachability, Reachability::Unknown);
        assert_eq!(status.last_heartbeat, None);
        assert_eq!(status.current_version, None);
        let json = serde_json::to_value(status).unwrap();
        assert!(json["lastHeartbeat"].is_null());
        assert!(json["currentVersion"].is_null());
    }
 }
--- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs
+++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs
--- a/fleet/harmony-fleet-operator/src/frontend/assets.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/assets.rs
@@ -2,10 +2,9 @@
 //!
 //! Tailwind CSS is built by `build.rs` into `$OUT_DIR/tailwind.css`
 //! (empty if the CLI was unavailable — dev uses `--css-from` instead).
-//! HTMX and its SSE extension are vendored under `vendor/` so the
+//! HTMX is vendored under `vendor/` so the container ships with no
-//! container ships with no external script dependencies.
+//! external script dependencies.
 pub const TAILWIND_CSS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/tailwind.css"));
 pub const HTMX_JS: &[u8] = include_bytes!("../../vendor/htmx.min.js");
 pub const HTMX_SSE_JS: &[u8] = include_bytes!("../../vendor/htmx-ext-sse.js");
 pub const APP_JS: &[u8] = include_bytes!("../../vendor/app.js");
--- a/fleet/harmony-fleet-operator/src/frontend/layout.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/layout.rs
@@ -8,7 +8,6 @@ const ICON_DASHBOARD: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="1
 const ICON_DEVICES: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="3" width="20" height="14" rx="2"/><line x1="8" y1="21" x2="16" y2="21"/><line x1="12" y1="17" x2="12" y2="21"/></svg>"#;
 const ICON_DEPLOY: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>"#;
 const ICON_BELL: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M6 8a6 6 0 0 1 12 0c0 7 3 9 3 9H3s3-2 3-9"/><path d="M10.3 21a1.94 1.94 0 0 0 3.4 0"/></svg>"#;
 const ICON_COG: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"/></svg>"#;
 const ICON_LOGOUT: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M9 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h4"/><polyline points="16 17 21 12 16 7"/><line x1="21" y1="12" x2="9" y2="12"/></svg>"#;
 const ICON_BRAND: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round"><path d="M4 4v16M20 4v16M4 12h16"/></svg>"#;
@@ -30,13 +29,12 @@ pub fn page(
                title { (title) " — Harmony Fleet" }
                link rel="stylesheet" href="/static/tailwind.css";
                script src="/static/htmx.min.js" defer {}
                script src="/static/htmx-ext-sse.js" defer {}
                script src="/static/app.js" defer {}
                @if live_reload {
                    script { (PreEscaped(LIVE_RELOAD_JS)) }
                }
            }
-            body class="min-h-screen" hx-ext="sse" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" {
+            body class="min-h-screen" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" {
                div class="flex h-screen overflow-hidden" style="background:var(--bg)" {
                    (sidebar(current_path, session, unacked_alerts))
                    main class="flex-1 min-w-0 flex flex-col overflow-hidden" {
@@ -44,7 +42,6 @@ pub fn page(
                        div class="flex-1 overflow-y-auto grid-bg" { (content) }
                    }
                }
                div id="modal-root" {}
            }
        }
    }
@@ -55,22 +52,21 @@ fn sidebar(
    session: Option<&DashboardSession>,
    unacked_alerts: usize,
 ) -> Markup {
-    let nav_items: [(&str, &str, &str, usize); 5] = [
+    let nav_items: [(&str, &str, &str, usize); 4] = [
        ("/", ICON_DASHBOARD, "Dashboard", 0),
        ("/devices", ICON_DEVICES, "Devices", 0),
        ("/deployments", ICON_DEPLOY, "Deployments", 0),
        ("/alerts", ICON_BELL, "Alerts", unacked_alerts),
        ("/settings", ICON_COG, "Settings", 0),
    ];
    html! {
-        aside class="shrink-0 flex flex-col border-r w-[224px]" style="border-color:var(--border); background:var(--bg)" {
+        aside class="shrink-0 flex flex-col border-r w-14 sm:w-[224px]" style="border-color:var(--border); background:var(--bg)" {
-            div class="flex items-center justify-between px-4 py-4 border-b" style="border-color:var(--border)" {
+            div class="flex items-center justify-center sm:justify-between px-2 sm:px-4 py-4 border-b" style="border-color:var(--border)" {
                div class="flex items-center gap-2" {
                    div class="relative w-6 h-6 rounded-md flex items-center justify-center" style="background:var(--accent); color:#0c0c0c" {
                        (PreEscaped(ICON_BRAND))
                    }
-                    span class="text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" }
+                    span class="hidden sm:inline text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" }
                }
            }
@@ -79,7 +75,9 @@ fn sidebar(
                    @let active = is_active(current_path, href);
                    a
                        href=(*href)
-                        class={"group w-full flex items-center gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative "
+                        title=(*label)
                        aria-label=(*label)
                        class={"group w-full flex items-center justify-center sm:justify-start gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative "
                            (if active { "text-slate-100 font-medium" } else { "text-slate-400 hover:text-slate-100" })}
                        style={(if active { "background:rgba(148,163,184,0.06)" } else { "background:transparent" })}
                    {
@@ -89,9 +87,9 @@ fn sidebar(
                        span class={(if active { "text-slate-100" } else { "text-slate-500 group-hover:text-slate-300" })} {
                            (PreEscaped(icon))
                        }
-                        span class="flex-1 text-left" { (label) }
+                        span class="hidden sm:inline flex-1 text-left" { (label) }
                        @if *badge > 0 {
-                            span class="inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" {
+                            span class="hidden sm:inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" {
                                (badge)
                            }
                        }
@@ -100,9 +98,12 @@ fn sidebar(
            }
            @if let Some(s) = session {
-                div class="border-t p-3" style="border-color:var(--border)" {
+                div class="hidden sm:block border-t p-3" style="border-color:var(--border)" {
                    (user_footer(s))
                }
                a href="/logout" class="sm:hidden border-t py-3 flex justify-center text-slate-500 hover:text-rose-400" style="border-color:var(--border)" title="Log out" {
                    (PreEscaped(ICON_LOGOUT))
                }
            }
        }
    }
@@ -170,14 +171,14 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup {
            div class="flex items-center gap-2" {
                div class="relative" {
                    input
-                        class="input w-64"
+                        class="hidden sm:block input w-40 lg:w-64"
                        type="text"
                        name="search"
                        placeholder="Search devices, deployments\u{2026}"
-                        hx-get="/devices/search"
+                        hx-get="/devices"
                        hx-trigger="keyup changed delay:300ms"
-                        hx-target="#device-table-wrapper"
+                        hx-target="body"
-                        hx-swap="innerHTML";
+                        hx-push-url="true";
                }
                a href="/alerts" class="relative btn btn-ghost py-1.5" {
                    (PreEscaped(ICON_BELL))
@@ -188,9 +189,6 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup {
                        }
                    }
                }
                a href="/settings" class="btn btn-ghost py-1.5" title="Settings" {
                    (PreEscaped(ICON_COG))
                }
            }
        }
    }
--- a/fleet/harmony-fleet-operator/src/frontend/server.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/server.rs
@@ -19,17 +19,18 @@ use maud::Markup;
 use serde::Deserialize;
 use tokio_stream::StreamExt;
-use super::assets::{APP_JS, HTMX_JS, HTMX_SSE_JS, TAILWIND_CSS};
+use super::assets::{APP_JS, HTMX_JS, TAILWIND_CSS};
 use super::layout::page;
 use super::views::{
    alerts as alerts_view, dashboard as dashboard_view, deployments as deployments_view,
-    devices as devices_view, settings as settings_view,
+    devices as devices_view,
 };
 use crate::frontend::auth::{self, DASHBOARD_SESSION_COOKIE, DashboardSession, JwksCache};
 use crate::service::FleetService;
 use harmony_zitadel_auth::ZitadelAuthConfig;
 pub const DEFAULT_PORT: u16 = 18080;
 const REQUIRED_ROLE: &str = "fleet-admin";
 #[derive(Clone)]
 pub struct AppState {
@@ -89,9 +90,9 @@ pub fn router(state: AppState) -> Router {
    let public_routes = Router::new()
        .route("/login", get(auth::login_handler))
        .route("/auth/callback", get(auth::callback_handler))
        .route("/logout", get(auth::logout_handler))
        .route("/static/tailwind.css", get(tailwind_css))
        .route("/static/htmx.min.js", get(htmx_js))
        .route("/static/htmx-ext-sse.js", get(htmx_sse_js))
        .route("/static/app.js", get(app_js));
    let private_routes = Router::new()
@@ -99,10 +100,7 @@ pub fn router(state: AppState) -> Router {
        .route("/", get(dashboard_handler))
        // Devices
        .route("/devices", get(devices_handler))
        .route("/devices/search", get(devices_search_handler))
        .route("/devices/{id}/blacklist", post(blacklist_handler))
        .route("/devices/{id}/logs", get(device_logs_handler))
        .route("/devices/{id}/logs/stream", get(device_logs_stream_handler))
        .route("/devices/{id}/exec", post(device_exec_handler))
        // Device detail
        .route("/device/{id}", get(device_detail_handler))
@@ -112,11 +110,6 @@ pub fn router(state: AppState) -> Router {
        // Alerts
        .route("/alerts", get(alerts_handler))
        .route("/alerts/{id}/ack", post(ack_alert_handler))
        // Settings
        .route("/settings", get(settings_handler))
        .route("/settings/toggle/{key}", post(settings_toggle_handler))
        // Logout
        .route("/logout", get(auth::logout_handler))
        .route_layer(middleware::from_fn_with_state(state.clone(), csrf_protect))
        .route_layer(middleware::from_fn_with_state(state.clone(), require_auth));
@@ -145,6 +138,13 @@ async fn require_auth(
    match state.jwks.verify(cookie.value(), &state.config).await {
        Ok(session) => {
            if !session.has_role(REQUIRED_ROLE) {
                tracing::warn!(
                    roles = ?session.roles,
                    "dashboard access denied: missing {REQUIRED_ROLE} role"
                );
                return forbidden_response();
            }
            req.extensions_mut().insert(session);
            next.run(req).await
        }
@@ -156,6 +156,27 @@ async fn require_auth(
    }
 }
 fn forbidden_response() -> Response {
    let body = maud::html! {
        (maud::DOCTYPE)
        html lang="en" {
            head {
                meta charset="utf-8";
                meta name="viewport" content="width=device-width, initial-scale=1";
                title { "Access denied - Harmony Fleet" }
            }
            body {
                main {
                    h1 { "Access denied" }
                    p { "The " code { (REQUIRED_ROLE) } " role is required to use this dashboard." }
                    p { "Ask your administrator to grant the role, or " a href="/logout" { "sign out" } "." }
                }
            }
        }
    };
    (StatusCode::FORBIDDEN, body).into_response()
 }
 async fn csrf_protect(State(state): State<AppState>, req: Request<Body>, next: Next) -> Response {
    if !is_mutating_method(req.method()) {
        return next.run(req).await;
@@ -371,44 +392,11 @@ async fn devices_handler(
    ))
 }
 async fn devices_search_handler(
    State(s): State<AppState>,
    Query(q): Query<DevicesQuery>,
 ) -> Result<Markup, AppError> {
    let status = q.status.as_deref().and_then(parse_device_status);
    let devices = s
        .fleet
        .filtered_devices(
            status,
            q.deployment.clone(),
            q.region.clone(),
            q.search.clone(),
        )
        .await?;
    Ok(devices_view::page(
        &devices,
        &[],
        &[],
        status,
        q.deployment.as_deref(),
        q.region.as_deref(),
        q.search.as_deref(),
    ))
 }
 // ── Device detail ──────────────────────────────────────────────────────
 #[derive(Deserialize, Default)]
 struct DeviceDetailQuery {
    tab: Option<String>,
 }
 async fn device_detail_handler(
    State(s): State<AppState>,
    Path(id): Path<String>,
    Query(q): Query<DeviceDetailQuery>,
    session: Option<Extension<DashboardSession>>,
 ) -> Result<Markup, AppError> {
    let device = s
@@ -423,18 +411,6 @@ async fn device_detail_handler(
        None
    };
    let tab = q.tab.as_deref().unwrap_or("overview");
    // Tab click (HTMX): return the tabs block (bar + content) so the
    // active highlight re-renders with the content.
    if q.tab.is_some() {
        return Ok(devices_view::device_tabs(
            &device,
            deployment_version.as_deref(),
            tab,
        ));
    }
    let unacked = s
        .fleet
        .list_alerts()
@@ -453,6 +429,60 @@ async fn device_detail_handler(
    ))
 }
 #[derive(Deserialize)]
 struct ExecForm {
    command: String,
 }
 async fn device_exec_handler(
    State(s): State<AppState>,
    Path(id): Path<String>,
    Extension(session): Extension<DashboardSession>,
    Form(form): Form<ExecForm>,
 ) -> Result<Markup, AppError> {
    if form.command.trim().is_empty() {
        return Ok(devices_view::command_output(
            &form.command,
            "Command failed: command must not be empty",
        ));
    }
    if form.command.len() > 16 * 1024 {
        return Ok(devices_view::command_output(
            "",
            "Command failed: command exceeds 16384 byte limit",
        ));
    }
    tracing::info!(
        operator = %session.subject,
        device = %id,
        command_bytes = form.command.len(),
        "dashboard device command requested"
    );
    let started = tokio::time::Instant::now();
    let output = match s.fleet.run_command(&id, &form.command).await {
        Ok(output) => {
            tracing::info!(
                operator = %session.subject,
                device = %id,
                duration_ms = started.elapsed().as_millis(),
                "dashboard device command completed"
            );
            output
        }
        Err(error) => {
            tracing::warn!(
                operator = %session.subject,
                device = %id,
                duration_ms = started.elapsed().as_millis(),
                %error,
                "dashboard device command failed"
            );
            format!("Command failed: {error}")
        }
    };
    Ok(devices_view::command_output(&form.command, &output))
 }
 // ── Deployments ────────────────────────────────────────────────────────
 async fn deployments_handler(
@@ -483,6 +513,8 @@ async fn deployments_handler(
 #[derive(Deserialize, Default)]
 struct DeploymentQuery {
    tab: Option<String>,
    #[serde(default)]
    summary: bool,
 }
 async fn deployment_handler(
@@ -497,6 +529,10 @@ async fn deployment_handler(
        .await?
        .ok_or_else(|| anyhow::anyhow!("deployment not found: {id}"))?;
    if q.summary {
        return Ok(deployments_view::summary(&deployment));
    }
    let devices = s.fleet.get_deployment_devices(&id).await?;
    let tab = q.tab.as_deref().unwrap_or("overview");
    let unacked = s
@@ -561,79 +597,14 @@ async fn ack_alert_handler(
    Ok(alerts_view::alert_row(alert))
 }
 // ── Settings ───────────────────────────────────────────────────────────
 async fn settings_handler(
    State(s): State<AppState>,
    session: Option<Extension<DashboardSession>>,
 ) -> Result<Markup, AppError> {
    let unacked = s
        .fleet
        .list_alerts()
        .await?
        .iter()
        .filter(|a| !a.acked)
        .count();
    Ok(page(
        "Settings",
        s.live_reload,
        "/settings",
        session.as_ref().map(|e| &e.0),
        unacked,
        settings_view::page(),
    ))
 }
 async fn settings_toggle_handler(Path(_key): Path<String>) -> Result<Markup, AppError> {
    // In a real app this would toggle a notification channel.
    // For the mock, we return the same static content.
    Ok(settings_view::page())
 }
 // ── Device logs ────────────────────────────────────────────────────────
 async fn device_logs_handler(Path(id): Path<String>) -> Result<Markup, AppError> {
    Ok(devices_view::logs_modal(&id))
 }
 async fn device_logs_stream_handler(
    Path(_id): Path<String>,
 ) -> Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>> {
    // One honest notice, then keep-alive. Real agent-log streaming (over
    // NATS) is pending — don't fabricate log lines.
    let html = r#"<div class="px-0 py-px italic text-slate-600">— live agent log streaming is not implemented yet —</div>"#;
    let stream = futures_util::stream::once(async move {
        Ok::<_, Infallible>(Event::default().event("log").data(html))
    });
    Sse::new(stream).keep_alive(KeepAlive::new().interval(Duration::from_secs(15)))
 }
 // ── Run command ────────────────────────────────────────────────────────
 #[derive(Deserialize)]
 struct ExecForm {
    command: String,
 }
 async fn device_exec_handler(
    State(s): State<AppState>,
    Path(id): Path<String>,
    Form(form): Form<ExecForm>,
 ) -> Result<Markup, AppError> {
    let output = s.fleet.run_command(&id, &form.command).await?;
    Ok(devices_view::command_output(&form.command, &output))
 }
 // ── Blacklist ──────────────────────────────────────────────────────────
 async fn blacklist_handler(
    State(s): State<AppState>,
    Path(id): Path<String>,
-) -> Result<Markup, AppError> {
+) -> Result<Response, AppError> {
-    let updated = s.fleet.blacklist_device(&id).await?;
+    s.fleet.blacklist_device(&id).await?;
-    Ok(devices_view::row(&updated))
+    Ok(Redirect::to(&format!("/device/{id}")).into_response())
 }
 // ── Helpers ────────────────────────────────────────────────────────────
@@ -670,13 +641,6 @@ async fn htmx_js() -> Response {
    static_response(HTMX_JS.to_vec(), "application/javascript; charset=utf-8")
 }
 async fn htmx_sse_js() -> Response {
    static_response(
        HTMX_SSE_JS.to_vec(),
        "application/javascript; charset=utf-8",
    )
 }
 async fn app_js() -> Response {
    static_response(APP_JS.to_vec(), "application/javascript; charset=utf-8")
 }
--- a/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs
@@ -10,8 +10,6 @@ pub fn page(alerts: &[Alert]) -> Markup {
            div class="flex items-center gap-2" {
                h2 class="text-[15px] font-semibold text-slate-200" { "Alerts" }
                span class="text-[11px] text-slate-500" { "\u{b7} " (unacked) " unacked" }
                div class="flex-1" {}
                button class="btn btn-ghost" { "Ack all" }
            }
            div class="card card-flush" {
                table class="tbl" {
--- a/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs
@@ -74,6 +74,20 @@ fn deployment_card(d: &DeploymentDetail) -> Markup {
 // ── Deployment detail page ─────────────────────────────────────────────
 pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup {
    html! {
        div class="p-6 space-y-4" {
            (summary(deployment))
            // The whole tabs block re-renders on switch so the active
            // highlight follows the content.
            div id="dep-tabs" {
                (tabs_and_content(deployment, devices, "overview"))
            }
        }
    }
 }
 pub fn summary(deployment: &DeploymentDetail) -> Markup {
    let pct = if deployment.target > 0 {
        ((deployment.healthy as f64 / deployment.target as f64) * 100.0).round() as u32
    } else {
@@ -81,9 +95,13 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
    };
    html! {
-        div class="p-6 space-y-4" {
+        div
-            // Header
+            id="deployment-summary"
-            div class="card p-5" {
+            class="card p-5"
            hx-get={"/deployment/" (&deployment.name) "?summary=true"}
            hx-trigger="every 3s"
            hx-target="this"
            hx-swap="outerHTML" {
                div class="flex items-start justify-between gap-6" {
                    div class="min-w-0" {
                        div class="flex items-center gap-3 flex-wrap" {
@@ -102,6 +120,18 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
                    }
                }
                @if let Some(error) = &deployment.last_error {
                    div class="mt-5 rounded border px-4 py-3" style="border-color:var(--bad); background:rgba(251,113,133,0.08)" {
                        div class="flex items-center gap-2 text-[11px] uppercase tracking-wider" style="color:var(--bad)" {
                            span { "Latest rollout error" }
                            span class="text-slate-600" { "\u{b7}" }
                            a href={"/device/" (&error.device)} class="font-mono normal-case tracking-normal hover:underline" { (&error.device) }
                            span class="text-slate-500 normal-case tracking-normal tabular-nums" { (&error.at) }
                        }
                        p class="mt-1 text-[13px] text-slate-200" { (&error.message) }
                    }
                }
                // Rollout progress
                div class="mt-5" {
                    div {
@@ -140,13 +170,6 @@ pub fn detail(deployment: &DeploymentDetail, devices: &[DeviceDetail]) -> Markup
                        }
                    }
                }
            }
            // The whole tabs block re-renders on switch so the active
            // highlight follows the content.
            div id="dep-tabs" {
                (tabs_and_content(deployment, devices, "overview"))
            }
        }
    }
 }
@@ -211,7 +234,7 @@ fn devices_tab(devices: &[DeviceDetail]) -> Markup {
                            td { (badges::device_status(d.status)) }
                            td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } }
                            td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" {
-                                @if let Some(inv) = &d.inventory { (&inv.agent_version) } @else { "\u{2014}" }
+                                @if let Some(version) = &d.current_version { (version) } @else { "\u{2014}" }
                            } }
                            td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } }
                        }
@@ -251,7 +274,7 @@ fn per_device_grid(devices: &[DeviceDetail]) -> Markup {
    html! {
        div class="card" {
            div class="flex items-center justify-between px-4 py-3 border-b" style="border-color:var(--border)" {
-                span class="section-title" { "Per-device rollout" }
+                span class="section-title" { "Device reachability" }
                span class="text-[10px] text-slate-600 font-mono" { (devices.len()) " devices" }
            }
            div class="p-4 grid grid-cols-10 gap-1.5" {
@@ -266,10 +289,9 @@ fn per_device_grid(devices: &[DeviceDetail]) -> Markup {
                }
            }
            div class="border-t px-4 py-2.5 flex items-center gap-3 text-[10px] text-slate-500 font-mono" style="border-color:var(--border)" {
-                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--ok)" {} " healthy" }
+                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--ok)" {} " reachable" }
                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--warn)" {} " pending" }
                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:var(--bad)" {} " failing" }
                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:rgba(251,113,133,0.6)" {} " stale" }
                span class="flex items-center gap-1" { span class="w-2 h-2 rounded-sm" style="background:#475569" {} " unknown or blacklisted" }
            }
        }
    }
@@ -315,3 +337,45 @@ fn time_ago(minutes: i64) -> String {
        format!("{}d ago", minutes / (60 * 24))
    }
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use crate::service::{DeploymentError, DeploymentStatus};
    #[test]
    fn summary_polls_itself_and_shows_actionable_error() {
        let deployment = DeploymentDetail {
            name: "edge".into(),
            rollout_revision: "deployment-uid:2".into(),
            version: "1.2.3".into(),
            status: DeploymentStatus::Failing,
            target: 2,
            healthy: 1,
            failing: 1,
            pending: 0,
            updated_at: "2026-07-27 10:15".into(),
            last_error: Some(DeploymentError {
                device: "pi-09".into(),
                message: "image pull denied".into(),
                at: "2026-07-27T10:15:00Z".into(),
            }),
        };
        let html = summary(&deployment).into_string();
        assert!(html.contains("hx-get=\"/deployment/edge?summary=true\""));
        assert!(html.contains("hx-trigger=\"every 3s\""));
        assert!(html.contains("hx-swap=\"outerHTML\""));
        assert!(html.contains("href=\"/device/pi-09\""));
        assert!(html.contains("image pull denied"));
        assert!(html.contains("2026-07-27T10:15:00Z"));
    }
    #[test]
    fn overview_describes_device_reachability() {
        let html = per_device_grid(&[]).into_string();
        assert!(html.contains("Device reachability"));
        assert!(html.contains("reachable"));
        assert!(!html.contains("Per-device rollout"));
    }
 }
--- a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs
@@ -6,13 +6,8 @@ use crate::service::{DeviceDetail, DeviceStatus};
 // ── Inline icons ────────────────────────────────────────────────────────
 const ICON_SEARCH: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>"#;
 const ICON_CHEVRON_DOWN: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="6 9 12 15 18 9"/></svg>"#;
 const ICON_POWER: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18.36 6.64a9 9 0 1 1-12.73 0"/><line x1="12" y1="2" x2="12" y2="12"/></svg>"#;
 const ICON_PAUSE: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="6" y="4" width="4" height="16"/><rect x="14" y="4" width="4" height="16"/></svg>"#;
 const ICON_BAN: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="4.93" y1="4.93" x2="19.07" y2="19.07"/></svg>"#;
 const ICON_EXPAND: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="15 3 21 3 21 9"/><polyline points="9 21 3 21 3 15"/><line x1="21" y1="3" x2="14" y2="10"/><line x1="3" y1="21" x2="10" y2="14"/></svg>"#;
 const ICON_EXTERNAL: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"/><polyline points="15 3 21 3 21 9"/><line x1="10" y1="14" x2="21" y2="3"/></svg>"#;
 const ICON_REFRESH: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="23 4 23 10 17 10"/><polyline points="1 20 1 14 7 14"/><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10"/><path d="M20.49 15a9 9 0 0 1-14.85 3.36L1 14"/></svg>"#;
 const ICON_COPY: &str = r#"<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>"#;
 // ── Devices list page ──────────────────────────────────────────────────
@@ -172,7 +167,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
        div class="p-6 space-y-4" {
            // Header
            div class="card p-5" {
-                div class="flex items-start justify-between gap-6" {
+                div class="flex flex-col sm:flex-row sm:items-start justify-between gap-4 sm:gap-6" {
                    div class="min-w-0" {
                        div class="flex items-center gap-3 flex-wrap" {
                            h1 class="text-[22px] font-semibold font-mono text-slate-50 truncate whitespace-nowrap" { (&device.id) }
@@ -189,10 +184,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
                            span { span class="text-slate-600" { "Last ping" } " " span class="text-slate-300 tabular-nums" { (time_ago(device.minutes_ago)) } }
                        }
                    }
-                    div class="flex items-center gap-2 shrink-0" {
+                    div class="flex items-center gap-2 shrink-0 self-start" {
                        button class="btn btn-ghost" { (PreEscaped(ICON_REFRESH)) " Reconcile" }
                        button class="btn btn-ghost" { (PreEscaped(ICON_POWER)) " Restart" }
                        button class="btn btn-ghost" { (PreEscaped(ICON_PAUSE)) " Suspend" }
                        @if device.status != DeviceStatus::Blacklisted {
                            button
                                class="btn btn-danger"
@@ -206,80 +198,28 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup
                }
            }
-            // The whole tabs block re-renders on switch so the active
+            (overview_tab(device, deployment_version))
-            // highlight follows (only the content swapping would leave it
+            (command_panel(device))
            // stuck on Overview).
            div id="device-tabs" {
                (device_tabs(device, deployment_version, "overview"))
            }
        }
    }
 }
 /// Tab bar (active highlighted) + the active tab's content. Swapped as a
 /// unit into `#device-tabs`.
 pub fn device_tabs(
    device: &DeviceDetail,
    deployment_version: Option<&str>,
    active: &str,
 ) -> Markup {
    let content = match active {
        "logs" => logs_tab(device),
        "command" => command_tab(&device.id),
        _ => overview_tab(device, deployment_version),
    };
    html! {
        div class="flex items-center gap-1 border-b" style="border-color:var(--border)" {
            (tab_button(&device.id, "Overview", "overview", active == "overview"))
            (tab_button(&device.id, "Logs", "logs", active == "logs"))
            (tab_button(&device.id, "Run command", "command", active == "command"))
            div class="flex-1" {}
            button
                class="btn btn-ghost mb-1"
                hx-get={"/devices/" (device.id) "/logs"}
                hx-target="#modal-root"
                hx-swap="innerHTML" {
                (PreEscaped(ICON_EXPAND)) " Pop-out logs"
            }
        }
        div { (content) }
    }
 }
 fn tab_button(device_id: &str, label: &str, tab: &str, active: bool) -> Markup {
    html! {
        button
            class={"px-3 py-2 text-[13px] font-medium relative "
                (if active { "text-slate-100" } else { "text-slate-500 hover:text-slate-300" })}
            hx-get={"/device/" (device_id) "?tab=" (tab)}
            hx-target="#device-tabs"
            hx-swap="innerHTML" {
            (label)
            @if active {
                span class="absolute left-0 right-0 -bottom-px h-0.5" style="background:var(--accent)" {}
            }
        }
    }
 }
 fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup {
    html! {
-        div class="grid grid-cols-12 gap-4" {
+        div class="space-y-4" {
            // Device info + current deployment
            div class="col-span-12 lg:col-span-5 space-y-4" {
                div class="card p-5" {
                    div class="section-title mb-3" { "Device info" }
-                    (definition("Device ID", &device.id, true, true))
+                    (definition("Device ID", &device.id, true))
-                    (definition("Region", &device.region, true, false))
+                    (definition("Region", &device.region, true))
-                    (definition("Last ping", &time_ago(device.minutes_ago), false, false))
+                    (definition("Last ping", &time_ago(device.minutes_ago), false))
                    (definition("Agent", agent_version(device), true))
                    @if let Some(inv) = &device.inventory {
-                        (definition("Hostname", &inv.hostname, true, false))
+                        (definition("Hostname", &inv.hostname, true))
-                        (definition("Arch", &inv.arch, true, false))
+                        (definition("Arch", &inv.arch, true))
-                        (definition("OS", &inv.os, false, false))
+                        (definition("OS", &inv.os, false))
-                        (definition("Kernel", &inv.kernel, true, false))
+                        (definition("Kernel", &inv.kernel, true))
-                        (definition("CPU cores", &inv.cpu_cores.to_string(), false, false))
+                        (definition("CPU cores", &inv.cpu_cores.to_string(), false))
-                        (definition("Memory", &format!("{} MB", inv.memory_mb), false, false))
+                        (definition("Memory", &format!("{} MB", inv.memory_mb), false))
                        (definition("Agent", &inv.agent_version, true, false))
                    } @else {
                        div class="text-[12px] text-slate-500 mt-2" { "No inventory reported yet" }
                    }
@@ -310,181 +250,48 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark
                        div class="text-[12px] text-slate-500" { "No deployment assigned" }
                    }
                }
            }
            // Recent logs (live stream)
            div class="col-span-12 lg:col-span-7" {
                div class="card overflow-hidden" {
                    div class="flex items-center justify-between px-4 py-3 border-b" style="border-color:var(--border)" {
                        span class="section-title" { "Recent logs" }
                        button
                            class="text-[11px] text-slate-400 hover:text-slate-100 flex items-center gap-1"
                            hx-get={"/devices/" (device.id) "/logs"}
                            hx-target="#modal-root"
                            hx-swap="innerHTML" {
                            "Pop out " (PreEscaped(ICON_EXTERNAL))
                        }
                    }
                    (log_stream(&device.id, "260px"))
                }
            }
        }
    }
 }
-fn logs_tab(device: &DeviceDetail) -> Markup {
+fn command_panel(device: &DeviceDetail) -> Markup {
    html! {
-        div class="card overflow-hidden mt-4" {
+        div class="card p-5" {
-            div class="flex items-center gap-2 px-4 py-2.5 border-b" style="border-color:var(--border)" {
+            div class="section-title mb-3" { "Run command" }
                span class="relative flex w-1.5 h-1.5" {
                    span class="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60" style="background:var(--accent)" {}
                    span class="relative inline-flex w-1.5 h-1.5 rounded-full" style="background:var(--accent)" {}
                }
                span class="text-[11px] font-mono text-slate-400" { "streaming" }
                span class="text-[11px] text-slate-600 font-mono" { "\u{b7} live" }
            }
            (log_stream(&device.id, "520px"))
        }
    }
 }
 /// Shared SSE log console. The stream endpoint is the seam the real
 /// agent-log transport plugs into; the markup is identical wherever a
 /// device's logs are shown (overview card, logs tab, pop-out modal).
 fn log_stream(device_id: &str, height: &str) -> Markup {
    html! {
        div
            class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto"
            style={"background:#050608; height:" (height)}
            hx-ext="sse"
            sse-connect={"/devices/" (device_id) "/logs/stream"}
            sse-swap="log"
            hx-swap="beforeend" {
            div class="px-0 py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" }
        }
    }
 }
 /// One-shot "run a shell command on the device" panel. Submits the
 /// command to the [`run_command`](crate::service::FleetService::run_command)
 /// seam and appends the response to the console. Live streaming and a
 /// full TTY are later refinements.
 fn command_tab(device_id: &str) -> Markup {
    html! {
        div class="card overflow-hidden mt-4" {
            form
-                class="flex items-center gap-2 px-4 py-3 border-b"
+                class="flex gap-2"
-                style="border-color:var(--border)"
+                hx-post={"/devices/" (device.id) "/exec"}
-                hx-post={"/devices/" (device_id) "/exec"}
+                hx-target="#command-output"
-                hx-target="#exec-output"
+                hx-swap="innerHTML" {
                hx-swap="beforeend"
                "hx-on::after-request"="this.reset()" {
                span class="font-mono text-slate-500 text-[13px]" { "$" }
                input
-                    class="input flex-1 font-mono text-[12px]"
+                    class="flex-1 rounded border px-3 py-2 font-mono text-[12px] text-slate-100"
                    style="background:#050608; border-color:var(--border)"
                    type="text"
                    name="command"
-                    placeholder="e.g. systemctl status harmony-agent"
+                    maxlength="16384"
                    autocomplete="off"
                    placeholder="uname -a"
                    required;
-                button class="btn btn-primary" type="submit" { "Run" }
+                button class="btn" type="submit" { "Run" }
            }
-            div
+            pre id="command-output" class="mt-3 min-h-16 overflow-auto whitespace-pre-wrap rounded p-3 font-mono text-[11.5px] text-slate-300" style="background:#050608" {
-                id="exec-output"
+                "Command output appears here."
                class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto"
                style="background:#050608; height:440px" {
                div class="px-0 py-px italic text-slate-700" {
                    "\u{2014} commands run here are sent to the device; output appears below \u{2014}"
                }
            }
        }
    }
 }
 /// Markup for one command's response, appended to the exec console.
 pub fn command_output(command: &str, output: &str) -> Markup {
    html! {
-        div class="py-1 border-t" style="border-color:var(--border)" {
+        span class="text-slate-500" { "$ " (command) "\n" }
-            div class="text-slate-300" { span class="text-slate-500" { "$ " } (command) }
+        (output)
            pre class="text-slate-400 whitespace-pre-wrap mt-0.5" { (output) }
        }
    }
 }
 // ── Logs modal (SSE streaming) ─────────────────────────────────────────
 pub fn logs_modal(device_id: &str) -> Markup {
    html! {
        dialog
            id="device-logs-modal"
            class="m-auto grid grid-rows-[auto_1fr] h-[88vh] w-[min(96vw,82rem)] overflow-hidden rounded-none border-t-2 border-x-0 border-b-0 p-0 text-slate-100 shadow-[0_32px_64px_rgba(0,0,0,0.9),0_0_0_1px_rgba(148,163,184,0.06)] backdrop:bg-black/85"
            style="border-color:var(--accent); background:#080a0c"
        {
            div class="flex items-center justify-between border-b px-5 py-3" style="background:#0c1018; border-color:var(--border)" {
                div class="flex items-center gap-3" {
                    span class="relative flex h-1.5 w-1.5 shrink-0" {
                        span class="absolute inline-flex h-full w-full animate-ping rounded-full bg-orange-400 opacity-60" {}
                        span class="relative inline-flex h-1.5 w-1.5 rounded-full" style="background:var(--accent)" {}
                    }
                    code class="text-sm font-medium text-slate-100" { (device_id) }
                    span class="text-[10px] font-semibold uppercase tracking-[0.15em] text-orange-500/60" { "\u{b7} logs" }
                }
                form method="dialog" {
                    button
                        type="submit"
                        class="flex items-center gap-1.5 text-slate-500 transition-colors hover:text-slate-200"
                        aria-label="Close"
                    {
                        kbd class="rounded border bg-slate-800/60 px-1.5 py-0.5 font-mono text-[10px] text-slate-400" style="border-color:var(--border-strong)" { "esc" }
                        span class="text-xs" { "close" }
                    }
                }
            }
            div
                class="overflow-y-auto py-3 font-mono text-[11.5px] leading-6 px-5"
                style="background:#050608"
                hx-ext="sse"
                sse-connect={"/devices/" (device_id) "/logs/stream"}
                sse-swap="log"
                hx-swap="beforeend" {
                div class="py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" }
            }
        }
    }
 }
 // ── Row (for blacklist response) ───────────────────────────────────────
 pub fn row(d: &DeviceDetail) -> Markup {
    html! {
        tr id={"device-" (d.id)} hx-get={"/device/" (d.id)} hx-target="body" hx-push-url="true" class="cursor-pointer" {
            td {
                span class="font-mono text-slate-100 hover:text-(--accent-fg) hover:underline underline-offset-2 whitespace-nowrap" {
                    (&d.id)
                }
            }
            td { (badges::device_status(d.status)) }
            td {
                @if let Some(dep) = &d.deployment { span class="font-mono text-[12px] text-slate-300 whitespace-nowrap" { (dep) } }
                @else { span class="text-slate-700" { "\u{2014}" } }
            }
            td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } }
            td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" { (agent_version(d)) } }
            td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } }
        }
    }
 }
 // ── Helpers ────────────────────────────────────────────────────────────
 /// Agent version from the device's inventory, or an em-dash placeholder
 /// when the agent hasn't reported inventory yet.
 fn agent_version(d: &DeviceDetail) -> &str {
-    d.inventory
+    d.current_version.as_deref().unwrap_or("\u{2014}")
        .as_ref()
        .map(|i| i.agent_version.as_str())
        .unwrap_or("\u{2014}")
 }
 fn time_ago(minutes: i64) -> String {
@@ -499,15 +306,12 @@ fn time_ago(minutes: i64) -> String {
    }
 }
-fn definition(label: &str, value: &str, mono: bool, copyable: bool) -> Markup {
+fn definition(label: &str, value: &str, mono: bool) -> Markup {
    html! {
        div class="flex items-center justify-between py-1.5 text-[12px] border-b last:border-b-0" style="border-color:var(--border)" {
            span class="text-slate-500" { (label) }
            span class={(if mono { "font-mono whitespace-nowrap" } else { "" }) " text-slate-200 flex items-center gap-1.5"} {
                (value)
                @if copyable {
                    button class="text-slate-600 hover:text-slate-300" title="Copy" { (PreEscaped(ICON_COPY)) }
                }
            }
        }
    }
@@ -527,6 +331,7 @@ mod tests {
            deployment: Some("edge-gateway".into()),
            region: "eu-paris-1".into(),
            tags: vec!["prod".into()],
            current_version: Some("v1.2.3".into()),
            inventory: Some(crate::service::InventorySnapshot {
                hostname: "hf-edge-001".into(),
                arch: "aarch64".into(),
@@ -540,28 +345,15 @@ mod tests {
    }
    #[test]
-    fn overview_shows_device_info_not_removed_mock() {
+    fn detail_shows_command_form_and_only_implemented_device_features() {
        let html = detail(&sample(), Some("v2.14.1")).into_string();
        assert!(html.contains("Device info"));
-        assert!(html.contains("v1.2.3"), "agent version from inventory");
+        assert!(html.contains("v1.2.3"), "agent version from heartbeat");
        assert!(html.contains("aarch64"));
        assert!(html.contains("Run command"));
        // Removed mock surfaces must be gone.
        assert!(!html.contains("MAC"));
        assert!(!html.contains("triggered reconcile"));
        assert!(!html.contains("Deployment history"));
    }
    #[test]
    fn command_tab_posts_to_exec_seam() {
        let html = device_tabs(&sample(), None, "command").into_string();
        assert!(html.contains("/devices/hf-edge-001/exec"));
-        assert!(html.contains(r#"name="command""#));
+        for unsupported in ["Reconcile", "Restart", "Suspend", "Recent logs"] {
-    }
+            assert!(!html.contains(unsupported));
-
+        }
    #[test]
    fn logs_tab_connects_to_stream() {
        let html = device_tabs(&sample(), None, "logs").into_string();
        assert!(html.contains("/devices/hf-edge-001/logs/stream"));
    }
 }
--- a/fleet/harmony-fleet-operator/src/frontend/views/mod.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/views/mod.rs
@@ -3,4 +3,3 @@ pub mod badges;
 pub mod dashboard;
 pub mod deployments;
 pub mod devices;
 pub mod settings;
--- a/fleet/harmony-fleet-operator/src/frontend/views/settings.rs
+++ b/fleet/harmony-fleet-operator/src/frontend/views/settings.rs
@@ -1,74 +0,0 @@
 use maud::{Markup, PreEscaped, html};
 pub fn page() -> Markup {
    html! {
        div class="p-6 max-w-3xl space-y-4" {
            div {
                h2 class="text-[15px] font-semibold text-slate-200" { "Notification channels" }
                p class="text-[12px] text-slate-500 mt-1" { "Where alerts get delivered when something needs your attention." }
            }
            (channel_row("Email", "email", "alerts@example.com", true))
            (channel_row("Slack", "slack", "#fleet-alerts", true))
            (channel_row("Discord", "discord", "https://discord.com/api/webhooks/\u{2026}", false))
            (channel_row("SMS", "sms", "+1 555 010 0001", true))
        }
    }
 }
 fn channel_row(name: &str, key: &str, placeholder: &str, enabled: bool) -> Markup {
    let enabled_val = if enabled {
        "var(--ok)"
    } else {
        "rgba(148,163,184,0.2)"
    };
    let translate = if enabled { "18px" } else { "2px" };
    let display_val = if enabled { placeholder } else { "disabled" };
    html! {
        div class="card p-5" {
            div class="flex items-center justify-between" {
                div class="flex items-center gap-3" {
                    span class="inline-flex items-center justify-center w-9 h-9 rounded-md" style="background:var(--bg-elev-2); color:var(--accent-fg)" {
                        (PreEscaped(channel_icon(key)))
                    }
                    div {
                        div class="text-[14px] text-slate-100 font-medium" { (name) }
                        div class="text-[11px] text-slate-500" { (display_val) }
                    }
                }
                button
                    class="relative w-9 h-5 rounded-full transition-colors"
                    style={"background:" (enabled_val)}
                    hx-post={"/settings/toggle/" (key)}
                    hx-target="closest .card"
                    hx-swap="outerHTML" {
                    span class="absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform" style={"transform:translateX(" (translate) ")"} {}
                }
            }
            div class={(if enabled { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3" } else { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3 max-h-0 opacity-0 overflow-hidden" })} {
                div {
                    label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Destination" }
                    input class="input mt-1 w-full" style="padding-left:10px" type="text" placeholder=(placeholder) value=(placeholder) {}
                }
                div {
                    label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Notify on" }
                    div class="mt-1 flex gap-1.5" {
                        span class="chip active" { "critical" }
                        span class="chip active" { "warning" }
                        span class="chip" { "info" }
                    }
                }
            }
        }
    }
 }
 fn channel_icon(key: &str) -> String {
    match key {
        "email" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 4h16c1.1 0 2 .9 2 2v12c0 1.1-.9 2-2 2H4c-1.1 0-2-.9-2-2V6c0-1.1.9-2 2-2z"/><polyline points="22,6 12,13 2,6"/></svg>"#.to_string(),
        "slack" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="currentColor" stroke="none"><path d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"/></svg>"#.to_string(),
        "discord" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="currentColor" stroke="none"><path d="M20.317 4.37a19.79 19.79 0 0 0-4.885-1.515.07.07 0 0 0-.07.035c-.21.375-.444.864-.608 1.25a18.27 18.27 0 0 0-5.487 0 12.64 12.64 0 0 0-.617-1.25.077.077 0 0 0-.07-.035 19.74 19.74 0 0 0-4.885 1.515.07.07 0 0 0-.032.027C.533 9.046-.32 13.58.099 18.057c.001.01.008.02.018.027a19.9 19.9 0 0 0 5.993 3.03.078.078 0 0 0 .084-.028c.462-.63.873-1.295 1.226-1.994a.076.076 0 0 0-.041-.106 13.107 13.107 0 0 1-1.872-.892.077.077 0 0 1-.008-.128 10.2 10.2 0 0 0 .372-.292.074.074 0 0 1 .077-.01c3.928 1.793 8.18 1.793 12.062 0a.074.074 0 0 1 .078.01c.12.098.246.198.373.292a.077.077 0 0 1-.006.127 12.299 12.299 0 0 1-1.873.892.077.077 0 0 0-.041.107c.36.698.772 1.362 1.225 1.993a.076.076 0 0 0 .084.028 19.84 19.84 0 0 0 6.002-3.03.077.077 0 0 0 .018-.026c.5-5.177-.838-9.674-3.548-13.66a.061.061 0 0 0-.031-.029zM8.02 15.33c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.956-2.419 2.157-2.419 1.21 0 2.176 1.095 2.157 2.42 0 1.333-.956 2.418-2.157 2.418zm7.975 0c-1.183 0-2.157-1.085-2.157-2.419 0-1.333.955-2.419 2.157-2.419 1.21 0 2.176 1.095 2.157 2.42 0 1.333-.946 2.418-2.157 2.418z"/></svg>"#.to_string(),
        "sms" => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"/></svg>"#.to_string(),
        _ => r#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/></svg>"#.to_string(),
    }
 }
--- a/fleet/harmony-fleet-operator/src/group_cache.rs
+++ b/fleet/harmony-fleet-operator/src/group_cache.rs
@@ -0,0 +1,93 @@
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use harmony_reconciler_contracts::DeviceGroupSource;
 use tokio::sync::RwLock;
 pub type DeviceGroups = HashMap<String, HashSet<String>>;
 #[derive(Default)]
 struct CachedGroups {
    groups: Option<Arc<DeviceGroups>>,
    updated_at: Option<DateTime<Utc>>,
 }
 /// Failed refreshes retain the last successful snapshot; before the first
 /// success there is no snapshot, so startup remains fail-closed.
 pub struct DeviceGroupCache {
    source: Arc<dyn DeviceGroupSource>,
    cached: RwLock<CachedGroups>,
 }
 impl DeviceGroupCache {
    pub fn new(source: Arc<dyn DeviceGroupSource>) -> Self {
        Self {
            source,
            cached: RwLock::new(CachedGroups::default()),
        }
    }
    pub async fn refresh(&self) -> Option<Arc<DeviceGroups>> {
        match self.source.device_groups().await {
            Ok(groups) => {
                let groups = Arc::new(groups);
                let mut cached = self.cached.write().await;
                cached.groups = Some(groups.clone());
                cached.updated_at = Some(Utc::now());
                Some(groups)
            }
            Err(error) => {
                let cached = self.cached.read().await;
                let age_seconds = cached
                    .updated_at
                    .map(|updated_at| Utc::now().signed_duration_since(updated_at).num_seconds());
                tracing::warn!(
                    %error,
                    ?age_seconds,
                    "device group refresh failed; retaining last known state"
                );
                cached.groups.clone()
            }
        }
    }
    pub async fn snapshot(&self) -> Option<Arc<DeviceGroups>> {
        self.cached.read().await.groups.clone()
    }
 }
 #[cfg(test)]
 mod tests {
    use std::sync::atomic::{AtomicUsize, Ordering};
    use async_trait::async_trait;
    use harmony_reconciler_contracts::GroupSourceError;
    use super::*;
    struct OneSuccess(AtomicUsize);
    #[async_trait]
    impl DeviceGroupSource for OneSuccess {
        async fn device_groups(&self) -> Result<DeviceGroups, GroupSourceError> {
            if self.0.fetch_add(1, Ordering::Relaxed) == 0 {
                Ok(HashMap::from([(
                    "device-1".into(),
                    HashSet::from(["edge-a".into()]),
                )]))
            } else {
                Err(GroupSourceError::Source("unavailable".into()))
            }
        }
    }
    #[tokio::test]
    async fn failed_refresh_retains_last_snapshot() {
        let cache = DeviceGroupCache::new(Arc::new(OneSuccess(AtomicUsize::new(0))));
        let first = cache.refresh().await.unwrap();
        let stale = cache.refresh().await.unwrap();
        assert!(Arc::ptr_eq(&first, &stale));
    }
 }
--- a/fleet/harmony-fleet-operator/src/lib.rs
+++ b/fleet/harmony-fleet-operator/src/lib.rs
@@ -10,13 +10,20 @@
 //! `harmony-fleet-deploy` when installing the operator.
 pub mod access;
 pub mod agent_upgrade;
 pub mod commands;
 pub mod crd;
 pub mod device_reconciler;
 pub mod device_status;
 pub mod fleet_aggregator;
 pub mod group_cache;
 pub mod rollout;
 pub mod task;
 pub mod task_run_controller;
 pub use crd::{
-    AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device,
+    AgentUpgradeTarget, AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec,
-    DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy,
+    DeploymentStatus, Device, DeviceSpec, DeviceStatus, DeviceUpgradeStatus, Reachability, Rollout,
    RolloutStrategy,
 };
 pub use task::{SystemUpgradeTaskV1, TaskRun, TaskRunPhase, TaskRunSpec, TaskRunStatus};
--- a/fleet/harmony-fleet-operator/src/main.rs
+++ b/fleet/harmony-fleet-operator/src/main.rs
@@ -6,7 +6,10 @@ mod frontend;
 mod service;
 use harmony_fleet_operator::access::StaticDeviceGroups;
-use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
+use harmony_fleet_operator::group_cache::DeviceGroupCache;
 use harmony_fleet_operator::{
    agent_upgrade, device_reconciler, device_status, fleet_aggregator, task_run_controller,
 };
 use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource};
 use harmony_secret::OpenBaoDeploymentSecretGrants;
 use harmony_zitadel_auth::ZitadelDeviceGroups;
@@ -159,33 +162,27 @@ async fn main() -> Result<()> {
            addr,
            css_from,
            live_reload,
-        } => serve_web(mock, addr, css_from, live_reload, &cli.tenant_namespace).await,
+        } => serve_web(mock, addr, css_from, live_reload).await,
    }
 }
-/// `serve-web` subcommand: dashboard on its own (mock data, or the live
+/// `serve-web` subcommand for frontend iteration with mock data. The
-/// CR-reading service). The deployed operator instead serves the
+/// live dashboard runs alongside the reconcile loop; see [`spawn_dashboard`].
 /// dashboard alongside the reconcile loop — see [`spawn_dashboard`].
 #[cfg(feature = "web-frontend")]
 async fn serve_web(
    mock: bool,
    addr: std::net::SocketAddr,
    css_from: Option<PathBuf>,
    live_reload: bool,
    tenant_namespace: &str,
 ) -> Result<()> {
    use std::sync::Arc;
-    use service::{FleetService, mock::MockFleetService, real::RealFleetService};
+    use service::{FleetService, mock::MockFleetService};
-    let fleet: Arc<dyn FleetService> = if mock {
+    if !mock {
-        Arc::new(MockFleetService::default())
+        anyhow::bail!("live dashboard runs in-process with the operator; use serve-web --mock");
-    } else {
+    }
-        Arc::new(RealFleetService::new(
+    let fleet: Arc<dyn FleetService> = Arc::new(MockFleetService::default());
            Client::try_default().await?,
            tenant_namespace,
        ))
    };
    serve_dashboard(fleet, addr, css_from, live_reload).await
 }
@@ -244,7 +241,12 @@ async fn serve_dashboard(
 /// (e.g. Zitadel not yet reachable for JWKS) is logged but never tears
 /// down reconcile — the read UI is best-effort, the controller is not.
 #[cfg(feature = "web-frontend")]
-fn spawn_dashboard(client: Client, tenant_namespace: &str) {
+fn spawn_dashboard(
    client: Client,
    tenant_namespace: &str,
    commands: harmony_fleet_operator::commands::FleetCommandsClient,
    fleet_state: fleet_aggregator::SharedFleetState,
 ) {
    use std::net::SocketAddr;
    use std::sync::Arc;
@@ -253,7 +255,12 @@ fn spawn_dashboard(client: Client, tenant_namespace: &str) {
    let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT));
    let tenant_namespace = tenant_namespace.to_string();
    tokio::spawn(async move {
-        let fleet = Arc::new(RealFleetService::new(client, tenant_namespace));
+        let fleet = Arc::new(RealFleetService::new(
            client,
            tenant_namespace,
            commands,
            fleet_state,
        ));
        if let Err(e) = serve_dashboard(fleet, addr, None, false).await {
            tracing::error!(error = %e, "dashboard server exited; reconcile continues");
        }
@@ -270,7 +277,7 @@ async fn run(
 ) -> Result<()> {
    let nats = connect_with_retry(nats_url, credentials_toml).await?;
    tracing::info!(url = %nats_url, "connected to NATS");
-    let js = jetstream::new(nats);
+    let js = jetstream::new(nats.clone());
    let desired_state_kv = js
        .create_key_value(jetstream::kv::Config {
            bucket: bucket.to_string(),
@@ -304,7 +311,7 @@ async fn run(
    };
    // Group membership (the scheduling gate): Zitadel role grants in
-    // prod, a static map for dev/e2e, or ungated when neither is set.
+    // prod or a static map for dev/e2e. Missing membership fails closed.
    let group_source: Option<Arc<dyn DeviceGroupSource>> = match (
        std::env::var("ZITADEL_URL"),
        std::env::var("ZITADEL_PAT"),
@@ -322,17 +329,27 @@ async fn run(
        _ => {
            tracing::warn!(
                "no device-group source (ZITADEL_URL/PAT/PROJECT_ID or FLEET_DEVICE_GROUPS); \
-                 deployments with allowedGroups will match no devices"
+                 deployments will match no devices"
            );
            None
        }
    };
-    // Serve the read-only dashboard in the same process (best-effort;
+    let group_cache = group_source.map(|source| Arc::new(DeviceGroupCache::new(source)));
-    // it reads CRs only, no NATS). Built only with the web-frontend
+    let fleet_state = Arc::new(tokio::sync::Mutex::new(
-    // feature; absent from the lean reconcile-only image.
+        fleet_aggregator::FleetState::default(),
    ));
    let rollout_plans = harmony_fleet_operator::rollout::open(&js).await?;
    // Dashboard associations use the aggregator's authorization decision;
    // interactive commands use NATS request/reply.
    #[cfg(feature = "web-frontend")]
-    spawn_dashboard(client.clone(), tenant_namespace);
+    spawn_dashboard(
        client.clone(),
        tenant_namespace,
        harmony_fleet_operator::commands::FleetCommandsClient::new(nats),
        fleet_state.clone(),
    );
    // Concurrent tasks:
    //   controller          — CR validation + finalizer-cleanup
@@ -344,10 +361,20 @@ async fn run(
    let ctl_client = client.clone();
    let dr_client = client.clone();
    let dr_js = js.clone();
    let ds_client = client.clone();
    let ds_js = js.clone();
    let upgrade_client = client.clone();
    let upgrade_js = js.clone();
    let task_client = client.clone();
    let task_js = js.clone();
    let task_group_cache = group_cache.clone();
    tokio::select! {
-        r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r,
+        r = controller::run(ctl_client, tenant_namespace, desired_state_kv, rollout_plans.clone()) => r,
        r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r,
-        r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r,
+        r = device_status::run(ds_client, tenant_namespace, ds_js) => r,
        r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js) => r,
        r = task_run_controller::run(task_client, tenant_namespace, task_js, task_group_cache, rollout_plans.clone()) => r,
        r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_cache, fleet_state, rollout_plans) => r,
    }
 }
--- a/fleet/harmony-fleet-operator/src/rollout.rs
+++ b/fleet/harmony-fleet-operator/src/rollout.rs
@@ -0,0 +1,331 @@
 use std::collections::BTreeMap;
 use anyhow::{Context, Result};
 use async_nats::jetstream::kv::{Operation, Store};
 use serde::{Deserialize, Serialize};
 use crate::AggregateLastError;
 const BUCKET: &str = "rollout-plan";
 const MAX_BYTES: i64 = 64 * 1024 * 1024;
 const MAX_PLAN_BYTES: i32 = 1024 * 1024;
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Target {
    pub device_id: String,
    pub canary: bool,
 }
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub struct Plan {
    pub revision: String,
    pub targets: Vec<Target>,
    #[serde(
        default,
        alias = "task_failures",
        skip_serializing_if = "Vec::is_empty"
    )]
    pub failures: Vec<Failure>,
    #[serde(default)]
    pub closed: bool,
 }
 impl<'de> Deserialize<'de> for Plan {
    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        #[derive(Deserialize)]
        struct StoredPlan {
            revision: String,
            targets: Vec<Target>,
            #[serde(default, alias = "task_failures")]
            failures: Vec<Failure>,
            #[serde(default)]
            failed: Vec<String>,
            #[serde(default)]
            closed: bool,
        }
        let mut stored = StoredPlan::deserialize(deserializer)?;
        for device_id in stored.failed {
            if stored
                .failures
                .iter()
                .all(|failure| failure.error.device_id != device_id)
            {
                stored.failures.push(Failure {
                    reason: "legacy rollout failure".into(),
                    error: AggregateLastError {
                        device_id,
                        message: "rollout failed before failure details were recorded".into(),
                        at: String::new(),
                    },
                });
            }
        }
        Ok(Self {
            revision: stored.revision,
            targets: stored.targets,
            failures: stored.failures,
            closed: stored.closed,
        })
    }
 }
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Failure {
    pub reason: String,
    pub error: AggregateLastError,
 }
 impl Plan {
    pub fn new(revision: String, mut targets: Vec<Target>) -> Self {
        targets.sort_by(|a, b| a.device_id.cmp(&b.device_id));
        Self {
            revision,
            targets,
            failures: Vec::new(),
            closed: false,
        }
    }
    pub fn has_canaries(&self) -> bool {
        self.targets.iter().any(|target| target.canary)
    }
    pub fn failure(&self, device_id: &str) -> Option<&Failure> {
        self.failures
            .iter()
            .find(|failure| failure.error.device_id == device_id)
    }
 }
 pub async fn record_failures(
    store: &Store,
    key: &str,
    revision: &str,
    failures: &[Failure],
 ) -> Result<Plan> {
    update_plan(store, key, revision, |plan| {
        plan.failures.extend_from_slice(failures);
        plan.failures
            .sort_by(|a, b| a.error.device_id.cmp(&b.error.device_id));
        plan.failures
            .dedup_by(|a, b| a.error.device_id == b.error.device_id);
    })
    .await
 }
 async fn update_plan(
    store: &Store,
    key: &str,
    revision: &str,
    update: impl FnOnce(&mut Plan),
 ) -> Result<Plan> {
    let entry = store
        .entry(key)
        .await?
        .filter(|entry| entry.operation == Operation::Put)
        .with_context(|| format!("rollout plan '{key}' missing"))?;
    let mut plan: Plan = serde_json::from_slice(&entry.value)
        .with_context(|| format!("invalid rollout plan '{key}'"))?;
    anyhow::ensure!(plan.revision == revision, "rollout plan revision changed");
    anyhow::ensure!(!plan.closed, "rollout plan '{key}' is closed");
    update(&mut plan);
    let value = serde_json::to_vec(&plan)?;
    anyhow::ensure!(
        value.len() <= MAX_PLAN_BYTES as usize,
        "rollout plan '{key}' progress is too large"
    );
    store.update(key, value.into(), entry.revision).await?;
    Ok(plan)
 }
 pub fn is_canary(labels: &BTreeMap<String, String>) -> bool {
    labels.get("canary").map(String::as_str) == Some("true")
 }
 pub async fn open(jetstream: &async_nats::jetstream::Context) -> Result<Store> {
    Ok(jetstream
        .create_key_value(async_nats::jetstream::kv::Config {
            bucket: BUCKET.into(),
            history: 1,
            max_bytes: MAX_BYTES,
            max_value_size: MAX_PLAN_BYTES,
            ..Default::default()
        })
        .await?)
 }
 pub async fn freeze(store: &Store, key: &str, desired: &Plan) -> Result<Plan> {
    let value = serde_json::to_vec(desired)?;
    anyhow::ensure!(
        value.len() <= MAX_PLAN_BYTES as usize / 2,
        "rollout plan '{key}' is too large"
    );
    let existing = store.entry(key).await?;
    if let Some(entry) = existing.filter(|entry| entry.operation == Operation::Put) {
        let plan: Plan = serde_json::from_slice(&entry.value)
            .with_context(|| format!("invalid rollout plan '{key}'"))?;
        anyhow::ensure!(!plan.closed, "rollout plan '{key}' is closed");
        if plan.revision == desired.revision {
            return Ok(plan);
        }
        if newer_generation(&plan.revision, &desired.revision) {
            return Ok(plan);
        }
        store.update(key, value.into(), entry.revision).await?;
    } else {
        store.create(key, value.into()).await?;
    }
    Ok(desired.clone())
 }
 pub async fn close(store: &Store, key: &str) -> Result<Option<Plan>> {
    loop {
        if let Some(entry) = store
            .entry(key)
            .await?
            .filter(|entry| entry.operation == Operation::Put)
        {
            let mut plan: Plan = serde_json::from_slice(&entry.value)
                .with_context(|| format!("invalid rollout plan '{key}'"))?;
            if !plan.closed {
                plan.closed = true;
                store
                    .update(key, serde_json::to_vec(&plan)?.into(), entry.revision)
                    .await?;
            }
            return Ok(Some(plan));
        }
        let mut sentinel = Plan::new(String::new(), Vec::new());
        sentinel.closed = true;
        match store
            .create(key, serde_json::to_vec(&sentinel)?.into())
            .await
        {
            Ok(_) => return Ok(Some(sentinel)),
            Err(error) => {
                if store.entry(key).await?.is_some() {
                    continue;
                }
                return Err(error.into());
            }
        }
    }
 }
 fn newer_generation(existing: &str, desired: &str) -> bool {
    let Some((existing_uid, existing_generation)) = existing.rsplit_once(':') else {
        return false;
    };
    let Some((desired_uid, desired_generation)) = desired.rsplit_once(':') else {
        return false;
    };
    existing_uid == desired_uid
        && matches!(
            (existing_generation.parse::<u64>(), desired_generation.parse::<u64>()),
            (Ok(existing), Ok(desired)) if existing > desired
        )
 }
 pub async fn load(store: &Store, key: &str) -> Result<Option<Plan>> {
    store
        .entry(key)
        .await?
        .filter(|entry| entry.operation == Operation::Put)
        .map(|entry| serde_json::from_slice(&entry.value).context("invalid rollout plan"))
        .transpose()
 }
 pub async fn delete(store: &Store, key: &str) -> Result<()> {
    if store
        .entry(key)
        .await?
        .is_some_and(|entry| entry.operation == Operation::Put)
    {
        store.delete(key).await?;
    }
    Ok(())
 }
 pub fn deployment_key(uid: &str) -> String {
    format!("deployment.{uid}")
 }
 pub fn task_key(uid: &str) -> String {
    format!("task.{uid}")
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn plan_sorts_and_detects_canaries() {
        let plan = Plan::new(
            "revision".into(),
            vec![
                Target {
                    device_id: "b".into(),
                    canary: false,
                },
                Target {
                    device_id: "a".into(),
                    canary: true,
                },
            ],
        );
        assert_eq!(plan.targets[0].device_id, "a");
        assert!(plan.has_canaries());
    }
    #[test]
    fn task_failure_checkpoint_round_trips_exactly() {
        let mut plan = Plan::new("revision".into(), Vec::new());
        plan.failures.push(Failure {
            reason: "RepairRequired".into(),
            error: AggregateLastError {
                device_id: "device-1".into(),
                message: "dpkg requires manual repair".into(),
                at: "2026-07-24T12:00:00+00:00".into(),
            },
        });
        let encoded = serde_json::to_vec(&plan).unwrap();
        assert_eq!(serde_json::from_slice::<Plan>(&encoded).unwrap(), plan);
        let legacy = serde_json::json!({
            "revision": "revision",
            "targets": [],
            "task_failures": [{
                "reason": "RepairRequired",
                "error": {
                    "deviceId": "device-1",
                    "message": "dpkg requires manual repair",
                    "at": "2026-07-24T12:00:00+00:00"
                }
            }],
            "closed": false
        });
        assert_eq!(serde_json::from_value::<Plan>(legacy).unwrap(), plan);
    }
    #[test]
    fn legacy_failed_devices_remain_failed() {
        let plan: Plan = serde_json::from_value(serde_json::json!({
            "revision": "revision",
            "targets": [],
            "failed": ["device-1"]
        }))
        .unwrap();
        assert!(plan.failure("device-1").is_some());
    }
    #[test]
    fn deployment_generation_cannot_move_backwards() {
        assert!(newer_generation("uid:3", "uid:2"));
        assert!(!newer_generation("uid:2", "uid:3"));
        assert!(!newer_generation("other:3", "uid:2"));
        assert!(!newer_generation("task-uid", "task-uid"));
    }
 }
--- a/fleet/harmony-fleet-operator/src/service/mock.rs
+++ b/fleet/harmony-fleet-operator/src/service/mock.rs
@@ -4,8 +4,8 @@ use std::sync::Mutex;
 use async_trait::async_trait;
 use super::{
-    Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
+    Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentError, DeploymentStatus,
-    DeviceStatus, FleetService, InventorySnapshot,
+    DeviceDetail, DeviceStatus, FleetService, InventorySnapshot,
 };
 pub struct MockFleetService {
@@ -168,6 +168,7 @@ fn seed_devices() -> Vec<DeviceDetail> {
            deployment,
            region,
            tags,
            current_version: inventory.as_ref().map(|i| i.agent_version.clone()),
            inventory,
        });
    }
@@ -190,6 +191,7 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
    vec![
        DeploymentDetail {
            name: "edge-gateway".into(),
            rollout_revision: "edge-gateway:1".into(),
            version: "v2.14.1".into(),
            status: DeploymentStatus::Active,
            target: 32,
@@ -197,9 +199,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 0,
            pending: 1,
            updated_at: "2026-05-19 04:12".into(),
            last_error: None,
        },
        DeploymentDetail {
            name: "sensor-firmware".into(),
            rollout_revision: "sensor-firmware:1".into(),
            version: "v0.9.3".into(),
            status: DeploymentStatus::Rolling,
            target: 41,
@@ -207,9 +211,15 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 1,
            pending: 12,
            updated_at: "2026-05-19 06:48".into(),
            last_error: Some(DeploymentError {
                device: "hf-sensor-042".into(),
                message: "image pull timed out".into(),
                at: "2026-05-19 06:48".into(),
            }),
        },
        DeploymentDetail {
            name: "ingest-pipeline".into(),
            rollout_revision: "ingest-pipeline:1".into(),
            version: "v1.7.0".into(),
            status: DeploymentStatus::Active,
            target: 8,
@@ -217,9 +227,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 0,
            pending: 0,
            updated_at: "2026-05-15 11:30".into(),
            last_error: None,
        },
        DeploymentDetail {
            name: "control-plane".into(),
            rollout_revision: "control-plane:1".into(),
            version: "v3.2.0".into(),
            status: DeploymentStatus::Failing,
            target: 6,
@@ -227,9 +239,15 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 2,
            pending: 1,
            updated_at: "2026-05-19 07:01".into(),
            last_error: Some(DeploymentError {
                device: "hf-gw-018".into(),
                message: "service failed its health check".into(),
                at: "2026-05-19 07:01".into(),
            }),
        },
        DeploymentDetail {
            name: "telemetry-collector".into(),
            rollout_revision: "telemetry-collector:1".into(),
            version: "v0.4.12".into(),
            status: DeploymentStatus::Active,
            target: 12,
@@ -237,9 +255,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 0,
            pending: 0,
            updated_at: "2026-05-12 09:22".into(),
            last_error: None,
        },
        DeploymentDetail {
            name: "gateway-proxy".into(),
            rollout_revision: "gateway-proxy:1".into(),
            version: "v1.0.5".into(),
            status: DeploymentStatus::Paused,
            target: 4,
@@ -247,9 +267,11 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 0,
            pending: 4,
            updated_at: "2026-05-18 18:14".into(),
            last_error: None,
        },
        DeploymentDetail {
            name: "media-relay".into(),
            rollout_revision: "media-relay:1".into(),
            version: "v2.0.0-rc.3".into(),
            status: DeploymentStatus::Rolling,
            target: 9,
@@ -257,6 +279,7 @@ fn seed_deployments() -> Vec<DeploymentDetail> {
            failing: 0,
            pending: 4,
            updated_at: "2026-05-19 06:55".into(),
            last_error: None,
        },
    ]
 }
@@ -429,6 +452,10 @@ impl FleetService for MockFleetService {
        Ok(dev.clone())
    }
    async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
        Ok(format!("[{device_id}] {command}\n[exit 0]"))
    }
    async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>> {
        Ok(self.alerts.lock().unwrap().clone())
    }
@@ -443,14 +470,6 @@ impl FleetService for MockFleetService {
        }
    }
    async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
        // Seam only: the real impl publishes the command to the device
        // and streams stdout/stderr back.
        Ok(format!(
            "$ {command}\n[{device_id}] command transport not yet implemented"
        ))
    }
    async fn filtered_devices(
        &self,
        status: Option<DeviceStatus>,
--- a/fleet/harmony-fleet-operator/src/service/mod.rs
+++ b/fleet/harmony-fleet-operator/src/service/mod.rs
@@ -16,6 +16,7 @@ pub trait FleetService: Send + Sync + 'static {
    async fn get_deployment(&self, name: &str) -> anyhow::Result<Option<DeploymentDetail>>;
    async fn get_deployment_devices(&self, name: &str) -> anyhow::Result<Vec<DeviceDetail>>;
    async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail>;
    async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String>;
    async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>>;
    async fn ack_alert(&self, id: &str) -> anyhow::Result<bool>;
    async fn filtered_devices(
@@ -25,10 +26,6 @@ pub trait FleetService: Send + Sync + 'static {
        region: Option<String>,
        search: Option<String>,
    ) -> anyhow::Result<Vec<DeviceDetail>>;
    /// Send a one-shot shell command to a device for administrative
    /// access. Returns the (eventual) output; streaming back live is a
    /// later refinement. The device round-trip is not wired yet.
    async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String>;
 }
 // ── Device ─────────────────────────────────────────────────────────────
@@ -42,6 +39,8 @@ pub struct DeviceDetail {
    pub deployment: Option<String>,
    pub region: String,
    pub tags: Vec<String>,
    /// Running version from the latest heartbeat.
    pub current_version: Option<String>,
    /// Hardware/OS facts from the agent. `None` until the first
    /// post-enrollment publish (mirrors `DeviceInfo.inventory`).
    pub inventory: Option<InventorySnapshot>,
@@ -76,6 +75,7 @@ impl DeviceStatus {
 #[derive(Debug, Clone, Serialize)]
 pub struct DeploymentDetail {
    pub name: String,
    pub rollout_revision: String,
    pub version: String,
    pub status: DeploymentStatus,
    pub target: u32,
@@ -83,6 +83,14 @@ pub struct DeploymentDetail {
    pub failing: u32,
    pub pending: u32,
    pub updated_at: String,
    pub last_error: Option<DeploymentError>,
 }
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub struct DeploymentError {
    pub device: String,
    pub message: String,
    pub at: String,
 }
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
--- a/fleet/harmony-fleet-operator/src/service/real.rs
+++ b/fleet/harmony-fleet-operator/src/service/real.rs
@@ -1,11 +1,10 @@
-//! Live [`FleetService`] as a read-only projection of Kubernetes CRs.
+//! Live [`FleetService`] backed by Kubernetes CRs and the NATS command channel.
 //!
 //! The operator is the write side: `device_reconciler` materializes
 //! `Device` CRs (labels + inventory), `device_status` reflects liveness
-//! onto `Device.status`, and `fleet_aggregator` writes
+//! onto `Device.status`, and `fleet_aggregator` owns target eligibility
-//! `Deployment.status.aggregate`. This dashboard is the read side — it
+//! and writes `Deployment.status.aggregate`. This dashboard projects
-//! only reads those CRs and projects them to view DTOs. No NATS: the
+//! those sources to view DTOs.
 //! CR is the single contract between the two sides.
 use std::collections::{BTreeMap, HashSet};
 use std::sync::Mutex;
@@ -17,15 +16,17 @@ use kube::api::{Api, ListParams, Patch, PatchParams};
 use kube::{Client, ResourceExt};
 use serde_json::json;
 use harmony_fleet_operator::commands::FleetCommandsClient;
 use harmony_fleet_operator::crd::{
-    Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability,
+    Deployment as DeploymentCr, DeploymentAggregate, Device as DeviceCr,
    DeviceStatus as DeviceLiveness, Reachability,
 };
-use harmony_fleet_operator::fleet_aggregator::selector_matches;
+use harmony_fleet_operator::fleet_aggregator::SharedFleetState;
-use harmony_reconciler_contracts::ReconcileScore;
+use harmony_reconciler_contracts::{ExecReply, ReconcileScore};
 use super::{
-    Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail,
+    Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentError, DeploymentStatus,
-    DeviceStatus, FleetService,
+    DeviceDetail, DeviceStatus, FleetService,
 };
 /// Label the operator UI sets to quarantine a device.
@@ -36,16 +37,25 @@ const REGION_LABEL: &str = "region";
 pub struct RealFleetService {
    kube: Client,
    namespace: String,
    commands: FleetCommandsClient,
    fleet_state: SharedFleetState,
    /// In-memory ack set. Alerts are derived from live CR state and
    /// have no store of their own, so acks don't survive a restart.
    acked_alerts: Mutex<HashSet<String>>,
 }
 impl RealFleetService {
-    pub fn new(kube: Client, namespace: impl Into<String>) -> Self {
+    pub fn new(
        kube: Client,
        namespace: impl Into<String>,
        commands: FleetCommandsClient,
        fleet_state: SharedFleetState,
    ) -> Self {
        Self {
            kube,
            namespace: namespace.into(),
            commands,
            fleet_state,
            acked_alerts: Mutex::new(HashSet::new()),
        }
    }
@@ -61,13 +71,23 @@ impl RealFleetService {
    }
    async fn devices(&self) -> anyhow::Result<Vec<DeviceDetail>> {
-        let deployments = self.deployment_crs().await?;
+        let eligible = self
            .fleet_state
            .lock()
            .await
            .eligible_deployments_by_device();
        let now = Utc::now();
        let mut devices: Vec<DeviceDetail> = self
            .device_crs()
            .await?
            .iter()
-            .map(|cr| map_device(cr, &deployments, now))
+            .map(|cr| {
                let deployment = eligible
                    .get(&cr.name_any())
                    .and_then(|deployments| deployments.first())
                    .cloned();
                map_device(cr, deployment, now)
            })
            .collect();
        devices.sort_by(|a, b| a.id.cmp(&b.id));
        Ok(devices)
@@ -85,7 +105,25 @@ impl RealFleetService {
    }
 }
-fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime<Utc>) -> DeviceDetail {
+fn format_exec(reply: ExecReply) -> String {
    let mut output = reply.stdout;
    if !reply.stderr.is_empty() {
        if !output.is_empty() && !output.ends_with('\n') {
            output.push('\n');
        }
        output.push_str(&reply.stderr);
    }
    if !output.is_empty() && !output.ends_with('\n') {
        output.push('\n');
    }
    output.push_str(&format!("[exit {}]", reply.exit_code));
    if reply.truncated {
        output.push_str(" [output truncated]");
    }
    output
 }
 fn map_device(cr: &DeviceCr, deployment: Option<String>, now: DateTime<Utc>) -> DeviceDetail {
    let labels = cr.metadata.labels.clone().unwrap_or_default();
    let blacklisted = labels.get(BLACKLIST_LABEL).map(String::as_str) == Some("true");
    let last_seen = cr
@@ -102,12 +140,13 @@ fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime<Utc>) -
        status: device_status(blacklisted, cr.status.as_ref()),
        last_seen,
        minutes_ago: (now - last_seen).num_minutes().max(0),
-        deployment: primary_deployment(&labels, deployments),
+        deployment,
        region: labels
            .get(REGION_LABEL)
            .cloned()
            .unwrap_or_else(|| "\u{2014}".to_string()),
        tags: tags_from_labels(&labels),
        current_version: cr.status.as_ref().and_then(|s| s.current_version.clone()),
        inventory: cr.spec.inventory.clone(),
    }
 }
@@ -126,22 +165,6 @@ fn device_status(blacklisted: bool, liveness: Option<&DeviceLiveness>) -> Device
    }
 }
 /// First deployment (by name) whose selector matches the device — the
 /// canonical [`selector_matches`] over CR labels, the same matcher the
 /// aggregator uses. No reconstruction.
 fn primary_deployment(
    labels: &BTreeMap<String, String>,
    deployments: &[DeploymentCr],
 ) -> Option<String> {
    let mut matched: Vec<String> = deployments
        .iter()
        .filter(|d| selector_matches(&d.spec.target_selector, labels))
        .map(ResourceExt::name_any)
        .collect();
    matched.sort();
    matched.into_iter().next()
 }
 /// Routing labels rendered as `k=v` chips, minus internal keys.
 fn tags_from_labels(labels: &BTreeMap<String, String>) -> Vec<String> {
    labels
@@ -152,20 +175,31 @@ fn tags_from_labels(labels: &BTreeMap<String, String>) -> Vec<String> {
 }
 fn map_deployment(cr: &DeploymentCr) -> DeploymentDetail {
    let current_revision = cr.rollout_revision();
    let agg = cr
        .status
        .as_ref()
        .filter(|status| {
            current_revision
                .as_deref()
                .is_some_and(|revision| status.rollout_revision.as_deref() == Some(revision))
        })
        .and_then(|s| s.aggregate.clone())
-        .unwrap_or_default();
+        .map(|agg| {
-    let status = if agg.failed > 0 {
+            let status = if agg.failed > 0 || agg.matched_device_count == 0 {
-        DeploymentStatus::Failing
+                DeploymentStatus::Failing
-    } else if agg.pending > 0 {
+            } else if agg.pending > 0 {
-        DeploymentStatus::Rolling
+                DeploymentStatus::Rolling
-    } else {
+            } else {
-        DeploymentStatus::Active
+                DeploymentStatus::Active
-    };
+            };
            (agg, status)
        });
    let (agg, status) =
        agg.unwrap_or_else(|| (DeploymentAggregate::default(), DeploymentStatus::Rolling));
    DeploymentDetail {
        name: cr.name_any(),
        rollout_revision: current_revision.unwrap_or_default(),
        version: deployment_version(&cr.spec.score),
        status,
        target: agg.matched_device_count,
@@ -176,6 +210,11 @@ fn map_deployment(cr: &DeploymentCr) -> DeploymentDetail {
            .creation_timestamp()
            .map(|t| t.0.format("%Y-%m-%d %H:%M").to_string())
            .unwrap_or_else(|| "\u{2014}".to_string()),
        last_error: agg.last_error.map(|error| DeploymentError {
            device: error.device_id,
            message: error.message,
            at: error.at,
        }),
    }
 }
@@ -203,16 +242,40 @@ fn derive_alerts(
 ) -> Vec<Alert> {
    let mut alerts = Vec::new();
    for d in deployments {
-        if d.failing > 0 {
+        if d.status == DeploymentStatus::Failing {
-            let id = format!("dep:{}:failing", d.name);
+            let id = format!("dep:{}:{}:failing", d.name, d.rollout_revision);
            let (title, device, at) = if d.target == 0 {
                (
                    format!("{} matched no devices", d.name),
                    None,
                    String::new(),
                )
            } else {
                d.last_error.as_ref().map_or_else(
                    || {
                        (
                            format!("{} has {} failing device(s)", d.name, d.failing),
                            None,
                            String::new(),
                        )
                    },
                    |error| {
                        (
                            format!("{}: {}", error.device, error.message),
                            Some(error.device.clone()),
                            error.at.clone(),
                        )
                    },
                )
            };
            alerts.push(Alert {
                acked: acked.contains(&id),
                id,
                severity: AlertSeverity::Critical,
-                title: format!("{} has {} failing device(s)", d.name, d.failing),
+                title,
                deployment: Some(d.name.clone()),
-                device: None,
+                device,
-                at: String::new(),
+                at,
            });
        }
    }
@@ -312,21 +375,26 @@ impl FleetService for RealFleetService {
    }
    async fn get_deployment_devices(&self, name: &str) -> anyhow::Result<Vec<DeviceDetail>> {
-        let deployments = self.deployment_crs().await?;
+        let eligible = self
-        let Some(cr) = deployments.iter().find(|c| c.name_any() == name) else {
+            .fleet_state
-            return Ok(Vec::new());
+            .lock()
-        };
+            .await
-        let selector = cr.spec.target_selector.clone();
+            .eligible_deployments_by_device();
        let now = Utc::now();
-        Ok(self
+        let mut devices: Vec<_> = self
            .device_crs()
            .await?
            .iter()
-            .filter(|dev| {
+            .filter_map(|device| {
-                selector_matches(&selector, &dev.metadata.labels.clone().unwrap_or_default())
+                let deployments = eligible.get(&device.name_any())?;
                deployments
                    .iter()
                    .any(|deployment| deployment == name)
                    .then(|| map_device(device, deployments.first().cloned(), now))
            })
-            .map(|dev| map_device(dev, &deployments, now))
+            .collect();
-            .collect())
+        devices.sort_by(|a, b| a.id.cmp(&b.id));
        Ok(devices)
    }
    async fn blacklist_device(&self, id: &str) -> anyhow::Result<DeviceDetail> {
@@ -340,6 +408,15 @@ impl FleetService for RealFleetService {
            .ok_or_else(|| anyhow::anyhow!("device {id} not found after blacklist"))
    }
    async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result<String> {
        Ok(format_exec(
            self.commands
                .exec(device_id, command)
                .await
                .with_context(|| format!("running command on device {device_id}"))?,
        ))
    }
    async fn list_alerts(&self) -> anyhow::Result<Vec<Alert>> {
        let devices = self.devices().await?;
        let deployments = self.deployments().await?;
@@ -351,14 +428,6 @@ impl FleetService for RealFleetService {
        Ok(self.acked_alerts.lock().unwrap().insert(id.to_string()))
    }
    async fn run_command(&self, _device_id: &str, command: &str) -> anyhow::Result<String> {
        // Seam only: the device round-trip (publish to the agent over
        // NATS, stream stdout/stderr back) needs agent-side support.
        Ok(format!(
            "$ {command}\n[device command transport not implemented yet]"
        ))
    }
    async fn filtered_devices(
        &self,
        status: Option<DeviceStatus>,
@@ -367,15 +436,15 @@ impl FleetService for RealFleetService {
        search: Option<String>,
    ) -> anyhow::Result<Vec<DeviceDetail>> {
        let search = search.map(|s| s.to_lowercase());
-        Ok(self
+        let devices = if let Some(deployment) = deployment.as_deref() {
-            .devices()
+            self.get_deployment_devices(deployment).await?
-            .await?
+        } else {
            self.devices().await?
        };
        Ok(devices
            .into_iter()
            .filter(|d| {
                status.is_none_or(|s| d.status == s)
                    && deployment
                        .as_deref()
                        .is_none_or(|dep| d.deployment.as_deref() == Some(dep))
                    && region.as_deref().is_none_or(|r| d.region == r)
                    && search.as_deref().is_none_or(|q| {
                        d.id.to_lowercase().contains(q)
@@ -392,12 +461,19 @@ impl FleetService for RealFleetService {
 #[cfg(test)]
 mod tests {
    use super::*;
    use harmony_fleet_operator::crd::{
        AggregateLastError, DeploymentAggregate, DeploymentSpec,
        DeploymentStatus as DeploymentCrStatus, Device, DeviceSpec, Rollout, RolloutStrategy,
    };
    use harmony_reconciler_contracts::{PodmanService, PodmanV0Score};
    use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
    fn liveness(r: Reachability) -> DeviceLiveness {
        DeviceLiveness {
            last_heartbeat: None,
            reachability: r,
            current_version: None,
            agent_upgrade: None,
        }
    }
@@ -418,12 +494,44 @@ mod tests {
        assert_eq!(device_status(false, None), DeviceStatus::Unknown);
    }
    #[test]
    fn device_mapping_uses_authoritative_deployment() {
        let device = Device::new(
            "pi-01",
            DeviceSpec {
                inventory: None,
                updater: None,
                agent_upgrade: None,
            },
        );
        assert_eq!(
            map_device(&device, Some("web".into()), Utc::now()).deployment,
            Some("web".into())
        );
        assert_eq!(map_device(&device, None, Utc::now()).deployment, None);
    }
    #[test]
    fn exec_output_includes_stderr_exit_and_truncation() {
        assert_eq!(
            format_exec(ExecReply {
                exit_code: 7,
                stdout: "out".into(),
                stderr: "err".into(),
                truncated: true,
            }),
            "out\nerr\n[exit 7] [output truncated]"
        );
    }
    #[test]
    fn version_from_image_tag() {
        let score = ReconcileScore::PodmanV0(PodmanV0Score {
            init_container: None,
            services: vec![PodmanService {
                name: "web".into(),
                image: "nginx:1.25".into(),
                image_pull_secret: None,
                ports: vec![],
                env: vec![],
                secret_env: vec![],
@@ -434,6 +542,91 @@ mod tests {
        assert_eq!(deployment_version(&score), "1.25");
    }
    #[test]
    fn deployment_mapping_projects_last_error() {
        let mut deployment = DeploymentCr::new(
            "edge",
            DeploymentSpec {
                allowed_groups: vec![],
                target_selector: LabelSelector::default(),
                score: ReconcileScore::PodmanV0(PodmanV0Score {
                    init_container: None,
                    services: vec![],
                }),
                rollout: Rollout {
                    strategy: RolloutStrategy::Immediate,
                },
            },
        );
        deployment.metadata.uid = Some("deployment-uid".into());
        deployment.metadata.generation = Some(2);
        deployment.status = Some(DeploymentCrStatus {
            rollout_revision: Some("deployment-uid:2".into()),
            aggregate: Some(DeploymentAggregate {
                matched_device_count: 1,
                failed: 1,
                last_error: Some(AggregateLastError {
                    device_id: "pi-09".into(),
                    message: "image pull denied".into(),
                    at: "2026-07-27T10:15:00Z".into(),
                }),
                ..Default::default()
            }),
        });
        assert_eq!(
            map_deployment(&deployment).last_error,
            Some(DeploymentError {
                device: "pi-09".into(),
                message: "image pull denied".into(),
                at: "2026-07-27T10:15:00Z".into(),
            })
        );
        assert_eq!(
            map_deployment(&deployment).status,
            DeploymentStatus::Failing
        );
    }
    #[test]
    fn deployment_mapping_ignores_absent_or_stale_aggregate() {
        let mut deployment = DeploymentCr::new(
            "edge",
            DeploymentSpec {
                allowed_groups: vec![],
                target_selector: LabelSelector::default(),
                score: ReconcileScore::PodmanV0(PodmanV0Score {
                    init_container: None,
                    services: vec![],
                }),
                rollout: Rollout {
                    strategy: RolloutStrategy::Immediate,
                },
            },
        );
        deployment.metadata.uid = Some("deployment-uid".into());
        deployment.metadata.generation = Some(2);
        deployment.status = Some(DeploymentCrStatus {
            rollout_revision: Some("deployment-uid:1".into()),
            aggregate: Some(DeploymentAggregate {
                matched_device_count: 3,
                succeeded: 3,
                ..Default::default()
            }),
        });
        let detail = map_deployment(&deployment);
        assert_eq!(detail.rollout_revision, "deployment-uid:2");
        assert_eq!(detail.status, DeploymentStatus::Rolling);
        assert_eq!((detail.target, detail.healthy), (0, 0));
        deployment.status = None;
        assert_eq!(
            map_deployment(&deployment).status,
            DeploymentStatus::Rolling
        );
    }
    #[test]
    fn alerts_from_failing_and_stale() {
        let devices = vec![DeviceDetail {
@@ -444,10 +637,12 @@ mod tests {
            deployment: None,
            region: "\u{2014}".into(),
            tags: vec![],
            current_version: None,
            inventory: None,
        }];
        let deployments = vec![DeploymentDetail {
            name: "edge".into(),
            rollout_revision: "deployment-uid:2".into(),
            version: "1.0".into(),
            status: DeploymentStatus::Failing,
            target: 3,
@@ -455,10 +650,34 @@ mod tests {
            failing: 2,
            pending: 0,
            updated_at: "\u{2014}".into(),
            last_error: Some(DeploymentError {
                device: "pi-10".into(),
                message: "image pull denied".into(),
                at: "2026-07-27T10:15:00Z".into(),
            }),
        }];
-        let alerts = derive_alerts(&devices, &deployments, &HashSet::new());
+        let alerts = derive_alerts(
            &devices,
            &deployments,
            &HashSet::from(["dep:edge:deployment-uid:1:failing".into()]),
        );
        assert_eq!(alerts.len(), 2);
-        assert!(alerts.iter().any(|a| a.severity == AlertSeverity::Critical));
+        let deployment_alert = alerts
            .iter()
            .find(|a| a.severity == AlertSeverity::Critical)
            .unwrap();
        assert_eq!(deployment_alert.device.as_deref(), Some("pi-10"));
        assert_eq!(deployment_alert.title, "pi-10: image pull denied");
        assert_eq!(deployment_alert.at, "2026-07-27T10:15:00Z");
        assert_eq!(deployment_alert.id, "dep:edge:deployment-uid:2:failing");
        assert!(!deployment_alert.acked);
        assert!(alerts.iter().any(|a| a.severity == AlertSeverity::Warning));
        let mut no_targets = deployments[0].clone();
        no_targets.target = 0;
        no_targets.failing = 0;
        no_targets.last_error = None;
        let alerts = derive_alerts(&[], &[no_targets], &HashSet::new());
        assert_eq!(alerts[0].title, "edge matched no devices");
    }
 }
--- a/fleet/harmony-fleet-operator/src/task.rs
+++ b/fleet/harmony-fleet-operator/src/task.rs
@@ -0,0 +1,108 @@
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, Time};
 use kube::{CustomResource, KubeSchema};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use crate::AggregateLastError;
 #[derive(CustomResource, Serialize, Deserialize, Clone, Debug, KubeSchema)]
 #[kube(
    group = "fleet.nationtech.io",
    version = "v1alpha1",
    kind = "TaskRun",
    plural = "taskruns",
    shortname = "fleettask",
    namespaced,
    status = "TaskRunStatus",
    validation = Rule::new("self.spec == oldSelf.spec").message("TaskRun spec is immutable")
 )]
 #[serde(rename_all = "camelCase")]
 pub struct TaskRunSpec {
    #[schemars(length(min = 1))]
    pub allowed_groups: Vec<String>,
    pub target_selector: LabelSelector,
    #[schemars(range(min = 60, max = 86400))]
    pub deadline_seconds: u32,
    pub system_upgrade_v1: SystemUpgradeTaskV1,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)]
 pub struct SystemUpgradeTaskV1 {}
 #[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, JsonSchema)]
 pub enum TaskRunPhase {
    Planning,
    Running,
    Complete,
    Failed,
 }
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct TaskRunStatus {
    pub phase: TaskRunPhase,
    pub target_count: u32,
    pub succeeded: u32,
    pub failed: u32,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub start_time: Option<Time>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub completion_time: Option<Time>,
    #[serde(skip_serializing_if = "Option::is_none")]
    #[schemars(length(max = 128))]
    pub reason: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub last_error: Option<AggregateLastError>,
 }
 #[cfg(test)]
 mod tests {
    use std::collections::BTreeMap;
    use kube::CustomResourceExt;
    use super::*;
    fn system_upgrade_run() -> TaskRun {
        TaskRun::new(
            "upgrade-device-1",
            TaskRunSpec {
                allowed_groups: vec!["production".into()],
                target_selector: LabelSelector {
                    match_labels: Some(BTreeMap::from([("device-id".into(), "device-1".into())])),
                    match_expressions: None,
                },
                deadline_seconds: 21_600,
                system_upgrade_v1: SystemUpgradeTaskV1 {},
            },
        )
    }
    #[test]
    fn system_upgrade_has_no_operator_control_over_apt() {
        let value = serde_json::to_value(system_upgrade_run()).unwrap();
        assert_eq!(
            value.pointer("/spec/systemUpgradeV1").unwrap(),
            &serde_json::json!({})
        );
    }
    #[test]
    fn task_run_crd_is_namespaced_and_immutable() {
        let crd = TaskRun::crd();
        assert_eq!(crd.spec.scope, "Namespaced");
        assert!(
            crd.spec.versions[0]
                .subresources
                .as_ref()
                .unwrap()
                .status
                .is_some()
        );
        let encoded = serde_json::to_string(&crd).unwrap();
        assert!(encoded.contains("self.spec == oldSelf.spec"));
        assert!(encoded.contains("minItems"));
        assert!(encoded.contains("maximum"));
    }
 }
--- a/fleet/harmony-fleet-operator/src/task_run_controller.rs
+++ b/fleet/harmony-fleet-operator/src/task_run_controller.rs
@@ -0,0 +1,843 @@
 use std::collections::HashSet;
 use std::sync::Arc;
 use std::time::Duration;
 use anyhow::{Context, Result};
 use async_nats::jetstream::kv::{Operation, Store};
 use chrono::Utc;
 use harmony_reconciler_contracts::{
    BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS, Id, SystemUpgradeAttempt,
    SystemUpgradePhase, SystemUpgradeStatus, system_upgrade_intent_key, system_upgrade_status_key,
 };
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
 use kube::api::{Api, ListParams, Patch, PatchParams};
 use kube::{Client, ResourceExt};
 use crate::crd::{AggregateLastError, Device};
 use crate::fleet_aggregator::{device_eligible, group_allows};
 use crate::group_cache::DeviceGroupCache;
 use crate::rollout::{self, Plan, Target};
 use crate::task::{TaskRun, TaskRunPhase, TaskRunStatus};
 const POLL_INTERVAL: Duration = Duration::from_secs(2);
 const MAX_ERROR_CHARS: usize = 512;
 const TASK_BUCKET_MAX_BYTES: i64 = 64 * 1024 * 1024;
 const FINALIZER: &str = "fleet.nationtech.io/task-run-finalizer";
 pub async fn run(
    client: Client,
    namespace: &str,
    jetstream: async_nats::jetstream::Context,
    group_cache: Option<Arc<DeviceGroupCache>>,
    rollout_plans: Store,
 ) -> Result<()> {
    let intents = jetstream
        .create_key_value(async_nats::jetstream::kv::Config {
            bucket: BUCKET_SYSTEM_UPGRADE_INTENT.into(),
            history: 1,
            max_age: Duration::from_secs(2 * 24 * 60 * 60),
            max_bytes: TASK_BUCKET_MAX_BYTES,
            max_value_size: 16 * 1024,
            ..Default::default()
        })
        .await?;
    let statuses = jetstream
        .create_key_value(async_nats::jetstream::kv::Config {
            bucket: BUCKET_SYSTEM_UPGRADE_STATUS.into(),
            history: 1,
            max_age: Duration::from_secs(30 * 24 * 60 * 60),
            max_bytes: TASK_BUCKET_MAX_BYTES,
            max_value_size: 16 * 1024,
            ..Default::default()
        })
        .await?;
    let runs: Api<TaskRun> = Api::namespaced(client.clone(), namespace);
    let devices: Api<Device> = Api::namespaced(client, namespace);
    let mut ticker = tokio::time::interval(POLL_INTERVAL);
    ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
    loop {
        ticker.tick().await;
        for task_run in runs.list(&ListParams::default()).await?.items {
            let name = task_run.name_any();
            if let Err(error) = reconcile(
                &runs,
                &devices,
                &intents,
                &statuses,
                group_cache.as_ref(),
                &rollout_plans,
                task_run,
            )
            .await
            {
                tracing::warn!(task_run = %name, error = %error, "TaskRun reconcile failed");
            }
        }
    }
 }
 async fn reconcile(
    runs: &Api<TaskRun>,
    devices: &Api<Device>,
    intents: &Store,
    statuses: &Store,
    group_cache: Option<&Arc<DeviceGroupCache>>,
    rollout_plans: &Store,
    task_run: TaskRun,
 ) -> Result<()> {
    let run_uid = task_run
        .metadata
        .uid
        .as_deref()
        .context("TaskRun metadata.uid missing")?;
    let plan_key = rollout::task_key(run_uid);
    if task_run.metadata.deletion_timestamp.is_some() {
        if let Some(plan) = rollout::close(rollout_plans, &plan_key).await? {
            cleanup_intents(intents, &plan, run_uid).await?;
            rollout::delete(rollout_plans, &plan_key).await?;
        }
        patch_finalizer(runs, &task_run, false).await?;
        return Ok(());
    }
    if !task_run
        .finalizers()
        .iter()
        .any(|finalizer| finalizer == FINALIZER)
    {
        patch_finalizer(runs, &task_run, true).await?;
        return Ok(());
    }
    if task_run
        .status
        .as_ref()
        .is_some_and(|status| matches!(status.phase, TaskRunPhase::Complete | TaskRunPhase::Failed))
    {
        if let Some(plan) = rollout::close(rollout_plans, &plan_key).await? {
            cleanup_intents(intents, &plan, run_uid).await?;
            rollout::delete(rollout_plans, &plan_key).await?;
        }
        return Ok(());
    }
    if task_run.status.is_none()
        || task_run
            .status
            .as_ref()
            .is_some_and(|status| status.phase == TaskRunPhase::Planning)
    {
        let Some(cache) = group_cache else {
            return patch_failed(runs, &task_run, 0, 0, None, "NoTargets", None).await;
        };
        let groups = cache
            .snapshot()
            .await
            .context("device group cache is not ready")?;
        let eligible = devices
            .list(&ListParams::default())
            .await?
            .items
            .into_iter()
            .filter(|device| {
                supports_system_upgrade(device)
                    && device_eligible(
                        &task_run.spec.allowed_groups,
                        &task_run.spec.target_selector,
                        &device.metadata.labels.clone().unwrap_or_default(),
                        groups.get(&device.name_any()),
                        groups.get("*"),
                    )
            })
            .map(|device| Target {
                canary: device
                    .metadata
                    .labels
                    .as_ref()
                    .is_some_and(rollout::is_canary),
                device_id: device.name_any(),
            })
            .collect::<Vec<_>>();
        if eligible.is_empty() {
            return patch_failed(runs, &task_run, 0, 0, None, "NoTargets", None).await;
        }
        let plan = rollout::freeze(
            rollout_plans,
            &plan_key,
            &Plan::new(run_uid.to_string(), eligible),
        )
        .await?;
        let start = Time(Utc::now());
        patch_status(
            runs,
            &task_run,
            &TaskRunStatus {
                phase: TaskRunPhase::Running,
                target_count: plan.targets.len().try_into().unwrap_or(u32::MAX),
                succeeded: 0,
                failed: 0,
                start_time: Some(start),
                completion_time: None,
                reason: None,
                last_error: None,
            },
        )
        .await?;
        return Ok(());
    }
    let status = task_run.status.as_ref().context("TaskRun status missing")?;
    let Some(start) = status.start_time.as_ref() else {
        return patch_failed(
            runs,
            &task_run,
            status.target_count,
            0,
            None,
            "InvalidPlan",
            None,
        )
        .await;
    };
    let Some(plan) = rollout::load(rollout_plans, &plan_key).await? else {
        return patch_failed(
            runs,
            &task_run,
            status.target_count,
            0,
            Some(start),
            "InvalidPlan",
            None,
        )
        .await;
    };
    if plan.revision != run_uid {
        return patch_failed(
            runs,
            &task_run,
            status.target_count,
            0,
            Some(start),
            "InvalidPlan",
            None,
        )
        .await;
    }
    let expires_at = start.0 + chrono::Duration::seconds(task_run.spec.deadline_seconds.into());
    reconcile_running(
        runs,
        devices,
        intents,
        statuses,
        rollout_plans,
        group_cache,
        &task_run,
        &plan,
        &plan_key,
        run_uid,
        start,
        expires_at,
    )
    .await
 }
 fn supports_system_upgrade(device: &Device) -> bool {
    device
        .spec
        .updater
        .as_ref()
        .is_some_and(|capabilities| capabilities.protocol == 1 && capabilities.apt_full_upgrade_v1)
 }
 fn authorized_for_release(
    device: &Device,
    allowed_groups: &[String],
    device_groups: Option<&HashSet<String>>,
    default_groups: Option<&HashSet<String>>,
 ) -> bool {
    supports_system_upgrade(device)
        && (group_allows(allowed_groups, device_groups)
            || group_allows(allowed_groups, default_groups))
 }
 #[derive(Debug)]
 enum TargetOutcome {
    Pending,
    Succeeded,
    Failed {
        reason: String,
        error: AggregateLastError,
    },
 }
 #[allow(clippy::too_many_arguments)]
 async fn reconcile_running(
    runs: &Api<TaskRun>,
    devices: &Api<Device>,
    intents: &Store,
    statuses: &Store,
    rollout_plans: &Store,
    group_cache: Option<&Arc<DeviceGroupCache>>,
    task_run: &TaskRun,
    plan: &Plan,
    plan_key: &str,
    run_uid: &str,
    start: &Time,
    expires_at: chrono::DateTime<Utc>,
 ) -> Result<()> {
    if plan.closed {
        return Ok(());
    }
    let persisted_failures = plan
        .failures
        .iter()
        .map(|failure| (failure.error.device_id.as_str(), failure))
        .collect::<std::collections::HashMap<_, _>>();
    let mut outcomes = Vec::with_capacity(plan.targets.len());
    for target in &plan.targets {
        outcomes.push(
            if let Some(failure) = persisted_failures.get(target.device_id.as_str()) {
                TargetOutcome::Failed {
                    reason: failure.reason.clone(),
                    error: failure.error.clone(),
                }
            } else {
                inspect_target(statuses, &target.device_id, run_uid, expires_at).await?
            },
        );
    }
    let canaries_succeeded = plan
        .targets
        .iter()
        .zip(&outcomes)
        .filter(|(target, _)| target.canary)
        .all(|(_, outcome)| matches!(outcome, TargetOutcome::Succeeded));
    let has_canaries = plan.has_canaries();
    let groups = if outcomes
        .iter()
        .any(|outcome| matches!(outcome, TargetOutcome::Pending))
    {
        Some(
            group_cache
                .context("device group cache is unavailable")?
                .snapshot()
                .await
                .context("device group cache is not ready")?,
        )
    } else {
        None
    };
    for (target, outcome) in plan.targets.iter().zip(&mut outcomes) {
        let released = !has_canaries || target.canary || canaries_succeeded;
        if !released || !matches!(outcome, TargetOutcome::Pending) || Utc::now() >= expires_at {
            continue;
        }
        let attempt = attempt(&target.device_id, run_uid, expires_at);
        let key = system_upgrade_intent_key(&target.device_id, run_uid);
        match intent_matches(intents, &key, &attempt).await? {
            Some(true) => continue,
            Some(false) => {
                *outcome = failed_outcome(&target.device_id, "AttemptMismatch", None, Utc::now());
                continue;
            }
            None => {}
        }
        let current_device = devices.get_opt(&target.device_id).await?;
        let still_eligible = groups.as_ref().is_some_and(|groups| {
            current_device.as_ref().is_some_and(|device| {
                authorized_for_release(
                    device,
                    &task_run.spec.allowed_groups,
                    groups.get(&target.device_id),
                    groups.get("*"),
                )
            })
        });
        if !still_eligible {
            *outcome = failed_outcome(&target.device_id, "TargetRevoked", None, Utc::now());
            continue;
        }
        if let Err(create_error) = intents
            .create(&key, serde_json::to_vec(&attempt)?.into())
            .await
        {
            match intent_matches(intents, &key, &attempt).await? {
                Some(true) => {}
                Some(false) => {
                    *outcome =
                        failed_outcome(&target.device_id, "AttemptMismatch", None, Utc::now());
                }
                None => return Err(create_error.into()),
            }
        }
    }
    let newly_failed = plan
        .targets
        .iter()
        .zip(&outcomes)
        .filter(|(target, outcome)| {
            matches!(outcome, TargetOutcome::Failed { .. })
                && !persisted_failures.contains_key(target.device_id.as_str())
        })
        .filter_map(|(_, outcome)| match outcome {
            TargetOutcome::Failed { reason, error } => Some(rollout::Failure {
                reason: reason.clone(),
                error: error.clone(),
            }),
            _ => None,
        })
        .collect::<Vec<_>>();
    if !newly_failed.is_empty() {
        rollout::record_failures(rollout_plans, plan_key, run_uid, &newly_failed).await?;
    }
    let succeeded = outcomes
        .iter()
        .filter(|outcome| matches!(outcome, TargetOutcome::Succeeded))
        .count() as u32;
    let known_failed = outcomes
        .iter()
        .filter(|outcome| matches!(outcome, TargetOutcome::Failed { .. }))
        .count() as u32;
    let deadline = Utc::now() >= expires_at;
    let canaries_finished = plan
        .targets
        .iter()
        .zip(&outcomes)
        .filter(|(target, _)| target.canary)
        .all(|(_, outcome)| !matches!(outcome, TargetOutcome::Pending));
    let canary_failed =
        plan.targets.iter().zip(&outcomes).any(|(target, outcome)| {
            target.canary && matches!(outcome, TargetOutcome::Failed { .. })
        });
    let all_finished = outcomes
        .iter()
        .all(|outcome| !matches!(outcome, TargetOutcome::Pending));
    let (phase, failed) = run_result(
        plan.targets.len() as u32,
        succeeded,
        known_failed,
        canary_failed,
        canaries_finished,
        all_finished,
        deadline,
    );
    let last_error = outcomes
        .iter()
        .filter_map(|outcome| match outcome {
            TargetOutcome::Failed { error, .. } => Some(error),
            _ => None,
        })
        .max_by(|a, b| a.at.cmp(&b.at))
        .cloned()
        .or_else(|| {
            task_run
                .status
                .as_ref()
                .and_then(|status| status.last_error.clone())
        });
    let reason = if phase == TaskRunPhase::Failed && deadline {
        Some("DeadlineExceeded".to_string())
    } else {
        outcomes.iter().find_map(|outcome| match outcome {
            TargetOutcome::Failed { reason, .. } => Some(reason.clone()),
            _ => None,
        })
    };
    let new_status = TaskRunStatus {
        phase,
        target_count: plan.targets.len().try_into().unwrap_or(u32::MAX),
        succeeded,
        failed,
        start_time: Some(start.clone()),
        completion_time: (phase != TaskRunPhase::Running).then(|| Time(Utc::now())),
        reason,
        last_error,
    };
    if task_run.status.as_ref() == Some(&new_status) {
        return Ok(());
    }
    patch_status(runs, task_run, &new_status).await
 }
 fn run_result(
    target_count: u32,
    succeeded: u32,
    known_failed: u32,
    canary_failed: bool,
    canaries_finished: bool,
    all_finished: bool,
    deadline: bool,
 ) -> (TaskRunPhase, u32) {
    if succeeded == target_count {
        (TaskRunPhase::Complete, 0)
    } else if deadline
        || canary_failed && canaries_finished
        || !canary_failed && known_failed > 0 && all_finished
    {
        (TaskRunPhase::Failed, target_count - succeeded)
    } else {
        (TaskRunPhase::Running, known_failed)
    }
 }
 async fn inspect_target(
    statuses: &Store,
    device_id: &str,
    run_uid: &str,
    expires_at: chrono::DateTime<Utc>,
 ) -> Result<TargetOutcome> {
    let attempt = attempt(device_id, run_uid, expires_at);
    let Some(entry) = statuses
        .entry(&system_upgrade_status_key(device_id, run_uid))
        .await?
        .filter(|entry| entry.operation == Operation::Put)
    else {
        return Ok(TargetOutcome::Pending);
    };
    let Some(received_at) = chrono::DateTime::from_timestamp(
        entry.created.unix_timestamp(),
        entry.created.nanosecond(),
    ) else {
        return Ok(failed_outcome(
            device_id,
            "StatusMismatch",
            Some("NATS returned an invalid status timestamp"),
            Utc::now(),
        ));
    };
    let status = match serde_json::from_slice::<SystemUpgradeStatus>(&entry.value) {
        Ok(status) if status_matches(&status, &attempt) => status,
        Ok(_) => {
            return Ok(failed_outcome(
                device_id,
                "StatusMismatch",
                None,
                received_at,
            ));
        }
        Err(error) => {
            return Ok(failed_outcome(
                device_id,
                "StatusMismatch",
                Some(&error.to_string()),
                received_at,
            ));
        }
    };
    match status.phase {
        SystemUpgradePhase::Complete if completion_verified(&status, expires_at) => {
            Ok(TargetOutcome::Succeeded)
        }
        SystemUpgradePhase::Complete => Ok(TargetOutcome::Pending),
        SystemUpgradePhase::Failed | SystemUpgradePhase::RepairRequired => {
            let reason = if received_at > expires_at {
                "DeadlineExceeded".to_string()
            } else {
                format!("{:?}", status.phase)
            };
            Ok(failed_outcome(
                device_id,
                &reason,
                status.error.as_deref(),
                status.updated_at,
            ))
        }
        _ => Ok(TargetOutcome::Pending),
    }
 }
 fn attempt(
    device_id: &str,
    run_uid: &str,
    expires_at: chrono::DateTime<Utc>,
 ) -> SystemUpgradeAttempt {
    SystemUpgradeAttempt {
        attempt_id: run_uid.to_string(),
        run_uid: run_uid.to_string(),
        device_id: Id::from(device_id),
        expires_at,
    }
 }
 fn failed_outcome(
    device_id: &str,
    reason: &str,
    message: Option<&str>,
    at: chrono::DateTime<Utc>,
 ) -> TargetOutcome {
    TargetOutcome::Failed {
        reason: reason.to_string(),
        error: AggregateLastError {
            device_id: device_id.to_string(),
            message: bounded(message.unwrap_or(reason)),
            at: at.to_rfc3339(),
        },
    }
 }
 async fn intent_matches(
    store: &Store,
    key: &str,
    expected: &SystemUpgradeAttempt,
 ) -> Result<Option<bool>> {
    Ok(store
        .entry(key)
        .await?
        .filter(|entry| entry.operation == Operation::Put)
        .map(|entry| attempt_matches(&entry.value, expected)))
 }
 fn attempt_matches(value: &[u8], expected: &SystemUpgradeAttempt) -> bool {
    serde_json::from_slice::<SystemUpgradeAttempt>(value).is_ok_and(|attempt| attempt == *expected)
 }
 fn status_matches(status: &SystemUpgradeStatus, attempt: &SystemUpgradeAttempt) -> bool {
    status.attempt_id == attempt.attempt_id && status.run_uid == attempt.run_uid
 }
 fn completion_verified(status: &SystemUpgradeStatus, deadline: chrono::DateTime<Utc>) -> bool {
    status
        .post_completion_heartbeat_at
        .is_some_and(|heartbeat| heartbeat > status.updated_at && heartbeat <= deadline)
 }
 async fn cleanup_intents(intents: &Store, plan: &Plan, run_uid: &str) -> Result<()> {
    for target in &plan.targets {
        let key = system_upgrade_intent_key(&target.device_id, run_uid);
        if intents
            .entry(&key)
            .await?
            .is_some_and(|entry| entry.operation == Operation::Put)
        {
            intents.delete(&key).await?;
        }
    }
    Ok(())
 }
 async fn patch_failed(
    runs: &Api<TaskRun>,
    task_run: &TaskRun,
    target_count: u32,
    failed: u32,
    start: Option<&Time>,
    reason: &str,
    error: Option<&str>,
 ) -> Result<()> {
    let now = Time(Utc::now());
    patch_status(
        runs,
        task_run,
        &TaskRunStatus {
            phase: TaskRunPhase::Failed,
            target_count,
            succeeded: 0,
            failed,
            start_time: start.cloned(),
            completion_time: Some(now.clone()),
            reason: Some(reason.to_string()),
            last_error: error.map(|message| AggregateLastError {
                device_id: String::new(),
                message: bounded(message),
                at: now.0.to_rfc3339(),
            }),
        },
    )
    .await
 }
 async fn patch_status(
    runs: &Api<TaskRun>,
    task_run: &TaskRun,
    status: &TaskRunStatus,
 ) -> Result<()> {
    let resource_version = task_run
        .metadata
        .resource_version
        .as_deref()
        .context("TaskRun metadata.resourceVersion missing")?;
    runs.patch_status(
        &task_run.name_any(),
        &PatchParams::default(),
        &Patch::Merge(serde_json::json!({
            "metadata": { "resourceVersion": resource_version },
            "status": status,
        })),
    )
    .await?;
    Ok(())
 }
 async fn patch_finalizer(runs: &Api<TaskRun>, task_run: &TaskRun, present: bool) -> Result<()> {
    let resource_version = task_run
        .metadata
        .resource_version
        .as_deref()
        .context("TaskRun metadata.resourceVersion missing")?;
    let mut finalizers = task_run.finalizers().to_vec();
    if present {
        finalizers.push(FINALIZER.to_string());
    } else {
        finalizers.retain(|finalizer| finalizer != FINALIZER);
    }
    runs.patch(
        &task_run.name_any(),
        &PatchParams::default(),
        &Patch::Merge(serde_json::json!({
            "metadata": {
                "resourceVersion": resource_version,
                "finalizers": finalizers,
            },
        })),
    )
    .await?;
    Ok(())
 }
 fn bounded(message: &str) -> String {
    message.chars().take(MAX_ERROR_CHARS).collect()
 }
 #[cfg(test)]
 mod tests {
    use std::collections::{BTreeMap, HashSet};
    use chrono::DateTime;
    use harmony_reconciler_contracts::UpdaterCapabilities;
    use super::*;
    fn attempt() -> SystemUpgradeAttempt {
        SystemUpgradeAttempt {
            attempt_id: "run-1".into(),
            run_uid: "run-1".into(),
            device_id: Id::from("device-1"),
            expires_at: DateTime::parse_from_rfc3339("2026-07-24T12:00:00Z")
                .unwrap()
                .with_timezone(&Utc),
        }
    }
    #[test]
    fn status_must_match_attempt_and_run() {
        let attempt = attempt();
        let mut status = SystemUpgradeStatus {
            attempt_id: attempt.attempt_id.clone(),
            run_uid: attempt.run_uid.clone(),
            phase: SystemUpgradePhase::Applying,
            started_at: attempt.expires_at,
            updated_at: attempt.expires_at,
            post_completion_heartbeat_at: None,
            error: None,
        };
        assert!(status_matches(&status, &attempt));
        status.attempt_id = "other".into();
        assert!(!status_matches(&status, &attempt));
        status.attempt_id = attempt.attempt_id.clone();
        status.run_uid = "other".into();
        assert!(!status_matches(&status, &attempt));
    }
    #[test]
    fn existing_intent_must_match_the_complete_attempt() {
        let expected = attempt();
        assert!(attempt_matches(
            &serde_json::to_vec(&expected).unwrap(),
            &expected
        ));
        let mut changed = expected.clone();
        changed.expires_at += chrono::Duration::seconds(1);
        assert!(!attempt_matches(
            &serde_json::to_vec(&changed).unwrap(),
            &expected
        ));
        assert!(!attempt_matches(b"not json", &expected));
    }
    #[test]
    fn release_authorization_ignores_current_selector_labels() {
        let mut device = Device::new(
            "device-1",
            crate::DeviceSpec {
                inventory: None,
                updater: Some(UpdaterCapabilities {
                    protocol: 1,
                    apt_full_upgrade_v1: true,
                }),
                agent_upgrade: None,
            },
        );
        device.metadata.labels = Some(BTreeMap::from([("site".into(), "changed".into())]));
        let allowed = vec!["production".to_string()];
        let groups = HashSet::from(["production".to_string()]);
        assert!(authorized_for_release(
            &device,
            &allowed,
            Some(&groups),
            None
        ));
        device.spec.updater = None;
        assert!(!authorized_for_release(
            &device,
            &allowed,
            Some(&groups),
            None
        ));
        device.spec.updater = Some(UpdaterCapabilities {
            protocol: 1,
            apt_full_upgrade_v1: true,
        });
        assert!(!authorized_for_release(&device, &allowed, None, None));
    }
    #[test]
    fn terminal_counts_cover_every_target_and_preserve_late_observed_success() {
        assert_eq!(
            run_result(3, 1, 1, false, false, false, false),
            (TaskRunPhase::Running, 1)
        );
        assert_eq!(
            run_result(3, 1, 1, true, true, false, false),
            (TaskRunPhase::Failed, 2)
        );
        assert_eq!(
            run_result(3, 3, 0, false, true, true, true),
            (TaskRunPhase::Complete, 0)
        );
        assert_eq!(
            run_result(3, 1, 0, false, false, false, true),
            (TaskRunPhase::Failed, 2)
        );
    }
    #[test]
    fn success_requires_durable_in_deadline_heartbeat_proof() {
        let terminal = DateTime::parse_from_rfc3339("2026-07-24T12:00:00Z")
            .unwrap()
            .with_timezone(&Utc);
        let deadline = terminal + chrono::Duration::minutes(1);
        let mut status = SystemUpgradeStatus {
            attempt_id: "run-1".into(),
            run_uid: "run-1".into(),
            phase: SystemUpgradePhase::Complete,
            started_at: terminal,
            updated_at: terminal,
            post_completion_heartbeat_at: None,
            error: None,
        };
        assert!(!completion_verified(&status, deadline));
        status.post_completion_heartbeat_at = Some(terminal);
        assert!(!completion_verified(&status, deadline));
        status.post_completion_heartbeat_at = Some(terminal + chrono::Duration::seconds(1));
        assert!(completion_verified(&status, deadline));
        status.post_completion_heartbeat_at = Some(deadline + chrono::Duration::seconds(1));
        assert!(!completion_verified(&status, deadline));
    }
 }
--- a/fleet/harmony-fleet-operator/vendor/app.js
+++ b/fleet/harmony-fleet-operator/vendor/app.js
@@ -1,27 +1,3 @@
 document.body.addEventListener('htmx:configRequest', (event) => {
  event.detail.headers['x-csrf-token'] = '1';
 });
 // Open a modal dialog swapped into #modal-root. Lives here (not inline)
 // because the production CSP forbids inline scripts/handlers.
 document.body.addEventListener('htmx:afterSwap', (event) => {
  if (!event.target || event.target.id !== 'modal-root') return;
  const dialog = event.target.querySelector('dialog');
  if (!dialog || typeof dialog.showModal !== 'function') return;
  dialog.showModal();
  // Backdrop click closes; closing clears the root so it can re-open.
  dialog.addEventListener('click', (e) => {
    if (e.target === dialog) dialog.close();
  });
  dialog.addEventListener('close', () => {
    event.target.innerHTML = '';
  });
  // Keep a streaming log body scrolled to the latest line.
  const body = dialog.querySelector('[sse-connect]');
  if (body) {
    new MutationObserver(() => {
      body.scrollTop = body.scrollHeight;
    }).observe(body, { childList: true });
  }
 });
--- a/fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js
+++ b/fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js
@@ -1,290 +0,0 @@
 /*
 Server Sent Events Extension
 ============================
 This extension adds support for Server Sent Events to htmx.  See /www/extensions/sse.md for usage instructions.
 */
 (function() {
  /** @type {import("../htmx").HtmxInternalApi} */
  var api
  htmx.defineExtension('sse', {
    /**
     * Init saves the provided reference to the internal HTMX API.
     *
     * @param {import("../htmx").HtmxInternalApi} api
     * @returns void
     */
    init: function(apiRef) {
      // store a reference to the internal API.
      api = apiRef
      // set a function in the public API for creating new EventSource objects
      if (htmx.createEventSource == undefined) {
        htmx.createEventSource = createEventSource
      }
    },
    getSelectors: function() {
      return ['[sse-connect]', '[data-sse-connect]', '[sse-swap]', '[data-sse-swap]']
    },
    /**
     * onEvent handles all events passed to this extension.
     *
     * @param {string} name
     * @param {Event} evt
     * @returns void
     */
    onEvent: function(name, evt) {
      var parent = evt.target || evt.detail.elt
      switch (name) {
        case 'htmx:beforeCleanupElement':
          var internalData = api.getInternalData(parent)
          // Try to remove remove an EventSource when elements are removed
          var source = internalData.sseEventSource
          if (source) {
            api.triggerEvent(parent, 'htmx:sseClose', {
              source,
              type: 'nodeReplaced',
            })
            internalData.sseEventSource.close()
          }
          return
        // Try to create EventSources when elements are processed
        case 'htmx:afterProcessNode':
          ensureEventSourceOnElement(parent)
      }
    }
  })
  /// ////////////////////////////////////////////
  // HELPER FUNCTIONS
  /// ////////////////////////////////////////////
  /**
   * createEventSource is the default method for creating new EventSource objects.
   * it is hoisted into htmx.config.createEventSource to be overridden by the user, if needed.
   *
   * @param {string} url
   * @returns EventSource
   */
  function createEventSource(url) {
    return new EventSource(url, { withCredentials: true })
  }
  /**
   * registerSSE looks for attributes that can contain sse events, right
   * now hx-trigger and sse-swap and adds listeners based on these attributes too
   * the closest event source
   *
   * @param {HTMLElement} elt
   */
  function registerSSE(elt) {
    // Add message handlers for every `sse-swap` attribute
    if (api.getAttributeValue(elt, 'sse-swap')) {
      // Find closest existing event source
      var sourceElement = api.getClosestMatch(elt, hasEventSource)
      if (sourceElement == null) {
        // api.triggerErrorEvent(elt, "htmx:noSSESourceError")
        return null // no eventsource in parentage, orphaned element
      }
      // Set internalData and source
      var internalData = api.getInternalData(sourceElement)
      var source = internalData.sseEventSource
      var sseSwapAttr = api.getAttributeValue(elt, 'sse-swap')
      var sseEventNames = sseSwapAttr.split(',')
      for (var i = 0; i < sseEventNames.length; i++) {
        const sseEventName = sseEventNames[i].trim()
        const listener = function(event) {
          // If the source is missing then close SSE
          if (maybeCloseSSESource(sourceElement)) {
            return
          }
          // If the body no longer contains the element, remove the listener
          if (!api.bodyContains(elt)) {
            source.removeEventListener(sseEventName, listener)
            return
          }
          // swap the response into the DOM and trigger a notification
          if (!api.triggerEvent(elt, 'htmx:sseBeforeMessage', event)) {
            return
          }
          swap(elt, event.data)
          api.triggerEvent(elt, 'htmx:sseMessage', event)
        }
        // Register the new listener
        api.getInternalData(elt).sseEventListener = listener
        source.addEventListener(sseEventName, listener)
      }
    }
    // Add message handlers for every `hx-trigger="sse:*"` attribute
    if (api.getAttributeValue(elt, 'hx-trigger')) {
      // Find closest existing event source
      var sourceElement = api.getClosestMatch(elt, hasEventSource)
      if (sourceElement == null) {
        // api.triggerErrorEvent(elt, "htmx:noSSESourceError")
        return null // no eventsource in parentage, orphaned element
      }
      // Set internalData and source
      var internalData = api.getInternalData(sourceElement)
      var source = internalData.sseEventSource
      var triggerSpecs = api.getTriggerSpecs(elt)
      triggerSpecs.forEach(function(ts) {
        if (ts.trigger.slice(0, 4) !== 'sse:') {
          return
        }
        var listener = function (event) {
          if (maybeCloseSSESource(sourceElement)) {
            return
          }
          if (!api.bodyContains(elt)) {
            source.removeEventListener(ts.trigger.slice(4), listener)
          }
          // Trigger events to be handled by the rest of htmx
          htmx.trigger(elt, ts.trigger, event)
          htmx.trigger(elt, 'htmx:sseMessage', event)
        }
        // Register the new listener
        api.getInternalData(elt).sseEventListener = listener
        source.addEventListener(ts.trigger.slice(4), listener)
      })
    }
  }
  /**
   * ensureEventSourceOnElement creates a new EventSource connection on the provided element.
   * If a usable EventSource already exists, then it is returned.  If not, then a new EventSource
   * is created and stored in the element's internalData.
   * @param {HTMLElement} elt
   * @param {number} retryCount
   * @returns {EventSource | null}
   */
  function ensureEventSourceOnElement(elt, retryCount) {
    if (elt == null) {
      return null
    }
    // handle extension source creation attribute
    if (api.getAttributeValue(elt, 'sse-connect')) {
      var sseURL = api.getAttributeValue(elt, 'sse-connect')
      if (sseURL == null) {
        return
      }
      ensureEventSource(elt, sseURL, retryCount)
    }
    registerSSE(elt)
  }
  function ensureEventSource(elt, url, retryCount) {
    var source = htmx.createEventSource(url)
    source.onerror = function(err) {
      // Log an error event
      api.triggerErrorEvent(elt, 'htmx:sseError', { error: err, source })
      // If parent no longer exists in the document, then clean up this EventSource
      if (maybeCloseSSESource(elt)) {
        return
      }
      // Otherwise, try to reconnect the EventSource
      if (source.readyState === EventSource.CLOSED) {
        retryCount = retryCount || 0
        retryCount = Math.max(Math.min(retryCount * 2, 128), 1)
        var timeout = retryCount * 500
        window.setTimeout(function() {
          ensureEventSourceOnElement(elt, retryCount)
        }, timeout)
      }
    }
    source.onopen = function(evt) {
      api.triggerEvent(elt, 'htmx:sseOpen', { source })
      if (retryCount && retryCount > 0) {
        const childrenToFix = elt.querySelectorAll("[sse-swap], [data-sse-swap], [hx-trigger], [data-hx-trigger]")
        for (let i = 0; i < childrenToFix.length; i++) {
          registerSSE(childrenToFix[i])
        }
        // We want to increase the reconnection delay for consecutive failed attempts only
        retryCount = 0
      }
    }
    api.getInternalData(elt).sseEventSource = source
    var closeAttribute = api.getAttributeValue(elt, "sse-close");
    if (closeAttribute) {
      // close eventsource when this message is received
      source.addEventListener(closeAttribute, function() {
        api.triggerEvent(elt, 'htmx:sseClose', {
          source,
          type: 'message',
        })
        source.close()
      });
    }
  }
  /**
   * maybeCloseSSESource confirms that the parent element still exists.
   * If not, then any associated SSE source is closed and the function returns true.
   *
   * @param {HTMLElement} elt
   * @returns boolean
   */
  function maybeCloseSSESource(elt) {
    if (!api.bodyContains(elt)) {
      var source = api.getInternalData(elt).sseEventSource
      if (source != undefined) {
        api.triggerEvent(elt, 'htmx:sseClose', {
          source,
          type: 'nodeMissing',
        })
        source.close()
        // source = null
        return true
      }
    }
    return false
  }
  /**
   * @param {HTMLElement} elt
   * @param {string} content
   */
  function swap(elt, content) {
    api.withExtensions(elt, function(extension) {
      content = extension.transformResponse(content, null, elt)
    })
    var swapSpec = api.getSwapSpecification(elt)
    var target = api.getTarget(elt)
    api.swap(target, content, swapSpec)
  }
  function hasEventSource(node) {
    return api.getInternalData(node).sseEventSource != null
  }
 })()
--- a/fleet/scripts/deploy-apache.sh
+++ b/fleet/scripts/deploy-apache.sh
@@ -1,4 +1,4 @@
-cargo run -q -p example_harmony_apply_deployment -- \
+cargo run -q -p harmony-fleet-deploy --bin harmony-fleet-example-deployment -- \
 	--namespace "fleet-demo" \
 	--name "testdeployment-apache" \
 	--target-device "paul" \
--- a/Show More
+++ b/Show More
		`@@ -0,0 +1,2 @@`
							`FROM busybox:1.37`
							`CMD ["sleep", "infinity"]`
		`@@ -0,0 +1,2 @@`
							`FROM busybox:1.37`
							`CMD ["httpd", "-f", "-p", "8080"]`