NationTech/harmony
5
0
Fork 2
Code Issues 40 Pull Requests 13 Actions Packages Projects Releases Wiki Activity

feat/iot-helm #275

Merged
johnride merged 52 commits from feat/iot-helm into feat/iot-walking-skeleton 2026-04-25 13:52:24 +00:00
Conversation 2 Commits 52 Files Changed 70 +6698 -956
70 changed files with 6698 additions and 956 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
Cargo.lock generated
View File

@@ -3166,7 +3166,36 @@ dependencies = [
]
[[package]]
name = "example_iot_vm_setup"
name = "example_fleet_load_test"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"chrono",
"clap",
"harmony-fleet-operator",
"harmony-reconciler-contracts",
"k8s-openapi",
"kube",
"rand 0.9.2",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "example_fleet_nats_install"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"harmony",
"tokio",
]
[[package]]
name = "example_fleet_vm_setup"
version = "0.1.0"
dependencies = [
"anyhow",
@@ -3178,6 +3207,20 @@ dependencies = [
"tokio",
]
[[package]]
name = "example_harmony_apply_deployment"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"harmony",
"harmony-fleet-operator",
"k8s-openapi",
"kube",
"serde_json",
"tokio",
]
[[package]]
name = "example_linux_vm"
version = "0.1.0"
@@ -3690,6 +3733,47 @@ dependencies = [
"walkdir",
]
[[package]]
name = "harmony-fleet-agent"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"chrono",
"clap",
"futures-util",
"harmony",
"harmony-reconciler-contracts",
"serde",
"serde_json",
"tokio",
"toml",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "harmony-fleet-operator"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"chrono",
"clap",
"futures-util",
"harmony",
"harmony-reconciler-contracts",
"k8s-openapi",
"kube",
"schemars 0.8.22",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "harmony-k8s"
version = "0.1.0"
@@ -3732,8 +3816,10 @@ version = "0.1.0"
dependencies = [
"chrono",
"harmony_types",
"schemars 0.8.22",
"serde",
"serde_json",
"thiserror 2.0.18",
]
[[package]]
@@ -4710,48 +4796,6 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "iot-agent-v0"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"chrono",
"clap",
"futures-util",
"harmony",
"harmony-reconciler-contracts",
"serde",
"serde_json",
"tokio",
"toml",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "iot-operator-v0"
version = "0.1.0"
dependencies = [
"anyhow",
"async-nats",
"async-trait",
"clap",
"futures-util",
"harmony",
"harmony-k8s",
"harmony-reconciler-contracts",
"k8s-openapi",
"kube",
"schemars 0.8.22",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -4910,6 +4954,7 @@ checksum = "aa60a41b57ae1a0a071af77dbcf89fc9819cfe66edaf2beeb204c34459dcf0b2"
dependencies = [
"base64 0.22.1",
"chrono",
"schemars 0.8.22",
"serde",
"serde_json",
]

6
Cargo.toml
View File

@@ -28,8 +28,8 @@ members = [
"harmony_node_readiness",
"harmony-k8s",
"harmony_assets", "opnsense-codegen", "opnsense-api",
"iot/iot-operator-v0",
"iot/iot-agent-v0",
"fleet/harmony-fleet-operator",
"fleet/harmony-fleet-agent",
"harmony-reconciler-contracts",
]
@@ -66,7 +66,7 @@ kube = { version = "1.1.0", features = [
"ws",
"jsonpatch",
] }
k8s-openapi = { version = "0.25", features = ["v1_30"] }
k8s-openapi = { version = "0.25", features = ["v1_30", "schemars"] }
# TODO replace with https://github.com/bourumir-wyngs/serde-saphyr as serde_yaml is deprecated https://github.com/sebastienrousseau/serde_yml
serde_yaml = "0.9"
serde-value = "0.7"

4
ROADMAP/12-code-review-april-2026.md
View File

@@ -99,7 +99,7 @@ Replace `kubectl exec bao ...` shell commands in `openbao/setup.rs` with typed `
`K8sAnywhereTopology` and `HAClusterTopology` have accumulated opinions — cert-manager install, tenant manager setup, helm probes, TLS passthrough, SSO wiring — that make them unfit for narrow, ad-hoc Score execution. Calling `ensure_ready()` on `K8sAnywhereTopology` to apply a single CRD installs a full product stack as a side effect; that's the opposite of what "make me ready" should mean.
Concrete example: `iot/iot-operator-v0/src/install.rs` needed a topology that satisfies `K8sclient` for a single `K8sResourceScore::<CustomResourceDefinition>` apply. `K8sAnywhereTopology` was wrong (too heavy); `HAClusterTopology` was wrong (bare-metal). Work-around: a 30-line inline `InstallTopology` that wraps a pre-built `K8sClient` and has a noop `ensure_ready`. That file flags the architectural smell in its doc comment and points back to this entry.
Concrete example: `fleet/harmony-fleet-operator/src/install.rs` needed a topology that satisfies `K8sclient` for a single `K8sResourceScore::<CustomResourceDefinition>` apply. `K8sAnywhereTopology` was wrong (too heavy); `HAClusterTopology` was wrong (bare-metal). Work-around: a 30-line inline `InstallTopology` that wraps a pre-built `K8sClient` and has a noop `ensure_ready`. That file flags the architectural smell in its doc comment and points back to this entry.
If every narrow Score ends up vendoring its own ad-hoc topology, we get exactly the proliferation this entry is meant to prevent.
@@ -113,4 +113,4 @@ If every narrow Score ends up vendoring its own ad-hoc topology, we get exactly
- Adding a new ad-hoc Score against k8s doesn't require inventing a new topology.
- `K8sAnywhereTopology` stops being the default reach and starts being a deliberate product choice.
- Test: can we delete the inline `InstallTopology` in `iot/iot-operator-v0/src/install.rs` by replacing it with a one-liner `K8sBareTopology::from_env()`? That's the smoke test for "we fixed the proliferation."
- Test: can we delete the inline `InstallTopology` in `fleet/harmony-fleet-operator/src/install.rs` by replacing it with a one-liner `K8sBareTopology::from_env()`? That's the smoke test for "we fixed the proliferation."

14
ROADMAP/iot_platform/arm_vm_plan.md → ROADMAP/fleet_platform/arm_vm_plan.md
View File

@@ -15,7 +15,7 @@ for CI) so:
- the VM runs the same Ubuntu 24.04 arm64 cloud image customers will
eventually flash onto a Pi;
- the iot-agent shipped to it is a real aarch64 binary produced by
- the fleet-agent shipped to it is a real aarch64 binary produced by
our existing cross-compile toolchain;
- apt/systemd/podman on the VM are the actual arm64 packages; and
- smoke-a3 exercises all of it end-to-end.
@@ -126,11 +126,11 @@ In `modules/iot/preflight.rs`, when the caller asks for arm64 VMs
### 6. Cross-compiled agent
smoke-a3.sh phase 2 currently does native `cargo build --release
-p iot-agent-v0`. When arch=aarch64:
-p fleet-agent-v0`. When arch=aarch64:
- `cargo build --release --target aarch64-unknown-linux-gnu
-p iot-agent-v0`
-p fleet-agent-v0`
- AGENT_BINARY points at `target/aarch64-unknown-linux-gnu/release/
iot-agent-v0`
fleet-agent-v0`
Opt-in via `--arch aarch64` CLI flag on both
`example_iot_vm_setup` and `smoke-a3.sh`. Default stays x86_64.
@@ -152,9 +152,9 @@ arch=aarch64. Smoke-a3's phase 5 reboot gate also lengthens.
| `harmony/src/modules/kvm/topology.rs` | Copy per-VM NVRAM template on ensure_vm; thread arch through to XML. |
| `harmony/src/modules/iot/assets.rs` | `ensure_ubuntu_2404_cloud_image_for_arch(arch)`; pin arm64 URL+sha256. |
| `harmony/src/modules/iot/preflight.rs` | Arch-aware preflight; qemu-system-aarch64 + firmware + qemu-version. |
| `examples/iot_vm_setup/src/main.rs` | `--arch x86_64|aarch64` CLI flag; resolve matching cloud image. |
| `iot/scripts/smoke-a3.sh` | Arch flag plumbing; cross-compile; extended timeouts; preflight. |
| `iot/scripts/smoke-a3-arm.sh` (new) | Dedicated arm smoke as the CI hook — `ARCH=aarch64 ./smoke-a3.sh`. |
| `examples/fleet_vm_setup/src/main.rs` | `--arch x86_64|aarch64` CLI flag; resolve matching cloud image. |
| `fleet/scripts/smoke-a3.sh` | Arch flag plumbing; cross-compile; extended timeouts; preflight. |
| `fleet/scripts/smoke-a3-arm.sh` (new) | Dedicated arm smoke as the CI hook — `ARCH=aarch64 ./smoke-a3.sh`. |
## Out of scope

453
ROADMAP/fleet_platform/chapter_4_aggregation_scale.md Normal file
View File

@@ -0,0 +1,453 @@
# Chapter 4 — Aggregation architecture at IoT scale
> **Status: SUPERSEDED (2026-04-23) — historical archaeology only.**
>
> This document proposed an event-stream CQRS architecture
> (`StateChangeEvent` on a JetStream stream, per-key `Revision`
> tracking, `LifecycleTransition::{Applied, Removed}` diff events,
> cold-start re-walk, durable consumer folding events into counters).
> The design was implemented, then entirely removed in favor of a
> simpler shape: the operator watches `device-state` KV directly
> via `bucket.watch_with_history(">")`, selector evaluation runs
> against a cluster-scoped `Device` CRD cache, and `desired-state`
> entries are diffed from the selector → matched-devices set on
> watch events. No event stream, no revisions, no transition
> enum.
>
> **What's still accurate in this doc:**
>
> - The per-concern KV split (`device-info`, `device-state`,
> `device-heartbeat`) and their cadences.
> - The operator's responsibilities: counter aggregation, dirty-set
> debouncing, 1 Hz CR patch cadence.
> - The scale target (10 000 devices × 1 000 deployments at
> 10 000 state writes/s — load-tested and green).
> - The `.status.aggregate` fields (succeeded / failed / pending /
> lastError, plus the new `matchedDeviceCount`).
>
> **What's no longer true:**
>
> - No `events.state.>` JetStream stream, no durable event consumer.
> - No per-key `Revision(agent_epoch, sequence)` — KV ordering is
> sufficient.
> - No `LifecycleTransition` diff enum on the wire — phase
> transitions are derived from cached vs. current state inside
> the operator.
> - No `events.log.>` stream, no `logs.<device>.query` request-
> reply protocol. Logs are deferred until a real consumer lands.
> - No cold-start event re-walk — KV watch with history replays
> current state, which covers restart-correctness for the
> device-state cache.
>
> **Where to look now:**
>
> - Shipped design: `v0_1_plan.md` Chapter 2 (marked SHIPPED 2026-04-23).
> - Source of truth: `fleet/harmony-fleet-operator/src/fleet_aggregator.rs`,
> `fleet/harmony-fleet-operator/src/device_reconciler.rs`,
> `harmony-reconciler-contracts/src/{fleet,kv,status}.rs`.
>
> Everything below is preserved verbatim as the decision trail of a
> path not taken. Useful as context for why the current design is
> shaped the way it is; not a spec for future work.
>
> ---
>
> (Original design draft begins here.)
## 1. Why now
We have no real deployment in the field yet. That's a liability when
shipping (no user, no revenue) but a gift when designing: we can move
the data model before customers depend on it. After a partner fleet
lands, changing the aggregation substrate is a multi-quarter
migration. Doing it now is days of work.
Chapter 2's aggregator was the right "make it work" design for a
walking-skeleton proof. It's the wrong "make it scale" design for a
partner deployment of even a few hundred devices, let alone the
fleet sizes the product thesis targets. This chapter replaces it.
## 2. What's wrong today
**Per-tick cost, current design.** Every 5 seconds, for each
Deployment CR, resolve the selector against the full device snapshot
and fold into an aggregate:
```
O(deployments × devices) per tick
+ 1 kube patch per CR per tick
```
At 10k deployments × 1M devices, that's 10^10 selector evaluations
and 10k apiserver patches every 5 s. Nothing resembles viable there.
**What else goes wrong at scale.**
- The operator holds the full fleet snapshot in memory. 1M `AgentStatus`
payloads × a few kB each = GB of heap, dominated by `recent_events`
rings.
- Agent heartbeats publish the whole `AgentStatus` every 30 s — a lot
of bytes on the wire whose only incremental content is usually a
timestamp update.
- `agent-status` is a KV bucket. KV is designed for "latest value per
key," not "stream of state changes." We've been using it for both
roles and paying the worst of each.
- Logs are nowhere yet (good — this is the moment to put them in the
right place before we're committed).
## 3. Design overview
Shift to a **CQRS-style architecture** where devices write their
authoritative state, and the operator maintains incrementally-updated
aggregates driven by state-change events.
```
device (N× agents) operator
────────────────── ────────
current state keys ───reads─▶ on cold-start:
(authoritative) walk keys → rebuild counters
then: stream consumer
state-change events ═ JS stream═▶ ± counters per event
(delta stream) ± update reverse index
on tick (1 Hz):
device_info keys ───reads─▶ patch .status for dirty deployments
(labels, inventory)
logs ───at-least-once NATS subj────▶ not stored centrally
(streamed on query)
```
Three substrates, each chosen for its fit:
- **JetStream KV, per-device keys** — device-authoritative state.
Cheap to read when needed, never scanned globally at scale.
- **JetStream stream, per-device events** — ordered delta feed.
Operator consumers replay on restart, consume incrementally during
steady state.
- **Plain NATS subjects, logs** — at-least-once pub/sub, device-side
buffering (~10k lines), streamed on query.
## 4. Data model
### 4.1 NATS KV buckets
**`device-info`** — static-ish facts per device, infrequent updates.
| Key | Value | Written by | Read by |
|-----|-------|------------|---------|
| `info.<device_id>` | `DeviceInfo` (labels, inventory, agent_version) | agent on startup + label change | operator (selector resolution, inventory display) |
**`device-state`** — current phase per deployment per device.
Authoritative source of truth for "what's running where."
| Key | Value | Written by | Read by |
|-----|-------|------------|---------|
| `state.<device_id>.<deployment_name>` | `DeploymentState` (phase, last_event_at, last_error) | agent on reconcile transition | operator on cold-start only |
One key per (device, deployment) pair. Natural TTL via JetStream KV
per-key history — lets us cap the keyspace.
**`device-heartbeat`** — liveness only. Tiny payload, frequent
updates.
| Key | Value | Written by | Read by |
|-----|-------|------------|---------|
| `heartbeat.<device_id>` | `{ timestamp }` (32 bytes) | agent every 30s | operator (stale detection) |
Separate from `device-state` so routine heartbeats don't churn the
state keys or emit spurious state-change events.
### 4.2 NATS JetStream stream
**`device-events`** — ordered delta feed for operator aggregation.
- Subject: `events.state.<device_id>.<deployment_name>`
- Payload: `StateChangeEvent { from: Phase, to: Phase, at, last_error }`
- Retention: time-based (e.g. 24h) — consumers that fall further
behind than retention rebuild from `device-state` KV on recovery.
- Agents emit one event per phase transition, **not** per heartbeat.
Separate stream for **event log** (user-facing reconcile log events):
- Subject: `events.log.<device_id>`
- Payload: `LogEvent { at, severity, message, deployment? }`
- Retention: time-based (1h, enough for "show me what happened the
last few minutes" queries; the device's in-memory ring holds the
rest).
### 4.3 Log transport (NOT JetStream)
- Subject: `logs.<device_id>` — plain pub/sub, at-least-once
- Not persisted by NATS
- Device buffers last ~10k lines in a ring buffer
- Query protocol: request-reply on `logs.<device_id>.query`
- Device responds with buffer contents, then streams live tail
until the query closes
This is a dedicated transport because structured logs at fleet scale
(1M devices × 1k lines/h = 1B messages/h) would crush JetStream's
per-subject storage without adding operator-visible value. Operators
only look at logs on-demand, per-device; device-side buffering
matches the access pattern.
### 4.4 CRD fields
Minimal change from Chapter 2:
- `.status.aggregate.succeeded | failed | pending` — now sourced
from counters, not per-tick fold.
- `.status.aggregate.last_error` — updated on `to: Failed` events.
- `.status.aggregate.last_heartbeat_at` — from the per-deployment
latest event.
- `.status.aggregate.recent_events` — bounded per-deployment ring,
updated on event arrival.
- **Drop** `.status.aggregate.unreported` (no meaningful definition
under selector-based targeting — already removed in the pre-chapter
cleanup).
- **Add** `.status.aggregate.stale: u32` — count of devices matching
the selector whose last heartbeat is older than a threshold
(default 5 min). This is the replacement for "unreported" that
makes sense at scale. Computed on tick from the operator's
reverse-indexed view, not per-device query.
### 4.5 Operator in-memory state
- **Counters** — `HashMap<DeploymentKey, PhaseCounters>`, one entry
per CR, updated atomically on event arrival.
- **Reverse index** — `HashMap<DeviceId, HashSet<DeploymentKey>>`,
updated when a device's labels change or when a CR's selector
changes. Lets a state-change event find affected deployments in
O(deployments-matching-this-device) rather than O(all-deployments).
- **Last-error rollup** — per deployment, the most recent error
keyed by timestamp.
- **Recent-events ring** — per deployment, bounded by N (e.g. 10).
- **Dirty set** — deployments whose aggregate has changed since last
patch. Tick reads + clears this set; only dirty deployments get
patched.
Operator heap is bounded by fleet + deployment count, not their
product.
## 5. Counter invariants (the contract)
Correctness rests on two rules:
### 5.1 Device publishes exactly one transition per reconcile outcome
Every reconcile results in a state. If the state differs from the
last published state for `(device, deployment)`, the agent:
1. Writes the new state to `state.<device>.<deployment>` KV (CAS
against expected-revision for multi-writer safety — only one
agent process per device, so contention is theoretical).
2. Publishes a `StateChangeEvent` to
`events.state.<device>.<deployment>`.
These two writes must be atomic from the agent's perspective — if
(1) succeeds and (2) fails (or vice versa), the agent retries until
both reach NATS. Worst case: a duplicate event on the stream;
counter handles duplicates via `from → to` structure (see 5.2).
### 5.2 Counters are driven by transitions, not snapshots
Each event carries `from: Phase, to: Phase`. Counter update is a
single atomic action:
```rust
counters[(deployment, from)] -= 1;
counters[(deployment, to)] += 1;
```
Duplicates (same `from → to` replayed) are a no-op if `from` ==
current phase for that (device, deployment) — the operator
cross-checks the device's current state in the reverse index before
applying. A duplicate past event is detected and ignored; a duplicate
current event is idempotent anyway (counters converge).
### 5.3 The bootstrap transition
A device's first-ever event for a deployment has `from: None` (or a
sentinel `Unassigned` variant): counter update is just `to`
increment.
### 5.4 Device leaves fleet
When a device's heartbeat goes stale past threshold + grace, OR
when its labels no longer match the deployment's selector:
- Counters are decremented for every deployment the device was
previously contributing to (via the reverse index).
- The device's state keys aren't touched — they're the authoritative
record; a device re-joining resumes from them.
### 5.5 CR created / selector changed
The reverse index + counters are rebuilt for the affected CR by
walking `device-info` + `device-state` once (O(devices + states)
local NATS KV reads). Cheap for a single CR; happens at CR-apply
time, not on every tick.
## 6. Cold-start protocol
On operator process start:
1. **Load CRs** — list `Deployment` CRs via kube API. Build the
reverse index skeleton (deployment → selector).
2. **Load device labels** — iterate `device-info` KV keys once.
Resolve each device against every CR's selector, populate the
reverse index device-side entries. O(devices × CRs), one-time,
in-memory. For 1M devices × 10k CRs this is 10^10 op but purely
local lookups (BTreeMap matches on label maps); back-of-envelope
has it at a few seconds to a minute on a modern CPU.
3. **Rebuild counters** — iterate `device-state` KV keys once.
For each `state.<device>.<deployment>`, look up the matching
deployments from the reverse index and increment counters.
4. **Attach stream consumer** — durable consumer on
`events.state.>`, starting from the newest sequence at cold-start
moment. The KV walk was the "past"; the stream is the "future."
5. **Begin tick loop** — patch dirty CRs on a 1 Hz schedule.
Cold-start time dominated by step 2, not step 3. An ArgoCD-style
"pause all reconciles during leader election / startup" envelope
keeps the CR patches from competing with the cold-start scans.
**What if the operator falls behind the stream's retention window?**
Reset to step 3 (re-walk `device-state`). The KV is authoritative;
the stream is an accelerator.
## 7. CR status patch cadence
- Counter updates happen in memory, instantly.
- The **dirty set** captures which deployments' aggregates changed
since the last patch.
- A 1 Hz ticker reads + clears the dirty set, patches those CRs.
- Individual CR patches are debounced to at most once per second
— avoids hammering the apiserver when a deployment is mid-rollout
and devices are transitioning in a burst.
Steady-state operator → apiserver traffic is proportional to the
rate of *interesting* changes, not to fleet size.
## 8. Failure modes
| Scenario | Detection | Recovery |
|---|---|---|
| Operator crash | k8s restarts the pod | Cold-start protocol §6 |
| Stream consumer falls behind retention | Stream API returns out-of-range | Re-run §6 step 3 (re-walk KV) |
| Agent publishes event but KV write fails | Agent-side local retry; event is replayed | Counter is idempotent per §5.2 |
| Agent writes KV but event publish fails | Agent-side local retry | Operator never sees the transition until retry succeeds; stale threshold catches the device if agent is permanently broken |
| Device's label change lost | Heartbeat carries current labels; stale entry aged out | Periodic sync (e.g. 1/h) re-scans `device-info` to catch drift |
| Duplicate event (retry) | `from == current` in reverse index | No-op (§5.2) |
| Out-of-order event (retry ordering) | Sequence number on event | Consumer tracks per-(device, deployment) last-applied sequence; old events ignored |
## 9. Scale back-of-envelope
**Target:** 1M devices, 10k deployments, p50 reconcile rate 1 event
per device per hour.
- **Event volume.** 1M × (1/3600s) = 278 events/s.
- **Operator event-processing cost.** Each event touches a bounded
number of in-memory counters (via reverse index). At 278 eps, this
is ~1 µs-equivalent of CPU, ~0 network (JetStream local to operator).
- **Operator → apiserver patches.** Deployments change at a rate
far below event rate; debounced dirty-set drains limit patches to
a few per second even during bursty rollouts.
- **Operator memory.** Reverse index entries (device_id + set of
deployment keys) ≈ 200 bytes × 1M = 200 MB. Counters ≈ 10k × few
fields = negligible. Last-error + recent-events rings ≈ 10k × 10
entries × 512 bytes = 50 MB. Total ~250 MB — fine.
- **Cold-start time.** 1M KV reads × amortized 0.1 ms (JetStream KV
is fast for key iteration) = 100 s. Acceptable for a
several-minute-once-per-release recovery window. If it becomes a
problem, chunk the walk and resume-from-checkpoint.
- **Stale device sweep.** On each tick, O(dirty set × reverse index
lookups). Stale detection itself is O(devices-whose-heartbeat-is-old);
a second, slower ticker (e.g. 30 s) scans the heartbeat KV for
entries older than threshold and emits synthetic "device went
stale" events that drive the same counter-decrement path.
## 10. Schema migration
`Deployment` CRD is still `v1alpha1`, not deployed anywhere, so no
migration machinery is needed for the CRD itself — we just change
the aggregate subtree definition.
`harmony-reconciler-contracts::AgentStatus` is deprecated by this
chapter. Replaced by narrower wire types:
- `DeviceInfo` — what `info.<device_id>` stores
- `DeploymentState` — what `state.<device>.<dep>` stores
- `HeartbeatPayload` — what `heartbeat.<device_id>` stores
- `StateChangeEvent` — what events stream emits
- `LogEvent` — what event-log stream emits
The old `AgentStatus` type goes away when the old aggregator
goes away. Clean break, same CRD version.
## 11. Implementation milestones
Landing order, each a reviewable increment:
1. **M1: new contracts crate shapes** — `DeviceInfo`,
`DeploymentState`, `HeartbeatPayload`, `StateChangeEvent`,
`LogEvent`. Round-trip serde tests. No runtime code changes yet.
2. **M2: agent-side rewrite** — agent writes the new KV shapes +
publishes state-change events + heartbeats. Old `AgentStatus`
publish path stays in parallel for the smoke to keep passing.
3. **M3: operator-side cold-start protocol** — new operator task
that walks the new KV buckets and builds in-memory counters.
Runs alongside the old aggregator; logs counter parity checks
against the legacy aggregator's output so we can verify
correctness before switching over.
4. **M4: operator-side event consumer** — attach the durable stream
consumer, drive counters incrementally. Parity checks still on.
5. **M5: flip CR patch source** — the new counter-backed aggregator
patches `.status.aggregate`, the legacy one goes read-only, then
deleted in the next commit.
6. **M6: logs subject + query protocol** — device-side ring buffer,
query API, a first CLI surface (`natiq logs device=X` or
equivalent) that drives it.
7. **M7: synthetic-scale test harness** — spin up 1k (then 10k) mock
agents in-process, drive a realistic event load through the
operator, measure + publish numbers.
8. **M8: delete legacy `AgentStatus`** — `harmony-reconciler-contracts`
cleanup, smoke-a4 updates.
M1-M5 can land on one branch; M6 is adjacent work; M7-M8 close out.
## 12. Open questions
- **Multi-operator HA.** The design assumes one operator at a time.
Adding HA means either (a) one active + one standby operator with
NATS-based leader election, or (b) shared counter state in KV
instead of in-memory. (a) is simpler; (b) scales better.
Defer until a specific availability target demands it.
- **Counter-KV snapshots.** Should we periodically snapshot the
in-memory counter state to a `counters` KV bucket so cold-start
can resume from a recent snapshot + a short stream tail, instead
of always re-walking `device-state`? Probably yes once cold-start
time becomes an operational concern, but not in the initial cut.
- **Stream retention tuning.** 24h for `events.state.>` is a guess.
Real number depends on observed operator downtime p99. Initial
setting, tune from operational data.
- **Compaction policy for `device-state` KV.** JetStream KV
per-key history can grow unbounded if phases churn. Set
`max_history_per_key = 1` (keep only latest value) unless there's
a reason to keep transition history (there isn't — that's what
the events stream is for).
- **Agent crash before publishing state-change event.** Transition
is durably captured in the agent's local podman state; on agent
restart the reconcile loop re-observes the phase and either
re-publishes (if it differs from `state.<device>.<dep>`) or stays
silent. Correctness preserved at the cost of event-stream ordering
ambiguity during the crash window — acceptable.
## 13. What this chapter deliberately does *not* change
- CRD `.spec.target_selector` semantics — stays exactly as shipped.
- Operator's kube-rs controller loop for CR reconcile — stays as is.
- Helm chart structure (Chapter 3) — orthogonal.
- Authentication (Chapter Auth) — orthogonal. When that chapter
lands, every subject + KV bucket above will be re-scoped under
device-specific NATS credentials; the topology above doesn't need
to change for that to slot in.

2
ROADMAP/iot_platform/context_conversation.md → ROADMAP/fleet_platform/context_conversation.md
View File

@@ -183,7 +183,7 @@ Drawing these out as they're load-bearing for judgment calls:
8. **The partner relationship is strategic.** Tuesday demo conversation is half the Tuesday deliverable. Framing the v0.1/v0.2/v0.3 roadmap to them matters as much as the running code.
9. **End-customer debuggability is a UX constraint.** Mechanical/electrical/chemical engineers will touch these devices. `systemctl status iot-agent` must tell them what's happening. `journalctl -u iot-agent` must be parseable by humans. Error messages must be understandable without Kubernetes knowledge.
9. **End-customer debuggability is a UX constraint.** Mechanical/electrical/chemical engineers will touch these devices. `systemctl status fleet-agent` must tell them what's happening. `journalctl -u fleet-agent` must be parseable by humans. Error messages must be understandable without Kubernetes knowledge.
10. **NATS is the long-term architectural commitment.** Everything on NATS — not as a queue, as a coordination fabric. The "decentralized cluster management" future depends on this choice. Implementation decisions that weaken this (e.g., "let's just put a database in the middle") should be pushed back on.

381
ROADMAP/fleet_platform/v0_1_plan.md Normal file
View File

@@ -0,0 +1,381 @@
# IoT Platform v0.1 and beyond — forward plan
Authoritative forward plan for the NationTech decentralized-infra /
IoT platform, written after the v0 walking skeleton shipped
(see `v0_walking_skeleton.md` for the historical diary). Organized as
five chapters in execution order.
## State of the world (as of 2026-04-23)
**Green, end-to-end:**
- CRD → operator → NATS JetStream KV write path (`smoke-a1.sh`).
- Agent watches KV, reconciles podman containers (`smoke-a1.sh`).
- VM-as-device provisioning: cloud-init + fleet-agent install + NATS
smoke (`smoke-a3.sh`), x86_64 (native KVM) and aarch64 (TCG).
- Power-cycle / reboot resilience (`smoke-a3.sh` phase 5).
- aarch64 cross-compile of the agent (no Harmony modules need to
feature-gate aarch64).
- Operator installed via a harmony Score (typed Rust, no yaml).
- `harmony-reconciler-contracts` crate — cross-boundary types
(bucket names, key helpers, `DeviceInfo`, `DeploymentState`,
`HeartbeatPayload`, `DeploymentName`, `Id` re-export).
**Chapter 1 shipped** (2026-04-21): composed end-to-end demo
(`smoke-a4.sh`) — operator in k3d + in-cluster NATS + ARM VM +
typed-Rust CR applier + hand-off menu + `--auto` regression. Green
on x86_64 (native KVM) and aarch64 (TCG).
**Chapter 2 shipped** (2026-04-23): selector-based targeting +
Device CRD + `.status.aggregate` reflect-back. `Deployment.spec.
targetSelector: LabelSelector` resolves against cluster-scoped
`Device` CRs materialized from NATS `device-info`. Operator writes
`desired-state` KV per matched pair, patches
`.status.aggregate` (matchedDeviceCount / succeeded / failed /
pending / lastError) at 1 Hz. Load-tested to 10 000 devices ×
1 000 Deployments at 10 000 KV writes/s sustained, zero errors.
**Not yet wired (real v0.1 work still to go):**
- Helm packaging of the operator (Chapter 3).
- Zitadel + OpenBao auth (per-device credentials, SSO for
operator users). Placeholder `CredentialSource` trait on the
agent side (Chapter 4).
- Any frontend (Chapter 5).
- Small quality items (not blockers): agent config-driven labels,
`matchExpressions` in selectors, `Device.status.conditions`
populated from heartbeat staleness.
**Verified during planning** (so future implementation doesn't
have to re-litigate):
- **Upgrade already works.** `reconciler.rs::apply` byte-compares
serialized score payloads; drift triggers re-reconcile.
`PodmanTopology::ensure_service_running` removes then re-creates
containers on spec drift. No "stale + new" window.
- **The polymorphism stays.** `ReconcileScore` is an externally-tagged
enum; adding `OkdApplyV0` later is additive.
**Surprises since v0 started** (for context, none architectural):
- Arch `edk2-aarch64-202602-2` shipped empty firmware blobs;
`202508-1` ships unpadded edk2 that needs 64 MiB pflash padding.
Fixed via runtime discovery + padding in `modules/kvm/firmware.rs`.
- MTTCG isn't default for cross-arch TCG on QEMU 10.2; force via
`qemu:commandline` override. `pauth-impdef=on` likewise a
qemu:commandline opt-in.
- `ensure_vm` is idempotent on "domain exists" — re-apply of a
changed XML requires manual `undefine --nvram --remove-all-storage`.
Noted as a follow-up in the code comments.
---
## Chapter 1 — Hands-on end-to-end demo (imminent)
**Goal:** the user runs one command, watches operator + NATS + ARM
VM come up, then drives a CRD through the full loop by hand:
`kubectl apply` it (manually or via a typed Rust applier), watch the
operator log "acquired," check the NATS KV store with `natsbox`,
SSH/console into the VM, `curl` the running nginx container from
the workstation.
### User-facing requirements (explicit)
- **No yaml fixtures.** Sample `Deployment` CRs constructed in
typed Rust using `DeploymentSpec` + `PodmanV0Score`. Same
discipline as the `install` Score that replaced `gen-crd | kubectl
apply`.
- **ArgoCD deferred.** User's production clusters have it; bringing
it into the smoke harness adds setup overhead without validating
anything `helm install` doesn't. Chapter 3 produces the chart;
ArgoCD integration is a later operational concern.
- **Operator logs every CR it acquires** — `controller.rs` already
does `tracing::info!(%ns, %name, "reconcile")`; verify the output
reads well in the command-menu hand-off.
- **natsbox debugging is first-class.** Script prints exact
natsbox one-liners at hand-off so the user can inspect KV state.
- **In-cluster NATS.** Not a side-by-side podman container (as
smoke-a1 does today). Expose to the libvirt VM via k3d
loadbalancer port mapping.
### Design decisions
- **Rust CR applier.** New binary `examples/harmony_apply_deployment/`.
CLI flags `--name --namespace --target-device --image --port
--delete`. Constructs the `Deployment` CR via
`kube::Api<Deployment>` + typed `DeploymentSpec`; calls
`api.apply(...)`. Can also `--print` the CR JSON to stdout so
`kubectl apply -f -` still works from the terminal.
- **smoke-a4.sh orchestration stays bash for now.** User agreed
this is test-harness scope, not framework path; converting it
to Rust is "not as important right now."
- **Hand-off is the default mode**, not `--keep`. The whole point
of Chapter 1 is that the user drives the last stage interactively.
`smoke-a4.sh` brings everything up, applies *nothing*, prints
the command menu, waits on `INT/TERM` to tear down. `--auto`
runs the full apply/curl/upgrade/delete regression for CI.
- **In-cluster NATS path.** Preferred: use `harmony::modules::nats`
if it has a lightweight single-node / no-supercluster mode.
Fallback: typed `K8sResourceScore` applying a minimal Deployment
+ NodePort Service. 15-min research task before committing.
### Composed smoke phases (`smoke-a4.sh`)
1. k3d cluster up with `-p "4222:4222@loadbalancer"` so the host
port 4222 forwards into the cluster. Reachable from the
libvirt VM via the gateway IP (typically `192.168.122.1:4222`).
2. NATS in-cluster via the chosen path (harmony module or direct
K8sResourceScore). Wait for readiness.
3. Install CRD via the operator's `install` subcommand (typed Rust).
4. Spawn operator as a host-side process (same pattern as
smoke-a1). Operator connects to `nats://localhost:4222`.
5. Provision ARM VM via `example_iot_vm_setup` (same entry point
smoke-a3 uses). Agent configured to connect to
`nats://<libvirt_gateway>:4222` — discover the gateway IP via
`virsh net-dumpxml default`, as smoke-a3 already does.
6. Sanity: `kubectl wait ... crd Established`, operator logged
"KV bucket ready", agent logged "watching KV keys",
`status.<device>` present in `agent-status` bucket.
7. Hand off. Print the command menu below. Exit 0 with a cleanup
trap on `INT/TERM`.
### Command menu at hand-off
- `kubectl get deployments.fleet.nationtech.io -A -w` — watch CR
reconcile reactively.
- `cargo run -q -p example_harmony_apply_deployment -- --image
nginx:latest --target-device $TARGET_DEVICE` — apply an nginx
deployment via typed Rust.
- `cargo run -q -p example_harmony_apply_deployment -- --print
--image nginx:latest --target-device $TARGET_DEVICE |
kubectl apply -f -` — same thing, through kubectl.
- `ssh -i $SSH_KEY fleet-admin@$VM_IP` — connect to the VM.
- `virsh console $VM_NAME --force` — serial console alternative.
- `podman --url unix://$VM_IP:... ps` or ssh + `podman ps`
— list containers on the VM from the workstation.
- `podman run --rm docker.io/natsio/nats-box nats --server
nats://localhost:4222 kv ls desired-state` — list desired
state keys (from the host).
- `podman run --rm ... nats kv get desired-state
'<device>.<deployment>' --raw` — dump a specific desired state.
- `podman run --rm ... nats kv get agent-status
'status.<device>' --raw` — dump the heartbeat.
- `curl http://$VM_IP:8080/` — hit the deployed nginx.
### `--auto` path (for regression)
1. Apply `nginx:latest`, wait for container on VM, `curl` 200.
2. Apply `nginx:1.26` (upgrade), wait for container *id* to change,
`curl` 200 against the new container.
3. Apply `--delete`, wait for container gone from VM.
### Files
- **NEW** `examples/harmony_apply_deployment/Cargo.toml` +
`src/main.rs` — typed applier.
- **NEW** `fleet/scripts/smoke-a4.sh`.
- **NO yaml fixtures.** Rust CLI flags cover the shape.
- Optional: factor shared smoke phases (NATS up, k3d up, operator
spawn, VM provision) into `fleet/scripts/lib/` if the duplication
across a1/a3/a4 becomes obvious. Don't force it.
### NATS exposure — implementation-time notes
- k3d `@loadbalancer` port mapping binds the host's `0.0.0.0:4222`
by default; libvirt VMs on `virbr0` can reach it via the gateway
IP. No special NAT config required.
- Fallback if environmental snag: keep the side-by-side podman
container on an opt-in `NATS_MODE=podman` flag. Don't default
to that — user explicitly asked for in-cluster.
### Verification
- Fresh host: `ARCH=aarch64 ./fleet/scripts/smoke-a4.sh` completes
in 8-15 min, prints the command menu.
- `ARCH=aarch64 ./fleet/scripts/smoke-a4.sh --auto` PASSes
end-to-end including upgrade id-change assertion.
- x86_64 (`ARCH=x86-64`) completes in 2-5 min.
### Explicitly out of scope
- `AgentStatus` / `DeploymentStatus` enrichment — Chapter 2.
- Helm chart, ArgoCD, auth, frontend — later chapters.
- Lifting the applier into a reusable `ApplyDeploymentScore` —
only if a second consumer appears.
---
## Chapter 2 — Status reflect-back + selector-based targeting **[SHIPPED 2026-04-23]**
**Goal:** CRD `.status` reflects fleet reality — per-deployment
success/failure/pending counts, last-error surface, freshness. The
Deployment CR targets devices by label selector, not by id list.
> The shipped design replaces the original `AgentStatus` + list-of-ids
> proposal wholesale. See `chapter_4_aggregation_scale.md` for the
> superseded design-doc archaeology. Commits:
> `refactor(iot): delete legacy AgentStatus path`,
> `refactor(iot): operator watches device-state KV directly; drop event stream`,
> `refactor(iot): Deployment.targetSelector + Device CRD (DaemonSet-like)`.
### What shipped
**Wire format** (in `harmony-reconciler-contracts`): four per-concern
payloads on dedicated NATS KV buckets. No monolithic per-device blob,
no separate event stream.
| Type | Bucket | Cadence |
|------|--------|---------|
| `DeviceInfo` | `device-info` | on startup + label/inventory change |
| `DeploymentState` | `device-state` | on reconcile phase transition |
| `HeartbeatPayload` | `device-heartbeat` | every 30 s |
**CRDs.** Two cluster resources:
- `Deployment` (namespaced) — `spec.targetSelector: LabelSelector`
(standard K8s `matchLabels` / `matchExpressions`). No device list
on spec. `.status.aggregate` carries `matchedDeviceCount`,
`succeeded`, `failed`, `pending`, `lastError`.
- `Device` (cluster-scoped, like `Node`) — `metadata.labels` carries
the device's routing labels; `spec.inventory` holds the hardware/OS
snapshot; `status.conditions` is reserved for liveness (populated
lazily by a future heartbeat-freshness reconciler, not every ping).
**Operator tasks** (three concurrent loops in one process):
1. `controller` — validates Deployment CR names, holds the finalizer
that cleans `desired-state.<device>.<deployment>` KV entries on
delete. No writes on apply (aggregator handles that).
2. `device_reconciler` — watches the `device-info` KV; server-side-
applies a `Device` CR per `DeviceInfo` payload, with label
sanitization. Agents remain kube-unaware.
3. `fleet_aggregator` — three caches driven by watches (Deployment
CRs, Device CRs, `device-state` KV). On any change, resolves
each selector against the Device cache, writes/deletes
`desired-state` KV entries for diffed matches, and patches
`.status.aggregate` at 1 Hz for the CRs whose counters moved.
**Agents** publish `device-id=<id>` as a default DeviceInfo label, so
targeting a single device with `matchLabels: {device-id: pi-42}` is
zero-config. User-defined labels layer on from agent config (scoped
out of this chapter; follow-up item).
### Scale proof
`fleet/scripts/load-test.sh` + `examples/fleet_load_test` simulate N
devices across M Deployments, driving `device-state` KV updates at a
configurable cadence while the full operator stack runs against a
local k3d apiserver. Verified:
- 100 devices / 10 groups / 1 Hz / 60 s — 100 writes/s sustained,
all 10 CR aggregates converge.
- 10 000 devices / 1 000 groups / 1 Hz / 120 s — ~10 000 writes/s
sustained, 0 errors, all 1 000 CR aggregates correct
(`matchedDeviceCount == expected`, `succeeded + failed + pending
== matched`). Same envelope before and after the selector rewrite.
### Out of scope in this chapter (follow-ups)
- Agent config-driven labels (`[labels]` in agent toml → DeviceInfo).
~30 lines; deferred until a concrete need lands.
- `matchExpressions` evaluator. Operator currently supports
`matchLabels` only and logs a warning for expression-bearing
selectors. ~50 lines; deferred.
- `Device.status.conditions` populated from heartbeat staleness
(Reachable / Stale transitions). Liveness is computable today by
reading `device-heartbeat` directly; CR-side reflection is a
convenience. ~100 lines; deferred.
- Full journald log streaming. The `.status.aggregate.lastError`
surface covers the user's reflect-back requirement for now.
- Multi-device regression smoke — defer until real hardware or a
second VM is around.
---
## Chapter 3 — Helm chart (ArgoCD deferred)
**Goal:** operator ships as a versioned helm chart with CRD
version-locked inside.
User clarified this session: ArgoCD exists in production; all it
does is apply resources from the chart. Standing up ArgoCD in the
smoke adds setup overhead with no incremental validation value.
Chapter 3 produces the chart + validates `helm install / helm
upgrade` lifecycles. ArgoCD consumption is a user operational
concern downstream.
### Sketch
- Chart location: `fleet/harmony-fleet-operator/chart/` (or sibling repo —
defer decision to implementation time).
- Templates: Namespace, SA, ClusterRole, ClusterRoleBinding,
Deployment (operator pod), CRD.
- **CRD yaml in the chart is generated at chart-publish time** from
the Rust `Deployment::crd()`. One-off release artifact, not
framework path — consistent with "no yaml in framework code."
- Values: operator image tag, NATS URL, log level.
- Smoke: `helm install` into k3d → CR apply → same assertions as
Chapter 1.
### Open questions
- Chart repo: subdir vs. separate git repo.
- CRD install mechanism: chart hook vs. templates directory.
Drives CRD upgrade story.
---
## Chapter 4 — Auth: Zitadel + OpenBao + per-device identity
**Goal:** per-device granular NATS credentials; SSO for operator
users; OpenBao policy per device; JWT bootstrap from Zitadel.
Zitadel + OpenBao are already ~99% integrated in harmony; this
chapter is wiring the IoT-specific flows.
### Sketch
- Agent's `CredentialSource` trait (already abstract in agent
`config.rs`) gets a Zitadel-JWT-backed implementation. Mints
short-lived NATS creds via OpenBao auth callout.
- Remove the shared-credentials `toml-shared` variant (v0 demo
leftover).
- Availability: auth-callout caches policies, tolerates OpenBao
outages.
- SSO for operator users (separate flow): Zitadel groups →
Kubernetes RBAC subjects on the `Deployment` CRD.
---
## Chapter 5 — Frontend (last)
**Goal:** operator-friendly UI for the decentralized platform.
Form factor undecided: Leptos web dashboard, CLI extension to
`harmony_cli`, or a TUI. Minimum viable product: read-only view of
fleet state (devices + deployments + aggregated status) powered by
the CRD `.status` from Chapter 2. Aspiration: write operations with
auth from Chapter 4.
---
## Principles — what we've learned and want to keep doing
- **No yaml in framework code paths.** Every kube-rs type is
typed; every Score apply goes through typed Rust. Yaml generation
happens only at chart-publish time, never at runtime.
- **Scores describe desired state; topologies expose capabilities.**
Prefer adding capability traits over thickening a single topology.
- **Minimal topologies for ad-hoc Score execution.** `K8sAnywhereTopology`
has too many opinions (cert-manager install, tenant-manager bootstrap,
helm probes) for narrow apply-a-CRD use cases. See ROADMAP
§12.6 — a lean shared `K8sBareTopology` is the durable fix.
- **Cross-boundary wire types in `harmony-reconciler-contracts`**,
everything else in its natural crate.
- **Never ship untested code.** Every commit that changes runtime
behavior is verified against a smoke script before landing.
Cargo check + unit tests aren't enough.
- **Prove claims about upstream before blaming upstream.** The
Arch edk2 investigation showed this matters; see
`memory/feedback_prove_before_blaming_upstream.md`.

104
ROADMAP/iot_platform/v0_walking_skeleton.md → ROADMAP/fleet_platform/v0_walking_skeleton.md
View File

@@ -1,5 +1,23 @@
# IoT Platform v0 — Walking Skeleton
> **Status: SHIPPED (2026-04-21)**
>
> This document is the historical design diary for the v0 walking skeleton
> work — it captures the decision trail, hour-by-hour plan, and risk
> analysis as they were written before the skeleton was built. It is
> preserved unchanged as an archaeology reference.
>
> The walking skeleton shipped end-to-end: CRD → operator → NATS KV →
> on-device agent → podman reconcile; VM-as-device flow (x86_64 + aarch64
> via TCG); power-cycle resilience; operator installed as a Score rather
> than kubectl-apply-a-yaml. See smoke-a1, smoke-a3, smoke-a3-arm for the
> executable proof.
>
> **Forward plan lives in `ROADMAP/fleet_platform/v0_1_plan.md`** — five
> chapters covering hands-on demo, status reflect-back, helm chart, SSO/
> secrets, and frontend. When a chapter grows scope it may move into its
> own `chapter_N_*.md`.
**Approach:** Walking skeleton (Cockburn). Thin end-to-end thread through every architectural component. Naive first, architecture emerges from running code, hardening follows real-world feedback.
## 1. Strategic framing
@@ -116,11 +134,11 @@ iot-workload-hello/
`deployment.yaml`:
```yaml
apiVersion: iot.nationtech.io/v1alpha1
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: hello-world
namespace: iot-demo
namespace: fleet-demo
spec:
targetDevices:
- pi-demo-01
@@ -138,10 +156,10 @@ spec:
### 5.2 Central cluster setup
Existing k8s cluster. Namespaces:
- `iot-system` — operator, NATS (single-node for v0)
- `iot-demo` — `Deployment` CRs
- `fleet-system` — operator, NATS (single-node for v0)
- `fleet-demo` — `Deployment` CRs
ArgoCD application pre-configured to sync `iot-workload-hello` repo into `iot-demo` namespace.
ArgoCD application pre-configured to sync `iot-workload-hello` repo into `fleet-demo` namespace.
### 5.3 Raspberry Pi 5 setup
@@ -151,9 +169,9 @@ Base OS: **Ubuntu Server 24.04 LTS ARM64** (ships Podman 4.9 in repos). Raspberr
Installed:
- `podman` (4.4+, ARM64) with `systemctl --user enable --now podman.socket` (required for `podman-api` crate)
- `iot-agent` binary (cross-compiled to aarch64 via existing Harmony aarch64 toolchain)
- `/etc/iot-agent/config.toml` with NATS URL + shared credential
- systemd unit `iot-agent.service`
- `fleet-agent` binary (cross-compiled to aarch64 via existing Harmony aarch64 toolchain)
- `/etc/fleet-agent/config.toml` with NATS URL + shared credential
- systemd unit `fleet-agent.service`
### 5.4 What the code does
@@ -227,7 +245,7 @@ trait CredentialSource: Send + Sync {
}
```
v0: `TomlFileCredentialSource` reading `/etc/iot-agent/config.toml`.
v0: `TomlFileCredentialSource` reading `/etc/fleet-agent/config.toml`.
v0.2: `ZitadelBootstrappedCredentialSource` — same trait, swapped via config.
30 minutes Friday. Saves 3 hours of refactor in v0.2.
@@ -258,7 +276,7 @@ device_id = "pi-demo-01"
[credentials]
type = "toml-shared"
nats_user = "iot-agent"
nats_user = "fleet-agent"
nats_pass = "dev-shared-password"
[nats]
@@ -306,9 +324,9 @@ Document findings in the Friday night log regardless of outcome. v0.1 work inclu
- Write 1-page `v0-demo.md`: demo script, success criteria, fallback plan.
- Decide Pi OS: Ubuntu 24.04 ARM64 (default) vs Raspberry Pi OS 64-bit. Don't agonize beyond 10 min.
*Dispatch agent A1 (operator):* "Create Rust crate `iot/iot-operator-v0/` using `kube-rs` implementing a Deployment CRD controller that writes to NATS KV. Exact spec in task card §9.A1. Self-verify: `kubectl apply` → `nats kv get` shows entry. Under 300 lines main.rs. No auth."
*Dispatch agent A1 (operator):* "Create Rust crate `fleet/harmony-fleet-operator/` using `kube-rs` implementing a Deployment CRD controller that writes to NATS KV. Exact spec in task card §9.A1. Self-verify: `kubectl apply` → `nats kv get` shows entry. Under 300 lines main.rs. No auth."
*Dispatch agent A2 (Pi provisioning, fallback-aware):* "Attempt Harmony-based Raspberry Pi 5 provisioning Score. Target: fresh Pi flashed via SD card, boots, static IP, Ubuntu 24.04 ARM64 with Podman 4.9, podman user socket enabled, user `iot-agent` with linger enabled, `/etc/iot-agent/` ready. If Harmony doesn't have Pi primitives, document the gap and produce a manual provisioning runbook instead (rpi-imager + cloud-init). Hard time limit: 90 min. Self-verify: `ssh iot-agent@<pi-ip> 'podman --version'` returns 4.4+."
*Dispatch agent A2 (Pi provisioning, fallback-aware):* "Attempt Harmony-based Raspberry Pi 5 provisioning Score. Target: fresh Pi flashed via SD card, boots, static IP, Ubuntu 24.04 ARM64 with Podman 4.9, podman user socket enabled, user `fleet-agent` with linger enabled, `/etc/fleet-agent/` ready. If Harmony doesn't have Pi primitives, document the gap and produce a manual provisioning runbook instead (rpi-imager + cloud-init). Hard time limit: 90 min. Self-verify: `ssh fleet-agent@<pi-ip> 'podman --version'` returns 4.4+."
**Hour 2 — your work: agent crate**
@@ -324,8 +342,8 @@ Crate in `harmony/src/modules/iot_agent/` or a new binary in the Harmony workspa
**Hour 3 — local integration**
- Review agent A1's operator. Deploy to central cluster `iot-system` namespace.
- Deploy NATS to `iot-system` if not already (single-node JetStream).
- Review agent A1's operator. Deploy to central cluster `fleet-system` namespace.
- Deploy NATS to `fleet-system` if not already (single-node JetStream).
- Review agent A2's Pi provisioning. If Harmony Score succeeded, note for demo; if manual runbook, accept and move on.
- Agent compiles on laptop. Connects to central NATS.
@@ -380,7 +398,7 @@ Named subsection: the most important class of failures for Pi-in-field deploymen
**Hour 3-4 — demo polish:**
- `./demo.sh` is one command, no manual steps.
- Output is clean: clear PASS/FAIL with per-phase timings.
- `kubectl get deployments.iot.nationtech.io` output is readable.
- `kubectl get deployments.fleet.nationtech.io` output is readable.
**Hour 5-6 — partner-facing polish:**
- README in workload repo: 4 lines. "Edit this, git push, done."
@@ -421,8 +439,8 @@ Each card is self-contained. Hand the entire card to an agent.
# Note: harmony is built with --no-default-features to exclude KVM (libvirt cannot cross-compile to aarch64).
# The 5 KVM examples (kvm_vm_examples, kvm_okd_ha_cluster, opnsense_vm_integration,
# opnsense_pair_integration, example_linux_vm) are x86_64-only by design.
cargo build --target x86_64-unknown-linux-gnu -p harmony -p harmony_agent -p iot-agent-v0 -p iot-operator-v0
cargo build --target aarch64-unknown-linux-gnu -p harmony --no-default-features -p harmony_agent -p iot-agent-v0 -p iot-operator-v0
cargo build --target x86_64-unknown-linux-gnu -p harmony -p harmony_agent -p fleet-agent-v0 -p harmony-fleet-operator
cargo build --target aarch64-unknown-linux-gnu -p harmony --no-default-features -p harmony_agent -p fleet-agent-v0 -p harmony-fleet-operator
```
All three must exit 0. Note: `cargo test --target aarch64-unknown-linux-gnu` cannot run on x86_64 (exec format error) — that's expected. Test execution is only for the host architecture via `./build/check.sh`. If any check fails, fix the issue before marking the task complete. Include the output in the PR description.
@@ -431,11 +449,11 @@ All three must exit 0. Note: `cargo test --target aarch64-unknown-linux-gnu` can
**Goal:** `kube-rs` operator that watches `Deployment` CRs and writes the Score to NATS KV.
**Deliverable:** Crate `iot/iot-operator-v0/`:
**Deliverable:** Crate `fleet/harmony-fleet-operator/`:
- `Cargo.toml`: `kube`, `k8s-openapi`, `async-nats`, `serde`, `serde_yaml`, `serde_json`, `tokio`, `tracing`, `tracing-subscriber`, `anyhow`.
- `src/main.rs` under 300 lines.
- `deploy/operator.yaml` — Deployment, ServiceAccount, ClusterRole, ClusterRoleBinding.
- `deploy/crd.yaml` — `Deployment` CRD for `iot.nationtech.io/v1alpha1`.
- `deploy/crd.yaml` — `Deployment` CRD for `fleet.nationtech.io/v1alpha1`.
**Behavior:**
1. Connect to NATS on startup (`NATS_URL` env, no auth).
@@ -462,7 +480,7 @@ status:
**Self-verification:**
```bash
cd iot/iot-operator-v0
cd fleet/harmony-fleet-operator
cargo build && cargo clippy -- -D warnings
# Test against k3d:
@@ -474,7 +492,7 @@ OP_PID=$!
sleep 3
kubectl apply -f - <<EOF
apiVersion: iot.nationtech.io/v1alpha1
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: test-deploy
@@ -496,7 +514,7 @@ sleep 5
nats --server nats://localhost:4222 kv get desired-state test-device-01.test-deploy
# Must print the Score JSON with type="PodmanV0"
kubectl get deployment.iot.nationtech.io test-deploy -o jsonpath='{.status.observedScoreString}'
kubectl get deployment.fleet.nationtech.io test-deploy -o jsonpath='{.status.observedScoreString}'
# Must print the stored string
kill $OP_PID
@@ -505,7 +523,7 @@ docker stop nats
```
**Forbidden:**
- Code outside `iot/iot-operator-v0/`.
- Code outside `fleet/harmony-fleet-operator/`.
- Zitadel, OpenBao, auth callout dependencies.
- Parsing `score.data`.
- Rollout logic beyond KV writes.
@@ -524,19 +542,19 @@ docker stop nats
- Ubuntu Server 24.04 LTS ARM64 (or Raspberry Pi OS 64-bit if Ubuntu fails).
- Static IP on lab network.
- Packages: `podman`, `systemd-container`, `openssh-server`, `curl`, `jq`.
- `systemctl --user enable --now podman.socket` for user `iot-agent`.
- User `iot-agent` with linger enabled (`loginctl enable-linger iot-agent`).
- `/etc/iot-agent/` (owned by iot-agent, 0750).
- `/var/lib/iot-agent/`.
- `systemctl --user enable --now podman.socket` for user `fleet-agent`.
- User `fleet-agent` with linger enabled (`loginctl enable-linger fleet-agent`).
- `/etc/fleet-agent/` (owned by fleet-agent, 0750).
- `/var/lib/fleet-agent/`.
**Self-verification:**
```bash
ssh iot-agent@<pi-ip> 'podman --version'
ssh fleet-agent@<pi-ip> 'podman --version'
# Must be 4.4+ (target 4.9+)
ssh iot-agent@<pi-ip> 'systemctl --user is-active podman.socket'
ssh fleet-agent@<pi-ip> 'systemctl --user is-active podman.socket'
# Must print "active"
ssh iot-agent@<pi-ip> 'loginctl show-user iot-agent | grep Linger=yes'
ssh iot-agent@<pi-ip> 'uname -m'
ssh fleet-agent@<pi-ip> 'loginctl show-user fleet-agent | grep Linger=yes'
ssh fleet-agent@<pi-ip> 'uname -m'
# Must print aarch64
```
@@ -550,13 +568,13 @@ ssh iot-agent@<pi-ip> 'uname -m'
**Prerequisites:** Agent binary exists (Sylvain writes Friday).
**Deliverable:** `iot/iot-agent-v0/scripts/install.sh`:
**Deliverable:** `iot/fleet-agent-v0/scripts/install.sh`:
1. Args: `--host <ip>`, `--device-id <id>`, `--nats-url <url>`, `--nats-user <u>`, `--nats-pass <p>`.
2. Cross-builds for aarch64 using existing Harmony aarch64 toolchain.
3. `scp` binary to Pi, `sudo mv` to `/usr/local/bin/iot-agent`.
4. Templates `/etc/iot-agent/config.toml` from args.
5. Installs `/etc/systemd/system/iot-agent.service`.
6. `systemctl daemon-reload && systemctl enable --now iot-agent`.
3. `scp` binary to Pi, `sudo mv` to `/usr/local/bin/fleet-agent`.
4. Templates `/etc/fleet-agent/config.toml` from args.
5. Installs `/etc/systemd/system/fleet-agent.service`.
6. `systemctl daemon-reload && systemctl enable --now fleet-agent`.
7. Waits up to 15s for "connected to NATS" in journal.
**systemd unit:**
@@ -568,8 +586,8 @@ Wants=network-online.target
[Service]
Type=simple
User=iot-agent
ExecStart=/usr/local/bin/iot-agent
User=fleet-agent
ExecStart=/usr/local/bin/fleet-agent
Restart=on-failure
RestartSec=5
StandardOutput=journal
@@ -584,9 +602,9 @@ WantedBy=multi-user.target
```bash
./install.sh --host <pi-ip> --device-id pi-demo-01 \
--nats-url nats://central:4222 \
--nats-user iot-agent --nats-pass dev-shared-password
ssh iot-agent@<pi-ip> 'sudo systemctl status iot-agent' # active (running)
ssh iot-agent@<pi-ip> 'sudo journalctl -u iot-agent --since "2 minutes ago"' | grep "connected to NATS"
--nats-user fleet-agent --nats-pass dev-shared-password
ssh fleet-agent@<pi-ip> 'sudo systemctl status fleet-agent' # active (running)
ssh fleet-agent@<pi-ip> 'sudo journalctl -u fleet-agent --since "2 minutes ago"' | grep "connected to NATS"
```
**Time limit:** 2 hours agent time.
@@ -595,7 +613,7 @@ ssh iot-agent@<pi-ip> 'sudo journalctl -u iot-agent --since "2 minutes ago"' | g
**Goal:** One command runs full demo flow.
**Deliverable:** `iot/scripts/demo.sh`:
**Deliverable:** `fleet/scripts/demo.sh`:
1. Verifies Pi reachable + agent running.
2. Applies `scripts/demo-deployment.yaml`.
3. Waits up to 120s for container on Pi (ssh + `podman ps`).
@@ -606,7 +624,7 @@ ssh iot-agent@<pi-ip> 'sudo journalctl -u iot-agent --since "2 minutes ago"' | g
**Self-verification:**
```bash
./iot/scripts/demo.sh
./fleet/scripts/demo.sh
# Ends with "PASS", total < 5 min
```

24
examples/fleet_load_test/Cargo.toml Normal file
View File

@@ -0,0 +1,24 @@
[package]
name = "example_fleet_load_test"
version.workspace = true
edition = "2024"
license.workspace = true
[[bin]]
name = "fleet_load_test"
path = "src/main.rs"
[dependencies]
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
async-nats = { workspace = true }
chrono = { workspace = true }
kube = { workspace = true, features = ["runtime", "derive"] }
k8s-openapi.workspace = true
serde_json = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true }
rand = { workspace = true }

552
examples/fleet_load_test/src/main.rs Normal file
View File

@@ -0,0 +1,552 @@
//! Load test for the IoT operator's `fleet_aggregator`.
//!
//! Simulates N devices across M Deployment CRs, each device pushing
//! a `DeploymentState` update to NATS every `--tick-ms`. Measures
//! throughput on both sides (devices → NATS and operator → kube
//! apiserver) and, at the end of the run, verifies each CR's
//! `.status.aggregate` counters sum to its expected group size (and
//! that `matched_device_count` equals that size — i.e. every
//! registered device got picked up by the CR's label selector).
//!
//! Assumes an already-running stack:
//! - NATS reachable at `--nats-url`
//! - k8s cluster with the operator's CRD installed (KUBECONFIG)
//! - the operator process running against the same NATS + cluster
//!
//! The `fleet/scripts/smoke-a4.sh` script brings all three up — pass
//! `--hold` to leave them running, then run this binary.
//!
//! Typical invocation:
//!
//! cargo run -q -p example_fleet_load_test -- \
//! --namespace fleet-load \
//! --groups 55,5,5,5,5,5,5,5,5,5 \
//! --tick-ms 1000 \
//! --duration-s 60
use anyhow::{Context, Result};
use async_nats::jetstream::{self, kv};
use chrono::Utc;
use clap::Parser;
use harmony_fleet_operator::crd::{
Deployment, DeploymentSpec, Rollout, RolloutStrategy, ScorePayload,
};
use harmony_reconciler_contracts::{
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
DeploymentState, DeviceInfo, HeartbeatPayload, Id, Phase, device_heartbeat_key,
device_info_key, device_state_key,
};
use k8s_openapi::api::core::v1::Namespace;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams, PostParams};
use rand::Rng;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
#[derive(Parser, Debug, Clone)]
#[command(
name = "fleet_load_test",
about = "Synthetic load for the IoT operator's fleet_aggregator"
)]
struct Cli {
/// NATS URL (same one the operator connects to).
#[arg(long, default_value = "nats://localhost:4222")]
nats_url: String,
/// k8s namespace for the load-test Deployment CRs. Created if
/// missing.
#[arg(long, default_value = "fleet-load")]
namespace: String,
/// Group shape — comma-separated device counts, one per CR.
/// Default: 100 devices over 10 groups (1 × 55 + 9 × 5).
#[arg(long, default_value = "55,5,5,5,5,5,5,5,5,5")]
groups: String,
/// Per-device tick in ms. Each tick publishes one DeploymentState.
#[arg(long, default_value_t = 1000)]
tick_ms: u64,
/// Heartbeat cadence in seconds (separate from the state tick).
#[arg(long, default_value_t = 30)]
heartbeat_s: u64,
/// Total run duration in seconds before tearing down.
#[arg(long, default_value_t = 60)]
duration_s: u64,
/// Report throughput every N seconds.
#[arg(long, default_value_t = 5)]
report_s: u64,
/// Keep the CRs + KV entries in place after the run instead of
/// deleting them. Useful with HOLD=1 to inspect the steady-state
/// aggregate after the load finishes.
#[arg(long)]
keep: bool,
}
/// Metrics collected across all device tasks.
#[derive(Default)]
struct Counters {
state_writes: AtomicU64,
heartbeat_writes: AtomicU64,
errors: AtomicU64,
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();
let cli = Cli::parse();
let group_sizes = parse_groups(&cli.groups)?;
let total: usize = group_sizes.iter().sum();
tracing::info!(
devices = total,
groups = group_sizes.len(),
shape = ?group_sizes,
tick_ms = cli.tick_ms,
duration_s = cli.duration_s,
"fleet_load_test starting"
);
// --- NATS setup ----------------------------------------------------------
let nc = async_nats::connect(&cli.nats_url)
.await
.with_context(|| format!("connecting to NATS at {}", cli.nats_url))?;
let js = jetstream::new(nc);
let info_bucket = open_bucket(&js, BUCKET_DEVICE_INFO).await?;
let state_bucket = open_bucket(&js, BUCKET_DEVICE_STATE).await?;
let heartbeat_bucket = open_bucket(&js, BUCKET_DEVICE_HEARTBEAT).await?;
// --- kube setup ----------------------------------------------------------
let client = Client::try_default().await.context("kube client")?;
ensure_namespace(&client, &cli.namespace).await?;
let deployments: Api<Deployment> = Api::namespaced(client.clone(), &cli.namespace);
// --- plan groups + device ids --------------------------------------------
let plan = build_plan(&group_sizes);
apply_crs(&deployments, &plan).await?;
publish_device_infos(&info_bucket, &plan).await?;
// --- spawn simulators ----------------------------------------------------
let counters = Arc::new(Counters::default());
let mut sims = JoinSet::new();
let tick = Duration::from_millis(cli.tick_ms);
let hb_tick = Duration::from_secs(cli.heartbeat_s);
for device in &plan.devices {
let device = Arc::new(device.clone());
sims.spawn(simulate_state_loop(
device.clone(),
state_bucket.clone(),
counters.clone(),
tick,
));
sims.spawn(simulate_heartbeat_loop(
device.clone(),
heartbeat_bucket.clone(),
counters.clone(),
hb_tick,
));
}
// --- metrics reporter ----------------------------------------------------
let report_tick = Duration::from_secs(cli.report_s);
let reporter_counters = counters.clone();
let reporter = tokio::spawn(async move {
let mut ticker = tokio::time::interval(report_tick);
ticker.tick().await; // skip immediate fire
let mut prev_state = 0u64;
let mut prev_hb = 0u64;
loop {
ticker.tick().await;
let s = reporter_counters.state_writes.load(Ordering::Relaxed);
let h = reporter_counters.heartbeat_writes.load(Ordering::Relaxed);
let e = reporter_counters.errors.load(Ordering::Relaxed);
let dt = report_tick.as_secs_f64();
let ss = (s - prev_state) as f64 / dt;
let hh = (h - prev_hb) as f64 / dt;
tracing::info!(
state_writes_total = s,
state_writes_per_s = format!("{ss:.1}"),
heartbeats_total = h,
heartbeats_per_s = format!("{hh:.1}"),
errors = e,
"load"
);
prev_state = s;
prev_hb = h;
}
});
// --- run for duration ----------------------------------------------------
let started = Instant::now();
tokio::time::sleep(Duration::from_secs(cli.duration_s)).await;
reporter.abort();
sims.shutdown().await;
let elapsed = started.elapsed();
let s = counters.state_writes.load(Ordering::Relaxed);
let h = counters.heartbeat_writes.load(Ordering::Relaxed);
let e = counters.errors.load(Ordering::Relaxed);
tracing::info!(
elapsed_s = format!("{:.1}", elapsed.as_secs_f64()),
state_writes_total = s,
state_writes_per_s = format!("{:.1}", s as f64 / elapsed.as_secs_f64()),
heartbeats_total = h,
errors = e,
"run complete"
);
// --- give the aggregator a second to drain --------------------------------
tokio::time::sleep(Duration::from_secs(2)).await;
// --- verify CR status aggregates -----------------------------------------
//
// With selector-based matching there's a second axis we want to check:
// `matched_device_count` must equal the expected group size (selector
// actually resolved every registered Device), AND the phase counters
// must sum to it.
let mut all_ok = true;
for group in &plan.groups {
let cr = deployments.get(&group.cr_name).await?;
let Some(status) = cr.status.as_ref().and_then(|s| s.aggregate.as_ref()) else {
tracing::warn!(cr = %group.cr_name, "aggregate missing on CR status");
all_ok = false;
continue;
};
let total_reported = status.succeeded + status.failed + status.pending;
let expected = group.devices.len() as u32;
let ok = status.matched_device_count == expected && total_reported == expected;
if !ok {
all_ok = false;
}
tracing::info!(
cr = %group.cr_name,
expected_devices = expected,
matched = status.matched_device_count,
succeeded = status.succeeded,
failed = status.failed,
pending = status.pending,
total = total_reported,
ok,
"cr status"
);
}
if !cli.keep {
tracing::info!("cleanup: deleting CRs + KV entries");
for group in &plan.groups {
let _ = deployments
.delete(&group.cr_name, &DeleteParams::default())
.await;
}
for device in &plan.devices {
let _ = state_bucket
.delete(&device_state_key(
&device.device_id,
&DeploymentName::try_new(&device.cr_name).unwrap(),
))
.await;
let _ = info_bucket
.delete(&device_info_key(&device.device_id))
.await;
let _ = heartbeat_bucket
.delete(&device_heartbeat_key(&device.device_id))
.await;
}
}
if all_ok {
tracing::info!("PASS — all CR aggregates match device counts");
Ok(())
} else {
anyhow::bail!("FAIL — at least one CR aggregate did not sum to its target device count")
}
}
fn parse_groups(s: &str) -> Result<Vec<usize>> {
let out: Vec<usize> = s
.split(',')
.map(|t| t.trim().parse::<usize>())
.collect::<Result<_, _>>()
.context("parsing --groups")?;
if out.is_empty() {
anyhow::bail!("--groups must have at least one size");
}
Ok(out)
}
/// A single simulated device and the CR it belongs to.
#[derive(Clone)]
struct DevicePlan {
device_id: String,
cr_name: String,
}
#[derive(Clone)]
struct GroupPlan {
cr_name: String,
devices: Vec<String>,
}
struct Plan {
devices: Vec<DevicePlan>,
groups: Vec<GroupPlan>,
}
fn build_plan(group_sizes: &[usize]) -> Plan {
// CR-name + device-id width scale with group count so large runs
// get zero-padded ids that sort sensibly in kubectl.
let cr_width = group_sizes.len().to_string().len().max(2);
let total: usize = group_sizes.iter().sum();
let dev_width = total.to_string().len().max(5);
let mut devices = Vec::new();
let mut groups = Vec::new();
let mut next_id = 1usize;
for (i, size) in group_sizes.iter().enumerate() {
let cr_name = format!("load-group-{i:0cr_width$}");
let mut ids = Vec::with_capacity(*size);
for _ in 0..*size {
let id = format!("load-dev-{next_id:0dev_width$}");
next_id += 1;
devices.push(DevicePlan {
device_id: id.clone(),
cr_name: cr_name.clone(),
});
ids.push(id);
}
groups.push(GroupPlan {
cr_name,
devices: ids,
});
}
Plan { devices, groups }
}
async fn open_bucket(js: &jetstream::Context, bucket: &'static str) -> Result<kv::Store> {
Ok(js
.create_key_value(kv::Config {
bucket: bucket.to_string(),
history: 1,
..Default::default()
})
.await?)
}
async fn ensure_namespace(client: &Client, name: &str) -> Result<()> {
let api: Api<Namespace> = Api::all(client.clone());
if api.get_opt(name).await?.is_some() {
return Ok(());
}
let ns = Namespace {
metadata: kube::api::ObjectMeta {
name: Some(name.to_string()),
..Default::default()
},
..Default::default()
};
match api.create(&PostParams::default(), &ns).await {
Ok(_) => Ok(()),
Err(kube::Error::Api(ae)) if ae.code == 409 => Ok(()),
Err(e) => Err(e.into()),
}
}
async fn apply_crs(api: &Api<Deployment>, plan: &Plan) -> Result<()> {
let params = PatchParams::apply("fleet-load-test").force();
let started = Instant::now();
// Cap concurrency so we don't overwhelm the apiserver on large
// fleets. 32 in-flight applies is well under typical apiserver
// QPS limits and keeps the startup latency predictable.
const CONCURRENCY: usize = 32;
let mut in_flight: JoinSet<Result<String>> = JoinSet::new();
let mut iter = plan.groups.iter();
for _ in 0..CONCURRENCY {
if let Some(group) = iter.next() {
in_flight.spawn(apply_one_cr(api.clone(), group.clone(), params.clone()));
}
}
while let Some(res) = in_flight.join_next().await {
res??;
if let Some(group) = iter.next() {
in_flight.spawn(apply_one_cr(api.clone(), group.clone(), params.clone()));
}
}
tracing::info!(
crs = plan.groups.len(),
elapsed_ms = started.elapsed().as_millis() as u64,
"applied Deployment CRs"
);
Ok(())
}
async fn apply_one_cr(
api: Api<Deployment>,
group: GroupPlan,
params: PatchParams,
) -> Result<String> {
// Selector-based targeting: every Device CR in this group carries
// a `group=<cr_name>` label (we publish that on DeviceInfo; the
// operator reflects it into Device.metadata.labels).
let mut match_labels = BTreeMap::new();
match_labels.insert("group".to_string(), group.cr_name.clone());
let cr = Deployment::new(
&group.cr_name,
DeploymentSpec {
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,
},
// Score content doesn't matter — no real agents consume
// the desired-state here. The aggregator still writes KV
// for each matched device; that's wire noise we accept
// as part of the realism.
score: ScorePayload {
type_: "PodmanV0".to_string(),
data: serde_json::json!({
"services": [{
"name": group.cr_name,
"image": "docker.io/library/nginx:alpine",
"ports": ["8080:80"],
}],
}),
},
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
);
api.patch(&group.cr_name, &params, &Patch::Apply(&cr))
.await
.with_context(|| format!("applying CR {}", group.cr_name))?;
Ok(group.cr_name)
}
async fn publish_device_infos(bucket: &kv::Store, plan: &Plan) -> Result<()> {
let started = Instant::now();
const CONCURRENCY: usize = 64;
let mut in_flight: JoinSet<Result<()>> = JoinSet::new();
let mut iter = plan.devices.iter();
for _ in 0..CONCURRENCY {
if let Some(device) = iter.next() {
in_flight.spawn(publish_one_info(bucket.clone(), device.clone()));
}
}
while let Some(res) = in_flight.join_next().await {
res??;
if let Some(device) = iter.next() {
in_flight.spawn(publish_one_info(bucket.clone(), device.clone()));
}
}
tracing::info!(
devices = plan.devices.len(),
elapsed_ms = started.elapsed().as_millis() as u64,
"seeded DeviceInfo"
);
Ok(())
}
async fn publish_one_info(bucket: kv::Store, device: DevicePlan) -> Result<()> {
let info = DeviceInfo {
device_id: Id::from(device.device_id.clone()),
labels: BTreeMap::from([("group".to_string(), device.cr_name.clone())]),
inventory: None,
updated_at: Utc::now(),
};
let key = device_info_key(&device.device_id);
let payload = serde_json::to_vec(&info)?;
bucket.put(&key, payload.into()).await?;
Ok(())
}
async fn simulate_state_loop(
device: Arc<DevicePlan>,
bucket: kv::Store,
counters: Arc<Counters>,
tick: Duration,
) {
let Ok(deployment) = DeploymentName::try_new(&device.cr_name) else {
return;
};
let state_key = device_state_key(&device.device_id, &deployment);
let mut ticker = tokio::time::interval(tick);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
let phase = pick_phase();
let ds = DeploymentState {
device_id: Id::from(device.device_id.clone()),
deployment: deployment.clone(),
phase,
last_event_at: Utc::now(),
last_error: matches!(phase, Phase::Failed)
.then(|| format!("synthetic failure @{}", device.device_id)),
};
match serde_json::to_vec(&ds) {
Ok(payload) => match bucket.put(&state_key, payload.into()).await {
Ok(_) => {
counters.state_writes.fetch_add(1, Ordering::Relaxed);
}
Err(_) => {
counters.errors.fetch_add(1, Ordering::Relaxed);
}
},
Err(_) => {
counters.errors.fetch_add(1, Ordering::Relaxed);
}
}
}
}
async fn simulate_heartbeat_loop(
device: Arc<DevicePlan>,
bucket: kv::Store,
counters: Arc<Counters>,
tick: Duration,
) {
let hb_key = device_heartbeat_key(&device.device_id);
let mut ticker = tokio::time::interval(tick);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
let hb = HeartbeatPayload {
device_id: Id::from(device.device_id.clone()),
at: Utc::now(),
};
if let Ok(payload) = serde_json::to_vec(&hb) {
if bucket.put(&hb_key, payload.into()).await.is_ok() {
counters.heartbeat_writes.fetch_add(1, Ordering::Relaxed);
} else {
counters.errors.fetch_add(1, Ordering::Relaxed);
}
}
}
}
/// Phase distribution mirroring a healthy-ish fleet: mostly Running,
/// a sprinkle of Failed + Pending to exercise the aggregator's
/// transition-handling + last_error logic.
fn pick_phase() -> Phase {
let n: u32 = rand::rng().random_range(0..100);
match n {
0..80 => Phase::Running,
80..90 => Phase::Failed,
_ => Phase::Pending,
}
}

15
examples/fleet_nats_install/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "example_fleet_nats_install"
version.workspace = true
edition = "2024"
license.workspace = true
[[bin]]
name = "fleet_nats_install"
path = "src/main.rs"
[dependencies]
harmony = { path = "../../harmony", default-features = false }
tokio.workspace = true
anyhow.workspace = true
clap.workspace = true

91
examples/fleet_nats_install/src/main.rs Normal file
View File

@@ -0,0 +1,91 @@
//! Install a single-node NATS server into the cluster `KUBECONFIG`
//! points at, using harmony's `NatsBasicScore` + `K8sBareTopology`.
//!
//! This binary is the glue between the smoke harness (`smoke-a4.sh`)
//! and the framework Score. Typical usage from a demo script:
//!
//! KUBECONFIG=$KUBECFG cargo run -q -p example_fleet_nats_install \
//! -- --namespace fleet-system --name fleet-nats --node-port 4222
//!
//! Behaviour:
//! - Ensures the target namespace exists
//! - Deploys a single-replica NATS server (JetStream on)
//! - Exposes it as a Service (NodePort by default so off-cluster
//! clients like a libvirt VM agent can reach it through the
//! k3d loadbalancer port mapping)
//!
//! For production / HA / TLS, graduate to `NatsK8sScore`.
use anyhow::{Context, Result};
use clap::Parser;
use harmony::inventory::Inventory;
use harmony::modules::k8s::K8sBareTopology;
use harmony::modules::nats::NatsBasicScore;
use harmony::score::Score;
#[derive(Parser, Debug)]
#[command(
name = "fleet_nats_install",
about = "Install single-node NATS (JetStream) via NatsBasicScore"
)]
struct Cli {
/// Target namespace. Created if missing.
#[arg(long, default_value = "fleet-system")]
namespace: String,
/// Resource name for the NATS Deployment + Service.
#[arg(long, default_value = "fleet-nats")]
name: String,
/// Service exposure mode. `load-balancer` pairs with k3d's
/// `-p PORT:PORT@loadbalancer` port mapping (direct service-
/// port routing). `node-port` demands a port in the apiserver's
/// nodeport range (default 30000-32767). `cluster-ip` keeps
/// NATS in-cluster only.
#[arg(long, value_enum, default_value_t = ExposeMode::LoadBalancer)]
expose: ExposeMode,
/// NodePort when `--expose=node-port`. Must be in the cluster's
/// nodeport range (default 30000-32767). Ignored otherwise.
#[arg(long, default_value_t = 30422)]
node_port: i32,
/// Override the NATS container image.
#[arg(long)]
image: Option<String>,
}
#[derive(Clone, Debug, clap::ValueEnum)]
enum ExposeMode {
ClusterIp,
NodePort,
LoadBalancer,
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
let topology = K8sBareTopology::from_kubeconfig("fleet-nats-install")
.await
.map_err(|e| anyhow::anyhow!(e))
.context("building K8sBareTopology from KUBECONFIG")?;
let mut score = NatsBasicScore::new(&cli.name, &cli.namespace);
match cli.expose {
ExposeMode::ClusterIp => {}
ExposeMode::NodePort => score = score.node_port(cli.node_port),
ExposeMode::LoadBalancer => score = score.load_balancer(),
}
if let Some(image) = cli.image {
score = score.image(image);
}
let interpret = Score::<K8sBareTopology>::create_interpret(&score);
let outcome = interpret
.execute(&Inventory::empty(), &topology)
.await
.map_err(|e| anyhow::anyhow!("execute NatsBasicScore: {e}"))?;
println!(
"NATS installed: namespace={}, name={}, expose={:?} outcome={outcome:?}",
cli.namespace, cli.name, cli.expose
);
Ok(())
}

4
examples/iot_vm_setup/Cargo.toml → examples/fleet_vm_setup/Cargo.toml
View File

@@ -1,11 +1,11 @@
[package]
name = "example_iot_vm_setup"
name = "example_fleet_vm_setup"
version.workspace = true
edition = "2024"
license.workspace = true
[[bin]]
name = "iot_vm_setup"
name = "fleet_vm_setup"
path = "src/main.rs"
[dependencies]

12
examples/iot_vm_setup/README.md → examples/fleet_vm_setup/README.md
View File

@@ -6,8 +6,8 @@ Harmony Scores in sequence:
1. **`KvmVmScore`** — provision a libvirt VM from an Ubuntu 24.04 cloud
image with a cloud-init seed ISO that authorizes one SSH key. Returns
the booted VM's IP.
2. **`IotDeviceSetupScore`** — SSH into the VM (via the Ansible-backed
`HostConfigurationProvider`) and install podman + the `iot-agent`
2. **`FleetDeviceSetupScore`** — SSH into the VM (via the Ansible-backed
`HostConfigurationProvider`) and install podman + the `fleet-agent`
binary, drop the TOML config, bring up the systemd unit.
After a successful run, the VM is a fleet member reporting to NATS under
@@ -42,21 +42,21 @@ sudo virsh net-autostart default
## Run
```bash
cargo build -p iot-agent-v0
cargo build -p fleet-agent-v0
cargo run -p example_iot_vm_setup -- \
--base-image /var/tmp/harmony-iot-smoke/ubuntu-24.04-server-cloudimg-amd64.img \
--ssh-pubkey /var/tmp/harmony-iot-smoke/ssh/id_ed25519.pub \
--ssh-privkey /var/tmp/harmony-iot-smoke/ssh/id_ed25519 \
--work-dir /var/tmp/harmony-iot-smoke \
--agent-binary target/debug/iot-agent-v0 \
--agent-binary target/debug/fleet-agent-v0 \
--nats-url nats://192.168.122.1:4222
```
## Changing groups
Re-running with a different `--group` rewrites
`/etc/iot-agent/config.toml` on the VM and restarts the agent. The VM
`/etc/fleet-agent/config.toml` on the VM and restarts the agent. The VM
itself is untouched.
```bash
@@ -65,5 +65,5 @@ cargo run -p example_iot_vm_setup -- ... --group group-b
## Full end-to-end via smoke test
See `iot/scripts/smoke-a3.sh` — stands up NATS in a podman container,
See `fleet/scripts/smoke-a3.sh` — stands up NATS in a podman container,
runs this example, asserts the agent's status lands in NATS.

110
examples/iot_vm_setup/src/main.rs → examples/fleet_vm_setup/src/main.rs
View File

@@ -5,15 +5,15 @@
//! capability. Here we satisfy it with `KvmVirtualMachineHost`
//! (libvirt). Swapping to VMware/Proxmox/cloud would be a
//! different topology injection with the same Score code.
//! 2. `IotDeviceSetupScore` — SSHes into the booted VM and installs
//! podman + iot-agent via the split Linux-host capabilities.
//! 2. `FleetDeviceSetupScore` — SSHes into the booted VM and installs
//! podman + fleet-agent via the split Linux-host capabilities.
use anyhow::{Context, Result};
use clap::Parser;
use harmony::inventory::Inventory;
use harmony::modules::iot::{
IotDeviceSetupConfig, IotDeviceSetupScore, ProvisionVmScore,
check_iot_smoke_preflight_for_arch, ensure_iot_ssh_keypair,
use harmony::modules::fleet::{
FleetDeviceSetupConfig, FleetDeviceSetupScore, ProvisionVmScore,
check_fleet_smoke_preflight_for_arch, ensure_fleet_ssh_keypair,
};
use harmony::modules::kvm::KvmVirtualMachineHost;
use harmony::modules::kvm::config::init_executor;
@@ -42,7 +42,7 @@ impl From<CliArch> for VmArchitecture {
#[derive(Parser, Debug)]
#[command(
name = "iot_vm_setup",
name = "fleet_vm_setup",
about = "Provision one VM + onboard it into the IoT fleet"
)]
struct Cli {
@@ -51,22 +51,34 @@ struct Cli {
#[arg(long, value_enum, default_value_t = CliArch::X86_64)]
arch: CliArch,
/// libvirt domain name for the VM.
#[arg(long, default_value = "iot-vm-01")]
#[arg(long, default_value = "fleet-vm-01")]
vm_name: String,
/// Device id the agent will announce to NATS. Defaults to a
/// fresh `Id` (hex timestamp + random suffix).
#[arg(long)]
device_id: Option<String>,
/// Fleet group label to write into the agent's TOML config.
#[arg(long, default_value = "group-a")]
group: String,
/// Routing labels to write into the agent's TOML config.
/// Comma-separated list of `key=value` pairs. Published in every
/// DeviceInfo heartbeat; the operator resolves Deployment
/// `spec.targetSelector` against this map. At least one label
/// is required so the device is targetable — the default
/// `group=group-a` satisfies that.
#[arg(long, default_value = "group=group-a")]
labels: String,
/// libvirt network name to attach the VM to.
#[arg(long, default_value = "default")]
network: String,
/// Admin username created on first boot.
#[arg(long, default_value = "iot-admin")]
#[arg(long, default_value = "fleet-admin")]
admin_user: String,
/// Path to the cross-compiled iot-agent binary.
/// Optional plaintext password for the admin user. Enables SSH
/// password auth on the guest — intended for interactive
/// debugging / reliability-testing sessions where the operator
/// wants to break things on purpose. Leave unset for key-only
/// auth (production default).
#[arg(long, env = "FLEET_VM_ADMIN_PASSWORD")]
admin_password: Option<String>,
/// Path to the cross-compiled fleet-agent binary.
/// Required unless `--bootstrap-only` is set.
#[arg(long)]
agent_binary: Option<PathBuf>,
@@ -84,6 +96,13 @@ struct Cli {
/// SSH key, libvirt pool) and exit.
#[arg(long)]
bootstrap_only: bool,
/// Virtual disk size in GiB. The stock Ubuntu cloud image has
/// only ~2 GiB of root — resized on first boot by
/// cloud-initramfs-growroot. Bump this to 16 GiB by default so
/// podman can sideload a couple of container images without
/// running out of space.
#[arg(long, default_value_t = 16)]
disk_size_gb: u32,
}
#[tokio::main]
@@ -92,7 +111,7 @@ async fn main() -> Result<()> {
let cli = Cli::parse();
let arch: VmArchitecture = cli.arch.into();
check_iot_smoke_preflight_for_arch(arch)
check_fleet_smoke_preflight_for_arch(arch)
.await
.map_err(|e| anyhow::anyhow!("{e}"))?;
@@ -100,13 +119,13 @@ async fn main() -> Result<()> {
harmony::modules::linux::ensure_ansible_venv()
.await
.map_err(|e| anyhow::anyhow!("ansible venv: {e}"))?;
harmony::modules::iot::ensure_ubuntu_2404_cloud_image_for_arch(arch)
harmony::modules::fleet::ensure_ubuntu_2404_cloud_image_for_arch(arch)
.await
.map_err(|e| anyhow::anyhow!("cloud image: {e}"))?;
ensure_iot_ssh_keypair()
ensure_fleet_ssh_keypair()
.await
.map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?;
harmony::modules::iot::ensure_harmony_iot_pool()
harmony::modules::fleet::ensure_harmony_fleet_pool()
.await
.map_err(|e| anyhow::anyhow!("libvirt pool: {e}"))?;
println!("bootstrap complete");
@@ -114,16 +133,16 @@ async fn main() -> Result<()> {
}
// --- Step 1: provision the VM ---
let base_image = harmony::modules::iot::ensure_ubuntu_2404_cloud_image_for_arch(arch)
let base_image = harmony::modules::fleet::ensure_ubuntu_2404_cloud_image_for_arch(arch)
.await
.map_err(|e| anyhow::anyhow!("cloud image: {e}"))?;
let pool = harmony::modules::iot::ensure_harmony_iot_pool()
let pool = harmony::modules::fleet::ensure_harmony_fleet_pool()
.await
.map_err(|e| anyhow::anyhow!("libvirt pool: {e}"))?;
let ssh = ensure_iot_ssh_keypair()
let ssh = ensure_fleet_ssh_keypair()
.await
.map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?;
let authorized_key = harmony::modules::iot::read_public_key(&ssh)
let authorized_key = harmony::modules::fleet::read_public_key(&ssh)
.await
.map_err(|e| anyhow::anyhow!("read ssh pubkey: {e}"))?;
@@ -142,12 +161,13 @@ async fn main() -> Result<()> {
architecture: arch,
cpus: 2,
memory_mib: 2048,
disk_size_gb: None,
disk_size_gb: Some(cli.disk_size_gb),
network: cli.network.clone(),
first_boot: Some(VmFirstBootConfig {
hostname: Some(cli.vm_name.clone()),
admin_user: Some(cli.admin_user.clone()),
authorized_keys: vec![authorized_key],
admin_password: cli.admin_password.clone(),
}),
},
};
@@ -162,7 +182,7 @@ async fn main() -> Result<()> {
let agent_binary = cli
.agent_binary
.clone()
.context("--agent-binary is required (e.g. target/release/iot-agent-v0)")?;
.context("--agent-binary is required (e.g. target/release/fleet-agent-v0)")?;
let device_id = cli
.device_id
.clone()
@@ -179,9 +199,16 @@ async fn main() -> Result<()> {
},
);
let setup_score = IotDeviceSetupScore::new(IotDeviceSetupConfig {
let labels = parse_labels(&cli.labels)?;
let labels_display = labels
.iter()
.map(|(k, v)| format!("{k}={v}"))
.collect::<Vec<_>>()
.join(",");
let setup_score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig {
device_id: device_id.clone(),
group: cli.group.clone(),
labels,
nats_urls: vec![cli.nats_url.clone()],
nats_user: cli.nats_user.clone(),
nats_pass: cli.nats_pass.clone(),
@@ -189,13 +216,33 @@ async fn main() -> Result<()> {
});
run_setup_score(&setup_score, &linux_topology).await?;
println!(
"device '{device_id}' (group '{}') onboarded via {vm_ip}",
cli.group
);
println!("device '{device_id}' ({labels_display}) onboarded via {vm_ip}");
Ok(())
}
/// Parse `key=value,key=value` into a BTreeMap. Errors on any
/// malformed chunk, empty keys/values, or an empty map overall —
/// a device with no labels is practically untargetable, so we'd
/// rather fail at the CLI than silently onboard a ghost.
fn parse_labels(raw: &str) -> anyhow::Result<std::collections::BTreeMap<String, String>> {
let mut out = std::collections::BTreeMap::new();
for piece in raw.split(',').map(str::trim).filter(|p| !p.is_empty()) {
let (k, v) = piece
.split_once('=')
.ok_or_else(|| anyhow::anyhow!("label chunk '{piece}' missing '='"))?;
let k = k.trim();
let v = v.trim();
if k.is_empty() || v.is_empty() {
anyhow::bail!("label chunk '{piece}' has empty key or value");
}
out.insert(k.to_string(), v.to_string());
}
if out.is_empty() {
anyhow::bail!("--labels must include at least one key=value pair");
}
Ok(out)
}
async fn run_vm_score(
score: &ProvisionVmScore,
topology: &KvmVirtualMachineHost,
@@ -215,14 +262,17 @@ async fn run_vm_score(
anyhow::bail!("ProvisionVmScore finished without reporting an IP: {outcome:?}")
}
async fn run_setup_score(score: &IotDeviceSetupScore, topology: &LinuxHostTopology) -> Result<()> {
async fn run_setup_score(
score: &FleetDeviceSetupScore,
topology: &LinuxHostTopology,
) -> Result<()> {
use harmony::score::Score;
let inventory = Inventory::empty();
let interpret = Score::<LinuxHostTopology>::create_interpret(score);
let outcome = interpret
.execute(&inventory, topology)
.await
.map_err(|e| anyhow::anyhow!("IotDeviceSetupScore execute: {e}"))?;
.map_err(|e| anyhow::anyhow!("FleetDeviceSetupScore execute: {e}"))?;
println!("setup: {} ({:?})", outcome.message, outcome.details);
Ok(())
}

19
examples/harmony_apply_deployment/Cargo.toml Normal file
View File

@@ -0,0 +1,19 @@
[package]
name = "example_harmony_apply_deployment"
version.workspace = true
edition = "2024"
license.workspace = true
[[bin]]
name = "harmony_apply_deployment"
path = "src/main.rs"
[dependencies]
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
kube = { workspace = true, features = ["runtime", "derive"] }
k8s-openapi = { workspace = true }
serde_json.workspace = true
tokio.workspace = true
anyhow.workspace = true
clap.workspace = true

178
examples/harmony_apply_deployment/src/main.rs Normal file
View File

@@ -0,0 +1,178 @@
//! Typed-Rust applier for the harmony fleet `Deployment` CR.
//!
//! Builds a `Deployment` CR via the typed `DeploymentSpec` +
//! `PodmanV0Score` + `kube::Api`, then either applies it directly
//! through the kube client or prints it to stdout so the user can
//! pipe into `kubectl apply -f -`.
//!
//! The CRD is domain-agnostic — it's "declarative reconcile intent
//! for a set of devices matched by label selector," which is the
//! same shape whether the fleet is Pi podman, OKD clusters, or
//! KVM VMs. The name `harmony_apply_deployment` reflects that
//! (not `iot_`-anything), in line with the review call to position
//! the operator as a generic fleet/reconcile tool.
//!
//! The CRD types live in `harmony_fleet_operator::crd`; the score types
//! live in `harmony::modules::podman` (PodmanV0 being the first
//! reconciler variant — future variants drop in alongside).
//!
//! Typical demo-driver usage:
//!
//! # apply an nginx deployment
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:latest
//!
//! # print the CR JSON (lets the user kubectl-apply it manually)
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:latest --print | kubectl apply -f -
//!
//! # upgrade the same deployment to a newer image
//! cargo run -q -p example_harmony_apply_deployment -- \
//! --target-device fleet-smoke-vm-arm \
//! --image nginx:1.26
//!
//! # delete the deployment
//! cargo run -q -p example_harmony_apply_deployment -- --delete
use anyhow::{Context, Result};
use clap::Parser;
use harmony::modules::podman::{PodmanService, PodmanV0Score};
use harmony_fleet_operator::crd::{
Deployment, DeploymentSpec, Rollout, RolloutStrategy, ScorePayload,
};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams};
use std::collections::BTreeMap;
#[derive(Parser, Debug)]
#[command(
name = "harmony_apply_deployment",
about = "Build + apply a harmony fleet Deployment CR from typed Rust (no yaml)"
)]
struct Cli {
/// Kubernetes namespace for the Deployment CR.
#[arg(long, default_value = "fleet-demo")]
namespace: String,
/// Deployment CR name. Also used as the KV key suffix and
/// podman container name on the device.
#[arg(long, default_value = "hello-world")]
name: String,
/// Shortcut: if set, picks a single device by id. Shorthand for
/// `--selector device-id=<target_device>` — the agent publishes
/// a `device-id=<id>` label on its DeviceInfo by default so this
/// works without any cluster-side label pre-wiring.
#[arg(long, default_value = "fleet-smoke-vm")]
target_device: String,
/// Repeatable `key=value` label selector. Takes precedence over
/// `--target-device` when provided. All pairs AND together.
#[arg(long = "selector", value_name = "KEY=VALUE")]
selectors: Vec<String>,
/// Container image to run.
#[arg(long, default_value = "docker.io/library/nginx:latest")]
image: String,
/// `host:container` port mapping exposed on the device.
#[arg(long, default_value = "8080:80")]
port: String,
/// Delete the Deployment CR instead of applying it.
#[arg(long)]
delete: bool,
/// Print the CR as JSON to stdout instead of applying it.
/// Useful for piping into `kubectl apply -f -`.
#[arg(long)]
print: bool,
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
let cr = build_cr(&cli);
if cli.print {
println!("{}", serde_json::to_string_pretty(&cr)?);
return Ok(());
}
let client = Client::try_default()
.await
.context("building kube client (is KUBECONFIG set?)")?;
let api: Api<Deployment> = Api::namespaced(client, &cli.namespace);
if cli.delete {
match api.delete(&cli.name, &DeleteParams::default()).await {
Ok(_) => println!("deleted deployment '{}/{}'", cli.namespace, cli.name),
Err(kube::Error::Api(ae)) if ae.code == 404 => {
println!(
"deployment '{}/{}' not found (already gone)",
cli.namespace, cli.name
)
}
Err(e) => anyhow::bail!("delete failed: {e}"),
}
return Ok(());
}
// Server-side apply so repeated invocations (upgrades) patch
// the existing CR instead of erroring with "already exists."
let params = PatchParams::apply("harmony-apply-deployment").force();
let applied = api
.patch(&cli.name, &params, &Patch::Apply(&cr))
.await
.context("applying Deployment CR")?;
let meta = applied.metadata;
println!(
"applied deployment '{}/{}' (resourceVersion={}, image={})",
cli.namespace,
meta.name.as_deref().unwrap_or("?"),
meta.resource_version.as_deref().unwrap_or("?"),
cli.image,
);
Ok(())
}
fn build_cr(cli: &Cli) -> Deployment {
let score = PodmanV0Score {
services: vec![PodmanService {
name: cli.name.clone(),
image: cli.image.clone(),
ports: vec![cli.port.clone()],
}],
};
let payload = ScorePayload {
type_: "PodmanV0".to_string(),
// `ScorePayload::data` is `serde_json::Value` by design
// (opaque payload routed to the agent). Serialize the typed
// score through serde_json — the agent's `ReconcileScore` enum
// accepts exactly this shape via `#[serde(tag, content)]`.
data: serde_json::to_value(&score).expect("PodmanV0Score is JSON-clean"),
};
let mut match_labels = BTreeMap::new();
if cli.selectors.is_empty() {
match_labels.insert("device-id".to_string(), cli.target_device.clone());
} else {
for kv in &cli.selectors {
let (k, v) = kv
.split_once('=')
.unwrap_or_else(|| panic!("--selector expects KEY=VALUE, got '{kv}'"));
match_labels.insert(k.to_string(), v.to_string());
}
}
Deployment::new(
&cli.name,
DeploymentSpec {
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,
},
score: payload,
rollout: Rollout {
strategy: RolloutStrategy::Immediate,
},
},
)
}

2
iot/iot-agent-v0/Cargo.toml → fleet/harmony-fleet-agent/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "iot-agent-v0"
name = "harmony-fleet-agent"
version = "0.1.0"
edition = "2024"
rust-version = "1.85"

55
iot/iot-agent-v0/src/config.rs → fleet/harmony-fleet-agent/src/config.rs
View File

@@ -1,5 +1,6 @@
use harmony_reconciler_contracts::Id;
use serde::Deserialize;
use std::collections::BTreeMap;
use std::path::Path;
#[derive(Debug, Clone, Deserialize)]
@@ -7,6 +8,14 @@ pub struct AgentConfig {
pub agent: AgentSection,
pub nats: NatsSection,
pub credentials: CredentialsSection,
/// Routing labels published verbatim in every DeviceInfo
/// heartbeat. The operator reflects them into
/// `Device.metadata.labels` so Deployment `spec.targetSelector`
/// resolves against them (K8s-Node-analogue flow). Empty by
/// default — a device with no labels is targetable only by its
/// auto-published `device-id` label.
#[serde(default)]
pub labels: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Deserialize)]
@@ -69,3 +78,49 @@ pub fn load_config(path: &Path) -> anyhow::Result<AgentConfig> {
let config: AgentConfig = toml::from_str(&content)?;
Ok(config)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_config_with_labels_section() {
let raw = r#"
[agent]
device_id = "pi-42"
[credentials]
type = "toml-shared"
nats_user = "u"
nats_pass = "p"
[nats]
urls = ["nats://nats:4222"]
[labels]
group = "site-a"
arch = "aarch64"
"#;
let cfg: AgentConfig = toml::from_str(raw).expect("valid config");
assert_eq!(cfg.labels.get("group"), Some(&"site-a".to_string()));
assert_eq!(cfg.labels.get("arch"), Some(&"aarch64".to_string()));
}
#[test]
fn labels_section_optional_defaults_empty() {
let raw = r#"
[agent]
device_id = "pi-42"
[credentials]
type = "toml-shared"
nats_user = "u"
nats_pass = "p"
[nats]
urls = ["nats://nats:4222"]
"#;
let cfg: AgentConfig = toml::from_str(raw).expect("valid config");
assert!(cfg.labels.is_empty());
}
}

126
fleet/harmony-fleet-agent/src/fleet_publisher.rs Normal file
View File

@@ -0,0 +1,126 @@
//! Agent-side publish surface.
//!
//! Thin wrapper around three KV buckets: [`BUCKET_DEVICE_INFO`],
//! [`BUCKET_DEVICE_STATE`], [`BUCKET_DEVICE_HEARTBEAT`].
//!
//! Failure mode: log and swallow. The KV is the source of truth —
//! a dropped put gets corrected on the next reconcile transition
//! or operator watch reconnection.
use async_nats::jetstream::{self, kv};
use harmony_reconciler_contracts::{
BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName,
DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, device_heartbeat_key,
device_info_key, device_state_key,
};
use std::collections::BTreeMap;
pub struct FleetPublisher {
device_id: Id,
info_bucket: kv::Store,
state_bucket: kv::Store,
heartbeat_bucket: kv::Store,
}
impl FleetPublisher {
/// Open every bucket the agent needs, creating those that don't
/// exist yet. Idempotent with operator-side creation.
pub async fn connect(client: async_nats::Client, device_id: Id) -> anyhow::Result<Self> {
let jetstream = jetstream::new(client);
let info_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_INFO.to_string(),
history: 1,
..Default::default()
})
.await?;
let state_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_STATE.to_string(),
history: 1,
..Default::default()
})
.await?;
let heartbeat_bucket = jetstream
.create_key_value(kv::Config {
bucket: BUCKET_DEVICE_HEARTBEAT.to_string(),
history: 1,
..Default::default()
})
.await?;
Ok(Self {
device_id,
info_bucket,
state_bucket,
heartbeat_bucket,
})
}
/// Publish the agent's static-ish facts. Called at startup and
/// on label change.
pub async fn publish_device_info(
&self,
labels: BTreeMap<String, String>,
inventory: Option<InventorySnapshot>,
) {
let info = DeviceInfo {
device_id: self.device_id.clone(),
labels,
inventory,
updated_at: chrono::Utc::now(),
};
let key = device_info_key(&self.device_id.to_string());
match serde_json::to_vec(&info) {
Ok(payload) => {
if let Err(e) = self.info_bucket.put(&key, payload.into()).await {
tracing::warn!(%key, error = %e, "publish_device_info: kv put failed");
}
}
Err(e) => tracing::warn!(error = %e, "publish_device_info: serialize failed"),
}
}
/// Tiny liveness ping. Called every 30s.
pub async fn publish_heartbeat(&self) {
let hb = HeartbeatPayload {
device_id: self.device_id.clone(),
at: chrono::Utc::now(),
};
let key = device_heartbeat_key(&self.device_id.to_string());
match serde_json::to_vec(&hb) {
Ok(payload) => {
if let Err(e) = self.heartbeat_bucket.put(&key, payload.into()).await {
tracing::debug!(%key, error = %e, "publish_heartbeat: kv put failed");
}
}
Err(e) => tracing::warn!(error = %e, "publish_heartbeat: serialize failed"),
}
}
/// Persist the authoritative current phase for a `(device,
/// deployment)` pair. The operator's watch on the `device-state`
/// bucket picks up this put and updates CR status counters.
pub async fn write_deployment_state(&self, state: &DeploymentState) {
let key = device_state_key(&self.device_id.to_string(), &state.deployment);
match serde_json::to_vec(state) {
Ok(payload) => {
if let Err(e) = self.state_bucket.put(&key, payload.into()).await {
tracing::warn!(%key, error = %e, "write_deployment_state: kv put failed");
}
}
Err(e) => tracing::warn!(error = %e, "write_deployment_state: serialize failed"),
}
}
/// Delete the authoritative current-phase entry, e.g. when the
/// Deployment CR is removed and the agent has torn down the
/// container.
pub async fn delete_deployment_state(&self, deployment: &DeploymentName) {
let key = device_state_key(&self.device_id.to_string(), deployment);
if let Err(e) = self.state_bucket.delete(&key).await {
tracing::debug!(%key, error = %e, "delete_deployment_state: kv delete failed");
}
}
}

112
iot/iot-agent-v0/src/main.rs → fleet/harmony-fleet-agent/src/main.rs
View File

@@ -1,4 +1,5 @@
mod config;
mod fleet_publisher;
mod reconciler;
use std::sync::Arc;
@@ -8,14 +9,13 @@ use anyhow::{Context, Result};
use clap::Parser;
use config::{AgentConfig, CredentialSource, TomlFileCredentialSource};
use futures_util::StreamExt;
use harmony_reconciler_contracts::{
AgentStatus, BUCKET_AGENT_STATUS, BUCKET_DESIRED_STATE, Id, status_key,
};
use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, Id, InventorySnapshot};
use harmony::inventory::Inventory;
use harmony::modules::podman::PodmanTopology;
use harmony::topology::Topology;
use crate::fleet_publisher::FleetPublisher;
use crate::reconciler::Reconciler;
/// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch
@@ -23,12 +23,12 @@ use crate::reconciler::Reconciler;
const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
#[derive(Parser)]
#[command(name = "iot-agent-v0", about = "IoT agent for Raspberry Pi devices")]
#[command(name = "fleet-agent-v0", about = "IoT agent for Raspberry Pi devices")]
struct Cli {
#[arg(
long,
env = "IOT_AGENT_CONFIG",
default_value = "/etc/iot-agent/config.toml"
env = "FLEET_AGENT_CONFIG",
default_value = "/etc/fleet-agent/config.toml"
)]
config: std::path::PathBuf,
}
@@ -85,31 +85,51 @@ async fn watch_desired_state(
Ok(())
}
async fn report_status(client: async_nats::Client, device_id: Id) -> Result<()> {
let jetstream = async_nats::jetstream::new(client);
let bucket = jetstream
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_AGENT_STATUS.to_string(),
..Default::default()
})
.await?;
let key = status_key(&device_id.to_string());
/// Tiny liveness-only loop: push a `HeartbeatPayload` into the
/// `device-heartbeat` bucket every N seconds. Stays separate from
/// per-deployment state writes so routine pings don't churn the
/// device-state bucket or its watch subscribers.
async fn publish_heartbeat_loop(fleet: Arc<FleetPublisher>) {
let mut interval = tokio::time::interval(Duration::from_secs(30));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
interval.tick().await;
let status = AgentStatus {
device_id: device_id.clone(),
status: "running".to_string(),
timestamp: chrono::Utc::now(),
};
let payload = serde_json::to_vec(&status)?;
bucket.put(&key, payload.into()).await?;
tracing::debug!(key = %key, "reported status");
fleet.publish_heartbeat().await;
}
}
/// Build a one-shot inventory snapshot at agent startup. Cheap,
/// published alongside every heartbeat until the agent restarts.
fn local_inventory(inventory: &Inventory) -> InventorySnapshot {
InventorySnapshot {
hostname: inventory.location.name.clone(),
arch: std::env::consts::ARCH.to_string(),
os: std::env::consts::OS.to_string(),
kernel: std::fs::read_to_string("/proc/sys/kernel/osrelease")
.map(|s| s.trim().to_string())
.unwrap_or_default(),
cpu_cores: std::thread::available_parallelism()
.map(|n| n.get() as u32)
.unwrap_or(0),
memory_mb: sys_memory_total_mb().unwrap_or(0),
agent_version: env!("CARGO_PKG_VERSION").to_string(),
}
}
/// Read total RAM from /proc/meminfo. Returns None on non-Linux or
/// if /proc isn't mounted. Small, avoids a sys-info crate dep for a
/// single field.
fn sys_memory_total_mb() -> Option<u64> {
let s = std::fs::read_to_string("/proc/meminfo").ok()?;
for line in s.lines() {
if let Some(rest) = line.strip_prefix("MemTotal:") {
let kb: u64 = rest.split_whitespace().next()?.parse().ok()?;
return Some(kb / 1024);
}
}
None
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
@@ -118,7 +138,7 @@ async fn main() -> Result<()> {
let cli = Cli::parse();
let cfg = config::load_config(&cli.config)?;
tracing::info!(device_id = %cfg.agent.device_id, "iot-agent-v0 starting");
tracing::info!(device_id = %cfg.agent.device_id, "fleet-agent-v0 starting");
let device_id = cfg.agent.device_id.clone();
@@ -134,11 +154,40 @@ async fn main() -> Result<()> {
let inventory = Arc::new(Inventory::from_localhost());
tracing::info!(hostname = %inventory.location.name, "inventory loaded");
let reconciler = Arc::new(Reconciler::new(topology, inventory));
let inventory_snapshot = local_inventory(&inventory);
let client = connect_nats(&cfg).await?;
// Publish surface. Opens the three KV buckets (idempotent
// creates). Must be live before the reconciler starts so
// writes on the first desired-state KV watch land on the wire.
let fleet = Arc::new(
FleetPublisher::connect(client.clone(), device_id.clone())
.await
.context("fleet publisher connect")?,
);
tracing::info!("fleet publisher ready");
// Publish DeviceInfo once at startup. Merge the config-declared
// labels with an always-on `device-id=<id>` default so every
// device is targetable by id even without explicit labels.
// Config labels win on key conflicts — operators can override
// `device-id` if they really want to (unusual but legal).
let mut startup_labels = cfg.labels.clone();
startup_labels
.entry("device-id".to_string())
.or_insert_with(|| device_id.to_string());
fleet
.publish_device_info(startup_labels, Some(inventory_snapshot.clone()))
.await;
let reconciler = Arc::new(Reconciler::new(
device_id.clone(),
topology,
inventory,
Some(fleet.clone()),
));
let ctrlc = async {
tokio::signal::ctrl_c().await.ok();
tracing::info!("received SIGINT, shutting down");
@@ -151,16 +200,17 @@ async fn main() -> Result<()> {
Ok::<(), anyhow::Error>(())
};
let watch = watch_desired_state(client.clone(), device_id.clone(), reconciler.clone());
let status = report_status(client, device_id);
let _ = inventory_snapshot; // consumed by the DeviceInfo publish above
let watch = watch_desired_state(client, device_id, reconciler.clone());
let reconcile = reconciler.clone().run_periodic(RECONCILE_INTERVAL);
let heartbeat = publish_heartbeat_loop(fleet);
tokio::select! {
_ = ctrlc => {},
r = sigterm => { r?; }
r = watch => { r?; }
r = status => { r?; }
_ = reconcile => {}
_ = heartbeat => {}
}
Ok(())

344
fleet/harmony-fleet-agent/src/reconciler.rs Normal file
View File

@@ -0,0 +1,344 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use chrono::Utc;
use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase};
use tokio::sync::Mutex;
use harmony::inventory::Inventory;
use harmony::modules::podman::{PodmanTopology, PodmanV0Score, ReconcileScore};
use harmony::score::Score;
use crate::fleet_publisher::FleetPublisher;
/// Cache key → last-seen state, populated by `apply` and consulted by the
/// 30-second periodic tick and the delete path.
struct CachedEntry {
/// Serialized score JSON. Used for string-compare idempotency per
/// ROADMAP §5.5 — cheaper and more deterministic than a hash.
serialized: String,
/// Parsed score. Cached so the periodic reconcile tick and delete
/// handlers don't have to re-parse the JSON.
score: PodmanV0Score,
}
pub struct Reconciler {
device_id: Id,
topology: Arc<PodmanTopology>,
inventory: Arc<Inventory>,
/// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per
/// KV key — in v0 there is no fan-out from one key to many scores.
state: Mutex<HashMap<String, CachedEntry>>,
/// Current phase per deployment, used to decide whether a new
/// write to the `device-state` KV is needed.
phases: Mutex<HashMap<DeploymentName, Phase>>,
/// Publish surface. Optional so unit tests without a live NATS
/// client still work; always populated in the real agent runtime.
fleet: Option<Arc<FleetPublisher>>,
}
impl Reconciler {
pub fn new(
device_id: Id,
topology: Arc<PodmanTopology>,
inventory: Arc<Inventory>,
fleet: Option<Arc<FleetPublisher>>,
) -> Self {
Self {
device_id,
topology,
inventory,
state: Mutex::new(HashMap::new()),
phases: Mutex::new(HashMap::new()),
fleet,
}
}
/// Record a new phase for a deployment and, if it changed, write
/// the updated [`DeploymentState`] to the KV. Same-phase
/// re-confirmations are no-ops so the periodic reconcile tick
/// doesn't churn the bucket.
async fn apply_phase(
&self,
deployment: &DeploymentName,
phase: Phase,
last_error: Option<String>,
) {
{
let mut phases = self.phases.lock().await;
if phases.get(deployment).copied() == Some(phase) {
return;
}
phases.insert(deployment.clone(), phase);
}
if let Some(publisher) = &self.fleet {
let state = DeploymentState {
device_id: self.device_id.clone(),
deployment: deployment.clone(),
phase,
last_event_at: Utc::now(),
last_error,
};
publisher.write_deployment_state(&state).await;
}
}
/// Clear the in-memory phase for a deployment and delete its KV
/// entry. Idempotent: a delete for a never-applied deployment is
/// a no-op in memory and a harmless tombstone write on the wire.
async fn drop_phase(&self, deployment: &DeploymentName) {
let was_known = {
let mut phases = self.phases.lock().await;
phases.remove(deployment).is_some()
};
if !was_known {
return;
}
if let Some(publisher) = &self.fleet {
publisher.delete_deployment_state(deployment).await;
}
}
/// Handle a Put event (new or updated score on NATS KV). No-ops if the
/// serialized score is byte-identical to the last-seen value for this
/// key.
pub async fn apply(&self, key: &str, value: &[u8]) -> Result<()> {
let deployment = deployment_from_key(key);
let incoming = match serde_json::from_slice::<ReconcileScore>(value) {
Ok(ReconcileScore::PodmanV0(s)) => s,
Err(e) => {
tracing::warn!(key, error = %e, "failed to deserialize score");
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(format!("bad payload: {e}")))
.await;
}
return Ok(());
}
};
let serialized = String::from_utf8_lossy(value).into_owned();
{
let state = self.state.lock().await;
if let Some(existing) = state.get(key) {
if existing.serialized == serialized {
tracing::debug!(key, "score unchanged — noop");
return Ok(());
}
}
}
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Pending, None).await;
}
match self.run_score(key, &incoming).await {
Ok(()) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Running, None).await;
}
}
Err(e) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
.await;
}
return Err(e);
}
}
let mut state = self.state.lock().await;
state.insert(
key.to_string(),
CachedEntry {
serialized,
score: incoming,
},
);
Ok(())
}
/// Handle a Delete/Purge event. Stops and removes every container
/// referenced by the last cached score for this key. Idempotent: if we
/// never saw a Put for this key (agent restart after delete), logs and
/// returns ok.
pub async fn remove(&self, key: &str) -> Result<()> {
let deployment = deployment_from_key(key);
let mut state = self.state.lock().await;
let Some(entry) = state.remove(key) else {
tracing::info!(key, "delete for unknown key — nothing to remove");
if let Some(name) = &deployment {
self.drop_phase(name).await;
}
return Ok(());
};
drop(state);
use harmony::topology::ContainerRuntime;
for service in &entry.score.services {
if let Err(e) = self.topology.remove_service(&service.name).await {
tracing::warn!(
key,
service = %service.name,
error = %e,
"failed to remove container"
);
} else {
tracing::info!(key, service = %service.name, "removed container");
}
}
if let Some(name) = &deployment {
self.drop_phase(name).await;
}
Ok(())
}
/// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of
/// event-driven PLEG. Agent polls podman every 30s as ground truth;
/// KV watch events are accelerators." Re-runs each cached score against
/// podman-api; the underlying `ensure_service_running` is idempotent
/// so a converged state produces no log noise.
pub async fn tick(&self) -> Result<()> {
let snapshot: Vec<(String, PodmanV0Score)> = {
let state = self.state.lock().await;
state
.iter()
.map(|(k, v)| (k.clone(), v.score.clone()))
.collect()
};
for (key, score) in snapshot {
let deployment = deployment_from_key(&key);
match self.run_score(&key, &score).await {
Ok(()) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Running, None).await;
}
}
Err(e) => {
tracing::warn!(key, error = %e, "periodic reconcile failed");
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
.await;
}
}
}
}
Ok(())
}
pub async fn run_periodic(self: Arc<Self>, interval: Duration) {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
if let Err(e) = self.tick().await {
tracing::warn!(error = %e, "reconcile tick error");
}
}
}
async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> {
let interpret = Score::<PodmanTopology>::create_interpret(score);
let outcome = interpret
.execute(&self.inventory, &self.topology)
.await
.map_err(|e| anyhow::anyhow!("PodmanV0Score interpret failed for {key}: {e}"))?;
tracing::info!(key, outcome = ?outcome, "reconciled");
Ok(())
}
}
/// Extract the deployment name from a NATS KV key of the form
/// `<device>.<deployment>`.
fn deployment_from_key(key: &str) -> Option<DeploymentName> {
let (_, rest) = key.split_once('.')?;
DeploymentName::try_new(rest).ok()
}
/// Truncate a long error message so the DeploymentState payload stays
/// comfortably below NATS JetStream's per-message limit.
fn short(s: &str) -> String {
const MAX: usize = 512;
if s.len() <= MAX {
s.to_string()
} else {
let mut cut = s[..MAX].to_string();
cut.push('…');
cut
}
}
#[cfg(test)]
mod tests {
//! Focused tests for transition detection. Drive `apply_phase` /
//! `drop_phase` directly with an inert topology (no real podman
//! socket) and a `None` FleetPublisher.
use super::*;
use harmony::inventory::Inventory;
use harmony::modules::podman::PodmanTopology;
use std::path::PathBuf;
fn reconciler() -> Reconciler {
let topology = Arc::new(
PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(),
);
let inventory = Arc::new(Inventory::empty());
Reconciler::new(
Id::from("test-device".to_string()),
topology,
inventory,
None,
)
}
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid test name")
}
#[tokio::test]
async fn apply_phase_records_new_phase() {
let r = reconciler();
r.apply_phase(&dn("hello"), Phase::Running, None).await;
let phases = r.phases.lock().await;
assert_eq!(phases.get(&dn("hello")), Some(&Phase::Running));
}
#[tokio::test]
async fn apply_phase_idempotent_for_same_phase() {
let r = reconciler();
r.apply_phase(&dn("hello"), Phase::Running, None).await;
r.apply_phase(&dn("hello"), Phase::Running, None).await;
let phases = r.phases.lock().await;
assert_eq!(phases.len(), 1);
}
#[tokio::test]
async fn apply_phase_transitions_update_phase() {
let r = reconciler();
r.apply_phase(&dn("hello"), Phase::Pending, None).await;
r.apply_phase(&dn("hello"), Phase::Running, None).await;
r.apply_phase(&dn("hello"), Phase::Failed, Some("oom".to_string()))
.await;
let phases = r.phases.lock().await;
assert_eq!(phases.get(&dn("hello")), Some(&Phase::Failed));
}
#[tokio::test]
async fn drop_phase_clears_known_deployment() {
let r = reconciler();
r.apply_phase(&dn("hello"), Phase::Running, None).await;
r.drop_phase(&dn("hello")).await;
let phases = r.phases.lock().await;
assert!(!phases.contains_key(&dn("hello")));
}
#[tokio::test]
async fn drop_phase_on_unknown_deployment_is_noop() {
let r = reconciler();
r.drop_phase(&dn("never-existed")).await;
let phases = r.phases.lock().await;
assert!(phases.is_empty());
}
}

5
iot/iot-operator-v0/Cargo.toml → fleet/harmony-fleet-operator/Cargo.toml
View File

@@ -1,14 +1,13 @@
[package]
name = "iot-operator-v0"
name = "harmony-fleet-operator"
version = "0.1.0"
edition = "2024"
rust-version = "1.85"
[dependencies]
harmony = { path = "../../harmony" }
harmony-k8s = { path = "../../harmony-k8s" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
async-trait.workspace = true
chrono = { workspace = true, features = ["serde"] }
kube = { workspace = true, features = ["runtime", "derive"] }
k8s-openapi.workspace = true
async-nats = { workspace = true }

26
fleet/harmony-fleet-operator/Dockerfile Normal file
View File

@@ -0,0 +1,26 @@
# Minimal runtime container for the IoT operator. Assumes
# `target/release/harmony-fleet-operator` has already been built on the
# host (the load-test harness does this). Base image is
# archlinux:base to guarantee the host's glibc (ABI-matched) —
# debian:bookworm-slim and similar distros ship older glibcs and
# would error at startup with "version `GLIBC_2.x' not found".
#
# When the operator gets its own release pipeline, swap this for a
# two-stage build that produces the binary inside a pinned Rust
# toolchain image.
FROM docker.io/library/archlinux:base
COPY target/release/harmony-fleet-operator /usr/local/bin/harmony-fleet-operator
# Non-root runtime. Pairs with the Pod's `securityContext.
# runAsNonRoot: true` in the helm chart — k8s admission rejects
# pods with that flag unless either the image declares a non-root
# USER or the Pod pins runAsUser. We deliberately don't pin
# runAsUser (OpenShift's restricted-v2 SCC assigns a namespace-
# specific UID and rejects fixed UIDs); the image's USER is the
# portable mechanism. 65532 is the `nonroot` UID convention used
# by distroless + many security-hardened base images; it's
# arbitrary but safe — no overlap with typical system UIDs.
USER 65532:65532
ENTRYPOINT ["/usr/local/bin/harmony-fleet-operator"]

342
fleet/harmony-fleet-operator/src/chart.rs Normal file
View File

@@ -0,0 +1,342 @@
//! Generate the operator's helm chart from typed Rust.
//!
//! Produces a self-contained chart directory that `helm install`
//! accepts as a path. Resources are constructed as typed k8s_openapi
//! values and serialized at chart-build time, matching ADR 018
//! (Template Hydration) — no hand-authored yaml in the source tree.
//!
//! The chart has no Helm templating (`{{ .Values.foo }}`); the caller
//! re-runs the generator whenever config changes. For a publishable
//! chart with user-facing values, layer a templating pass on top of
//! this output.
//!
//! Parity with `install` subcommand: both install the same two CRDs
//! (`Deployment`, `Device`). `install` applies the CRDs only, for
//! the host-side-operator path; `chart` packages CRDs + RBAC + the
//! operator Deployment into a helm chart the cluster runs itself.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
use k8s_openapi::api::apps::v1::{
Deployment as K8sDeployment, DeploymentSpec as K8sDeploymentSpec,
};
use k8s_openapi::api::core::v1::{
Capabilities, Container, EnvVar, PodSpec, PodTemplateSpec, SeccompProfile, SecurityContext,
ServiceAccount,
};
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::CustomResourceExt;
use kube::api::ObjectMeta;
use crate::crd::{Deployment, Device};
/// Inputs for chart generation. Default values are aimed at a
/// local-dev k3d install; override via the `chart` subcommand flags.
pub struct ChartOptions {
/// Where to write the chart directory. The chart is created as a
/// subdirectory `harmony-fleet-operator` inside this path.
pub output_dir: PathBuf,
/// Container image tag the operator Deployment should pull. For
/// k3d with sideloaded images, `IfNotPresent` + a tag that's
/// already in the cluster store is enough.
pub image: String,
/// `Always` for registry-backed dev loops, `IfNotPresent` for
/// sideloaded k3d images, `Never` if the image must already be
/// present.
pub image_pull_policy: String,
/// Namespace the operator Deployment runs in. `helm install
/// --create-namespace` creates it if absent; the chart itself
/// doesn't include a Namespace resource so the chart stays
/// reusable across namespaces.
pub namespace: String,
/// NATS URL the operator connects to. For in-cluster NATS at
/// `fleet-nats.fleet-system` the default `nats://fleet-nats.fleet-system:4222`
/// works with no config.
pub nats_url: String,
/// `RUST_LOG` value for the operator process.
pub log_level: String,
}
impl Default for ChartOptions {
fn default() -> Self {
Self {
output_dir: PathBuf::from("/tmp/fleet-load-test/chart"),
image: "localhost/harmony-fleet-operator:latest".to_string(),
image_pull_policy: "IfNotPresent".to_string(),
namespace: "fleet-system".to_string(),
nats_url: "nats://fleet-nats.fleet-system:4222".to_string(),
log_level: "info,kube_runtime=warn".to_string(),
}
}
}
const RELEASE_NAME: &str = "harmony-fleet-operator";
const SERVICE_ACCOUNT: &str = "harmony-fleet-operator";
const CLUSTER_ROLE: &str = "harmony-fleet-operator";
const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator";
/// Build + write the chart to `opts.output_dir`. Returns the full
/// path to the generated chart directory (which is what `helm
/// install <path>` wants).
pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
std::fs::create_dir_all(&opts.output_dir)
.with_context(|| format!("creating {:?}", opts.output_dir))?;
let mut chart = HelmChart::new(
RELEASE_NAME.to_string(),
env!("CARGO_PKG_VERSION").to_string(),
);
chart.description = "IoT operator — Deployment CRD → NATS KV".to_string();
chart.add_resource(HelmResourceKind::Crd(crd_with_keep_annotation(
Deployment::crd(),
)));
chart.add_resource(HelmResourceKind::Crd(crd_with_keep_annotation(
Device::crd(),
)));
chart.add_resource(HelmResourceKind::ServiceAccount(service_account(
&opts.namespace,
)));
chart.add_resource(HelmResourceKind::ClusterRole(cluster_role()));
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
&opts.namespace,
)));
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
let written = chart
.write_to(Path::new(&opts.output_dir))
.map_err(|e| anyhow::anyhow!("writing chart: {e}"))?;
Ok(written)
}
/// Annotate a CRD with `helm.sh/resource-policy: keep` so
/// `helm uninstall` **does not** cascade-delete the CRD and its
/// CRs. Without this, uninstall wipes every `Deployment` + `Device`
/// CR in the cluster via the GC → agents notice the desired-state
/// KV deletes → the whole fleet tears down its containers. One
/// typo on uninstall would be catastrophic. `keep` makes uninstall
/// idempotent and data-preserving; the user explicitly `kubectl
/// delete crd …` if they actually want to wipe.
fn crd_with_keep_annotation(mut crd: CustomResourceDefinition) -> CustomResourceDefinition {
let annotations = crd.metadata.annotations.get_or_insert_with(BTreeMap::new);
annotations.insert("helm.sh/resource-policy".to_string(), "keep".to_string());
crd
}
fn service_account(namespace: &str) -> ServiceAccount {
ServiceAccount {
metadata: ObjectMeta {
name: Some(SERVICE_ACCOUNT.to_string()),
namespace: Some(namespace.to_string()),
..Default::default()
},
..Default::default()
}
}
/// Verbs the operator actually uses — nothing aspirational. Tightening
/// later is a matter of deleting a line.
fn cluster_role() -> ClusterRole {
let group = "fleet.nationtech.io".to_string();
ClusterRole {
metadata: ObjectMeta {
name: Some(CLUSTER_ROLE.to_string()),
..Default::default()
},
rules: Some(vec![
// Deployments: controller lists + watches + patches
// (finalizer metadata); aggregator lists + watches +
// patches status.
PolicyRule {
api_groups: Some(vec![group.clone()]),
resources: Some(vec!["deployments".to_string()]),
verbs: vec!["get", "list", "watch", "patch", "update"]
.into_iter()
.map(String::from)
.collect(),
..Default::default()
},
PolicyRule {
api_groups: Some(vec![group.clone()]),
resources: Some(vec![
"deployments/status".to_string(),
"deployments/finalizers".to_string(),
]),
verbs: vec!["get", "update", "patch"]
.into_iter()
.map(String::from)
.collect(),
..Default::default()
},
// Devices: reconciler server-side-applies + deletes;
// aggregator lists + watches.
PolicyRule {
api_groups: Some(vec![group]),
resources: Some(vec!["devices".to_string()]),
verbs: vec![
"get", "list", "watch", "create", "update", "patch", "delete",
]
.into_iter()
.map(String::from)
.collect(),
..Default::default()
},
]),
..Default::default()
}
}
fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding {
ClusterRoleBinding {
metadata: ObjectMeta {
name: Some(CLUSTER_ROLE_BINDING.to_string()),
..Default::default()
},
role_ref: RoleRef {
api_group: "rbac.authorization.k8s.io".to_string(),
kind: "ClusterRole".to_string(),
name: CLUSTER_ROLE.to_string(),
},
subjects: Some(vec![Subject {
kind: "ServiceAccount".to_string(),
name: SERVICE_ACCOUNT.to_string(),
namespace: Some(namespace.to_string()),
..Default::default()
}]),
}
}
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
let mut match_labels = BTreeMap::new();
match_labels.insert(
"app.kubernetes.io/name".to_string(),
RELEASE_NAME.to_string(),
);
K8sDeployment {
metadata: ObjectMeta {
name: Some(RELEASE_NAME.to_string()),
namespace: Some(opts.namespace.clone()),
labels: Some(match_labels.clone()),
..Default::default()
},
spec: Some(K8sDeploymentSpec {
replicas: Some(1),
selector: LabelSelector {
match_labels: Some(match_labels.clone()),
match_expressions: None,
},
template: PodTemplateSpec {
metadata: Some(ObjectMeta {
labels: Some(match_labels),
..Default::default()
}),
spec: Some(PodSpec {
service_account_name: Some(SERVICE_ACCOUNT.to_string()),
containers: vec![Container {
name: "operator".to_string(),
image: Some(opts.image.clone()),
image_pull_policy: Some(opts.image_pull_policy.clone()),
env: Some(vec![
EnvVar {
name: "NATS_URL".to_string(),
value: Some(opts.nats_url.clone()),
..Default::default()
},
EnvVar {
name: "RUST_LOG".to_string(),
value: Some(opts.log_level.clone()),
..Default::default()
},
]),
security_context: Some(container_security_context()),
..Default::default()
}],
..Default::default()
}),
},
..Default::default()
}),
..Default::default()
}
}
/// Minimum-privilege container security context.
///
/// - `runAsNonRoot: true` — a compromised operator pod with
/// cluster-scoped write on Deployment + Device CRs is enough to
/// tear down the fleet; running as non-root limits blast radius.
/// - `readOnlyRootFilesystem: true` — the Rust operator logs to
/// stdout only; it never writes to `/`.
/// - `allowPrivilegeEscalation: false` — no setuid binaries, no
/// capability gain under any child exec.
/// - `capabilities: drop [ALL]` — no kernel capabilities retained.
/// - `seccompProfile: RuntimeDefault` — runtime's default syscall
/// filter (blocks the obscure/dangerous ones).
///
/// **Deliberately no `runAsUser`** — OpenShift's `restricted-v2`
/// SCC assigns namespace-specific UIDs and rejects pods that pin
/// a fixed UID outside its range. Relying on the image's USER
/// directive (see Dockerfile) lets vanilla k8s and OpenShift pick
/// a compatible UID without custom SCC bindings.
fn container_security_context() -> SecurityContext {
SecurityContext {
run_as_non_root: Some(true),
read_only_root_filesystem: Some(true),
allow_privilege_escalation: Some(false),
capabilities: Some(Capabilities {
add: None,
drop: Some(vec!["ALL".to_string()]),
}),
seccomp_profile: Some(SeccompProfile {
type_: "RuntimeDefault".to_string(),
localhost_profile: None,
}),
..Default::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn crds_carry_keep_annotation() {
let crd = crd_with_keep_annotation(Deployment::crd());
assert_eq!(
crd.metadata
.annotations
.as_ref()
.and_then(|a| a.get("helm.sh/resource-policy"))
.map(String::as_str),
Some("keep"),
"CRDs must carry the keep annotation so helm uninstall doesn't \
cascade-delete CRs and wipe the fleet"
);
}
#[test]
fn security_context_is_locked_down() {
let sc = container_security_context();
assert_eq!(sc.run_as_non_root, Some(true));
assert_eq!(sc.read_only_root_filesystem, Some(true));
assert_eq!(sc.allow_privilege_escalation, Some(false));
assert_eq!(
sc.capabilities.as_ref().and_then(|c| c.drop.as_ref()),
Some(&vec!["ALL".to_string()])
);
assert_eq!(
sc.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
Some("RuntimeDefault")
);
// OpenShift SCC compatibility: no fixed runAsUser, let the
// image/SCC negotiate.
assert!(sc.run_as_user.is_none());
}
}

136
fleet/harmony-fleet-operator/src/controller.rs Normal file
View File

@@ -0,0 +1,136 @@
//! Deployment controller.
//!
//! With the selector-based model, the controller's job shrank to:
//! - validate that the CR name is a valid `DeploymentName`
//! (apiserver already validates RFC 1123 — this is the
//! additional NATS-subject-safety check),
//! - hold a finalizer so delete is synchronous with desired-state
//! KV cleanup.
//!
//! The aggregator owns:
//! - resolving `spec.targetSelector` against Device CRs,
//! - writing `desired-state.<device>.<deployment>` KV entries,
//! - patching `.status.aggregate`.
//!
//! So on `apply` this function is a no-op past validation; the
//! aggregator notices the new CR via its own kube watch and
//! materializes KV entries for matched devices on the next tick.
//!
//! On `cleanup` we still need to remove every KV entry for this
//! deployment synchronously so agents stop reconciling before the
//! CR disappears. KV doesn't support prefix delete; we scan the
//! bucket and drop keys with the matching `.<deployment_name>`
//! suffix.
use std::sync::Arc;
use std::time::Duration;
use async_nats::jetstream::kv::Store;
use futures_util::StreamExt;
use harmony_reconciler_contracts::DeploymentName;
use kube::runtime::Controller;
use kube::runtime::controller::Action;
use kube::runtime::finalizer::{Event as FinalizerEvent, finalizer};
use kube::runtime::watcher::Config as WatcherConfig;
use kube::{Api, Client, ResourceExt};
use crate::crd::Deployment;
const FINALIZER: &str = "fleet.nationtech.io/finalizer";
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("kube api: {0}")]
Kube(#[from] kube::Error),
#[error("nats kv: {0}")]
Kv(String),
#[error("missing namespace on resource")]
MissingNamespace,
#[error("invalid deployment name '{0}': {1}")]
InvalidName(String, String),
}
pub struct Context {
pub client: Client,
pub kv: Store,
}
pub async fn run(client: Client, kv: Store) -> anyhow::Result<()> {
let api: Api<Deployment> = Api::all(client.clone());
let ctx = Arc::new(Context { client, kv });
tracing::info!("starting Deployment controller");
Controller::new(api, WatcherConfig::default())
.run(reconcile, error_policy, ctx)
.for_each(|res| async move {
match res {
Ok((obj, _)) => tracing::debug!(?obj, "reconciled"),
Err(e) => tracing::warn!(error = %e, "reconcile error"),
}
})
.await;
Ok(())
}
async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Error> {
let ns = obj.namespace().ok_or(Error::MissingNamespace)?;
let name = obj.name_any();
// Validation pass: apiserver accepts any RFC 1123 name; we need
// the additional NATS-subject-safety properties before anything
// downstream tries to use it as a KV key fragment.
DeploymentName::try_new(&name).map_err(|e| Error::InvalidName(name.clone(), e.to_string()))?;
let api: Api<Deployment> = Api::namespaced(ctx.client.clone(), &ns);
finalizer(&api, FINALIZER, obj, |event| async {
match event {
// No work on apply — the aggregator picks up the CR via
// its own kube watch and writes KV entries for matching
// devices. Long requeue so we're not pointlessly polling.
FinalizerEvent::Apply(_) => Ok(Action::requeue(Duration::from_secs(300))),
FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv).await,
}
})
.await
.map_err(|e| match e {
kube::runtime::finalizer::Error::ApplyFailed(e)
| kube::runtime::finalizer::Error::CleanupFailed(e) => e,
kube::runtime::finalizer::Error::AddFinalizer(e)
| kube::runtime::finalizer::Error::RemoveFinalizer(e) => Error::Kube(e),
kube::runtime::finalizer::Error::UnnamedObject => Error::Kv("unnamed object".into()),
kube::runtime::finalizer::Error::InvalidFinalizer => Error::Kv("invalid finalizer".into()),
})
}
async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
let name = obj.name_any();
let deployment_name =
DeploymentName::try_new(&name).map_err(|e| Error::InvalidName(name, e.to_string()))?;
let suffix = format!(".{}", deployment_name.as_str());
let mut removed = 0u64;
let mut keys = kv
.keys()
.await
.map_err(|e| Error::Kv(format!("listing keys: {e}")))?;
while let Some(key_res) = keys.next().await {
let key = key_res.map_err(|e| Error::Kv(format!("reading key: {e}")))?;
if key.ends_with(&suffix) {
kv.delete(&key)
.await
.map_err(|e| Error::Kv(format!("deleting {key}: {e}")))?;
removed += 1;
}
}
tracing::info!(
deployment = %deployment_name,
removed,
"cleanup: deleted desired-state entries"
);
Ok(Action::await_change())
}
fn error_policy(_obj: Arc<Deployment>, err: &Error, _ctx: Arc<Context>) -> Action {
tracing::warn!(error = %err, "requeueing after error");
Action::requeue(Duration::from_secs(30))
}

81
iot/iot-operator-v0/src/crd.rs → fleet/harmony-fleet-operator/src/crd.rs
View File

@@ -1,3 +1,5 @@
use harmony_reconciler_contracts::InventorySnapshot;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::CustomResource;
use schemars::JsonSchema;
use schemars::schema::{
@@ -5,19 +7,25 @@ use schemars::schema::{
};
use serde::{Deserialize, Serialize};
/// Deployment intent. Targets devices by label selector — identical
/// to the pattern K8s itself uses for DaemonSet nodeSelector, Service
/// pod selector, etc. The operator resolves the selector against
/// `Device` CRs at reconcile time; no list of device ids on spec.
#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[kube(
group = "iot.nationtech.io",
group = "fleet.nationtech.io",
version = "v1alpha1",
kind = "Deployment",
plural = "deployments",
shortname = "iotdep",
shortname = "fleetdep",
namespaced,
status = "DeploymentStatus"
)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentSpec {
pub target_devices: Vec<String>,
/// Which devices this deployment targets. matches against
/// `Device.metadata.labels`.
pub target_selector: LabelSelector,
#[schemars(schema_with = "score_payload_schema")]
pub score: ScorePayload,
pub rollout: Rollout,
@@ -35,13 +43,11 @@ pub struct ScorePayload {
///
/// 1. `x-kubernetes-preserve-unknown-fields: true` on `data` — the payload
/// is routed opaquely; its shape is enforced on-device by the agent's
/// typed `IotScore` deserialization, not by the apiserver.
/// typed `ReconcileScore` deserialization, not by the apiserver.
/// 2. An `x-kubernetes-validations` CEL rule on the enclosing `score` object
/// requiring `type` to be a valid Rust identifier, so typos (`"pdoman"`)
/// are rejected at `kubectl apply` time rather than silently reaching
/// the agent. This validates the *shape* of the discriminator without
/// listing the known variant catalog — the operator stays a generic
/// router (v0.3+ can add `OkdApplyV0` etc. without an operator release).
/// the agent.
fn score_payload_schema(_: &mut schemars::r#gen::SchemaGenerator) -> Schema {
let type_schema = Schema::Object(SchemaObject {
instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))),
@@ -100,6 +106,65 @@ pub enum RolloutStrategy {
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentStatus {
/// Per-deployment rollup. Present once the aggregator has
/// evaluated the selector at least once.
#[serde(skip_serializing_if = "Option::is_none")]
pub observed_score_string: Option<String>,
pub aggregate: Option<DeploymentAggregate>,
}
/// Rollup of per-device deployment phases for this Deployment CR.
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentAggregate {
/// How many Device CRs currently match `spec.targetSelector`.
/// The three phase counters below sum to this; targeted-but-
/// unreported devices are folded into `pending`.
pub matched_device_count: u32,
pub succeeded: u32,
pub failed: u32,
pub pending: u32,
/// Device id of the most recent device reporting a failure, with
/// its short error message. Cleared when that device transitions
/// back to Running.
#[serde(skip_serializing_if = "Option::is_none")]
pub last_error: Option<AggregateLastError>,
}
#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct AggregateLastError {
pub device_id: String,
pub message: String,
pub at: String,
}
/// A physical/virtual device registered with the fleet. Cluster-scoped
/// because devices aren't tenant-isolated by namespace — they're
/// infrastructure, the same way K8s Nodes are cluster-scoped.
///
/// Created by the operator from `DeviceInfo` entries in the NATS
/// `device-info` bucket. Agents never touch the kube apiserver
/// directly; they publish DeviceInfo to NATS and the operator
/// reflects it here.
///
/// `metadata.labels` carries the device's routing labels. `spec.
/// inventory` holds the hardware/OS snapshot. No status subresource
/// today — liveness is queried from the NATS `device-heartbeat`
/// bucket directly; when a CR-side reflection (Reachable / Stale
/// conditions) becomes useful, it'll land with its own reconciler
/// rather than sitting here as speculative surface.
#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)]
#[kube(
group = "fleet.nationtech.io",
version = "v1alpha1",
kind = "Device",
plural = "devices",
shortname = "fleetdev"
)]
#[serde(rename_all = "camelCase")]
pub struct DeviceSpec {
/// Hardware + OS facts reported by the agent at registration.
/// Rarely changes after first publish.
#[serde(skip_serializing_if = "Option::is_none")]
pub inventory: Option<InventorySnapshot>,
}

165
fleet/harmony-fleet-operator/src/device_reconciler.rs Normal file
View File

@@ -0,0 +1,165 @@
//! DeviceInfo (NATS `device-info` KV) → Device CR (kube).
//!
//! Agents publish a `DeviceInfo` payload to NATS on startup + on
//! label/inventory change. This reconciler watches that bucket and
//! materializes each entry as a cluster-scoped `Device` custom
//! resource, so label selectors and `kubectl get devices -l …`
//! work the way they do for K8s Nodes.
//!
//! Failure mode: idempotent server-side apply with a fixed field
//! manager, so repeated writes don't accumulate revisions and
//! concurrent edits from other sources stay merged safely.
use anyhow::Result;
use async_nats::jetstream::kv::{Operation, Store};
use futures_util::StreamExt;
use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo};
use kube::Client;
use kube::api::{Api, DeleteParams, Patch, PatchParams};
use std::collections::BTreeMap;
use crate::crd::{Device, DeviceSpec};
const FIELD_MANAGER: &str = "harmony-fleet-operator-device-reconciler";
pub async fn run(client: Client, js: async_nats::jetstream::Context) -> Result<()> {
let bucket = js
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DEVICE_INFO.to_string(),
..Default::default()
})
.await?;
run_loop(client, bucket).await
}
async fn run_loop(client: Client, bucket: Store) -> Result<()> {
let devices: Api<Device> = Api::all(client);
// `watch_with_history` replays every current entry then streams
// live updates. Matches the aggregator's pattern and means we
// don't need a separate cold-start KV scan here.
let mut watch = bucket.watch_with_history(">").await?;
tracing::info!("device-reconciler: watching device-info KV");
while let Some(entry_res) = watch.next().await {
let entry = match entry_res {
Ok(e) => e,
Err(e) => {
tracing::warn!(error = %e, "device-reconciler: watch delivery error");
continue;
}
};
match entry.operation {
Operation::Put => {
let info: DeviceInfo = match serde_json::from_slice(&entry.value) {
Ok(d) => d,
Err(e) => {
tracing::warn!(key = %entry.key, error = %e, "device-reconciler: bad DeviceInfo payload");
continue;
}
};
if let Err(e) = upsert_device(&devices, &info).await {
tracing::warn!(
device = %info.device_id,
error = %e,
"device-reconciler: upsert failed"
);
}
}
Operation::Delete | Operation::Purge => {
let Some(device_id) = entry.key.strip_prefix("info.") else {
continue;
};
if let Err(e) = delete_device(&devices, device_id).await {
tracing::warn!(%device_id, error = %e, "device-reconciler: delete failed");
}
}
}
}
Ok(())
}
async fn upsert_device(api: &Api<Device>, info: &DeviceInfo) -> Result<()> {
let name = info.device_id.to_string();
let mut device = Device::new(
&name,
DeviceSpec {
inventory: info.inventory.clone(),
},
);
device.metadata.labels = Some(clean_labels(&info.labels));
api.patch(
&name,
&PatchParams::apply(FIELD_MANAGER).force(),
&Patch::Apply(&device),
)
.await?;
tracing::debug!(%name, "device-reconciler: upserted");
Ok(())
}
async fn delete_device(api: &Api<Device>, name: &str) -> Result<()> {
match api.delete(name, &DeleteParams::default()).await {
Ok(_) => {
tracing::debug!(%name, "device-reconciler: deleted");
Ok(())
}
Err(kube::Error::Api(ae)) if ae.code == 404 => Ok(()),
Err(e) => Err(e.into()),
}
}
/// Drop labels whose keys or values violate k8s label-syntax rules.
/// Agents could in theory publish arbitrary strings; kube will reject
/// a whole apply if even one is malformed, which would take out that
/// device's registration. Skip-and-log beats block-everything.
fn clean_labels(raw: &BTreeMap<String, String>) -> BTreeMap<String, String> {
raw.iter()
.filter(|(k, v)| is_label_key(k) && is_label_value(v))
.map(|(k, v)| (k.clone(), v.clone()))
.collect()
}
fn is_label_key(s: &str) -> bool {
// Simplified: DNS-subdomain-like prefix + name ≤ 63 chars alnum/-/./_.
if s.is_empty() || s.len() > 253 {
return false;
}
let name = s.rsplit_once('/').map(|(_, n)| n).unwrap_or(s);
!name.is_empty()
&& name.len() <= 63
&& name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.' || c == '_')
}
fn is_label_value(s: &str) -> bool {
if s.len() > 63 {
return false;
}
s.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.' || c == '_')
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn label_cleaner_accepts_common_cases() {
assert!(is_label_key("group"));
assert!(is_label_key("arch"));
assert!(is_label_key("fleet.nationtech.io/region"));
assert!(is_label_value("aarch64"));
assert!(is_label_value("site-01"));
}
#[test]
fn label_cleaner_rejects_bad_cases() {
assert!(!is_label_key(""));
assert!(!is_label_key("has space"));
assert!(!is_label_value("has space"));
assert!(!is_label_value(&"x".repeat(64)));
}
}

831
fleet/harmony-fleet-operator/src/fleet_aggregator.rs Normal file
View File

@@ -0,0 +1,831 @@
//! Operator-side aggregator + desired-state writer.
//!
//! Maintains three in-memory caches driven by watches:
//! - Deployment CRs (kube watch) → what we want to run
//! - Device CRs (kube watch) → where we could run it
//! - DeploymentState KV (NATS watch) → what's actually running
//!
//! Outputs:
//! - Writes `desired-state.<device>.<deployment>` KV entries when a
//! Deployment's selector matches a Device. Deletes them when the
//! match goes away.
//! - Patches `Deployment.status.aggregate` at 1 Hz for every CR
//! whose matched-device set or phase counts changed.
//!
//! No separate event stream, no per-key revision tracking: KV watches
//! are ordered and last-writer-wins, and the dirty set naturally
//! coalesces high-frequency state churn into one patch per tick.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
use async_nats::jetstream::kv::{Operation, Store};
use futures_util::{StreamExt, TryStreamExt};
use harmony_reconciler_contracts::{
BUCKET_DESIRED_STATE, BUCKET_DEVICE_STATE, DeploymentName, DeploymentState, Phase,
desired_state_key,
};
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use kube::api::{Api, Patch, PatchParams};
use kube::runtime::watcher::{self, Config as WatcherConfig, Event};
use kube::{Client, ResourceExt};
use serde_json::json;
use tokio::sync::Mutex;
use crate::crd::{AggregateLastError, Deployment, DeploymentAggregate, Device};
const PATCH_TICK: Duration = Duration::from_secs(1);
// ---------------------------------------------------------------------------
// State
// ---------------------------------------------------------------------------
/// (namespace, name) identifying a Deployment CR.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct DeploymentKey {
pub namespace: String,
pub name: String,
}
impl DeploymentKey {
pub fn from_cr(cr: &Deployment) -> Option<Self> {
Some(Self {
namespace: cr.namespace()?,
name: cr.name_any(),
})
}
}
/// One `(device, deployment)` pair.
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct DevicePair {
pub device_id: String,
pub deployment: DeploymentName,
}
/// Thin projection of a Deployment CR — everything we need for
/// selector evaluation + desired-state writes + status aggregation,
/// without borrowing the full kube object.
#[derive(Debug, Clone)]
pub struct CachedDeployment {
key: DeploymentKey,
deployment_name: DeploymentName,
selector: LabelSelector,
/// JSON-serialized score payload ready to `put` into
/// desired-state. Cached because the same bytes are written to
/// every matched device's KV entry.
score_json: Vec<u8>,
}
#[derive(Debug, Default)]
pub struct FleetState {
/// Cached Deployment CRs, keyed by (namespace, name).
deployments: HashMap<DeploymentKey, CachedDeployment>,
/// Cached Device labels, keyed by `metadata.name`.
devices: HashMap<String, BTreeMap<String, String>>,
/// Latest DeploymentState per (device, deployment) pair.
states: HashMap<DevicePair, DeploymentState>,
/// Which devices have we pushed desired-state for, per deployment?
/// Diff against recomputed targets on any change. Keyed by
/// `DeploymentName` (not `DeploymentKey`) because the
/// `desired-state` KV key space doesn't carry namespace —
/// deployment names are globally unique at the NATS level. This
/// lets cold-start seeding from the KV populate the map
/// correctly without having to guess namespaces.
owned_targets: HashMap<DeploymentName, HashSet<String>>,
/// Per-deployment latest-failure surface for the CR status.
last_error: HashMap<DeploymentKey, AggregateLastError>,
/// CR keys whose status needs re-patching on the next tick.
dirty: HashSet<DeploymentKey>,
}
pub type SharedFleetState = Arc<Mutex<FleetState>>;
// ---------------------------------------------------------------------------
// Selector evaluation
// ---------------------------------------------------------------------------
/// Does `selector` match this label set? matchLabels only for MVP —
/// matchExpressions logs a warning once and is treated as "no match"
/// until we need it.
pub fn selector_matches(selector: &LabelSelector, labels: &BTreeMap<String, String>) -> bool {
if let Some(match_labels) = &selector.match_labels {
for (k, v) in match_labels {
if labels.get(k) != Some(v) {
return false;
}
}
}
if selector
.match_expressions
.as_ref()
.is_some_and(|v| !v.is_empty())
{
tracing::warn!(
"LabelSelector.matchExpressions is not yet supported; treating CR as empty-selector (matches nothing)"
);
return false;
}
true
}
/// Set of Device names currently matching `selector`.
fn matched_devices(
selector: &LabelSelector,
devices: &HashMap<String, BTreeMap<String, String>>,
) -> HashSet<String> {
devices
.iter()
.filter(|(_, labels)| selector_matches(selector, labels))
.map(|(name, _)| name.clone())
.collect()
}
// ---------------------------------------------------------------------------
// Top-level run
// ---------------------------------------------------------------------------
pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::Result<()> {
let state_bucket = js
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DEVICE_STATE.to_string(),
..Default::default()
})
.await?;
let desired_bucket = js
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DESIRED_STATE.to_string(),
..Default::default()
})
.await?;
// Cold-start: initialize owned_targets from the current contents
// of the desired-state bucket so we don't orphan entries written
// by a previous operator run.
let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default()));
seed_owned_targets(&desired_bucket, &state).await?;
let deployments_api: Api<Deployment> = Api::all(client.clone());
let devices_api: Api<Device> = Api::all(client.clone());
let patch_api: Api<Deployment> = Api::all(client);
tracing::info!(
owned = state
.lock()
.await
.owned_targets
.values()
.map(|s| s.len())
.sum::<usize>(),
"aggregator: startup complete"
);
let state_watcher_handle = {
let state = state.clone();
let bucket = state_bucket.clone();
tokio::spawn(async move {
if let Err(e) = run_state_kv_watcher(bucket, state).await {
tracing::warn!(error = %e, "aggregator: state watcher exited");
}
})
};
let deployment_watcher_handle = {
let state = state.clone();
let desired = desired_bucket.clone();
tokio::spawn(async move {
if let Err(e) = run_deployment_watcher(deployments_api.clone(), state, desired).await {
tracing::warn!(error = %e, "aggregator: deployment watcher exited");
}
})
};
let device_watcher_handle = {
let state = state.clone();
let desired = desired_bucket.clone();
tokio::spawn(async move {
if let Err(e) = run_device_watcher(devices_api, state, desired).await {
tracing::warn!(error = %e, "aggregator: device watcher exited");
}
})
};
let patch_state = state.clone();
let patch_loop = async move {
let mut ticker = tokio::time::interval(PATCH_TICK);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
if let Err(e) = patch_tick(&patch_api, &patch_state).await {
tracing::warn!(error = %e, "aggregator: patch tick failed");
}
}
};
tokio::select! {
_ = patch_loop => Ok(()),
_ = state_watcher_handle => Ok(()),
_ = deployment_watcher_handle => Ok(()),
_ = device_watcher_handle => Ok(()),
}
}
// ---------------------------------------------------------------------------
// Device-state KV watcher (unchanged path)
// ---------------------------------------------------------------------------
fn parse_state_key(key: &str) -> Option<DevicePair> {
let rest = key.strip_prefix("state.")?;
let (device, deployment) = rest.split_once('.')?;
Some(DevicePair {
device_id: device.to_string(),
deployment: DeploymentName::try_new(deployment).ok()?,
})
}
async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow::Result<()> {
let mut watch = bucket.watch_with_history(">").await?;
while let Some(entry_res) = watch.next().await {
let entry = match entry_res {
Ok(e) => e,
Err(e) => {
tracing::warn!(error = %e, "aggregator: state watch delivery error");
continue;
}
};
let Some(pair) = parse_state_key(&entry.key) else {
continue;
};
match entry.operation {
Operation::Put => {
let ds: DeploymentState = match serde_json::from_slice(&entry.value) {
Ok(d) => d,
Err(e) => {
tracing::warn!(key = %entry.key, error = %e, "aggregator: bad device_state payload");
continue;
}
};
let mut guard = state.lock().await;
apply_state(&mut guard, pair, ds);
}
Operation::Delete | Operation::Purge => {
let mut guard = state.lock().await;
drop_state(&mut guard, &pair);
}
}
}
Ok(())
}
/// Record a device's latest state, dedup against older timestamps,
/// maintain last_error, mark the deployment dirty.
pub fn apply_state(state: &mut FleetState, pair: DevicePair, ds: DeploymentState) {
if let Some(prev) = state.states.get(&pair) {
if prev.last_event_at > ds.last_event_at {
return;
}
}
let phase = ds.phase;
let device_id = ds.device_id.to_string();
let last_error_msg = ds.last_error.clone();
let at = ds.last_event_at.to_rfc3339();
state.states.insert(pair.clone(), ds);
for key in matching_deployment_keys(state, &pair.deployment) {
match phase {
Phase::Failed => {
if let Some(msg) = last_error_msg.as_deref() {
state.last_error.insert(
key.clone(),
AggregateLastError {
device_id: device_id.clone(),
message: msg.to_string(),
at: at.clone(),
},
);
}
}
Phase::Running => {
if let Some(existing) = state.last_error.get(&key) {
if existing.device_id == device_id {
state.last_error.remove(&key);
}
}
}
Phase::Pending => {}
}
state.dirty.insert(key);
}
}
pub fn drop_state(state: &mut FleetState, pair: &DevicePair) {
let Some(removed) = state.states.remove(pair) else {
return;
};
let device_id = removed.device_id.to_string();
for key in matching_deployment_keys(state, &pair.deployment) {
if let Some(existing) = state.last_error.get(&key) {
if existing.device_id == device_id {
state.last_error.remove(&key);
}
}
state.dirty.insert(key);
}
}
/// CR keys that carry a given deployment name. Deployment names are
/// globally unique at the KV level, so typically 0 or 1 entry here;
/// Vec lets us surface a warning rather than panic if a misconfigured
/// cluster has duplicates across namespaces.
fn matching_deployment_keys(state: &FleetState, deployment: &DeploymentName) -> Vec<DeploymentKey> {
state
.deployments
.values()
.filter(|d| &d.deployment_name == deployment)
.map(|d| d.key.clone())
.collect()
}
// ---------------------------------------------------------------------------
// Deployment CR watcher
// ---------------------------------------------------------------------------
async fn run_deployment_watcher(
api: Api<Deployment>,
state: SharedFleetState,
desired: Store,
) -> anyhow::Result<()> {
let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed();
while let Some(event) = stream.try_next().await? {
match event {
Event::Apply(cr) | Event::InitApply(cr) => {
on_deployment_upsert(&state, &desired, cr).await;
}
Event::Delete(cr) => {
on_deployment_delete(&state, &desired, cr).await;
}
Event::Init | Event::InitDone => {}
}
}
Ok(())
}
async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Deployment) {
let Some(key) = DeploymentKey::from_cr(&cr) else {
return;
};
let Ok(deployment_name) = DeploymentName::try_new(&key.name) else {
tracing::warn!(name = %key.name, "aggregator: CR name is not a valid DeploymentName, skipping");
return;
};
let selector = cr.spec.target_selector.clone();
let score_json = match serde_json::to_vec(&cr.spec.score) {
Ok(v) => v,
Err(e) => {
tracing::warn!(namespace = %key.namespace, name = %key.name, error = %e, "aggregator: score payload not serializable");
return;
}
};
let (new_targets, previous_targets) = {
let mut guard = state.lock().await;
let new_targets = matched_devices(&selector, &guard.devices);
guard.deployments.insert(
key.clone(),
CachedDeployment {
key: key.clone(),
deployment_name: deployment_name.clone(),
selector: selector.clone(),
score_json: score_json.clone(),
},
);
let previous = guard
.owned_targets
.remove(&deployment_name)
.unwrap_or_default();
guard
.owned_targets
.insert(deployment_name.clone(), new_targets.clone());
guard.dirty.insert(key.clone());
(new_targets, previous)
};
reconcile_kv(
desired,
&deployment_name,
&new_targets,
&previous_targets,
&score_json,
)
.await;
}
async fn on_deployment_delete(state: &SharedFleetState, desired: &Store, cr: Deployment) {
let Some(key) = DeploymentKey::from_cr(&cr) else {
return;
};
let Ok(deployment_name) = DeploymentName::try_new(&key.name) else {
return;
};
let previous = {
let mut guard = state.lock().await;
guard.deployments.remove(&key);
guard.last_error.remove(&key);
guard.dirty.remove(&key);
guard
.owned_targets
.remove(&deployment_name)
.unwrap_or_default()
};
// Every previously-owned target becomes a KV delete. Controller
// finalizer does a belt-and-suspenders scan, but we pull our own
// entries here too so agents react immediately.
for device in &previous {
let k = desired_state_key(device, &deployment_name);
if let Err(e) = desired.delete(&k).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed");
}
}
}
// ---------------------------------------------------------------------------
// Device CR watcher
// ---------------------------------------------------------------------------
async fn run_device_watcher(
api: Api<Device>,
state: SharedFleetState,
desired: Store,
) -> anyhow::Result<()> {
let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed();
while let Some(event) = stream.try_next().await? {
match event {
Event::Apply(dev) | Event::InitApply(dev) => {
on_device_upsert(&state, &desired, dev).await;
}
Event::Delete(dev) => {
on_device_delete(&state, &desired, dev).await;
}
Event::Init | Event::InitDone => {}
}
}
Ok(())
}
async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) {
let name = dev.name_any();
let labels: BTreeMap<String, String> = dev.metadata.labels.clone().unwrap_or_default();
// For every deployment, compute whether this single device now
// matches vs. previously matched; diff against owned_targets;
// collect the KV writes/deletes to perform after the lock is
// released.
let per_deployment: Vec<(CachedDeployment, bool, bool)> = {
let mut guard = state.lock().await;
let snapshot: Vec<CachedDeployment> = guard.deployments.values().cloned().collect();
let previously_matched: HashMap<DeploymentName, bool> = snapshot
.iter()
.map(|d| {
let was = guard
.owned_targets
.get(&d.deployment_name)
.is_some_and(|set| set.contains(&name));
(d.deployment_name.clone(), was)
})
.collect();
guard.devices.insert(name.clone(), labels.clone());
let mut out = Vec::with_capacity(snapshot.len());
for d in snapshot {
let was = previously_matched
.get(&d.deployment_name)
.copied()
.unwrap_or(false);
let now = selector_matches(&d.selector, &labels);
if was != now {
let targets = guard
.owned_targets
.entry(d.deployment_name.clone())
.or_default();
if now {
targets.insert(name.clone());
} else {
targets.remove(&name);
}
guard.dirty.insert(d.key.clone());
}
out.push((d, was, now));
}
out
};
for (cached, was, now) in per_deployment {
match (was, now) {
(false, true) => {
let k = desired_state_key(&name, &cached.deployment_name);
if let Err(e) = desired.put(&k, cached.score_json.clone().into()).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed");
}
}
(true, false) => {
let k = desired_state_key(&name, &cached.deployment_name);
if let Err(e) = desired.delete(&k).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed");
}
}
_ => {}
}
}
}
async fn on_device_delete(state: &SharedFleetState, desired: &Store, dev: Device) {
let name = dev.name_any();
let removed_from: Vec<DeploymentName> = {
let mut guard = state.lock().await;
guard.devices.remove(&name);
let mut out = Vec::new();
let deployments_snapshot: Vec<CachedDeployment> =
guard.deployments.values().cloned().collect();
for cached in deployments_snapshot {
if let Some(set) = guard.owned_targets.get_mut(&cached.deployment_name) {
if set.remove(&name) {
out.push(cached.deployment_name.clone());
guard.dirty.insert(cached.key.clone());
}
}
}
out
};
for deployment_name in removed_from {
let k = desired_state_key(&name, &deployment_name);
if let Err(e) = desired.delete(&k).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on device delete failed");
}
}
}
// ---------------------------------------------------------------------------
// Diff helper: write/delete desired-state entries for one deployment
// ---------------------------------------------------------------------------
async fn reconcile_kv(
desired: &Store,
deployment_name: &DeploymentName,
new_targets: &HashSet<String>,
previous_targets: &HashSet<String>,
score_json: &[u8],
) {
// Writes: new_targets, unconditionally — idempotent put; agents
// byte-compare and no-op on unchanged content.
for device in new_targets {
let k = desired_state_key(device, deployment_name);
if let Err(e) = desired.put(&k, score_json.to_vec().into()).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed");
}
}
// Deletes: anything we owned previously but no longer target.
for device in previous_targets.difference(new_targets) {
let k = desired_state_key(device, deployment_name);
if let Err(e) = desired.delete(&k).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed");
}
}
}
/// Initialize `owned_targets` from the current contents of the
/// `desired-state` KV. After a restart, we need to know what was
/// previously written so we can diff correctly on the first
/// watch-driven reconcile (otherwise we'd leak orphans when a
/// selector change causes a deployment to stop targeting a device).
async fn seed_owned_targets(bucket: &Store, state: &SharedFleetState) -> anyhow::Result<()> {
let mut guard = state.lock().await;
let mut keys = bucket.keys().await?;
while let Some(key_res) = keys.next().await {
let key = key_res?;
// Keys are `<device>.<deployment>`. The KV key space carries
// no namespace — names are globally unique at this layer —
// which is exactly why `owned_targets` keys by DeploymentName.
let Some((device, deployment)) = key.split_once('.') else {
continue;
};
let Ok(deployment_name) = DeploymentName::try_new(deployment) else {
continue;
};
guard
.owned_targets
.entry(deployment_name)
.or_default()
.insert(device.to_string());
}
Ok(())
}
// ---------------------------------------------------------------------------
// Patch tick
// ---------------------------------------------------------------------------
async fn patch_tick(api: &Api<Deployment>, state: &SharedFleetState) -> anyhow::Result<()> {
let dirty: Vec<(DeploymentKey, DeploymentAggregate)> = {
let mut guard = state.lock().await;
let keys: Vec<DeploymentKey> = guard.dirty.drain().collect();
keys.iter()
.filter_map(|k| {
let cached = guard.deployments.get(k)?.clone();
let agg = compute_aggregate(&guard, &cached);
Some((k.clone(), agg))
})
.collect()
};
for (key, aggregate) in dirty {
let ns_api: Api<Deployment> = Api::namespaced(api.clone().into_client(), &key.namespace);
let status = json!({ "status": { "aggregate": aggregate } });
if let Err(e) = ns_api
.patch_status(&key.name, &PatchParams::default(), &Patch::Merge(&status))
.await
{
tracing::warn!(
namespace = %key.namespace,
name = %key.name,
error = %e,
"aggregator: status patch failed"
);
} else {
tracing::debug!(
namespace = %key.namespace,
name = %key.name,
matched = aggregate.matched_device_count,
succeeded = aggregate.succeeded,
failed = aggregate.failed,
pending = aggregate.pending,
"aggregator: status patched"
);
}
}
Ok(())
}
/// Compute the aggregate for one Deployment from current caches.
/// `owned_targets` is the authoritative "currently selector-matched"
/// set for the deployment, as maintained by the watchers.
pub fn compute_aggregate(state: &FleetState, cached: &CachedDeployment) -> DeploymentAggregate {
let empty = HashSet::new();
let targets = state
.owned_targets
.get(&cached.deployment_name)
.unwrap_or(&empty);
let mut agg = DeploymentAggregate {
matched_device_count: targets.len() as u32,
..Default::default()
};
for device_id in targets {
let pair = DevicePair {
device_id: device_id.clone(),
deployment: cached.deployment_name.clone(),
};
match state.states.get(&pair).map(|s| s.phase) {
Some(Phase::Running) => agg.succeeded += 1,
Some(Phase::Failed) => agg.failed += 1,
Some(Phase::Pending) | None => agg.pending += 1,
}
}
agg.last_error = state.last_error.get(&cached.key).cloned();
agg
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use harmony_reconciler_contracts::Id;
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid test name")
}
fn state(device: &str, deployment: &str, phase: Phase, seconds: i64) -> DeploymentState {
DeploymentState {
device_id: Id::from(device.to_string()),
deployment: dn(deployment),
phase,
last_event_at: Utc.timestamp_opt(1_700_000_000 + seconds, 0).unwrap(),
last_error: None,
}
}
fn cached(namespace: &str, name: &str, match_key: &str, match_val: &str) -> CachedDeployment {
let mut ml = BTreeMap::new();
ml.insert(match_key.to_string(), match_val.to_string());
CachedDeployment {
key: DeploymentKey {
namespace: namespace.to_string(),
name: name.to_string(),
},
deployment_name: dn(name),
selector: LabelSelector {
match_labels: Some(ml),
match_expressions: None,
},
score_json: b"{}".to_vec(),
}
}
fn pair(device: &str, deployment: &str) -> DevicePair {
DevicePair {
device_id: device.to_string(),
deployment: dn(deployment),
}
}
#[test]
fn selector_match_labels_only() {
let mut ml = BTreeMap::new();
ml.insert("group".to_string(), "edge-a".to_string());
let sel = LabelSelector {
match_labels: Some(ml),
match_expressions: None,
};
let mut matching = BTreeMap::new();
matching.insert("group".to_string(), "edge-a".to_string());
matching.insert("arch".to_string(), "aarch64".to_string());
assert!(selector_matches(&sel, &matching));
let mut non_matching = BTreeMap::new();
non_matching.insert("group".to_string(), "edge-b".to_string());
assert!(!selector_matches(&sel, &non_matching));
let empty = BTreeMap::new();
assert!(!selector_matches(&sel, &empty));
}
#[test]
fn empty_selector_matches_everything() {
let sel = LabelSelector::default();
let mut labels = BTreeMap::new();
labels.insert("anything".to_string(), "goes".to_string());
assert!(selector_matches(&sel, &labels));
assert!(selector_matches(&sel, &BTreeMap::new()));
}
#[test]
fn compute_aggregate_counts_matched_devices() {
let cached = cached("fleet-demo", "hello", "group", "edge-a");
let key = cached.key.clone();
let mut s = FleetState::default();
s.deployments.insert(key, cached.clone());
// Three devices already in owned_targets (selector resolution
// is separate from the aggregate; aggregate reads owned_targets).
s.owned_targets.insert(
cached.deployment_name.clone(),
["pi-01", "pi-02", "pi-03"]
.iter()
.map(|s| s.to_string())
.collect(),
);
s.states.insert(
pair("pi-01", "hello"),
state("pi-01", "hello", Phase::Running, 0),
);
s.states.insert(
pair("pi-02", "hello"),
state("pi-02", "hello", Phase::Failed, 0),
);
// pi-03 has no state entry → pending
let agg = compute_aggregate(&s, &cached);
assert_eq!(agg.matched_device_count, 3);
assert_eq!(agg.succeeded, 1);
assert_eq!(agg.failed, 1);
assert_eq!(agg.pending, 1);
}
#[test]
fn matched_devices_picks_by_label() {
let mut ml = BTreeMap::new();
ml.insert("group".to_string(), "edge-a".to_string());
let sel = LabelSelector {
match_labels: Some(ml),
match_expressions: None,
};
let mut devices: HashMap<String, BTreeMap<String, String>> = HashMap::new();
let mut a = BTreeMap::new();
a.insert("group".to_string(), "edge-a".to_string());
devices.insert("pi-01".to_string(), a);
let mut b = BTreeMap::new();
b.insert("group".to_string(), "edge-b".to_string());
devices.insert("pi-02".to_string(), b);
let matched = matched_devices(&sel, &devices);
assert_eq!(matched.len(), 1);
assert!(matched.contains("pi-01"));
}
}

46
fleet/harmony-fleet-operator/src/install.rs Normal file
View File

@@ -0,0 +1,46 @@
//! Install the operator's CRD into a target Kubernetes cluster
//! via a harmony Score — no yaml generation, no kubectl shell-out.
//!
//! The Score is just [`K8sResourceScore`] over `Deployment::crd()`;
//! the topology is the shared `K8sBareTopology`, which exposes a
//! `K8sclient` backed by the caller's `KUBECONFIG` without dragging
//! in `K8sAnywhereTopology`'s product-level `ensure_ready`.
use anyhow::{Context, Result};
use harmony::inventory::Inventory;
use harmony::modules::k8s::K8sBareTopology;
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::score::Score;
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
use crate::crd::{Deployment, Device};
/// Apply the operator's CRDs to whatever cluster `KUBECONFIG` points
/// at. Returns once the apply call completes — does **not** wait for
/// the apiserver to mark the CRD `Established`; the caller does that
/// (e.g. with `kubectl wait --for=condition=Established`) if it
/// cares.
pub async fn install_crds() -> Result<()> {
let topology = K8sBareTopology::from_kubeconfig("harmony-fleet-operator-install")
.await
.map_err(|e| anyhow::anyhow!(e))
.context("building K8sBareTopology from KUBECONFIG")?;
let inventory = Inventory::empty();
let crds: Vec<CustomResourceDefinition> = vec![Deployment::crd(), Device::crd()];
let score = K8sResourceScore::<CustomResourceDefinition> {
resource: crds,
namespace: None,
};
let interpret = Score::<K8sBareTopology>::create_interpret(&score);
let outcome = interpret
.execute(&inventory, &topology)
.await
.map_err(|e| anyhow::anyhow!("install CRD: {e}"))
.context("executing K8sResourceScore for Deployment CRD")?;
tracing::info!(?outcome, "CRD installed");
Ok(())
}

11
fleet/harmony-fleet-operator/src/lib.rs Normal file
View File

@@ -0,0 +1,11 @@
//! Library surface of the IoT operator crate.
//!
//! Most of the crate is a binary (reconcile loop, install subcommand).
//! The CRD type definitions are exposed here as a library so external
//! consumers — tooling that applies CRs, tests, documentation generators
//! — can import the typed `Deployment`, `DeploymentSpec`,
//! `ScorePayload`, etc. without duplicating them.
pub mod crd;
pub mod device_reconciler;
pub mod fleet_aggregator;

146
fleet/harmony-fleet-operator/src/main.rs Normal file
View File

@@ -0,0 +1,146 @@
mod chart;
mod controller;
mod install;
use harmony_fleet_operator::{crd, device_reconciler, fleet_aggregator};
use anyhow::Result;
use async_nats::jetstream;
use clap::{Parser, Subcommand};
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
use kube::Client;
use std::path::PathBuf;
#[derive(Parser)]
#[command(
name = "harmony-fleet-operator",
about = "IoT operator — Deployment CRD → NATS KV"
)]
struct Cli {
#[command(subcommand)]
command: Option<Command>,
#[arg(
long,
env = "NATS_URL",
default_value = "nats://localhost:4222",
global = true
)]
nats_url: String,
#[arg(
long,
env = "KV_BUCKET",
default_value = BUCKET_DESIRED_STATE,
global = true
)]
kv_bucket: String,
}
#[derive(Subcommand)]
enum Command {
/// Run the controller (default when no subcommand is given).
Run,
/// Apply the operator's CRDs to the cluster `KUBECONFIG` points
/// at. Uses harmony's typed k8s client — no yaml, no kubectl.
Install,
/// Generate a helm chart directory that installs the operator
/// in-cluster (Deployment + RBAC + CRDs). Prints the written
/// chart path on success; `helm install <path>` takes it from
/// there. No registry publish — the chart lives on disk.
Chart {
#[arg(long, default_value = "/tmp/fleet-load-test/chart")]
output: PathBuf,
#[arg(long, default_value = "localhost/harmony-fleet-operator:latest")]
image: String,
#[arg(long, default_value = "IfNotPresent")]
image_pull_policy: String,
#[arg(long, default_value = "fleet-system")]
namespace: String,
#[arg(long, default_value = "nats://fleet-nats.fleet-system:4222")]
nats_url: String,
#[arg(long, default_value = "info,kube_runtime=warn")]
log_level: String,
},
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();
let cli = Cli::parse();
match cli.command.unwrap_or(Command::Run) {
Command::Install => install::install_crds().await,
Command::Run => run(&cli.nats_url, &cli.kv_bucket).await,
Command::Chart {
output,
image,
image_pull_policy,
namespace,
nats_url,
log_level,
} => {
let written = chart::build_chart(&chart::ChartOptions {
output_dir: output,
image,
image_pull_policy,
namespace,
nats_url,
log_level,
})?;
println!("{}", written.display());
Ok(())
}
}
}
async fn run(nats_url: &str, bucket: &str) -> Result<()> {
// Retry on the initial connect — startup races against the NATS
// server becoming fully ready.
let nats = connect_with_retry(nats_url).await?;
tracing::info!(url = %nats_url, "connected to NATS");
let js = jetstream::new(nats);
let desired_state_kv = js
.create_key_value(jetstream::kv::Config {
bucket: bucket.to_string(),
..Default::default()
})
.await?;
tracing::info!(bucket = %bucket, "KV bucket ready");
let client = Client::try_default().await?;
// Three concurrent tasks:
// controller — CR validation + finalizer-cleanup
// device_reconciler — NATS device-info → Device CR
// fleet_aggregator — watches Deployments + Devices + states,
// writes desired-state KV, patches CR status
// Any failing tears the process down; kube-rs Controller swallows
// its own transient reconcile errors.
let ctl_client = client.clone();
let dr_client = client.clone();
let dr_js = js.clone();
tokio::select! {
r = controller::run(ctl_client, desired_state_kv) => r,
r = device_reconciler::run(dr_client, dr_js) => r,
r = fleet_aggregator::run(client, js) => r,
}
}
async fn connect_with_retry(nats_url: &str) -> Result<async_nats::Client> {
use std::time::Duration;
let mut last_err: Option<anyhow::Error> = None;
for attempt in 0..15 {
match async_nats::connect(nats_url).await {
Ok(c) => return Ok(c),
Err(e) => {
tracing::warn!(attempt, error = %e, "NATS connect failed; retrying");
last_err = Some(e.into());
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
}
Err(last_err.unwrap_or_else(|| anyhow::anyhow!("NATS connect failed after retries")))
}

301
fleet/scripts/load-test.sh Executable file
View File

@@ -0,0 +1,301 @@
#!/usr/bin/env bash
# Load-test harness for the Harmony fleet operator's fleet_aggregator.
#
# Brings up the minimum stack (k3d + in-cluster NATS + CRD + operator)
# with no VM or real agent, then runs the `fleet_load_test` binary
# which simulates N devices pushing DeploymentState to NATS.
#
# All stable paths under $WORK_DIR (default /tmp/fleet-load-test) so you
# can point kubectl / tail at them while the test is running.
#
# Quick usage:
# fleet/scripts/load-test.sh # 100-device default (55 + 9×5)
# HOLD=1 fleet/scripts/load-test.sh # leave stack running for exploration
# DEVICES=10000 GROUP_SIZES=5500,500,500,500,500,500,500,500,500,500 \
# DURATION=90 fleet/scripts/load-test.sh
#
# While it's running, in another terminal:
# export KUBECONFIG=/tmp/fleet-load-test/kubeconfig
# kubectl get deployments.fleet.nationtech.io -A -w
# kubectl get deployments.fleet.nationtech.io -A \
# -o custom-columns=NAME:.metadata.name,RUN:.status.aggregate.succeeded,FAIL:.status.aggregate.failed,PEND:.status.aggregate.pending
# tail -f /tmp/fleet-load-test/operator.log
#
# Set DEBUG=1 to bump RUST_LOG so the operator logs every status patch.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
OPERATOR_DIR="$REPO_ROOT/fleet/harmony-fleet-operator"
# ---- config -----------------------------------------------------------------
K3D_BIN="${K3D_BIN:-$HOME/.local/share/harmony/k3d/k3d}"
CLUSTER_NAME="${CLUSTER_NAME:-fleet-load}"
NATS_NAMESPACE="${NATS_NAMESPACE:-fleet-system}"
NATS_NAME="${NATS_NAME:-fleet-nats}"
NATS_NODE_PORT="${NATS_NODE_PORT:-4222}"
NATS_IMAGE="${NATS_IMAGE:-docker.io/library/nats:2.10-alpine}"
DEVICES="${DEVICES:-100}"
GROUP_SIZES="${GROUP_SIZES:-55,5,5,5,5,5,5,5,5,5}"
TICK_MS="${TICK_MS:-1000}"
DURATION="${DURATION:-60}"
NAMESPACE="${NAMESPACE:-fleet-load}"
# Keep the stack alive after the test completes so the user can poke
# at CRs + NATS interactively. Ctrl-C to tear everything down.
HOLD="${HOLD:-0}"
# Stable working dir so kubectl + tail targets are predictable.
WORK_DIR="${WORK_DIR:-/tmp/fleet-load-test}"
mkdir -p "$WORK_DIR"
KUBECONFIG_FILE="$WORK_DIR/kubeconfig"
OPERATOR_LOG="$WORK_DIR/operator.log"
CHART_DIR="$WORK_DIR/chart"
OPERATOR_IMAGE="${OPERATOR_IMAGE:-localhost/harmony-fleet-operator:latest}"
OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-fleet-system}"
OPERATOR_RELEASE="${OPERATOR_RELEASE:-harmony-fleet-operator}"
OPERATOR_PID="" # unused in the helm path; kept so older trap-cleanup logic doesn't choke.
log() { printf '\033[1;34m[load-test]\033[0m %s\n' "$*"; }
fail() { printf '\033[1;31m[load-test FAIL]\033[0m %s\n' "$*" >&2; exit 1; }
dump_operator_log() {
[[ -n "$KUBECONFIG" && -f "$KUBECONFIG" ]] || return 0
kubectl -n "$OPERATOR_NAMESPACE" logs "deployment/$OPERATOR_RELEASE" \
--tail=1000 >"$OPERATOR_LOG" 2>/dev/null || true
}
cleanup() {
local rc=$?
log "cleanup…"
# Capture the operator's in-cluster log before we kill the
# cluster, so the tail-on-failure hook has something to show.
dump_operator_log
"$K3D_BIN" cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true
if [[ $rc -ne 0 && -s "$OPERATOR_LOG" ]]; then
log "operator log at $OPERATOR_LOG (kept for inspection)"
echo "----- operator log tail -----"
tail -n 60 "$OPERATOR_LOG" 2>/dev/null || true
elif [[ -s "$OPERATOR_LOG" ]]; then
log "operator log at $OPERATOR_LOG"
fi
exit $rc
}
trap cleanup EXIT INT TERM
require() { command -v "$1" >/dev/null 2>&1 || fail "missing required tool: $1"; }
require cargo
require kubectl
require podman
require docker
require helm
[[ -x "$K3D_BIN" ]] || fail "k3d binary not executable at $K3D_BIN"
# ---- phase 1: k3d cluster ---------------------------------------------------
log "phase 1: create k3d cluster '$CLUSTER_NAME' (host port $NATS_NODE_PORT → loadbalancer)"
"$K3D_BIN" cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true
"$K3D_BIN" cluster create "$CLUSTER_NAME" \
--wait --timeout 90s \
-p "${NATS_NODE_PORT}:${NATS_NODE_PORT}@loadbalancer" \
>/dev/null
"$K3D_BIN" kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG_FILE"
export KUBECONFIG="$KUBECONFIG_FILE"
# ---- phase 2: NATS in-cluster ------------------------------------------------
log "phase 2a: sideload NATS image ($NATS_IMAGE)"
if ! docker image inspect "$NATS_IMAGE" >/dev/null 2>&1; then
if ! podman image inspect "$NATS_IMAGE" >/dev/null 2>&1; then
podman pull "$NATS_IMAGE" >/dev/null || fail "podman pull $NATS_IMAGE failed"
fi
tmptar="$(mktemp -t nats-image.XXXXXX.tar)"
podman save "$NATS_IMAGE" -o "$tmptar" >/dev/null
docker load -i "$tmptar" >/dev/null
rm -f "$tmptar"
fi
"$K3D_BIN" image import "$NATS_IMAGE" -c "$CLUSTER_NAME" >/dev/null
log "phase 2b: install NATS via NatsBasicScore"
(
cd "$REPO_ROOT"
cargo run -q --release -p example_fleet_nats_install -- \
--namespace "$NATS_NAMESPACE" \
--name "$NATS_NAME" \
--expose load-balancer
)
# The upstream nats/nats helm chart provisions a StatefulSet, not a
# Deployment. Waiting on the pod-label condition works across both
# shapes without hardcoding a workload kind.
kubectl -n "$NATS_NAMESPACE" wait --for=condition=Ready \
"pod" -l "app.kubernetes.io/name=nats" --timeout=180s >/dev/null
log "probing nats://localhost:$NATS_NODE_PORT end-to-end"
for _ in $(seq 1 60); do
(echo >"/dev/tcp/127.0.0.1/$NATS_NODE_PORT") 2>/dev/null && break
sleep 1
done
(echo >"/dev/tcp/127.0.0.1/$NATS_NODE_PORT") 2>/dev/null \
|| fail "TCP localhost:$NATS_NODE_PORT never came up"
# ---- phase 3: operator container image + helm install ---------------------
log "phase 3a: build operator release binary"
(
cd "$REPO_ROOT"
cargo build -q --release -p harmony-fleet-operator
)
log "phase 3b: build container image $OPERATOR_IMAGE"
# The workspace's top-level .dockerignore excludes target/, which is
# the right default for most container builds but exactly what we
# need here. Stage the release binary into a dedicated clean build
# context so the Dockerfile's COPY sees it.
IMAGE_CTX="$WORK_DIR/image-ctx"
rm -rf "$IMAGE_CTX"
mkdir -p "$IMAGE_CTX/target/release"
cp "$REPO_ROOT/target/release/harmony-fleet-operator" "$IMAGE_CTX/target/release/harmony-fleet-operator"
cp "$REPO_ROOT/fleet/harmony-fleet-operator/Dockerfile" "$IMAGE_CTX/Dockerfile"
podman build -q -t "$OPERATOR_IMAGE" "$IMAGE_CTX" >/dev/null
log "phase 3c: sideload operator image into k3d cluster"
tmptar="$(mktemp -t harmony-fleet-operator-image.XXXXXX.tar)"
podman save "$OPERATOR_IMAGE" -o "$tmptar" >/dev/null
docker load -i "$tmptar" >/dev/null
rm -f "$tmptar"
"$K3D_BIN" image import "$OPERATOR_IMAGE" -c "$CLUSTER_NAME" >/dev/null
log "phase 3d: generate helm chart + install operator in-cluster"
# DEBUG=1 bumps operator logging so `kubectl logs` prints every
# status patch + transition.
if [[ "${DEBUG:-0}" == "1" ]]; then
OPERATOR_RUST_LOG="debug,async_nats=warn,hyper=warn,rustls=warn,kube=info"
else
OPERATOR_RUST_LOG="info,kube_runtime=warn"
fi
rm -rf "$CHART_DIR"
mkdir -p "$CHART_DIR"
(
cd "$OPERATOR_DIR"
cargo run -q -- chart \
--output "$CHART_DIR" \
--image "$OPERATOR_IMAGE" \
--image-pull-policy IfNotPresent \
--namespace "$OPERATOR_NAMESPACE" \
--nats-url "nats://${NATS_NAME}.${NATS_NAMESPACE}:4222" \
--log-level "$OPERATOR_RUST_LOG"
) >/dev/null
helm upgrade --install "$OPERATOR_RELEASE" "$CHART_DIR/$OPERATOR_RELEASE" \
--namespace "$OPERATOR_NAMESPACE" \
--create-namespace \
--wait --timeout 120s >/dev/null
kubectl wait --for=condition=Established \
"crd/deployments.fleet.nationtech.io" --timeout=30s >/dev/null
kubectl wait --for=condition=Established \
"crd/devices.fleet.nationtech.io" --timeout=30s >/dev/null
kubectl -n "$OPERATOR_NAMESPACE" wait --for=condition=Available \
"deployment/$OPERATOR_RELEASE" --timeout=120s >/dev/null
# Seed the operator log file from the pod so HOLD=1 banner + final
# summary both have something to read. We re-dump on cleanup.
dump_operator_log
# ---- explore banner (before the load run so the user can start watching) ----
print_banner() {
cat <<EOF
$(printf '\033[1;32m[load-test]\033[0m stack ready. In another terminal:')
$(printf '\033[1mPoint kubectl at the k3d cluster:\033[0m')
export KUBECONFIG=$KUBECONFIG_FILE
$(printf '\033[1mWatch CRs as they update:\033[0m')
kubectl -n $NAMESPACE get deployments.fleet.nationtech.io -w
$(printf '\033[1mSnapshot aggregate columns:\033[0m')
kubectl -n $NAMESPACE get deployments.fleet.nationtech.io \\
-o custom-columns=NAME:.metadata.name,MATCHED:.status.aggregate.matchedDeviceCount,OK:.status.aggregate.succeeded,FAIL:.status.aggregate.failed,PEND:.status.aggregate.pending,LAST_ERR:.status.aggregate.lastError.message
$(printf '\033[1mInspect a Deployment spec (no device list — selector only):\033[0m')
kubectl -n $NAMESPACE get deployments.fleet.nationtech.io/load-group-00 -o jsonpath='{.spec}' | jq
$(printf '\033[1mFull CR status JSON for one CR:\033[0m')
kubectl -n $NAMESPACE get deployments.fleet.nationtech.io/load-group-00 -o jsonpath='{.status.aggregate}' | jq
$(printf '\033[1mList Devices + filter by label:\033[0m')
kubectl get devices.fleet.nationtech.io | head -20
kubectl get devices.fleet.nationtech.io -l group=load-group-00 | head -10
kubectl get device.fleet.nationtech.io load-dev-00001 -o yaml
$(printf '\033[1mOperator log (in-cluster pod):\033[0m')
kubectl -n $OPERATOR_NAMESPACE logs -f deployment/$OPERATOR_RELEASE
# or the last snapshot dumped by the harness:
tail -F $OPERATOR_LOG
$(printf '\033[1mPeek at NATS KV directly (natsbox):\033[0m')
alias natsbox='podman run --rm docker.io/natsio/nats-box:latest nats --server nats://host.containers.internal:$NATS_NODE_PORT'
natsbox kv ls device-state
natsbox kv get device-state 'state.load-dev-00001.load-group-00' --raw
natsbox kv ls device-heartbeat
natsbox kv get device-heartbeat 'heartbeat.load-dev-00001' --raw
EOF
}
print_banner
# ---- phase 5: load test ------------------------------------------------------
log "phase 5: run fleet_load_test (devices=$DEVICES, tick=${TICK_MS}ms, duration=${DURATION}s)"
(
cd "$REPO_ROOT"
cargo build -q --release -p example_fleet_load_test
)
# `--no-cleanup` keeps the CRs + KV entries around after the run so
# you can inspect steady-state aggregate numbers after duration elapses.
LOAD_ARGS=(
--nats-url "nats://localhost:$NATS_NODE_PORT"
--namespace "$NAMESPACE"
--groups "$GROUP_SIZES"
--tick-ms "$TICK_MS"
--duration-s "$DURATION"
)
if [[ "$HOLD" == "1" ]]; then
LOAD_ARGS+=(--keep)
fi
RUST_LOG="info" "$REPO_ROOT/target/release/fleet_load_test" "${LOAD_ARGS[@]}"
# ---- phase 6: operator log stats --------------------------------------------
log "phase 6: operator log summary"
dump_operator_log
patches="$(grep -c "aggregator: status patched" "$OPERATOR_LOG" 2>/dev/null || echo 0)"
warnings="$(grep -c " WARN " "$OPERATOR_LOG" 2>/dev/null || echo 0)"
errors="$(grep -c " ERROR " "$OPERATOR_LOG" 2>/dev/null || echo 0)"
log " CR status patches logged (DEBUG-level; use DEBUG=1 to surface): $patches"
log " operator warnings: $warnings errors: $errors"
if [[ "$errors" -gt 0 ]]; then
echo "----- operator error lines -----"
grep " ERROR " "$OPERATOR_LOG" | tail -20
fi
# ---- hold open (optional) ---------------------------------------------------
if [[ "$HOLD" == "1" ]]; then
print_banner
log "HOLD=1 — stack is still running. Ctrl-C to tear down."
# Block until user interrupts; cleanup trap does the teardown.
while true; do sleep 60; done
fi
log "PASS"

42
iot/scripts/smoke-a1.sh → fleet/scripts/smoke-a1.sh
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# End-to-end smoke test for the IoT walking skeleton (ROADMAP/iot_platform/
# End-to-end smoke test for the IoT walking skeleton (ROADMAP/fleet_platform/
# v0_walking_skeleton.md §9.A1 and §5.4 agent dispatch).
#
# Deployment CR ─apply─▶ operator ─KV put─▶ NATS ◀─watch─ agent ─podman─▶ nginx
@@ -22,25 +22,25 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
OPERATOR_DIR="$REPO_ROOT/iot/iot-operator-v0"
AGENT_DIR="$REPO_ROOT/iot/iot-agent-v0"
OPERATOR_DIR="$REPO_ROOT/fleet/harmony-fleet-operator"
AGENT_DIR="$REPO_ROOT/fleet/harmony-fleet-agent"
K3D_BIN="${K3D_BIN:-$HOME/.local/share/harmony/k3d/k3d}"
CLUSTER_NAME="${CLUSTER_NAME:-iot-smoke}"
NATS_CONTAINER="${NATS_CONTAINER:-iot-smoke-nats}"
NATS_NET_NAME="${NATS_NET_NAME:-iot-smoke-net}"
CLUSTER_NAME="${CLUSTER_NAME:-fleet-smoke}"
NATS_CONTAINER="${NATS_CONTAINER:-fleet-smoke-nats}"
NATS_NET_NAME="${NATS_NET_NAME:-fleet-smoke-net}"
NATS_IMAGE="${NATS_IMAGE:-docker.io/library/nats:2.10-alpine}"
NATSBOX_IMAGE="${NATSBOX_IMAGE:-docker.io/natsio/nats-box:latest}"
NATS_PORT="${NATS_PORT:-4222}"
TARGET_DEVICE="${TARGET_DEVICE:-pi-demo-01}"
DEPLOY_NAME="${DEPLOY_NAME:-hello-world}"
DEPLOY_NS="${DEPLOY_NS:-iot-demo}"
DEPLOY_NS="${DEPLOY_NS:-fleet-demo}"
HELLO_CONTAINER="${HELLO_CONTAINER:-hello}"
HELLO_PORT="${HELLO_PORT:-8080}"
OPERATOR_LOG="$(mktemp -t iot-operator.XXXXXX.log)"
OPERATOR_LOG="$(mktemp -t harmony-fleet-operator.XXXXXX.log)"
OPERATOR_PID=""
AGENT_LOG="$(mktemp -t iot-agent.XXXXXX.log)"
AGENT_LOG="$(mktemp -t fleet-agent.XXXXXX.log)"
AGENT_PID=""
AGENT_CONFIG_FILE=""
KUBECONFIG_FILE=""
@@ -126,13 +126,13 @@ log "phase 2: create k3d cluster '$CLUSTER_NAME'"
"$K3D_BIN" cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true
"$K3D_BIN" cluster create "$CLUSTER_NAME" --wait --timeout 90s >/dev/null
KUBECONFIG_FILE="$(mktemp -t iot-smoke-kubeconfig.XXXXXX)"
KUBECONFIG_FILE="$(mktemp -t fleet-smoke-kubeconfig.XXXXXX)"
"$K3D_BIN" kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG_FILE"
export KUBECONFIG="$KUBECONFIG_FILE"
log "install CRD via operator's install subcommand (typed Rust — no yaml, no kubectl apply)"
( cd "$OPERATOR_DIR" && cargo run -q -- install ) >/dev/null
kubectl wait --for=condition=Established "crd/deployments.iot.nationtech.io" --timeout=30s >/dev/null
kubectl wait --for=condition=Established "crd/deployments.fleet.nationtech.io" --timeout=30s >/dev/null
kubectl get ns "$DEPLOY_NS" >/dev/null 2>&1 || kubectl create namespace "$DEPLOY_NS" >/dev/null
@@ -142,7 +142,7 @@ kubectl get ns "$DEPLOY_NS" >/dev/null 2>&1 || kubectl create namespace "$DEPLOY
###############################################################################
log "phase 2b: apiserver rejects invalid score.type"
BAD_CR=$(cat <<EOF
apiVersion: iot.nationtech.io/v1alpha1
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: bad-discriminator
@@ -163,8 +163,8 @@ else
fail "expected CEL rejection for score.type='has spaces'; got: $BAD_OUT"
fi
# Belt-and-braces: make sure nothing was persisted
if kubectl -n "$DEPLOY_NS" get deployment.iot.nationtech.io bad-discriminator >/dev/null 2>&1; then
kubectl -n "$DEPLOY_NS" delete deployment.iot.nationtech.io bad-discriminator >/dev/null 2>&1 || true
if kubectl -n "$DEPLOY_NS" get deployment.fleet.nationtech.io bad-discriminator >/dev/null 2>&1; then
kubectl -n "$DEPLOY_NS" delete deployment.fleet.nationtech.io bad-discriminator >/dev/null 2>&1 || true
fail "apiserver should have rejected 'bad-discriminator' but it was persisted"
fi
@@ -179,7 +179,7 @@ log "phase 3: start operator"
NATS_URL="nats://127.0.0.1:$NATS_PORT" \
KV_BUCKET="desired-state" \
RUST_LOG="info,kube_runtime=warn" \
"$REPO_ROOT/target/debug/iot-operator-v0" \
"$REPO_ROOT/target/debug/harmony-fleet-operator" \
>"$OPERATOR_LOG" 2>&1 &
OPERATOR_PID=$!
log "operator pid=$OPERATOR_PID (log: $OPERATOR_LOG)"
@@ -207,7 +207,7 @@ log "phase 3b: build + start agent"
# doesn't occupy the host port before we even start.
podman rm -f "$HELLO_CONTAINER" >/dev/null 2>&1 || true
AGENT_CONFIG_FILE="$(mktemp -t iot-agent-config.XXXXXX.toml)"
AGENT_CONFIG_FILE="$(mktemp -t fleet-agent-config.XXXXXX.toml)"
cat >"$AGENT_CONFIG_FILE" <<EOF
[agent]
device_id = "$TARGET_DEVICE"
@@ -221,9 +221,9 @@ nats_pass = "smoke"
urls = ["nats://127.0.0.1:$NATS_PORT"]
EOF
IOT_AGENT_CONFIG="$AGENT_CONFIG_FILE" \
FLEET_AGENT_CONFIG="$AGENT_CONFIG_FILE" \
RUST_LOG="info,async_nats=warn" \
"$REPO_ROOT/target/debug/iot-agent-v0" \
"$REPO_ROOT/target/debug/harmony-fleet-agent" \
>"$AGENT_LOG" 2>&1 &
AGENT_PID=$!
log "agent pid=$AGENT_PID (log: $AGENT_LOG)"
@@ -241,7 +241,7 @@ grep -q "watching KV keys" "$AGENT_LOG" \
###############################################################################
log "phase 4: apply Deployment CR"
cat <<EOF | kubectl apply -f - >/dev/null
apiVersion: iot.nationtech.io/v1alpha1
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: $DEPLOY_NAME
@@ -276,7 +276,7 @@ echo "$KV_VALUE" | grep -q '"image":"docker.io/library/nginx:alpine"' \
log "wait for .status.observedScoreString"
OBSERVED=""
for _ in $(seq 1 30); do
OBSERVED="$(kubectl -n "$DEPLOY_NS" get deployment.iot.nationtech.io "$DEPLOY_NAME" \
OBSERVED="$(kubectl -n "$DEPLOY_NS" get deployment.fleet.nationtech.io "$DEPLOY_NAME" \
-o jsonpath='{.status.observedScoreString}' 2>/dev/null || true)"
[[ -n "$OBSERVED" ]] && break
sleep 1
@@ -315,7 +315,7 @@ log "nginx responded"
# phase 5 — delete CR, expect cleanup via finalizer + agent
###############################################################################
log "phase 5: delete Deployment CR — finalizer + agent should remove KV and container"
kubectl -n "$DEPLOY_NS" delete deployment.iot.nationtech.io "$DEPLOY_NAME" --wait=true >/dev/null
kubectl -n "$DEPLOY_NS" delete deployment.fleet.nationtech.io "$DEPLOY_NAME" --wait=true >/dev/null
log "wait for KV key removal"
for _ in $(seq 1 30); do

8
iot/scripts/smoke-a3-arm.sh → fleet/scripts/smoke-a3-arm.sh
View File

@@ -4,7 +4,7 @@
# native KVM when the host is already arm64).
#
# This is exactly equivalent to:
# ARCH=aarch64 VM_NAME=iot-smoke-vm-arm ./smoke-a3.sh
# ARCH=aarch64 VM_NAME=fleet-smoke-vm-arm ./smoke-a3.sh
# with the VM name defaulted so it can live alongside an x86-64
# smoke run on the same host without clobbering libvirt state.
@@ -13,9 +13,9 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export ARCH=aarch64
export VM_NAME="${VM_NAME:-iot-smoke-vm-arm}"
export VM_NAME="${VM_NAME:-fleet-smoke-vm-arm}"
export DEVICE_ID="${DEVICE_ID:-$VM_NAME}"
export NATS_CONTAINER="${NATS_CONTAINER:-iot-smoke-nats-a3-arm}"
export NATS_NET_NAME="${NATS_NET_NAME:-iot-smoke-net-a3-arm}"
export NATS_CONTAINER="${NATS_CONTAINER:-fleet-smoke-nats-a3-arm}"
export NATS_NET_NAME="${NATS_NET_NAME:-fleet-smoke-net-a3-arm}"
exec "$SCRIPT_DIR/smoke-a3.sh" "$@"

38
iot/scripts/smoke-a3.sh → fleet/scripts/smoke-a3.sh
View File

@@ -6,7 +6,7 @@
# ssh+Ansible ◀────┘
# │
# ▼
# IotDeviceSetupScore ──▶ podman + iot-agent on VM
# FleetDeviceSetupScore ──▶ podman + fleet-agent on VM
# │
# ▼
# existing operator ──NATS────────┘ (agent joins fleet, reconciles CR)
@@ -32,7 +32,7 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
VM_NAME="${VM_NAME:-iot-smoke-vm}"
VM_NAME="${VM_NAME:-fleet-smoke-vm}"
DEVICE_ID="${DEVICE_ID:-$VM_NAME}"
GROUP="${GROUP:-group-a}"
LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"
@@ -43,8 +43,8 @@ LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"
# target, phase 4 timeout.
ARCH="${ARCH:-x86-64}"
NATS_CONTAINER="${NATS_CONTAINER:-iot-smoke-nats-a3}"
NATS_NET_NAME="${NATS_NET_NAME:-iot-smoke-net-a3}"
NATS_CONTAINER="${NATS_CONTAINER:-fleet-smoke-nats-a3}"
NATS_NET_NAME="${NATS_NET_NAME:-fleet-smoke-net-a3}"
NATS_IMAGE="${NATS_IMAGE:-docker.io/library/nats:2.10-alpine}"
NATS_PORT="${NATS_PORT:-4222}"
@@ -99,20 +99,20 @@ NAT_GW="$(virsh --connect "$LIBVIRT_URI" net-dumpxml default \
log "libvirt network gateway = $NAT_GW (VM will dial NATS at nats://$NAT_GW:$NATS_PORT)"
# ---------------------------- phase 2: build ---------------------------
log "phase 2: build iot-agent-v0 for guest arch=$ARCH (release — debug binary fills cloud rootfs)"
log "phase 2: build harmony-fleet-agent for guest arch=$ARCH (release — debug binary fills cloud rootfs)"
(
cd "$REPO_ROOT"
if [[ -n "$AGENT_TARGET" ]]; then
rustup target add "$AGENT_TARGET" >/dev/null
cargo build -q --release --target "$AGENT_TARGET" -p iot-agent-v0
cargo build -q --release --target "$AGENT_TARGET" -p harmony-fleet-agent
else
cargo build -q --release -p iot-agent-v0
cargo build -q --release -p harmony-fleet-agent
fi
)
if [[ -n "$AGENT_TARGET" ]]; then
AGENT_BINARY="$REPO_ROOT/target/$AGENT_TARGET/release/iot-agent-v0"
AGENT_BINARY="$REPO_ROOT/target/$AGENT_TARGET/release/harmony-fleet-agent"
else
AGENT_BINARY="$REPO_ROOT/target/release/iot-agent-v0"
AGENT_BINARY="$REPO_ROOT/target/release/harmony-fleet-agent"
fi
[[ -f "$AGENT_BINARY" ]] || fail "agent binary missing after build: $AGENT_BINARY"
@@ -120,7 +120,7 @@ fi
log "phase 3: bootstrap assets + provision VM + onboard device (arch=$EXAMPLE_ARCH)"
(
cd "$REPO_ROOT"
cargo run -q --release -p example_iot_vm_setup -- \
cargo run -q --release -p example_fleet_vm_setup -- \
--arch "$EXAMPLE_ARCH" \
--vm-name "$VM_NAME" \
--device-id "$DEVICE_ID" \
@@ -136,34 +136,34 @@ case "$ARCH" in
aarch64|arm64) STATUS_TIMEOUT=300 ;;
*) STATUS_TIMEOUT=60 ;;
esac
log "phase 4: wait for agent to report status to NATS (timeout=${STATUS_TIMEOUT}s)"
log "phase 4: wait for agent to report heartbeat to NATS (timeout=${STATUS_TIMEOUT}s)"
wait_for_status() {
local timeout=$1
for _ in $(seq 1 "$timeout"); do
if podman run --rm --network "$NATS_NET_NAME" \
docker.io/natsio/nats-box:latest \
nats --server "nats://$NATS_CONTAINER:4222" kv get agent-status \
"status.$DEVICE_ID" --raw >/dev/null 2>&1; then
nats --server "nats://$NATS_CONTAINER:4222" kv get device-heartbeat \
"heartbeat.$DEVICE_ID" --raw >/dev/null 2>&1; then
return 0
fi
sleep 1
done
return 1
}
wait_for_status "$STATUS_TIMEOUT" || fail "agent-status never appeared for $DEVICE_ID"
log "agent status present on NATS"
wait_for_status "$STATUS_TIMEOUT" || fail "device-heartbeat never appeared for $DEVICE_ID"
log "agent heartbeat present on NATS"
# ---------------------------- phase 5: hard power-cycle, expect recovery ----------------------------
log "phase 5: power-cycle VM (virsh destroy + start) → agent must reconnect to NATS"
nats_status_timestamp() {
# Prints the "timestamp" field of the status.<device> entry, or "".
# Prints the "at" field of the heartbeat.<device> entry, or "".
# Never errors (for `set -e` safety).
podman run --rm --network "$NATS_NET_NAME" \
docker.io/natsio/nats-box:latest \
nats --server "nats://$NATS_CONTAINER:4222" kv get agent-status \
"status.$DEVICE_ID" --raw 2>/dev/null \
| grep -oE '"timestamp":"[^"]+"' \
nats --server "nats://$NATS_CONTAINER:4222" kv get device-heartbeat \
"heartbeat.$DEVICE_ID" --raw 2>/dev/null \
| grep -oE '"at":"[^"]+"' \
| head -1 | cut -d'"' -f4 || true
}

529
fleet/scripts/smoke-a4.sh Executable file
View File

@@ -0,0 +1,529 @@
#!/usr/bin/env bash
# End-to-end hands-on demo: operator + in-cluster NATS + ARM VM agent.
#
# [k3d cluster]
# ├── NATS (single-node, NodePort 4222)
# └── CRD: fleet.nationtech.io/v1alpha1/Deployment
# ▲
# │ kubectl apply / harmony_apply_deployment
# │
# [host]
# ├── operator (cargo run) ──▶ NATS KV desired-state
# └── libvirt VM
# └── fleet-agent ──▶ NATS KV (watch) ──▶ podman container
#
# By default the script brings the whole stack up, applies no
# Deployment CR, prints a "command menu" of user-runnable one-liners,
# and blocks on Ctrl-C. With `--auto`, it also drives an apply +
# upgrade + delete cycle for regression coverage.
#
# Prereqs on the runner host (one-time, generic):
# 1. podman (rootless), cargo, kubectl, virsh, xorriso, python3,
# libvirt, qemu-system-x86_64/aarch64 + edk2 firmware for the
# chosen ARCH.
# 2. Be in the `libvirt` group.
# 3. `sudo virsh net-start default` (once per boot unless autostart).
# 4. Rootless podman user socket running:
# `systemctl --user start podman.socket`.
# 5. k3d binary at $K3D_BIN (defaults to Harmony's downloaded copy).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
OPERATOR_DIR="$REPO_ROOT/fleet/harmony-fleet-operator"
# ---- config -----------------------------------------------------------------
K3D_BIN="${K3D_BIN:-$HOME/.local/share/harmony/k3d/k3d}"
CLUSTER_NAME="${CLUSTER_NAME:-fleet-demo}"
ARCH="${ARCH:-x86-64}"
VM_NAME="${VM_NAME:-fleet-demo-vm}"
DEVICE_ID="${DEVICE_ID:-$VM_NAME}"
GROUP="${GROUP:-group-a}"
LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"
NATS_NAMESPACE="${NATS_NAMESPACE:-fleet-system}"
NATS_NAME="${NATS_NAME:-fleet-nats}"
NATS_NODE_PORT="${NATS_NODE_PORT:-4222}"
DEPLOY_NS="${DEPLOY_NS:-fleet-demo}"
DEPLOY_NAME="${DEPLOY_NAME:-hello-world}"
DEPLOY_PORT="${DEPLOY_PORT:-8080:80}"
# Source image we sideload into the VM's podman. Defaults to the
# `nginx:alpine` variant (~60 MB) which is almost always cached on
# dev boxes and keeps TCG-aarch64 boot budgets sane. The tarball
# transport + podman IfNotPresent semantics mean the agent never
# hits a public registry for this image.
SRC_IMAGE="${SRC_IMAGE:-docker.io/library/nginx:alpine}"
AUTO=0
[[ "${1:-}" == "--auto" ]] && AUTO=1
OPERATOR_LOG="$(mktemp -t harmony-fleet-operator.XXXXXX.log)"
OPERATOR_PID=""
KUBECONFIG_FILE=""
# ---- arch demux -------------------------------------------------------------
case "$ARCH" in
x86-64|x86_64)
EXAMPLE_ARCH=x86-64
AGENT_TARGET=
# Native-KVM x86: podman pull + layer unpack is seconds.
CONTAINER_WAIT_STEPS=90 # 180 s
;;
aarch64|arm64)
EXAMPLE_ARCH=aarch64
AGENT_TARGET=aarch64-unknown-linux-gnu
# TCG aarch64: network stack + userns layer unpack run
# ~3-5× slower than native. An `nginx:latest` pull (~250 MB)
# on a cold image takes 4-8 min observed here. Give it 15.
CONTAINER_WAIT_STEPS=450 # 900 s
;;
*) printf '[smoke-a4 FAIL] unsupported ARCH=%s (expected: x86-64 | aarch64)\n' "$ARCH" >&2; exit 1 ;;
esac
log() { printf '\033[1;34m[smoke-a4]\033[0m %s\n' "$*"; }
fail() { printf '\033[1;31m[smoke-a4 FAIL]\033[0m %s\n' "$*" >&2; exit 1; }
cleanup() {
local rc=$?
log "cleanup…"
if [[ -n "$OPERATOR_PID" ]] && kill -0 "$OPERATOR_PID" 2>/dev/null; then
kill "$OPERATOR_PID" 2>/dev/null || true
wait "$OPERATOR_PID" 2>/dev/null || true
fi
if [[ "${KEEP:-0}" != "1" ]]; then
virsh --connect "$LIBVIRT_URI" destroy "$VM_NAME" 2>/dev/null || true
virsh --connect "$LIBVIRT_URI" undefine --nvram \
--remove-all-storage "$VM_NAME" 2>/dev/null || true
"$K3D_BIN" cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true
[[ -n "$KUBECONFIG_FILE" ]] && rm -f "$KUBECONFIG_FILE"
else
log "KEEP=1 — leaving cluster '$CLUSTER_NAME' and VM '$VM_NAME' running"
[[ -n "$KUBECONFIG_FILE" ]] && log "KUBECONFIG=$KUBECONFIG_FILE"
fi
if [[ $rc -ne 0 && -s "$OPERATOR_LOG" ]]; then
log "operator log at $OPERATOR_LOG"
echo "----- operator log tail -----"
tail -n 40 "$OPERATOR_LOG" 2>/dev/null || true
else
rm -f "$OPERATOR_LOG"
fi
exit $rc
}
trap cleanup EXIT INT TERM
require() { command -v "$1" >/dev/null 2>&1 || fail "missing required tool: $1"; }
require cargo
require kubectl
require virsh
require podman
require docker # cross-runtime image transfer for k3d sideload
[[ -x "$K3D_BIN" ]] || fail "k3d binary not executable at $K3D_BIN (set K3D_BIN=…)"
# ---- phase 1: k3d cluster with NATS port exposed ----------------------------
log "phase 1: create k3d cluster '$CLUSTER_NAME' (host port $NATS_NODE_PORT → loadbalancer)"
"$K3D_BIN" cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true
"$K3D_BIN" cluster create "$CLUSTER_NAME" \
--wait --timeout 90s \
-p "${NATS_NODE_PORT}:${NATS_NODE_PORT}@loadbalancer" \
>/dev/null
KUBECONFIG_FILE="$(mktemp -t fleet-demo-kubeconfig.XXXXXX)"
"$K3D_BIN" kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG_FILE"
export KUBECONFIG="$KUBECONFIG_FILE"
# ---- phase 2: NATS in-cluster via NatsBasicScore ----------------------------
NATS_IMAGE="${NATS_IMAGE:-docker.io/library/nats:2.10-alpine}"
# Sideload the NATS image into k3d so the install doesn't race the
# Docker Hub rate limiter. `docker inspect` + `podman save` + `docker
# load` is the cross-runtime bridge on hosts that have both (rootful
# docker for k3d, rootless podman for IoT smokes). Cheap when the
# image is already in podman's store; a one-time Hub pull when not.
log "phase 2a: sideload NATS image ($NATS_IMAGE) into k3d cluster"
if ! docker image inspect "$NATS_IMAGE" >/dev/null 2>&1; then
if ! podman image inspect "$NATS_IMAGE" >/dev/null 2>&1; then
log "NATS image not cached locally — pulling from Docker Hub"
podman pull "$NATS_IMAGE" >/dev/null || fail "podman pull $NATS_IMAGE failed"
fi
tmptar="$(mktemp -t nats-image.XXXXXX.tar)"
podman save "$NATS_IMAGE" -o "$tmptar" >/dev/null
docker load -i "$tmptar" >/dev/null
rm -f "$tmptar"
fi
"$K3D_BIN" image import "$NATS_IMAGE" -c "$CLUSTER_NAME" >/dev/null
log "phase 2b: install NATS in-cluster via NatsBasicScore (namespace=$NATS_NAMESPACE, expose=load-balancer)"
(
cd "$REPO_ROOT"
cargo run -q --release -p example_fleet_nats_install -- \
--namespace "$NATS_NAMESPACE" \
--name "$NATS_NAME" \
--expose load-balancer
)
log "waiting for NATS Deployment to be Available"
kubectl -n "$NATS_NAMESPACE" wait --for=condition=Available \
"deployment/$NATS_NAME" --timeout=120s >/dev/null
# kubectl "Available" reports on pod readiness — k3d's klipper-lb
# takes a further few seconds to wire the host loadbalancer port to
# the Service endpoints. Probe the actual TCP port from the host
# before declaring NATS routable, else the operator's connect will
# race and die with "expected INFO, got nothing."
log "probing nats://localhost:$NATS_NODE_PORT end-to-end"
for _ in $(seq 1 60); do
if (echo >"/dev/tcp/127.0.0.1/$NATS_NODE_PORT") 2>/dev/null; then
break
fi
sleep 1
done
(echo >"/dev/tcp/127.0.0.1/$NATS_NODE_PORT") 2>/dev/null \
|| fail "TCP localhost:$NATS_NODE_PORT never came up after Deployment Available"
# ---- phase 3: install Deployment CRD via operator's Score-based install -----
log "phase 3: install Deployment CRD via operator \`install\` subcommand"
(
cd "$OPERATOR_DIR"
cargo run -q -- install
)
kubectl wait --for=condition=Established \
"crd/deployments.fleet.nationtech.io" --timeout=30s >/dev/null
kubectl get ns "$DEPLOY_NS" >/dev/null 2>&1 || \
kubectl create namespace "$DEPLOY_NS" >/dev/null
# ---- phase 4: operator running host-side ------------------------------------
log "phase 4: start operator (host-side) connected to nats://localhost:$NATS_NODE_PORT"
(
cd "$OPERATOR_DIR"
cargo build -q --release
)
NATS_URL="nats://localhost:$NATS_NODE_PORT" \
KV_BUCKET="desired-state" \
RUST_LOG="info,kube_runtime=warn" \
"$REPO_ROOT/target/release/harmony-fleet-operator" \
>"$OPERATOR_LOG" 2>&1 &
OPERATOR_PID=$!
log "operator pid=$OPERATOR_PID (log: $OPERATOR_LOG)"
for _ in $(seq 1 30); do
if grep -q "starting Deployment controller" "$OPERATOR_LOG"; then break; fi
if ! kill -0 "$OPERATOR_PID" 2>/dev/null; then fail "operator exited early"; fi
sleep 0.5
done
grep -q "starting Deployment controller" "$OPERATOR_LOG" \
|| fail "operator never logged 'starting Deployment controller'"
grep -q "KV bucket ready" "$OPERATOR_LOG" \
|| fail "operator never confirmed KV bucket ready"
# ---- phase 4.5: export the workload image to a tarball ----------------------
# Instead of running a local OCI registry (which needs `registry:2` from
# Docker Hub — rate-limited!), sideload the image straight into the VM's
# podman via `podman save`/`scp`/`podman load`. Paired with harmony's
# `PodmanTopology::ensure_image_present` (IfNotPresent semantics: present
# = skip pull), the agent never touches a public registry for known
# images. This is the same compounding-framework-value move as the k3d
# NATS sideload in phase 2a.
NAT_GW="$(virsh --connect "$LIBVIRT_URI" net-dumpxml default \
| grep -oP "ip address='\K[^']+" | head -1)"
[[ -n "$NAT_GW" ]] || fail "couldn't determine libvirt 'default' gateway IP"
log "libvirt network gateway = $NAT_GW (VM agent will dial nats://$NAT_GW:$NATS_NODE_PORT)"
log "phase 4.5: export $SRC_IMAGE to a local tarball for VM sideload"
# Arch the VM expects.
case "$ARCH" in
x86-64|x86_64) EXPECTED_IMAGE_ARCH=amd64 ;;
aarch64|arm64) EXPECTED_IMAGE_ARCH=arm64 ;;
esac
if ! podman image inspect "$SRC_IMAGE" >/dev/null 2>&1; then
log "source image $SRC_IMAGE not cached — attempting pull (platform=$EXPECTED_IMAGE_ARCH)"
podman pull --platform="linux/$EXPECTED_IMAGE_ARCH" "$SRC_IMAGE" >/dev/null || \
fail "podman pull $SRC_IMAGE failed (Docker Hub rate limit?). \
Pre-pull it when the quota is available (\`podman pull --platform=linux/$EXPECTED_IMAGE_ARCH $SRC_IMAGE\`), then re-run."
fi
# Verify arch matches. A podman cache shared across ARCH= runs can
# end up with a tag pointing at the wrong arch (pulling
# \`nginx:alpine\` for arm64 overwrites the tag's arm64/amd64
# binding). Better to fail loudly here than ship the VM an image
# it can't exec.
IMAGE_ACTUAL_ARCH="$(podman inspect "$SRC_IMAGE" --format '{{.Architecture}}' 2>/dev/null || true)"
if [[ "$IMAGE_ACTUAL_ARCH" != "$EXPECTED_IMAGE_ARCH" ]]; then
fail "$SRC_IMAGE is arch '$IMAGE_ACTUAL_ARCH' but ARCH=$ARCH needs '$EXPECTED_IMAGE_ARCH'. \
Either pre-pull the right platform (\`podman pull --platform=linux/$EXPECTED_IMAGE_ARCH $SRC_IMAGE\`) \
or point SRC_IMAGE at a locally-tagged variant."
fi
# The smoke upgrade test asserts container id change on image-tag
# change, so we'll expose two distinct local tag names pointing at
# the same bits. Tagging happens on the VM side after `podman load`
# so we stay compatible with older podman versions that don't grok
# the multi-image archive format (`podman save -m`).
V1_IMAGE="localdev/nginx:v1"
V2_IMAGE="localdev/nginx:v2"
IMAGE_TARBALL="$(mktemp -t fleet-demo-images.XXXXXX.tar)"
podman save -o "$IMAGE_TARBALL" "$SRC_IMAGE" >/dev/null \
|| fail "podman save failed"
log "exported $SRC_IMAGE → $IMAGE_TARBALL ($(du -h "$IMAGE_TARBALL" | cut -f1))"
# ---- phase 5: provision VM + install agent ----------------------------------
log "phase 5: build harmony-fleet-agent for arch=$ARCH + provision VM"
(
cd "$REPO_ROOT"
if [[ -n "$AGENT_TARGET" ]]; then
rustup target add "$AGENT_TARGET" >/dev/null
cargo build -q --release --target "$AGENT_TARGET" -p harmony-fleet-agent
else
cargo build -q --release -p harmony-fleet-agent
fi
)
if [[ -n "$AGENT_TARGET" ]]; then
AGENT_BINARY="$REPO_ROOT/target/$AGENT_TARGET/release/harmony-fleet-agent"
else
AGENT_BINARY="$REPO_ROOT/target/release/harmony-fleet-agent"
fi
[[ -f "$AGENT_BINARY" ]] || fail "agent binary missing: $AGENT_BINARY"
(
cd "$REPO_ROOT"
# Pass through FLEET_VM_ADMIN_PASSWORD if set so the VM admin user
# accepts SSH password auth. Useful for chaos / reliability
# testing sessions where the operator wants to log in and break
# things on purpose. Unset by default = key-only auth.
cargo run -q --release -p example_fleet_vm_setup -- \
--arch "$EXAMPLE_ARCH" \
--vm-name "$VM_NAME" \
--device-id "$DEVICE_ID" \
--group "$GROUP" \
--agent-binary "$AGENT_BINARY" \
--nats-url "nats://$NAT_GW:$NATS_NODE_PORT"
)
VM_IP="$(virsh --connect "$LIBVIRT_URI" domifaddr "$VM_NAME" \
| awk '/ipv4/ { print $4 }' | head -1 | cut -d/ -f1)"
[[ -n "$VM_IP" ]] || fail "couldn't resolve VM IP"
# ---- phase 5c: sideload workload images into fleet-agent's podman -------------
log "phase 5c: sideload $V1_IMAGE + $V2_IMAGE into fleet-agent's podman on VM"
# scp the tarball (ssh as the admin user, the only one with sshd
# access), then `podman load` inside an fleet-agent user session.
# Post-load the fleet-agent's podman has both tags locally, so
# `ensure_image_present` in harmony's PodmanTopology takes the
# "already present, skip pull" branch — no Docker Hub hit.
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "$HOME/.local/share/harmony/fleet/ssh/id_ed25519" \
"$IMAGE_TARBALL" "fleet-admin@$VM_IP:/tmp/fleet-demo-images.tar" >/dev/null \
|| fail "scp image tarball to VM failed"
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "$HOME/.local/share/harmony/fleet/ssh/id_ed25519" \
"fleet-admin@$VM_IP" -- \
"sudo chown fleet-agent:fleet-agent /tmp/fleet-demo-images.tar && \
sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman load -i /tmp/fleet-demo-images.tar' && \
sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman tag $SRC_IMAGE $V1_IMAGE' && \
sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman tag $SRC_IMAGE $V2_IMAGE' && \
sudo rm -f /tmp/fleet-demo-images.tar" >/dev/null \
|| fail "podman load + tag on VM failed"
rm -f "$IMAGE_TARBALL"
log "sideload complete — fleet-agent's podman has $V1_IMAGE + $V2_IMAGE"
# ---- phase 6: sanity --------------------------------------------------------
log "phase 6: sanity — operator + agent + KV"
for _ in $(seq 1 60); do
if kubectl -n "$NATS_NAMESPACE" get pod -l app="$NATS_NAME" \
-o jsonpath='{.items[0].status.phase}' 2>/dev/null \
| grep -q Running; then
break
fi
sleep 1
done
# NATS box one-liner we'll reuse in the hand-off too. Uses the host
# loadbalancer port so no pod-network plumbing needed.
NATSBOX_HOST="podman run --rm docker.io/natsio/nats-box:latest \
nats --server nats://host.containers.internal:$NATS_NODE_PORT"
log "checking agent heartbeat in NATS KV (device-heartbeat bucket)"
for _ in $(seq 1 30); do
if $NATSBOX_HOST kv get device-heartbeat "heartbeat.$DEVICE_ID" --raw \
>/dev/null 2>&1; then
break
fi
sleep 2
done
$NATSBOX_HOST kv get device-heartbeat "heartbeat.$DEVICE_ID" --raw >/dev/null \
|| fail "agent never published heartbeat to NATS"
log "agent heartbeat present: heartbeat.$DEVICE_ID"
# ---- phase 7: either hand off to user, or drive regression ------------------
if [[ "$AUTO" == "1" ]]; then
log "phase 7 (--auto): apply nginx via typed CR, verify, upgrade, delete"
log "applying $V1_IMAGE deployment"
(
cd "$REPO_ROOT"
cargo run -q -p example_harmony_apply_deployment -- \
--namespace "$DEPLOY_NS" \
--name "$DEPLOY_NAME" \
--target-device "$DEVICE_ID" \
--image "$V1_IMAGE" \
--port "$DEPLOY_PORT"
)
log "waiting for container on VM (up to $((CONTAINER_WAIT_STEPS * 2))s)"
CONTAINER_ID_V1=""
for _ in $(seq 1 "$CONTAINER_WAIT_STEPS"); do
id="$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "$HOME/.local/share/harmony/fleet/ssh/id_ed25519" \
"fleet-admin@$VM_IP" -- \
"sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman ps -q --filter name=$DEPLOY_NAME'" \
2>/dev/null | head -1)" || true
if [[ -n "$id" ]]; then CONTAINER_ID_V1="$id"; break; fi
sleep 2
done
[[ -n "$CONTAINER_ID_V1" ]] || fail "nginx container never appeared on VM"
log "container id (v1): $CONTAINER_ID_V1"
log "curl http://$VM_IP:${DEPLOY_PORT%%:*}/"
for _ in $(seq 1 30); do
if curl -sf "http://$VM_IP:${DEPLOY_PORT%%:*}/" >/dev/null; then
log "nginx responded (v1)"; break
fi
sleep 2
done
log "waiting for operator to aggregate .status.aggregate.succeeded == 1"
for _ in $(seq 1 30); do
got="$(kubectl -n "$DEPLOY_NS" get deployment.fleet.nationtech.io "$DEPLOY_NAME" \
-o jsonpath='{.status.aggregate.succeeded}' 2>/dev/null || true)"
if [[ "$got" == "1" ]]; then
log ".status.aggregate.succeeded = 1 — aggregator reflected agent state"
break
fi
sleep 2
done
got="$(kubectl -n "$DEPLOY_NS" get deployment.fleet.nationtech.io "$DEPLOY_NAME" \
-o jsonpath='{.status.aggregate.succeeded}' 2>/dev/null || true)"
[[ "$got" == "1" ]] || fail ".status.aggregate.succeeded never reached 1 (got '$got')"
log "upgrading to $V2_IMAGE"
(
cd "$REPO_ROOT"
cargo run -q -p example_harmony_apply_deployment -- \
--namespace "$DEPLOY_NS" \
--name "$DEPLOY_NAME" \
--target-device "$DEVICE_ID" \
--image "$V2_IMAGE" \
--port "$DEPLOY_PORT"
)
log "waiting for container id to change (upgrade, up to $((CONTAINER_WAIT_STEPS * 2))s)"
CONTAINER_ID_V2=""
for _ in $(seq 1 "$CONTAINER_WAIT_STEPS"); do
id="$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "$HOME/.local/share/harmony/fleet/ssh/id_ed25519" \
"fleet-admin@$VM_IP" -- \
"sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman ps -q --filter name=$DEPLOY_NAME'" \
2>/dev/null | head -1)" || true
if [[ -n "$id" && "$id" != "$CONTAINER_ID_V1" ]]; then
CONTAINER_ID_V2="$id"; break
fi
sleep 2
done
[[ -n "$CONTAINER_ID_V2" ]] || fail "container id did not change after upgrade"
log "container id (v2): $CONTAINER_ID_V2 — upgrade confirmed"
log "deleting deployment"
(
cd "$REPO_ROOT"
cargo run -q -p example_harmony_apply_deployment -- \
--namespace "$DEPLOY_NS" \
--name "$DEPLOY_NAME" \
--target-device "$DEVICE_ID" \
--delete
)
for _ in $(seq 1 60); do
if ! ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "$HOME/.local/share/harmony/fleet/ssh/id_ed25519" \
"fleet-admin@$VM_IP" -- podman ps -q --filter "name=$DEPLOY_NAME" 2>/dev/null \
| grep -q .; then
log "container removed from VM"
break
fi
sleep 2
done
log "PASS (--auto)"
exit 0
fi
# ---- hand-off mode ----------------------------------------------------------
SSH_KEY="$HOME/.local/share/harmony/fleet/ssh/id_ed25519"
cat <<EOF
$(printf '\033[1;32m[smoke-a4]\033[0m full stack is up. Your playground:\n')
KUBECONFIG=$KUBECONFIG_FILE
VM IP: $VM_IP
device id: $DEVICE_ID
libvirt NAT gateway (VM's view of the host): $NAT_GW
NATS URL (from host): nats://localhost:$NATS_NODE_PORT
NATS URL (from the VM): nats://$NAT_GW:$NATS_NODE_PORT
$(printf '\033[1mWatch CRs reconcile:\033[0m\n')
kubectl get deployments.fleet.nationtech.io -A -w
$(printf '\033[1mApply an nginx deployment (typed Rust):\033[0m\n')
cargo run -q -p example_harmony_apply_deployment -- \\
--namespace $DEPLOY_NS \\
--name $DEPLOY_NAME \\
--target-device $DEVICE_ID \\
--image docker.io/library/nginx:latest
$(printf '\033[1mUpgrade it:\033[0m\n')
cargo run -q -p example_harmony_apply_deployment -- \\
--namespace $DEPLOY_NS --name $DEPLOY_NAME --target-device $DEVICE_ID \\
--image docker.io/library/nginx:1.26
$(printf '\033[1mPreview the CR as JSON (and apply via kubectl):\033[0m\n')
cargo run -q -p example_harmony_apply_deployment -- \\
--name $DEPLOY_NAME --target-device $DEVICE_ID \\
--image docker.io/library/nginx:latest --print | kubectl apply -f -
$(printf '\033[1mConnect to the device:\033[0m\n')
ssh -i $SSH_KEY fleet-admin@$VM_IP
virsh --connect $LIBVIRT_URI console $VM_NAME --force # alternative
# list containers (agent runs rootless as fleet-agent, not fleet-admin):
ssh -i $SSH_KEY fleet-admin@$VM_IP "sudo su - fleet-agent -c 'XDG_RUNTIME_DIR=/run/user/\$(id -u) podman ps'"
$(printf '\033[1mInspect NATS KV (natsbox):\033[0m\n')
alias natsbox='podman run --rm docker.io/natsio/nats-box:latest nats --server nats://host.containers.internal:$NATS_NODE_PORT'
natsbox kv ls desired-state
natsbox kv get desired-state '$DEVICE_ID.$DEPLOY_NAME' --raw
natsbox kv ls device-state
natsbox kv ls device-heartbeat
natsbox kv get device-heartbeat 'heartbeat.$DEVICE_ID' --raw
$(printf '\033[1mHit the deployed nginx:\033[0m\n')
curl http://$VM_IP:${DEPLOY_PORT%%:*}/
$(printf '\033[1mOperator log:\033[0m\n')
tail -F $OPERATOR_LOG
$(printf '\033[1;33mCtrl-C to tear everything down.\033[0m\n')
EOF
# Block until user interrupts; cleanup trap handles teardown.
while true; do sleep 60; done

2
harmony-reconciler-contracts/Cargo.toml
View File

@@ -16,5 +16,7 @@ license.workspace = true
[dependencies]
chrono = { workspace = true, features = ["serde"] }
harmony_types = { path = "../harmony_types" }
schemars = "0.8.22"
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }

272
harmony-reconciler-contracts/src/fleet.rs Normal file
View File

@@ -0,0 +1,272 @@
//! Fleet-scale wire-format types.
//!
//! Per-concern payloads on dedicated NATS KV buckets:
//!
//! | Type | Bucket | Cadence |
//! |------|--------|---------|
//! | [`DeviceInfo`] | KV `device-info` | on startup + label/inventory change |
//! | [`DeploymentState`] | KV `device-state` | on reconcile phase transition |
//! | [`HeartbeatPayload`] | KV `device-heartbeat` | every 30 s |
//!
//! The operator watches `device-state` directly — KV watch deliveries
//! are ordered and last-writer-wins, so there's no separate event
//! stream or per-write revision to track.
use std::collections::BTreeMap;
use std::fmt;
use chrono::{DateTime, Utc};
use harmony_types::id::Id;
use serde::{Deserialize, Deserializer, Serialize};
use crate::status::{InventorySnapshot, Phase};
/// Deployment CR `metadata.name`, validated for NATS-subject safety.
///
/// Scope: what identifies a Deployment to the agent. Appears in KV
/// keys (`state.<device>.<deployment>`) and every in-memory map
/// keyed by "which deployment." A raw `String` here would let an
/// invalid name (containing a `.`, splitting into extra subject
/// tokens) break routing at runtime.
///
/// Validation:
/// - Not empty.
/// - No `.` (would alias an extra subject token).
/// - No `*` / `>` (NATS wildcards).
/// - No ASCII whitespace.
/// - ≤ 253 bytes (RFC 1123 max, matches Kubernetes name limit).
///
/// The constructor is fallible; deserialization runs the same
/// validation so malformed payloads are rejected at the wire.
#[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize)]
#[serde(transparent)]
pub struct DeploymentName(String);
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum InvalidDeploymentName {
#[error("deployment name must not be empty")]
Empty,
#[error("deployment name must not exceed 253 bytes")]
TooLong,
#[error("deployment name must not contain '.' (would alias an extra NATS subject token)")]
ContainsDot,
#[error("deployment name must not contain NATS wildcards '*' or '>'")]
ContainsWildcard,
#[error("deployment name must not contain whitespace")]
ContainsWhitespace,
}
impl DeploymentName {
pub fn try_new(s: impl Into<String>) -> Result<Self, InvalidDeploymentName> {
let s = s.into();
if s.is_empty() {
return Err(InvalidDeploymentName::Empty);
}
if s.len() > 253 {
return Err(InvalidDeploymentName::TooLong);
}
if s.contains('.') {
return Err(InvalidDeploymentName::ContainsDot);
}
if s.contains('*') || s.contains('>') {
return Err(InvalidDeploymentName::ContainsWildcard);
}
if s.chars().any(|c| c.is_ascii_whitespace()) {
return Err(InvalidDeploymentName::ContainsWhitespace);
}
Ok(Self(s))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for DeploymentName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl<'de> Deserialize<'de> for DeploymentName {
fn deserialize<D: Deserializer<'de>>(de: D) -> Result<Self, D::Error> {
let s = String::deserialize(de)?;
Self::try_new(s).map_err(serde::de::Error::custom)
}
}
/// Static-ish per-device facts: routing labels, hardware, agent
/// version. Written to KV key `info.<device_id>` in
/// [`crate::BUCKET_DEVICE_INFO`]. Rewritten by the agent on startup
/// and whenever its labels change — **not** on every heartbeat.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DeviceInfo {
pub device_id: Id,
/// Routing labels. Operator resolves Deployment
/// `targetSelector.matchLabels` against this map.
#[serde(default)]
pub labels: BTreeMap<String, String>,
/// Hardware / OS snapshot. `None` until the first post-startup
/// publish.
#[serde(default)]
pub inventory: Option<InventorySnapshot>,
/// RFC 3339 UTC timestamp of this publish.
pub updated_at: DateTime<Utc>,
}
/// Authoritative current phase for one `(device, deployment)` pair.
/// Written to KV key `state.<device_id>.<deployment>` in
/// [`crate::BUCKET_DEVICE_STATE`]. Deleted when the deployment is
/// removed from the device.
///
/// The operator's KV watch sees every write + delete in order, so
/// this value alone — plus the operator's in-memory belief about
/// the last phase for the pair — is enough to drive the aggregate
/// counters. No separate event stream, no per-write revision.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DeploymentState {
pub device_id: Id,
pub deployment: DeploymentName,
pub phase: Phase,
pub last_event_at: DateTime<Utc>,
#[serde(default)]
pub last_error: Option<String>,
}
/// Tiny liveness ping. Written to KV key `heartbeat.<device_id>` in
/// [`crate::BUCKET_DEVICE_HEARTBEAT`].
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct HeartbeatPayload {
pub device_id: Id,
pub at: DateTime<Utc>,
}
#[cfg(test)]
mod tests {
use super::*;
fn ts(s: &str) -> DateTime<Utc> {
DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc)
}
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid")
}
#[test]
fn deployment_name_accepts_rfc1123() {
assert!(DeploymentName::try_new("hello-world").is_ok());
assert!(DeploymentName::try_new("a").is_ok());
assert!(DeploymentName::try_new("a-b-c-1-2-3").is_ok());
}
#[test]
fn deployment_name_rejects_dot() {
assert_eq!(
DeploymentName::try_new("hello.world"),
Err(InvalidDeploymentName::ContainsDot)
);
}
#[test]
fn deployment_name_rejects_nats_wildcards() {
assert_eq!(
DeploymentName::try_new("hello*"),
Err(InvalidDeploymentName::ContainsWildcard)
);
assert_eq!(
DeploymentName::try_new("hello>"),
Err(InvalidDeploymentName::ContainsWildcard)
);
}
#[test]
fn deployment_name_rejects_empty_and_too_long() {
assert_eq!(
DeploymentName::try_new(""),
Err(InvalidDeploymentName::Empty)
);
assert_eq!(
DeploymentName::try_new("x".repeat(254)),
Err(InvalidDeploymentName::TooLong)
);
}
#[test]
fn deployment_name_rejects_whitespace() {
assert_eq!(
DeploymentName::try_new("hello world"),
Err(InvalidDeploymentName::ContainsWhitespace)
);
assert_eq!(
DeploymentName::try_new("hello\tworld"),
Err(InvalidDeploymentName::ContainsWhitespace)
);
}
#[test]
fn deployment_name_deserialization_validates() {
let json = r#""bad.name""#;
let result: Result<DeploymentName, _> = serde_json::from_str(json);
assert!(result.is_err());
}
#[test]
fn deployment_name_roundtrip() {
let name = dn("hello-world");
let json = serde_json::to_string(&name).unwrap();
assert_eq!(json, r#""hello-world""#);
let back: DeploymentName = serde_json::from_str(&json).unwrap();
assert_eq!(name, back);
}
#[test]
fn deployment_state_roundtrip() {
let original = DeploymentState {
device_id: Id::from("pi-01".to_string()),
deployment: dn("hello-web"),
phase: Phase::Failed,
last_event_at: ts("2026-04-22T10:05:00Z"),
last_error: Some("image pull 429".to_string()),
};
let json = serde_json::to_string(&original).unwrap();
let back: DeploymentState = serde_json::from_str(&json).unwrap();
assert_eq!(original, back);
}
#[test]
fn heartbeat_is_tiny() {
let hb = HeartbeatPayload {
device_id: Id::from("pi-01".to_string()),
at: ts("2026-04-22T10:00:30Z"),
};
let bytes = serde_json::to_vec(&hb).unwrap();
assert!(
bytes.len() < 96,
"heartbeat payload grew to {} bytes: {}",
bytes.len(),
String::from_utf8_lossy(&bytes),
);
}
#[test]
fn device_info_roundtrip() {
let original = DeviceInfo {
device_id: Id::from("pi-01".to_string()),
labels: BTreeMap::from([("group".to_string(), "site-a".to_string())]),
inventory: Some(InventorySnapshot {
hostname: "pi-01".to_string(),
arch: "aarch64".to_string(),
os: "Ubuntu 24.04".to_string(),
kernel: "6.8.0".to_string(),
cpu_cores: 4,
memory_mb: 8192,
agent_version: "0.1.0".to_string(),
}),
updated_at: ts("2026-04-22T10:00:00Z"),
};
let json = serde_json::to_string(&original).unwrap();
let back: DeviceInfo = serde_json::from_str(&json).unwrap();
assert_eq!(original, back);
}
}

81
harmony-reconciler-contracts/src/kv.rs
View File

@@ -7,47 +7,88 @@
//! here; agent + operator consume the constants directly, and smoke
//! scripts grep for the literal values locked in the tests below.
use crate::fleet::DeploymentName;
/// Operator-written bucket. One entry per `(device, deployment)` pair.
/// Values are the JSON-serialized Score envelope — today
/// `harmony::modules::podman::IotScore`, tomorrow any variant of
/// `harmony::modules::podman::ReconcileScore`, tomorrow any variant of
/// a polymorphic `Score` enum the framework ships.
pub const BUCKET_DESIRED_STATE: &str = "desired-state";
/// Agent-written bucket. One entry per device at `status.<device_id>`.
/// Values are JSON-serialized [`crate::AgentStatus`].
pub const BUCKET_AGENT_STATUS: &str = "agent-status";
/// Static-ish per-device facts: routing labels, inventory, agent
/// version. Agent rewrites the entry on startup and whenever its
/// labels change. Key format: `info.<device_id>`.
pub const BUCKET_DEVICE_INFO: &str = "device-info";
/// Current reconcile phase for each `(device, deployment)` pair.
/// Agent writes on phase transition; operator watches this bucket
/// to drive CR `.status.aggregate`. Authoritative source of truth
/// for "what's running where." Key format:
/// `state.<device_id>.<deployment>`.
pub const BUCKET_DEVICE_STATE: &str = "device-state";
/// Tiny liveness ping from each device every N seconds. Separate
/// from [`BUCKET_DEVICE_STATE`] so routine heartbeats don't churn
/// the state bucket. Key format: `heartbeat.<device_id>`.
pub const BUCKET_DEVICE_HEARTBEAT: &str = "device-heartbeat";
/// KV key for a `(device, deployment)` pair in [`BUCKET_DESIRED_STATE`].
/// Format: `<device>.<deployment>`.
pub fn desired_state_key(device_id: &str, deployment_name: &str) -> String {
format!("{device_id}.{deployment_name}")
pub fn desired_state_key(device_id: &str, deployment_name: &DeploymentName) -> String {
format!("{device_id}.{}", deployment_name.as_str())
}
/// KV key for a device's last-known status in [`BUCKET_AGENT_STATUS`].
/// Format: `status.<device_id>`.
pub fn status_key(device_id: &str) -> String {
format!("status.{device_id}")
/// KV key for a device's `DeviceInfo` entry in [`BUCKET_DEVICE_INFO`].
/// Format: `info.<device_id>`.
pub fn device_info_key(device_id: &str) -> String {
format!("info.{device_id}")
}
/// KV key for a `(device, deployment)` state entry in
/// [`BUCKET_DEVICE_STATE`]. Format: `state.<device_id>.<deployment>`.
pub fn device_state_key(device_id: &str, deployment_name: &DeploymentName) -> String {
format!("state.{device_id}.{}", deployment_name.as_str())
}
/// KV key for a device's liveness entry in
/// [`BUCKET_DEVICE_HEARTBEAT`]. Format: `heartbeat.<device_id>`.
pub fn device_heartbeat_key(device_id: &str) -> String {
format!("heartbeat.{device_id}")
}
#[cfg(test)]
mod tests {
use super::*;
fn dn(s: &str) -> crate::DeploymentName {
crate::DeploymentName::try_new(s).expect("valid")
}
#[test]
fn desired_state_key_format() {
assert_eq!(desired_state_key("pi-01", "hello-web"), "pi-01.hello-web");
assert_eq!(
desired_state_key("pi-01", &dn("hello-web")),
"pi-01.hello-web"
);
}
#[test]
fn status_key_format() {
assert_eq!(status_key("pi-01"), "status.pi-01");
}
#[test]
fn bucket_names_match_smoke_scripts() {
// These strings are also grepped by iot/scripts/smoke-*.sh —
// flipping them here must be paired with a script update.
fn bucket_names_stable() {
// Flipping these is a cross-component break — operator,
// agent, and smoke scripts all grep for the literal values.
assert_eq!(BUCKET_DESIRED_STATE, "desired-state");
assert_eq!(BUCKET_AGENT_STATUS, "agent-status");
assert_eq!(BUCKET_DEVICE_INFO, "device-info");
assert_eq!(BUCKET_DEVICE_STATE, "device-state");
assert_eq!(BUCKET_DEVICE_HEARTBEAT, "device-heartbeat");
}
#[test]
fn key_formats() {
assert_eq!(device_info_key("pi-01"), "info.pi-01");
assert_eq!(
device_state_key("pi-01", &dn("hello-web")),
"state.pi-01.hello-web"
);
assert_eq!(device_heartbeat_key("pi-01"), "heartbeat.pi-01");
}
}

27
harmony-reconciler-contracts/src/lib.rs
View File

@@ -3,28 +3,31 @@
//! Harmony's "reconciler" pattern is: a central **operator** writes
//! desired state into NATS JetStream KV; a remote **agent** watches
//! the KV, deserializes each entry as a Score, and drives the host
//! toward that state. This split lets one operator orchestrate a
//! fleet of agents across network boundaries it can't reach
//! directly — IoT devices today, OKD cluster agents or edge-compute
//! reconcilers tomorrow.
//! toward that state. The agent writes back per-device info and
//! per-deployment state into separate KV buckets; the operator reads
//! those to aggregate `.status.aggregate` onto the CR.
//!
//! This crate holds the wire-format bits both sides must agree on:
//! NATS bucket names, KV key formats, and the `AgentStatus`
//! heartbeat payload. The Score types themselves (`PodmanV0Score`,
//! future variants) live in their respective harmony modules —
//! consumers import them from there and serialize them over the
//! transport this crate describes.
//! NATS bucket names, KV key formats, and the typed payloads
//! (`DeviceInfo`, `DeploymentState`, `HeartbeatPayload`). The Score
//! types themselves live in their respective harmony modules.
//!
//! **Deliberately lean** — no tokio, no async-nats, no harmony.
//! The on-device agent build pulls it in alongside a minimal
//! async-nats client; the operator pulls it alongside kube-rs.
//! Neither should pay for the other's dependencies.
pub mod fleet;
pub mod kv;
pub mod status;
pub use kv::{BUCKET_AGENT_STATUS, BUCKET_DESIRED_STATE, desired_state_key, status_key};
pub use status::AgentStatus;
pub use fleet::{
DeploymentName, DeploymentState, DeviceInfo, HeartbeatPayload, InvalidDeploymentName,
};
pub use kv::{
BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE,
desired_state_key, device_heartbeat_key, device_info_key, device_state_key,
};
pub use status::{InventorySnapshot, Phase};
// Re-exports so consumers (agent, operator) don't need a direct
// harmony_types dependency purely to name the cross-boundary types.

99
harmony-reconciler-contracts/src/status.rs
View File

@@ -1,74 +1,37 @@
//! Agent → NATS KV status payload.
//!
//! The agent publishes a heartbeat + rollup status to the
//! `agent-status` bucket every 30 s (see
//! [`crate::BUCKET_AGENT_STATUS`]). Today the payload is intentionally
//! minimal — a single `"running"` state + a timestamp — so the
//! operator can implement §12 v0.1 "Status aggregation in operator"
//! without waiting on richer per-workload reporting.
//!
//! When the agent grows richer status (per-container state, rollout
//! progress) this struct gains fields with `#[serde(default)]`; old
//! operators keep working against newer agents.
//! Shared status primitives reused across the fleet wire format.
use chrono::{DateTime, Utc};
use harmony_types::id::Id;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// A single heartbeat published by the agent at
/// `status.<device_id>` in the `agent-status` bucket.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AgentStatus {
/// Echoed from the agent's own config so the operator can
/// cross-check which device it came from if the KV key is ever
/// ambiguous. Serializes transparently as a plain string.
pub device_id: Id,
/// Coarse rollup state. v0 only ever writes `"running"`; richer
/// variants are a v0.1+ concern. A String (not an enum) so old
/// operators parsing this payload don't fail on a new variant.
pub status: String,
/// RFC 3339 UTC timestamp. Used by the smoke test's reboot-
/// detection gate — any timestamp strictly greater than the gate
/// is evidence of a post-reboot write. `chrono::DateTime<Utc>`
/// serde-serializes as RFC 3339, so the wire format stays
/// lex-comparable (the smoke's string `>` still works).
pub timestamp: DateTime<Utc>,
/// Coarse state of a single reconcile on one device.
///
/// Deliberately coarse — richer granularity (ImagePulling,
/// ContainerCreating, …) is agent-internal; the operator's
/// aggregation only needs success/failure/pending counts.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
pub enum Phase {
/// Agent has applied the Score and the container is up.
Running,
/// Reconcile hit an error. See paired `last_error` for the message.
Failed,
/// Reconcile is in flight or waiting on an external dependency
/// (image pull, network, etc.). Agents may also report this
/// between a CR apply and the first reconcile tick.
Pending,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn status_roundtrip() {
let s = AgentStatus {
device_id: Id::from("pi-01".to_string()),
status: "running".to_string(),
timestamp: DateTime::parse_from_rfc3339("2026-04-21T18:15:42Z")
.unwrap()
.with_timezone(&Utc),
};
let json = serde_json::to_string(&s).unwrap();
let back: AgentStatus = serde_json::from_str(&json).unwrap();
assert_eq!(s, back);
}
#[test]
fn status_has_expected_wire_keys() {
let s = AgentStatus {
device_id: Id::from("pi-01".to_string()),
status: "running".to_string(),
timestamp: DateTime::parse_from_rfc3339("2026-04-21T18:15:42Z")
.unwrap()
.with_timezone(&Utc),
};
let json = serde_json::to_string(&s).unwrap();
// device_id must serialize as a flat string (not {"value": …}).
// Relies on `#[serde(transparent)]` on `harmony_types::id::Id`.
assert!(json.contains("\"device_id\":\"pi-01\""), "got {json}");
assert!(json.contains("\"status\":\"running\""));
// RFC 3339 output — the smoke script greps a `"timestamp":"<rfc3339>"`
// literal and compares lexicographically against a gate.
assert!(json.contains("\"timestamp\":\"2026-04-21T18:15:42Z\""));
}
/// Static-ish facts about the device. Embedded in
/// [`crate::DeviceInfo`]; republished on change.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct InventorySnapshot {
pub hostname: String,
pub arch: String,
pub os: String,
pub kernel: String,
pub cpu_cores: u32,
pub memory_mb: u64,
/// Agent semver (e.g. `"0.1.0"`). Lets the operator flag
/// agents that are behind the current release.
pub agent_version: String,
}

4
harmony/src/domain/interpret/mod.rs
View File

@@ -39,7 +39,7 @@ pub enum InterpretName {
K8sIngress,
PodmanV0,
KvmVm,
IotDeviceSetup,
FleetDeviceSetup,
}
impl std::fmt::Display for InterpretName {
@@ -75,7 +75,7 @@ impl std::fmt::Display for InterpretName {
InterpretName::K8sIngress => f.write_str("K8sIngress"),
InterpretName::PodmanV0 => f.write_str("PodmanV0"),
InterpretName::KvmVm => f.write_str("KvmVm"),
InterpretName::IotDeviceSetup => f.write_str("IotDeviceSetup"),
InterpretName::FleetDeviceSetup => f.write_str("FleetDeviceSetup"),
}
}
}

2
harmony/src/domain/topology/host_configuration.rs
View File

@@ -89,7 +89,7 @@ pub trait SystemdManager: Send + Sync {
) -> Result<ChangeReport, ExecutorError>;
/// Enable+start a user-scoped unit (e.g. `podman.socket` under
/// `iot-agent`). Assumes [`UnixUserManager::ensure_linger`] has
/// `fleet-agent`). Assumes [`UnixUserManager::ensure_linger`] has
/// already been called for the user.
async fn ensure_user_unit_active(
&self,

8
harmony/src/domain/topology/virtualization.rs
View File

@@ -119,6 +119,14 @@ pub struct VmFirstBootConfig {
/// Public SSH keys (OpenSSH single-line format) to authorize for
/// the admin user.
pub authorized_keys: Vec<String>,
/// Optional plaintext password for the admin user. When set,
/// the account is unlocked + SSH password auth is enabled on
/// the guest. Intended for interactive debugging / chaos
/// testing where the operator wants to log in and break things
/// manually. Leave `None` for production deployments — key-only
/// auth is the default.
#[serde(default)]
pub admin_password: Option<String>,
}
/// Observed runtime info for a VM.

145
harmony/src/modules/application/helm/mod.rs
View File

@@ -2,10 +2,12 @@
pub use k8s_openapi::api::{
apps::v1::{Deployment, DeploymentSpec},
core::v1::{
Container, ContainerPort, EnvVar, PodSpec, PodTemplateSpec, Service as K8sService,
ServicePort, ServiceSpec,
Container, ContainerPort, EnvVar, Namespace, PodSpec, PodTemplateSpec,
Service as K8sService, ServiceAccount, ServicePort, ServiceSpec,
},
rbac::v1::{ClusterRole, ClusterRoleBinding},
};
pub use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
use kube::core::ObjectMeta;
@@ -14,16 +16,36 @@ use crate::modules::application::config::{ApplicationNetworkPort, NetworkProtoco
use std::fs;
use std::path::{Path, PathBuf};
/// Enum representing all supported Kubernetes resource types for Helm charts.
/// Supports built-in typed resources and custom CRDs via YAML strings.
/// A rendered Kubernetes resource ready to drop into a helm chart's
/// `templates/` directory.
///
/// Each variant wraps a strongly-typed `k8s_openapi` struct — the chart
/// writer serializes via `serde_yaml` at package time, keeping the
/// `templates/` directory a pure data-transfer format (ADR 018
/// template hydration). The `CustomYaml` escape hatch is here for
/// resources we haven't typed yet; **prefer adding a typed variant
/// over using it**.
pub enum HelmResourceKind {
/// Built-in typed Service resource
/// `v1` Service (namespaced).
Service(K8sService),
/// Built-in typed Deployment resource
/// `apps/v1` Deployment (namespaced).
Deployment(Deployment),
/// Custom resource as pre-serialized YAML (e.g., CRDs, custom types)
/// `v1` Namespace (cluster-scoped).
Namespace(Namespace),
/// `v1` ServiceAccount (namespaced).
ServiceAccount(ServiceAccount),
/// `rbac.authorization.k8s.io/v1` ClusterRole (cluster-scoped).
ClusterRole(ClusterRole),
/// `rbac.authorization.k8s.io/v1` ClusterRoleBinding (cluster-scoped).
ClusterRoleBinding(ClusterRoleBinding),
/// `apiextensions.k8s.io/v1` CustomResourceDefinition
/// (cluster-scoped). Expected to be produced by
/// `kube::CustomResourceExt::crd()` on a derive-built type —
/// never hand-authored.
Crd(CustomResourceDefinition),
/// Escape hatch for resources without a typed variant yet.
/// Adding a typed variant above is always preferred.
CustomYaml { filename: String, content: String },
// Can add more typed variants as needed: ConfigMap, Secret, Ingress, etc.
}
impl HelmResourceKind {
@@ -31,6 +53,23 @@ impl HelmResourceKind {
match self {
HelmResourceKind::Service(_) => "service.yaml".to_string(),
HelmResourceKind::Deployment(_) => "deployment.yaml".to_string(),
HelmResourceKind::Namespace(_) => "namespace.yaml".to_string(),
HelmResourceKind::ServiceAccount(sa) => format!(
"serviceaccount-{}.yaml",
sa.metadata.name.as_deref().unwrap_or("unnamed")
),
HelmResourceKind::ClusterRole(cr) => format!(
"clusterrole-{}.yaml",
cr.metadata.name.as_deref().unwrap_or("unnamed")
),
HelmResourceKind::ClusterRoleBinding(crb) => format!(
"clusterrolebinding-{}.yaml",
crb.metadata.name.as_deref().unwrap_or("unnamed")
),
HelmResourceKind::Crd(c) => format!(
"crd-{}.yaml",
c.metadata.name.as_deref().unwrap_or("unnamed")
),
HelmResourceKind::CustomYaml { filename, .. } => filename.clone(),
}
}
@@ -39,6 +78,11 @@ impl HelmResourceKind {
match self {
HelmResourceKind::Service(s) => serde_yaml::to_string(s),
HelmResourceKind::Deployment(d) => serde_yaml::to_string(d),
HelmResourceKind::Namespace(n) => serde_yaml::to_string(n),
HelmResourceKind::ServiceAccount(sa) => serde_yaml::to_string(sa),
HelmResourceKind::ClusterRole(cr) => serde_yaml::to_string(cr),
HelmResourceKind::ClusterRoleBinding(crb) => serde_yaml::to_string(crb),
HelmResourceKind::Crd(c) => serde_yaml::to_string(c),
HelmResourceKind::CustomYaml { content, .. } => Ok(content.clone()),
}
}
@@ -65,7 +109,8 @@ impl HelmResourceKind {
}
}
/// Add a custom resource from any type that implements Serialize
/// Add a custom resource from any type that implements Serialize.
/// Prefer a typed variant constructor over this where one exists.
pub fn from_serializable<T: serde::Serialize>(
filename: impl Into<String>,
resource: &T,
@@ -444,3 +489,85 @@ pub fn create_service_from_ports(
..Default::default()
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn typed_variants_have_unique_filenames() {
let ns = Namespace {
metadata: ObjectMeta {
name: Some("fleet-system".to_string()),
..Default::default()
},
..Default::default()
};
let sa = ServiceAccount {
metadata: ObjectMeta {
name: Some("harmony-fleet-operator".to_string()),
namespace: Some("fleet-system".to_string()),
..Default::default()
},
..Default::default()
};
let cr = ClusterRole {
metadata: ObjectMeta {
name: Some("harmony-fleet-operator".to_string()),
..Default::default()
},
rules: None,
..Default::default()
};
let crb = ClusterRoleBinding {
metadata: ObjectMeta {
name: Some("harmony-fleet-operator".to_string()),
..Default::default()
},
role_ref: k8s_openapi::api::rbac::v1::RoleRef {
api_group: "rbac.authorization.k8s.io".to_string(),
kind: "ClusterRole".to_string(),
name: "harmony-fleet-operator".to_string(),
},
subjects: None,
};
let crd = CustomResourceDefinition {
metadata: ObjectMeta {
name: Some("widgets.example.io".to_string()),
..Default::default()
},
..Default::default()
};
let resources = [
HelmResourceKind::Namespace(ns),
HelmResourceKind::ServiceAccount(sa),
HelmResourceKind::ClusterRole(cr),
HelmResourceKind::ClusterRoleBinding(crb),
HelmResourceKind::Crd(crd),
];
let mut seen = std::collections::HashSet::new();
for r in &resources {
let f = r.filename();
assert!(seen.insert(f.clone()), "duplicate filename {f}");
// Make sure it serializes cleanly — catches any missing
// arm in `serialize_to_yaml`.
let yaml = r.serialize_to_yaml().expect("serialize");
assert!(!yaml.is_empty());
}
}
#[test]
fn crd_filename_carries_crd_name() {
let crd = CustomResourceDefinition {
metadata: ObjectMeta {
name: Some("deployments.fleet.nationtech.io".to_string()),
..Default::default()
},
..Default::default()
};
assert_eq!(
HelmResourceKind::Crd(crd).filename(),
"crd-deployments.fleet.nationtech.io.yaml"
);
}
}

26
harmony/src/modules/iot/assets.rs → harmony/src/modules/fleet/assets.rs
View File

@@ -1,7 +1,7 @@
//! Bootstrapped assets shared across IoT workflows.
//!
//! Everything here follows the `ensure_*` pattern — idempotent, caches
//! results under [`HARMONY_DATA_DIR`]`/iot/…`, and runs at most once per
//! results under [`HARMONY_DATA_DIR`]`/fleet/…`, and runs at most once per
//! process (enforced by a `tokio::sync::OnceCell`). The goal is that an
//! operator can run the IoT smoke test against a freshly-installed host
//! with nothing but `libvirt + qemu + xorriso + python3 + cargo +
@@ -127,7 +127,7 @@ async fn ensure_cloud_image(
return Err(exec(format!(
"downloaded image sha256 mismatch: expected {expected_sha256}, got {actual}. \
Ubuntu may have rotated the 'current release' pointer — bump the pin in \
modules::iot::assets.rs."
modules::fleet::assets.rs."
)));
}
// World-readable so libvirt-qemu can open it without a chmod ritual.
@@ -195,7 +195,7 @@ async fn sha256_of_file(path: &Path) -> Result<String, ExecutorError> {
}
fn cloud_images_dir() -> PathBuf {
HARMONY_DATA_DIR.join("iot").join("cloud-images")
HARMONY_DATA_DIR.join("fleet").join("cloud-images")
}
// ---------------------------------------------------------------------
@@ -206,20 +206,20 @@ fn cloud_images_dir() -> PathBuf {
/// same key identifies every VM we provision for smoke/integration
/// testing — cheap to reuse, easy to discard (just `rm -rf` the dir).
#[derive(Debug, Clone)]
pub struct IotSshKeypair {
pub struct FleetSshKeypair {
pub private_key: PathBuf,
pub public_key: PathBuf,
}
/// Ensure `$HARMONY_DATA_DIR/iot/ssh/id_ed25519[.pub]` exists. Runs
/// Ensure `$HARMONY_DATA_DIR/fleet/ssh/id_ed25519[.pub]` exists. Runs
/// `ssh-keygen` once; subsequent calls return the existing paths.
pub async fn ensure_iot_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
static CELL: OnceCell<IotSshKeypair> = OnceCell::const_new();
pub async fn ensure_fleet_ssh_keypair() -> Result<FleetSshKeypair, ExecutorError> {
static CELL: OnceCell<FleetSshKeypair> = OnceCell::const_new();
CELL.get_or_try_init(provision_ssh_keypair).await.cloned()
}
async fn provision_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
let dir = HARMONY_DATA_DIR.join("iot").join("ssh");
async fn provision_ssh_keypair() -> Result<FleetSshKeypair, ExecutorError> {
let dir = HARMONY_DATA_DIR.join("fleet").join("ssh");
tokio::fs::create_dir_all(&dir)
.await
.map_err(|e| exec(format!("create ssh dir {dir:?}: {e}")))?;
@@ -231,7 +231,7 @@ async fn provision_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
let pub_path = dir.join("id_ed25519.pub");
if priv_path.exists() && pub_path.exists() {
info!("ssh keypair cache hit at {priv_path:?}");
return Ok(IotSshKeypair {
return Ok(FleetSshKeypair {
private_key: priv_path,
public_key: pub_path,
});
@@ -248,7 +248,7 @@ async fn provision_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
"-N",
"", // no passphrase
"-C",
"harmony-iot-smoke",
"harmony-fleet-smoke",
"-f",
])
.arg(&priv_path) // PathBuf — kept separate so we don't force &str conversion
@@ -263,7 +263,7 @@ async fn provision_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
String::from_utf8_lossy(&status.stderr).trim()
)));
}
Ok(IotSshKeypair {
Ok(FleetSshKeypair {
private_key: priv_path,
public_key: pub_path,
})
@@ -271,7 +271,7 @@ async fn provision_ssh_keypair() -> Result<IotSshKeypair, ExecutorError> {
/// Read the generated public key (one line, openssh format) into a string
/// suitable for cloud-init's `authorized_keys`.
pub async fn read_public_key(kp: &IotSshKeypair) -> Result<String, ExecutorError> {
pub async fn read_public_key(kp: &FleetSshKeypair) -> Result<String, ExecutorError> {
let content = tokio::fs::read_to_string(&kp.public_key)
.await
.map_err(|e| exec(format!("read {:?}: {e}", kp.public_key)))?;

22
harmony/src/modules/iot/libvirt_pool.rs → harmony/src/modules/fleet/libvirt_pool.rs
View File

@@ -4,14 +4,14 @@
//! writable place to drop per-VM overlay disks + cloud-init seed ISOs.
//! Rather than ask the operator to set that up, we create a user-
//! owned dir-backed libvirt pool at
//! `$HARMONY_DATA_DIR/iot/kvm/pool/` and let libvirt handle:
//! `$HARMONY_DATA_DIR/fleet/kvm/pool/` and let libvirt handle:
//!
//! - **Perms**: dir contents get chowned to libvirt-qemu on VM start
//! via dynamic-ownership (default-on), and back to us on VM stop
//! (via remember_owner, also default-on). No `chmod 644` gymnastics.
//! - **Visibility**: `virsh vol-list harmony-iot` shows every
//! - **Visibility**: `virsh vol-list harmony-fleet` shows every
//! artifact we've created.
//! - **Cleanup**: `virsh vol-delete <name> harmony-iot` removes
//! - **Cleanup**: `virsh vol-delete <name> harmony-fleet` removes
//! managed volumes alongside `virsh undefine --remove-all-storage`.
//!
//! We *don't* rewrite the VM XML to use `<source pool="…" volume="…"/>`
@@ -30,11 +30,11 @@ use virt::storage_pool::StoragePool;
use crate::domain::config::HARMONY_DATA_DIR;
use crate::executors::ExecutorError;
pub const HARMONY_IOT_POOL_NAME: &str = "harmony-iot";
pub const HARMONY_FLEET_POOL_NAME: &str = "harmony-fleet";
/// Filesystem path + libvirt name of the managed pool.
#[derive(Debug, Clone)]
pub struct HarmonyIotPool {
pub struct HarmonyFleetPool {
pub name: String,
pub path: PathBuf,
}
@@ -46,13 +46,13 @@ pub struct HarmonyIotPool {
/// **Requires libvirt-group membership**. When the user isn't in the
/// group, libvirt rejects the `qemu:///system` connection — the
/// preflight check catches that upstream.
pub async fn ensure_harmony_iot_pool() -> Result<HarmonyIotPool, ExecutorError> {
static CELL: OnceCell<HarmonyIotPool> = OnceCell::const_new();
pub async fn ensure_harmony_fleet_pool() -> Result<HarmonyFleetPool, ExecutorError> {
static CELL: OnceCell<HarmonyFleetPool> = OnceCell::const_new();
CELL.get_or_try_init(provision_pool).await.cloned()
}
async fn provision_pool() -> Result<HarmonyIotPool, ExecutorError> {
let pool_dir = HARMONY_DATA_DIR.join("iot").join("kvm").join("pool");
async fn provision_pool() -> Result<HarmonyFleetPool, ExecutorError> {
let pool_dir = HARMONY_DATA_DIR.join("fleet").join("kvm").join("pool");
tokio::fs::create_dir_all(&pool_dir)
.await
.map_err(|e| exec(format!("create pool dir {pool_dir:?}: {e}")))?;
@@ -66,7 +66,7 @@ async fn provision_pool() -> Result<HarmonyIotPool, ExecutorError> {
.map_err(|e| exec(format!("chmod pool dir: {e}")))?;
let pool_path = pool_dir.clone();
let pool_name = HARMONY_IOT_POOL_NAME.to_string();
let pool_name = HARMONY_FLEET_POOL_NAME.to_string();
// virt-rs is blocking C bindings — bounce into spawn_blocking.
let pool_name_blocking = pool_name.clone();
@@ -106,7 +106,7 @@ async fn provision_pool() -> Result<HarmonyIotPool, ExecutorError> {
.await
.map_err(|e| exec(format!("spawn_blocking pool setup: {e}")))??;
Ok(HarmonyIotPool {
Ok(HarmonyFleetPool {
name: pool_name,
path: pool_path,
})

40
harmony/src/modules/fleet/mod.rs Normal file
View File

@@ -0,0 +1,40 @@
//! Harmony-side Scores for fleet device onboarding.
//!
//! Today this module exposes [`FleetDeviceSetupScore`] — a customer
//! runs it against a freshly-booted device (Pi, VM, bare-metal node
//! later) to install podman, place the `fleet-agent` binary, drop
//! the TOML config, and bring up the agent under systemd. Re-running
//! with a changed config (different labels, new NATS URL, new
//! credentials) is how a device is moved between fleet partitions.
//!
//! The operator + agent crates live outside `harmony/` under
//! `fleet/harmony-fleet-operator/` and `fleet/harmony-fleet-agent/`.
//! What belongs here is the harmony-framework side: the Scores a
//! customer runs through `harmony_cli::run` to provision devices
//! before they ever talk to NATS.
//!
//! "Fleet" is deliberately domain-agnostic — IoT was the first
//! customer's use case but the reconciler pattern (operator → NATS
//! KV → agent → target) applies equally to Pi podman, OKD apply,
//! KVM VMs, etc.
pub mod assets;
#[cfg(feature = "kvm")]
pub mod libvirt_pool;
pub mod preflight;
mod setup_score;
#[cfg(feature = "kvm")]
mod vm_score;
pub use assets::{
FleetSshKeypair, UBUNTU_2404_CLOUDIMG_ARM64_FILENAME, UBUNTU_2404_CLOUDIMG_ARM64_SHA256,
UBUNTU_2404_CLOUDIMG_ARM64_URL, UBUNTU_2404_CLOUDIMG_FILENAME, UBUNTU_2404_CLOUDIMG_SHA256,
UBUNTU_2404_CLOUDIMG_URL, ensure_fleet_ssh_keypair, ensure_ubuntu_2404_cloud_image,
ensure_ubuntu_2404_cloud_image_for_arch, read_public_key,
};
#[cfg(feature = "kvm")]
pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony_fleet_pool};
pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch};
pub use setup_score::{FleetDeviceSetupConfig, FleetDeviceSetupScore};
#[cfg(feature = "kvm")]
pub use vm_score::ProvisionVmScore;

10
harmony/src/modules/iot/preflight.rs → harmony/src/modules/fleet/preflight.rs
View File

@@ -19,18 +19,20 @@ use crate::executors::ExecutorError;
use crate::modules::kvm::firmware::discover_aarch64_firmware;
/// Run every preflight check for an x86_64 smoke run — equivalent
/// to [`check_iot_smoke_preflight_for_arch`] with
/// to [`check_fleet_smoke_preflight_for_arch`] with
/// [`VmArchitecture::X86_64`]. Kept as a distinct function so
/// existing callers don't need to thread an arch through yet.
pub async fn check_iot_smoke_preflight() -> Result<(), ExecutorError> {
check_iot_smoke_preflight_for_arch(VmArchitecture::X86_64).await
pub async fn check_fleet_smoke_preflight() -> Result<(), ExecutorError> {
check_fleet_smoke_preflight_for_arch(VmArchitecture::X86_64).await
}
/// Arch-aware preflight. On top of the host-generic checks
/// (virsh, qemu-img, xorriso, python3, ssh-keygen, libvirt group,
/// default network), an aarch64 target requires
/// `qemu-system-aarch64` and a usable AAVMF firmware pair.
pub async fn check_iot_smoke_preflight_for_arch(arch: VmArchitecture) -> Result<(), ExecutorError> {
pub async fn check_fleet_smoke_preflight_for_arch(
arch: VmArchitecture,
) -> Result<(), ExecutorError> {
check_tool_on_path("virsh", "libvirt client").await?;
check_tool_on_path("qemu-img", "qemu-utils").await?;
check_tool_on_path("xorriso", "ISO image builder").await?;

188
harmony/src/modules/iot/setup_score.rs → harmony/src/modules/fleet/setup_score.rs
View File

@@ -1,8 +1,9 @@
//! [`IotDeviceSetupScore`] — install podman + the iot-agent, wire the
//! [`FleetDeviceSetupScore`] — install podman + the fleet-agent, wire the
//! agent's TOML config, enable the systemd unit. Idempotent: re-running
//! with a changed config (e.g. a different `group`) updates only what
//! differs and restarts the agent once.
//! with a changed config (different labels, new NATS url, etc.) updates
//! only what differs and restarts the agent once.
use std::collections::BTreeMap;
use std::path::PathBuf;
use async_trait::async_trait;
@@ -25,43 +26,46 @@ use crate::score::Score;
/// User-visible configuration for the setup Score. Everything a customer
/// needs to tell us to bring a device into the fleet.
///
/// **On `group`.** For v0 the group is a *label*, written into the
/// agent's TOML config and reported back via the status bucket. It does
/// not yet drive deployment routing — `Deployment.spec.targetDevices`
/// still takes explicit device IDs. `targetGroups` is a v0.1+ item
/// (ROADMAP §6.5). Running this Score twice against the same device
/// with different `group` values is how a device is moved between
/// fleet partitions once group routing lands.
/// **On `labels`.** The label map is published verbatim in every
/// DeviceInfo heartbeat so the operator can resolve a Deployment's
/// `spec.targetSelector` against this device (K8s-Node-analogue flow).
/// `group` is the conventional primary label but any key/value pair
/// is legal. Re-running this Score with a changed label map is how a
/// device is moved between fleet partitions: the config file is
/// regenerated, byte-compare idempotency fires, the agent restarts,
/// new labels propagate.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IotDeviceSetupConfig {
pub struct FleetDeviceSetupConfig {
/// Stable device identifier. Written into the agent's TOML and
/// used as the KV key prefix (`<device_id>.<deployment>`). Harmony
/// `Id` values are sortable-by-creation-time and collision-safe
/// at up to ~10k devices/sec, which matches the feel of a fleet
/// registry.
pub device_id: Id,
/// Fleet partition this device belongs to.
pub group: String,
/// Routing labels. Published in every DeviceInfo heartbeat; the
/// operator reflects them into `Device.metadata.labels` so
/// Deployment selectors can match. Typical keys: `group`,
/// `arch`, `role`, `region`.
pub labels: BTreeMap<String, String>,
/// NATS URLs the agent should connect to. Typically one entry.
pub nats_urls: Vec<String>,
/// Shared v0 credentials (Zitadel-issued per-device tokens in v0.2).
pub nats_user: String,
pub nats_pass: String,
/// Local filesystem path to the cross-compiled `iot-agent-v0`
/// Local filesystem path to the cross-compiled `fleet-agent-v0`
/// binary. The Score uploads it to the device and installs to
/// `/usr/local/bin/iot-agent`. Future v0.1: this becomes a
/// `/usr/local/bin/fleet-agent`. Future v0.1: this becomes a
/// `DownloadableAsset` pointing at CI-published artifacts.
pub agent_binary_path: PathBuf,
}
impl IotDeviceSetupConfig {
/// Render the agent's `/etc/iot-agent/config.toml` content.
impl FleetDeviceSetupConfig {
/// Render the agent's `/etc/fleet-agent/config.toml` content.
pub fn render_toml(&self) -> String {
// Raw-string template with format! — the TOML escape rules for
// double-quoted strings are just `\` and `"`, handled by
// [`toml_escape`].
let device_id = toml_escape(&self.device_id.to_string());
let group = toml_escape(&self.group);
let nats_user = toml_escape(&self.nats_user);
let nats_pass = toml_escape(&self.nats_pass);
let urls = self
@@ -70,10 +74,18 @@ impl IotDeviceSetupConfig {
.map(|u| format!("\"{}\"", toml_escape(u)))
.collect::<Vec<_>>()
.join(", ");
// BTreeMap iteration is ordered — same labels render to
// byte-identical TOML across runs, which is what the
// Score's byte-compare idempotency relies on.
let labels = self
.labels
.iter()
.map(|(k, v)| format!("{} = \"{}\"", toml_escape(k), toml_escape(v)))
.collect::<Vec<_>>()
.join("\n");
format!(
r#"[agent]
device_id = "{device_id}"
group = "{group}"
[credentials]
type = "toml-shared"
@@ -82,6 +94,9 @@ nats_pass = "{nats_pass}"
[nats]
urls = [{urls}]
[labels]
{labels}
"#
)
}
@@ -95,10 +110,10 @@ Wants=network-online.target
[Service]
Type=simple
User=iot-agent
Environment=IOT_AGENT_CONFIG=/etc/iot-agent/config.toml
User=fleet-agent
Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml
Environment=RUST_LOG=info
ExecStart=/usr/local/bin/iot-agent
ExecStart=/usr/local/bin/fleet-agent
Restart=on-failure
RestartSec=5
StandardOutput=journal
@@ -115,23 +130,23 @@ fn toml_escape(s: &str) -> String {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IotDeviceSetupScore {
pub config: IotDeviceSetupConfig,
pub struct FleetDeviceSetupScore {
pub config: FleetDeviceSetupConfig,
}
impl IotDeviceSetupScore {
pub fn new(config: IotDeviceSetupConfig) -> Self {
impl FleetDeviceSetupScore {
pub fn new(config: FleetDeviceSetupConfig) -> Self {
Self { config }
}
}
impl<T: Topology + LinuxHostConfiguration> Score<T> for IotDeviceSetupScore {
impl<T: Topology + LinuxHostConfiguration> Score<T> for FleetDeviceSetupScore {
fn name(&self) -> String {
format!("IotDeviceSetupScore({})", self.config.device_id)
format!("FleetDeviceSetupScore({})", self.config.device_id)
}
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(IotDeviceSetupInterpret {
Box::new(FleetDeviceSetupInterpret {
config: self.config.clone(),
version: Version::from("0.1.0").expect("static version"),
status: InterpretStatus::QUEUED,
@@ -140,16 +155,16 @@ impl<T: Topology + LinuxHostConfiguration> Score<T> for IotDeviceSetupScore {
}
#[derive(Debug)]
struct IotDeviceSetupInterpret {
config: IotDeviceSetupConfig,
struct FleetDeviceSetupInterpret {
config: FleetDeviceSetupConfig,
version: Version,
status: InterpretStatus,
}
#[async_trait]
impl<T: Topology + LinuxHostConfiguration> Interpret<T> for IotDeviceSetupInterpret {
impl<T: Topology + LinuxHostConfiguration> Interpret<T> for FleetDeviceSetupInterpret {
fn get_name(&self) -> InterpretName {
InterpretName::IotDeviceSetup
InterpretName::FleetDeviceSetup
}
fn get_version(&self) -> Version {
self.version.clone()
@@ -179,33 +194,38 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for IotDeviceSetupInterp
log_change(&mut change_log, format!("package:{pkg}"), r);
}
// 2. iot-agent system user. Lingered so its user-systemd survives
// logout (needed for the user podman.socket we'll enable below).
// No explicit primary group — useradd on Debian-family systems
// defaults to `USERGROUPS_ENAB yes` which auto-creates a group
// matching the username. Setting `group:` here would require a
// separate `ensure_group` step to pre-create it.
// 2. fleet-agent user. Not `--system`: Ubuntu's useradd skips
// subuid/subgid auto-allocation for system users on the
// assumption that service accounts don't run user namespaces.
// Rootless podman needs those ranges in /etc/subuid +
// /etc/subgid before the container runtime ever starts. A
// regular useradd auto-allocates a non-overlapping range, so
// we get correct behavior for free and can coexist with any
// other user on the host that also runs rootless containers.
//
// Lingered so the user-systemd instance survives logout —
// required for the user podman.socket we enable below.
let user_spec = UserSpec {
name: "iot-agent".to_string(),
name: "fleet-agent".to_string(),
group: None,
supplementary_groups: vec![],
shell: Some("/bin/bash".to_string()),
system: true,
system: false,
create_home: true,
};
let r = UnixUserManager::ensure_user(topology, &user_spec)
.await
.map_err(wrap)?;
log_change(&mut change_log, "user:iot-agent", r);
log_change(&mut change_log, "user:fleet-agent", r);
let r = UnixUserManager::ensure_linger(topology, "iot-agent")
let r = UnixUserManager::ensure_linger(topology, "fleet-agent")
.await
.map_err(wrap)?;
log_change(&mut change_log, "linger:iot-agent", r);
log_change(&mut change_log, "linger:fleet-agent", r);
// 3. User-scoped podman socket. Required by `PodmanTopology` on
// the agent so it reaches /run/user/<uid>/podman/podman.sock.
let r = SystemdManager::ensure_user_unit_active(topology, "iot-agent", "podman.socket")
let r = SystemdManager::ensure_user_unit_active(topology, "fleet-agent", "podman.socket")
.await
.map_err(wrap)?;
log_change(&mut change_log, "user-unit:podman.socket", r);
@@ -218,7 +238,7 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for IotDeviceSetupInterp
let binary_r = FileDelivery::ensure_file(
topology,
&FileSpec {
path: "/usr/local/bin/iot-agent".to_string(),
path: "/usr/local/bin/fleet-agent".to_string(),
source: FileSource::LocalPath(cfg.agent_binary_path.clone()),
owner: Some("root".to_string()),
group: Some("root".to_string()),
@@ -227,25 +247,25 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for IotDeviceSetupInterp
)
.await
.map_err(wrap)?;
log_change(&mut change_log, "file:/usr/local/bin/iot-agent", binary_r);
log_change(&mut change_log, "file:/usr/local/bin/fleet-agent", binary_r);
// 5. /etc/iot-agent/ + config.toml
// 5. /etc/fleet-agent/ + config.toml
let config_toml = cfg.render_toml();
let toml_spec = FileSpec {
path: "/etc/iot-agent/config.toml".to_string(),
path: "/etc/fleet-agent/config.toml".to_string(),
source: FileSource::Content(config_toml),
owner: Some("iot-agent".to_string()),
group: Some("iot-agent".to_string()),
owner: Some("fleet-agent".to_string()),
group: Some("fleet-agent".to_string()),
mode: Some(0o600),
};
let toml_r = FileDelivery::ensure_file(topology, &toml_spec)
.await
.map_err(wrap)?;
log_change(&mut change_log, "file:/etc/iot-agent/config.toml", toml_r);
log_change(&mut change_log, "file:/etc/fleet-agent/config.toml", toml_r);
// 6. systemd unit for the agent itself.
let unit = SystemdUnitSpec {
name: "iot-agent".to_string(),
name: "fleet-agent".to_string(),
unit_content: cfg.render_systemd_unit().to_string(),
scope: SystemdScope::System,
start_immediately: true,
@@ -253,18 +273,18 @@ impl<T: Topology + LinuxHostConfiguration> Interpret<T> for IotDeviceSetupInterp
let unit_r = SystemdManager::ensure_systemd_unit(topology, &unit)
.await
.map_err(wrap)?;
log_change(&mut change_log, "unit:iot-agent", unit_r);
log_change(&mut change_log, "unit:fleet-agent", unit_r);
// 7. Restart the agent iff anything that affects it changed.
let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed;
if needs_restart {
SystemdManager::restart_service(topology, "iot-agent", SystemdScope::System)
SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System)
.await
.map_err(wrap)?;
change_log.push("restart:iot-agent".to_string());
info!("iot-agent restarted to pick up config/unit change");
change_log.push("restart:fleet-agent".to_string());
info!("fleet-agent restarted to pick up config/unit change");
} else {
debug!("iot-agent config + unit unchanged; no restart");
debug!("fleet-agent config + unit unchanged; no restart");
}
let outcome = if change_log.is_empty() {
@@ -292,3 +312,55 @@ fn log_change(change_log: &mut Vec<String>, what: impl Into<String>, r: ChangeRe
change_log.push(what.into());
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_config(labels: BTreeMap<String, String>) -> FleetDeviceSetupConfig {
FleetDeviceSetupConfig {
device_id: Id::from("pi-42".to_string()),
labels,
nats_urls: vec!["nats://nats:4222".to_string()],
nats_user: "admin".to_string(),
nats_pass: "pw".to_string(),
agent_binary_path: PathBuf::from("/dev/null"),
}
}
#[test]
fn render_toml_includes_labels_section() {
let mut labels = BTreeMap::new();
labels.insert("group".to_string(), "site-a".to_string());
labels.insert("arch".to_string(), "aarch64".to_string());
let toml = base_config(labels).render_toml();
assert!(toml.contains("[labels]"));
// BTreeMap sorts keys: `arch` before `group`.
let labels_block = toml.split("[labels]").nth(1).unwrap();
let arch_idx = labels_block.find("arch").unwrap();
let group_idx = labels_block.find("group").unwrap();
assert!(arch_idx < group_idx, "labels must render sorted");
assert!(labels_block.contains(r#"arch = "aarch64""#));
assert!(labels_block.contains(r#"group = "site-a""#));
}
#[test]
fn render_toml_same_labels_yields_identical_output() {
// Core idempotency invariant: two structurally-identical
// configs render byte-identical TOML. The Score's change
// detection relies on this.
let mut labels = BTreeMap::new();
labels.insert("group".to_string(), "site-a".to_string());
let a = base_config(labels.clone()).render_toml();
let b = base_config(labels).render_toml();
assert_eq!(a, b);
}
#[test]
fn render_toml_escapes_label_values() {
let mut labels = BTreeMap::new();
labels.insert("group".to_string(), r#"has"quote"#.to_string());
let toml = base_config(labels).render_toml();
assert!(toml.contains(r#"group = "has\"quote""#));
}
}

0
harmony/src/modules/iot/vm_score.rs → harmony/src/modules/fleet/vm_score.rs
View File

33
harmony/src/modules/iot/mod.rs
View File

@@ -1,33 +0,0 @@
//! IoT fleet primitives exposed to customers.
//!
//! Right now that's the single [`IotDeviceSetupScore`] — a customer runs
//! it against a freshly-booted device (Pi or VM) to install podman,
//! place the iot-agent binary, drop the TOML config, and bring up the
//! agent under systemd. Re-running with a different config (e.g.
//! different `group`) is what moves a device between fleet partitions.
//!
//! The operator + agent crates live outside of `harmony/` in `iot/`.
//! This module is where *Harmony Scores* that target IoT fleets live —
//! they run inside the Harmony framework proper, driven by the same
//! `harmony_cli::run` story every other Score uses.
pub mod assets;
#[cfg(feature = "kvm")]
pub mod libvirt_pool;
pub mod preflight;
mod setup_score;
#[cfg(feature = "kvm")]
mod vm_score;
pub use assets::{
IotSshKeypair, UBUNTU_2404_CLOUDIMG_ARM64_FILENAME, UBUNTU_2404_CLOUDIMG_ARM64_SHA256,
UBUNTU_2404_CLOUDIMG_ARM64_URL, UBUNTU_2404_CLOUDIMG_FILENAME, UBUNTU_2404_CLOUDIMG_SHA256,
UBUNTU_2404_CLOUDIMG_URL, ensure_iot_ssh_keypair, ensure_ubuntu_2404_cloud_image,
ensure_ubuntu_2404_cloud_image_for_arch, read_public_key,
};
#[cfg(feature = "kvm")]
pub use libvirt_pool::{HARMONY_IOT_POOL_NAME, HarmonyIotPool, ensure_harmony_iot_pool};
pub use preflight::{check_iot_smoke_preflight, check_iot_smoke_preflight_for_arch};
pub use setup_score::{IotDeviceSetupConfig, IotDeviceSetupScore};
#[cfg(feature = "kvm")]
pub use vm_score::ProvisionVmScore;

98
harmony/src/modules/k8s/bare_topology.rs Normal file
View File

@@ -0,0 +1,98 @@
//! Minimal Kubernetes topology for ad-hoc Score execution.
//!
//! Harmony's opinionated topologies (`K8sAnywhereTopology`,
//! `HAClusterTopology`) do a lot of product-level setup inside
//! `ensure_ready` — cert-manager install, tenant-manager bootstrap,
//! helm probes, TLS routing. That's appropriate when a caller is
//! standing up an entire NationTech-style product stack. It is
//! **not** appropriate when a caller just wants to apply a typed
//! resource (a CRD, a Deployment, a Secret, …) against an existing
//! Kubernetes cluster.
//!
//! `K8sBareTopology` is what that narrow use case needs: it carries
//! a single [`K8sClient`], implements [`K8sclient`] by handing it
//! out, and its `ensure_ready` is a noop. No helm, no certs, no
//! tenant-manager, no PLEG. Compose it with whichever
//! `K8sResourceScore<K>` / domain score needs a cluster client and
//! nothing more.
//!
//! History: this type is the promotion of a three-dozen-line
//! `InstallTopology` that lived inside `harmony-fleet-operator`'s
//! `install.rs`. When the NATS single-node install work added a
//! second consumer wanting the same shape, the extraction became
//! obvious (see ROADMAP/12-code-review-april-2026.md §12.6).
use std::process::Command;
use std::sync::Arc;
use async_trait::async_trait;
use harmony_k8s::K8sClient;
use crate::domain::topology::{HelmCommand, PreparationError, PreparationOutcome, Topology};
use crate::topology::K8sclient;
/// Minimal `Topology` that only knows how to hand out a pre-built
/// `K8sClient`. Use for Scores that need `K8sclient` but nothing
/// else from their topology.
///
/// Construct via [`K8sBareTopology::from_kubeconfig`] or
/// [`K8sBareTopology::from_client`].
#[derive(Clone)]
pub struct K8sBareTopology {
name: String,
client: Arc<K8sClient>,
}
impl K8sBareTopology {
/// Wrap a pre-built `K8sClient`. Caller is responsible for
/// having loaded it from the right place (KUBECONFIG, explicit
/// path, in-cluster service account, …).
pub fn from_client(name: impl Into<String>, client: Arc<K8sClient>) -> Self {
Self {
name: name.into(),
client,
}
}
/// Build a client from the standard kube client config
/// resolution (`KUBECONFIG` env var → `~/.kube/config` →
/// in-cluster service account, in that order).
pub async fn from_kubeconfig(name: impl Into<String>) -> Result<Self, String> {
let kube_client = kube::Client::try_default()
.await
.map_err(|e| format!("building kube client: {e}"))?;
Ok(Self::from_client(
name,
Arc::new(K8sClient::new(kube_client)),
))
}
}
#[async_trait]
impl Topology for K8sBareTopology {
fn name(&self) -> &str {
&self.name
}
async fn ensure_ready(&self) -> Result<PreparationOutcome, PreparationError> {
Ok(PreparationOutcome::Noop)
}
}
#[async_trait]
impl K8sclient for K8sBareTopology {
async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
Ok(self.client.clone())
}
}
/// Run the host's `helm` binary with whatever KUBECONFIG resolution
/// was used to build the `K8sBareTopology`. No extra context / ns
/// args — callers pass those on the command line. Lets NATS +
/// operator-install flows go through `HelmChartScore` against the
/// same cluster the bare topology already targets.
impl HelmCommand for K8sBareTopology {
fn get_helm_command(&self) -> Command {
Command::new("helm")
}
}

3
harmony/src/modules/k8s/mod.rs
View File

@@ -1,7 +1,10 @@
pub mod apps;
pub mod bare_topology;
pub mod coredns;
pub mod deployment;
mod failover;
pub mod ingress;
pub mod namespace;
pub mod resource;
pub use bare_topology::K8sBareTopology;

90
harmony/src/modules/kvm/cloudinit.rs
View File

@@ -48,6 +48,13 @@ pub struct CloudInitSeedConfig<'a> {
pub authorized_key: &'a str,
/// Local username to create with passwordless sudo.
pub user: &'a str,
/// Optional plaintext password for the admin user. `None` keeps
/// the account SSH-key-only (the default). Setting a password
/// unlocks the account *and* enables `ssh_pwauth: true` on the
/// guest — intended for interactive debugging / chaos-testing
/// workflows where the operator wants console or SSH password
/// access to break things on purpose.
pub admin_password: Option<&'a str>,
/// Extra `runcmd` lines to append to the user-data. Mostly useful
/// for no-op debugging; keep empty in production paths.
pub extra_runcmd: Vec<String>,
@@ -144,6 +151,21 @@ fn render_user_data(cfg: &CloudInitSeedConfig<'_>) -> String {
}
s
};
// Password handling is split into user-level (lock_passwd +
// plain_text_passwd) and daemon-level (ssh_pwauth). When a
// password is provided, cloud-init hashes + sets the password and
// we allow SSH password auth. When it isn't, the account stays
// locked and sshd denies password logins — the production default.
let (lock_passwd, plain_text_passwd_line, ssh_pwauth) = match cfg.admin_password {
Some(pw) => (
"false",
format!(" plain_text_passwd: \"{}\"\n", yaml_escape(pw)),
"true",
),
None => ("true", String::new(), "false"),
};
format!(
r#"#cloud-config
hostname: {hostname}
@@ -153,10 +175,10 @@ users:
- name: {user}
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
lock_passwd: true
ssh_authorized_keys:
lock_passwd: {lock_passwd}
{plain_text_passwd_line} ssh_authorized_keys:
- {authorized_key}
ssh_pwauth: false
ssh_pwauth: {ssh_pwauth}
disable_root: true
{runcmd}"#,
hostname = cfg.hostname,
@@ -165,6 +187,11 @@ disable_root: true
)
}
fn yaml_escape(s: &str) -> String {
// Double-quoted YAML: backslash and double-quote need escaping.
s.replace('\\', "\\\\").replace('"', "\\\"")
}
async fn write_file(path: &Path, content: &str) -> Result<(), KvmError> {
let mut f = tokio::fs::File::create(path).await.map_err(KvmError::Io)?;
f.write_all(content.as_bytes())
@@ -188,3 +215,60 @@ async fn which_xorriso() -> Option<PathBuf> {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_password_locks_account_and_disables_ssh_pwauth() {
let cfg = CloudInitSeedConfig {
hostname: "pi-01",
authorized_key: "ssh-ed25519 AAAA test",
user: "fleet-admin",
admin_password: None,
extra_runcmd: vec![],
};
let out = render_user_data(&cfg);
assert!(out.contains("lock_passwd: true"), "got:\n{out}");
assert!(out.contains("ssh_pwauth: false"), "got:\n{out}");
assert!(
!out.contains("plain_text_passwd"),
"password leaked into cloud-init without admin_password set:\n{out}"
);
}
#[test]
fn with_password_unlocks_account_and_enables_ssh_pwauth() {
let cfg = CloudInitSeedConfig {
hostname: "pi-01",
authorized_key: "ssh-ed25519 AAAA test",
user: "fleet-admin",
admin_password: Some("break-things-123"),
extra_runcmd: vec![],
};
let out = render_user_data(&cfg);
assert!(out.contains("lock_passwd: false"), "got:\n{out}");
assert!(out.contains("ssh_pwauth: true"), "got:\n{out}");
assert!(
out.contains("plain_text_passwd: \"break-things-123\""),
"password not inlined in cloud-init:\n{out}"
);
}
#[test]
fn password_with_quotes_is_yaml_escaped() {
let cfg = CloudInitSeedConfig {
hostname: "pi-01",
authorized_key: "ssh-ed25519 AAAA",
user: "fleet-admin",
admin_password: Some("he said \"hi\""),
extra_runcmd: vec![],
};
let out = render_user_data(&cfg);
assert!(
out.contains(r#"plain_text_passwd: "he said \"hi\"""#),
"got:\n{out}"
);
}
}

46
harmony/src/modules/kvm/topology.rs
View File

@@ -35,7 +35,7 @@ pub const DEFAULT_ADMIN_USER: &str = "harmony-admin";
///
/// Composes with a caller-chosen storage pool directory where per-VM
/// overlays + seed ISOs are placed. Harmony's IoT workflows use
/// [`crate::modules::iot::ensure_harmony_iot_pool`] to populate that
/// [`crate::modules::fleet::ensure_harmony_fleet_pool`] to populate that
/// dir; other callers can point at any user-owned libvirt pool root.
pub struct KvmVirtualMachineHost {
name: String,
@@ -120,7 +120,7 @@ impl VirtualMachineHost for KvmVirtualMachineHost {
.await
.map_err(|e| exec(format!("remove stale overlay: {e}")))?;
}
create_overlay(&self.base_image_path, &overlay_path).await?;
create_overlay(&self.base_image_path, &overlay_path, spec.disk_size_gb).await?;
info!(
"created overlay disk {overlay_path:?} backed by {:?}",
self.base_image_path
@@ -297,21 +297,36 @@ async fn ensure_vm_firmware(
async fn create_overlay(
base: &std::path::Path,
overlay: &std::path::Path,
size_gb: Option<u32>,
) -> Result<(), ExecutorError> {
let base_str = base
.to_str()
.ok_or_else(|| exec("base image path is not valid UTF-8"))?;
let overlay_str = overlay
.to_str()
.ok_or_else(|| exec("overlay path is not valid UTF-8"))?;
// qemu-img takes an optional trailing SIZE. Without it, the
// overlay inherits the backing image's virtual size (2-3 GiB
// for the stock Ubuntu cloud image) which is tight as soon as
// a couple of container images land. Ubuntu cloud-init ships
// `cloud-initramfs-growroot`, so a larger virtual size is
// resized on first boot without extra glue.
let size_arg = size_gb.filter(|g| *g > 0).map(|g| format!("{g}G"));
let mut args: Vec<&str> = vec![
"create",
"-f",
"qcow2",
"-F",
"qcow2",
"-b",
base_str,
overlay_str,
];
if let Some(s) = size_arg.as_deref() {
args.push(s);
}
let output = Command::new("qemu-img")
.args([
"create",
"-f",
"qcow2",
"-F",
"qcow2",
"-b",
base.to_str()
.ok_or_else(|| exec("base image path is not valid UTF-8"))?,
overlay
.to_str()
.ok_or_else(|| exec("overlay path is not valid UTF-8"))?,
])
.args(&args)
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()
@@ -349,6 +364,7 @@ async fn build_cloud_init_seed(
hostname: &hostname,
authorized_key: &authorized_key,
user: &admin_user,
admin_password: first_boot.admin_password.as_deref(),
extra_runcmd: vec![],
},
pool_dir,

4
harmony/src/modules/linux/ansible_configurator.rs
View File

@@ -57,7 +57,7 @@ impl AnsibleHostConfigurator {
// encapsulation we want. Callers say "install podman"; we
// pick apt/dnf/pacman/apk. Debian-family is the only dispatch
// currently wired because it's our first concrete target (IoT
// runs on Raspbian/Ubuntu per ROADMAP/iot_platform/
// runs on Raspbian/Ubuntu per ROADMAP/fleet_platform/
// v0_walking_skeleton.md §5.3). Extending to RHEL/Fedora/
// Alpine is a matter of detecting the family here and picking
// `ansible.builtin.dnf` / `community.general.pacman` /
@@ -112,7 +112,7 @@ impl AnsibleHostConfigurator {
spec: &FileSpec,
) -> Result<ChangeReport, ExecutorError> {
// Ansible's `copy` module doesn't auto-create parent dirs, so
// writes into fresh paths like `/etc/iot-agent/config.toml`
// writes into fresh paths like `/etc/fleet-agent/config.toml`
// fail with "Destination directory … does not exist". Create
// the parent first via the `file` module; state=directory is
// idempotent so this is a cheap noop on re-run.

2
harmony/src/modules/mod.rs
View File

@@ -5,10 +5,10 @@ pub mod cert_manager;
pub mod dhcp;
pub mod dns;
pub mod dummy;
pub mod fleet;
pub mod helm;
pub mod http;
pub mod inventory;
pub mod iot;
pub mod k3d;
pub mod k8s;
#[cfg(feature = "kvm")]

185
harmony/src/modules/nats/helm_chart.rs Normal file
View File

@@ -0,0 +1,185 @@
//! Shared helm-chart primitive for every NATS deployment shape.
//!
//! The upstream `nats/nats` helm chart is the single source of truth
//! for how a NATS pod / STS is actually built: probes, resource
//! shapes, RBAC, stateful-set options, JetStream storage volumes,
//! clustering, TLS, gateways, leaf nodes. Every high-level NATS
//! Score — `NatsBasicScore` for single-node, `NatsK8sScore` for
//! supercluster — delegates here. Differences between shapes are
//! expressed as `values_yaml`, not as parallel resource constructors.
//!
//! Why this is the right primitive:
//!
//! - The NATS project's chart tracks upstream server features
//! automatically; we get new knobs (`websocket.enabled`,
//! `gateway.merge.advertise`, …) without shipping code.
//! - One helm release per NATS deployment means `helm upgrade` /
//! `helm uninstall` / `helm list` all work naturally.
//! - Chapter 4 of the harmony review learned this the hard way: a
//! parallel k8s_openapi-based NATS primitive diverged on probe
//! shape + pod-anti-affinity and was deleted.
use std::str::FromStr;
use async_trait::async_trait;
use harmony_macros::hurl;
use harmony_types::id::Id;
use non_blank_string_rs::NonBlankString;
use serde::Serialize;
use crate::data::Version;
use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use crate::inventory::Inventory;
use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
use crate::score::Score;
use crate::topology::{HelmCommand, Topology};
/// The NATS-IO project's published helm chart. `hurl!` needs a
/// literal so the URL is inlined at the one call site below rather
/// than being a `const &str`.
const CHART_NAME: &str = "nats/nats";
const REPO_NAME: &str = "nats";
/// Thin preset over [`HelmChartScore`] that pins the NATS chart +
/// repository and leaves `values_yaml` as the one parameter.
///
/// Callers should almost never construct this directly — build a
/// high-level preset (`NatsBasicScore`, `NatsK8sScore`) instead.
/// The type is `pub` so those presets across different files can
/// share a single definition.
#[derive(Debug, Clone, Serialize)]
pub struct NatsHelmChartScore {
pub namespace: NonBlankString,
pub release_name: NonBlankString,
/// Helm values YAML specific to this shape. Build with the
/// preset's dedicated rendering function; `values_overrides`
/// style is intentionally not exposed — values_yaml is readable
/// + diffable, overrides are not.
pub values_yaml: String,
/// Whether helm should create the target namespace if missing.
pub create_namespace: bool,
/// `true` = `helm install` (fail on re-apply), `false` =
/// `helm upgrade --install` (idempotent). Presets default to
/// upgrade-install so re-running a Score is safe.
pub install_only: bool,
}
impl NatsHelmChartScore {
/// Build a score targeting the upstream NATS chart at the given
/// release name + namespace with the caller's values yaml.
pub fn new(
release_name: impl Into<String>,
namespace: impl Into<String>,
values_yaml: String,
) -> Self {
Self {
release_name: NonBlankString::from_str(&release_name.into())
.expect("non-blank release_name"),
namespace: NonBlankString::from_str(&namespace.into()).expect("non-blank namespace"),
values_yaml,
create_namespace: true,
install_only: false,
}
}
/// Convert into the underlying [`HelmChartScore`]. Exists for the
/// rare callers that need to hand the result to a non-NATS
/// pipeline (e.g. `ArgoCD`-backed deploy wrappers); presets
/// normally just use the `Score` impl.
pub fn into_helm_chart_score(self) -> HelmChartScore {
HelmChartScore {
namespace: Some(self.namespace),
release_name: self.release_name,
chart_name: NonBlankString::from_str(CHART_NAME).expect("chart name const is valid"),
chart_version: None,
values_overrides: None,
values_yaml: Some(self.values_yaml),
create_namespace: self.create_namespace,
install_only: self.install_only,
repository: Some(HelmRepository::new(
REPO_NAME.to_string(),
hurl!("https://nats-io.github.io/k8s/helm/charts/"),
true,
)),
}
}
}
impl<T: Topology + HelmCommand> Score<T> for NatsHelmChartScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(NatsHelmChartInterpret {
score: self.clone(),
})
}
fn name(&self) -> String {
format!("NatsHelmChartScore({})", self.release_name)
}
}
#[derive(Debug)]
pub struct NatsHelmChartInterpret {
score: NatsHelmChartScore,
}
#[async_trait]
impl<T: Topology + HelmCommand> Interpret<T> for NatsHelmChartInterpret {
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
self.score
.clone()
.into_helm_chart_score()
.create_interpret()
.execute(inventory, topology)
.await
}
fn get_name(&self) -> InterpretName {
InterpretName::HelmChart
}
fn get_version(&self) -> Version {
Version::from("0.1.0").expect("static version literal")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::QUEUED
}
fn get_children(&self) -> Vec<Id> {
vec![]
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn into_helm_chart_score_pins_chart_and_repo() {
let s = NatsHelmChartScore::new(
"fleet-nats",
"fleet-system",
"replicaCount: 1\n".to_string(),
);
let hc = s.into_helm_chart_score();
assert_eq!(hc.chart_name.to_string(), CHART_NAME);
let repo = hc.repository.expect("repo must be pinned");
// We're not inspecting the fields further — HelmRepository's
// fields are private — but pinning `repository = Some(..)`
// at all is what matters: without it `helm install` would
// try the release-name as a local path.
let _ = repo;
assert_eq!(hc.values_yaml.as_deref(), Some("replicaCount: 1\n"));
}
#[test]
fn defaults_are_upgrade_install_with_namespace_creation() {
let s = NatsHelmChartScore::new("n", "ns", "".to_string());
assert!(s.create_namespace, "presets expect namespace creation");
assert!(!s.install_only, "presets expect upgrade-install semantics");
}
}

5
harmony/src/modules/nats/mod.rs
View File

@@ -1,5 +1,10 @@
pub mod capability;
pub mod decentralized;
pub mod helm_chart;
pub mod pki;
pub mod score_nats_basic;
pub mod score_nats_k8s;
pub mod score_nats_supercluster;
pub use helm_chart::NatsHelmChartScore;
pub use score_nats_basic::{NatsBasicScore, NatsServiceType};

307
harmony/src/modules/nats/score_nats_basic.rs Normal file
View File

@@ -0,0 +1,307 @@
//! Single-node NATS — high-level preset over [`NatsHelmChartScore`].
//!
johnride commented 2026-04-23 14:02:35 +00:00
Outdated
Review
Copy Link

The feature is correct but the implementation of the nats node primitive is wrong, see previous comment.

The feature is correct but the implementation of the nats node primitive is wrong, see previous comment.
//! The shape this Score covers: one NATS server pod in a cluster,
//! JetStream on by default, exposed via ClusterIP / NodePort /
//! LoadBalancer. No TLS, no clustering, no auth. For any of those,
//! graduate to `NatsK8sScore` (supercluster + TLS + gateways).
//!
//! Everything concrete — probes, resource limits, statefulset
//! options — comes from the upstream `nats/nats` helm chart.
//! This Score just picks the chart values that select a minimal
//! single-node install.
//!
//! Typical usage:
//!
//! ```ignore
//! use harmony::modules::k8s::K8sBareTopology;
//! use harmony::modules::nats::NatsBasicScore;
//! use harmony::score::Score;
//! use harmony::inventory::Inventory;
//!
//! let topology = K8sBareTopology::from_kubeconfig("nats-install").await?;
//! let score = NatsBasicScore::new("fleet-nats", "fleet-system").load_balancer();
//! score.create_interpret().execute(&Inventory::empty(), &topology).await?;
//! ```
use async_trait::async_trait;
use harmony_types::id::Id;
use serde::Serialize;
use crate::data::Version;
use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use crate::inventory::Inventory;
use crate::modules::nats::helm_chart::NatsHelmChartScore;
use crate::score::Score;
use crate::topology::{HelmCommand, Topology};
/// How the NATS client port is exposed.
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
pub enum NatsServiceType {
/// In-cluster only. Caller reaches NATS via
/// `<release>.<namespace>.svc.cluster.local:4222`.
ClusterIp,
/// NodePort on the given host port — must fall in the cluster's
/// configured service-node-port range (default 30000-32767).
NodePort(i32),
/// LoadBalancer service. On k3d this uses the built-in
/// `klipper-lb`, which pairs naturally with
/// `k3d cluster create -p PORT:PORT@loadbalancer`.
LoadBalancer,
}
/// Declarative single-node NATS. Construct via [`new`], tune with
/// the builder-style methods, hand to a topology that implements
/// [`HelmCommand`].
#[derive(Debug, Clone, Serialize)]
pub struct NatsBasicScore {
release_name: String,
namespace: String,
jetstream: bool,
service_type: NatsServiceType,
/// Optional image override (`repository:tag` or full ref).
/// `None` = use the chart's default image.
image: Option<String>,
}
impl NatsBasicScore {
/// Build a single-node NATS score with JetStream on and
/// ClusterIP exposure. Use the builder methods to change the
/// exposure or image.
pub fn new(release_name: impl Into<String>, namespace: impl Into<String>) -> Self {
Self {
release_name: release_name.into(),
namespace: namespace.into(),
jetstream: true,
service_type: NatsServiceType::ClusterIp,
image: None,
}
}
pub fn jetstream(mut self, enabled: bool) -> Self {
self.jetstream = enabled;
self
}
pub fn node_port(mut self, port: i32) -> Self {
self.service_type = NatsServiceType::NodePort(port);
self
}
pub fn load_balancer(mut self) -> Self {
self.service_type = NatsServiceType::LoadBalancer;
self
}
pub fn image(mut self, image: impl Into<String>) -> Self {
self.image = Some(image.into());
self
}
/// Render the chart values for this preset. Public so tests +
/// downstream tools (e.g. `helm template` diffs) can inspect
/// exactly what the Score will install.
pub fn render_values(&self) -> String {
let mut y = String::new();
y.push_str(&format!("fullnameOverride: {}\n", self.release_name));
y.push_str("replicaCount: 1\n");
y.push_str("config:\n");
y.push_str(" cluster:\n");
y.push_str(" enabled: false\n");
y.push_str(" jetstream:\n");
y.push_str(&format!(" enabled: {}\n", self.jetstream));
if self.jetstream {
y.push_str(" fileStorage:\n");
y.push_str(" enabled: true\n");
y.push_str(" size: 10Gi\n");
}
match self.service_type {
NatsServiceType::ClusterIp => {
// Chart default. No overrides needed.
}
NatsServiceType::NodePort(port) => {
y.push_str("service:\n");
y.push_str(" merge:\n");
y.push_str(" spec:\n");
y.push_str(" type: NodePort\n");
y.push_str(" ports:\n");
y.push_str(" nats:\n");
y.push_str(" merge:\n");
y.push_str(&format!(" nodePort: {port}\n"));
}
NatsServiceType::LoadBalancer => {
y.push_str("service:\n");
y.push_str(" merge:\n");
y.push_str(" spec:\n");
y.push_str(" type: LoadBalancer\n");
}
}
if let Some(img) = &self.image {
let (repo, tag) = split_image_ref(img);
y.push_str("container:\n");
y.push_str(" image:\n");
y.push_str(&format!(" repository: {repo}\n"));
if let Some(tag) = tag {
y.push_str(&format!(" tag: {tag}\n"));
}
}
y
}
/// Name accessors — used by downstream presets + tests that
/// need to reference what this Score will name its resources.
pub fn release_name(&self) -> &str {
&self.release_name
}
pub fn namespace(&self) -> &str {
&self.namespace
}
}
fn split_image_ref(image: &str) -> (String, Option<String>) {
// Split on the *last* colon that isn't part of a registry port
// (`registry.io:5000/foo:v1`). Good enough for the shapes we
// see in practice (`nats:2.10-alpine`, `ghcr.io/nats-io/nats:v2`).
match image.rsplit_once(':') {
Some((r, t)) if !t.contains('/') => (r.to_string(), Some(t.to_string())),
_ => (image.to_string(), None),
}
}
impl<T: Topology + HelmCommand> Score<T> for NatsBasicScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(NatsBasicInterpret {
score: self.clone(),
})
}
fn name(&self) -> String {
"NatsBasicScore".to_string()
}
}
#[derive(Debug)]
pub struct NatsBasicInterpret {
score: NatsBasicScore,
}
#[async_trait]
impl<T: Topology + HelmCommand> Interpret<T> for NatsBasicInterpret {
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let values_yaml = self.score.render_values();
NatsHelmChartScore::new(&self.score.release_name, &self.score.namespace, values_yaml)
.create_interpret()
.execute(inventory, topology)
.await
}
fn get_name(&self) -> InterpretName {
InterpretName::Custom("NatsBasicInterpret")
}
fn get_version(&self) -> Version {
Version::from("0.1.0").expect("static version literal")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::QUEUED
}
fn get_children(&self) -> Vec<Id> {
vec![]
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn defaults_are_clusterip_jetstream_on() {
let s = NatsBasicScore::new("n", "ns");
assert_eq!(s.service_type, NatsServiceType::ClusterIp);
assert!(s.jetstream);
assert!(s.image.is_none());
}
#[test]
fn render_values_includes_fullname_and_replica() {
let y = NatsBasicScore::new("fleet-nats", "fleet-system").render_values();
assert!(y.contains("fullnameOverride: fleet-nats"));
assert!(y.contains("replicaCount: 1"));
// cluster.enabled stays false for a single-node shape.
assert!(y.contains("cluster:\n enabled: false"));
}
#[test]
fn render_values_enables_jetstream_with_storage_by_default() {
let y = NatsBasicScore::new("n", "ns").render_values();
assert!(y.contains("jetstream:\n enabled: true"));
assert!(y.contains("fileStorage:\n enabled: true"));
}
#[test]
fn render_values_omits_storage_when_jetstream_off() {
let y = NatsBasicScore::new("n", "ns")
.jetstream(false)
.render_values();
assert!(y.contains("jetstream:\n enabled: false"));
assert!(!y.contains("fileStorage"));
}
#[test]
fn render_values_node_port_patches_service_and_port() {
let y = NatsBasicScore::new("n", "ns")
.node_port(30222)
.render_values();
assert!(y.contains("type: NodePort"));
assert!(y.contains("nodePort: 30222"));
}
#[test]
fn render_values_load_balancer_sets_service_type() {
let y = NatsBasicScore::new("n", "ns")
.load_balancer()
.render_values();
assert!(y.contains("type: LoadBalancer"));
// LoadBalancer doesn't specify a nodePort — let kube assign.
assert!(!y.contains("nodePort:"));
}
#[test]
fn render_values_clusterip_has_no_service_block() {
let y = NatsBasicScore::new("n", "ns").render_values();
assert!(!y.contains("service:"));
}
#[test]
fn render_values_image_override_splits_repo_and_tag() {
let y = NatsBasicScore::new("n", "ns")
.image("registry.io/custom/nats:2.10-alpine")
.render_values();
assert!(y.contains("repository: registry.io/custom/nats"));
assert!(y.contains("tag: 2.10-alpine"));
}
#[test]
fn render_values_image_without_tag_omits_tag_line() {
let y = NatsBasicScore::new("n", "ns")
.image("my.internal/nats-no-tag")
.render_values();
assert!(y.contains("repository: my.internal/nats-no-tag"));
assert!(!y.contains("tag:"));
}
#[test]
fn setters_return_self_for_chaining() {
let s = NatsBasicScore::new("n", "ns")
.jetstream(true)
.load_balancer()
.image("nats:latest");
assert_eq!(s.release_name(), "n");
assert_eq!(s.namespace(), "ns");
}
}

27
harmony/src/modules/nats/score_nats_k8s.rs
View File

@@ -1,14 +1,12 @@
use std::{collections::BTreeMap, str::FromStr};
use std::collections::BTreeMap;
use async_trait::async_trait;
use harmony_k8s::KubernetesDistribution;
use harmony_macros::hurl;
use harmony_secret::{Secret, SecretManager};
use harmony_types::id::Id;
use k8s_openapi::{ByteString, api::core::v1::Secret as K8sSecret};
use kube::api::ObjectMeta;
use log::{debug, info};
use non_blank_string_rs::NonBlankString;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -17,9 +15,11 @@ use crate::{
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
inventory::Inventory,
modules::{
helm::chart::{HelmChartScore, HelmRepository},
k8s::{ingress::K8sIngressScore, resource::K8sResourceScore},
nats::capability::{Nats, NatsCluster, NatsEndpoint},
nats::{
capability::{Nats, NatsCluster, NatsEndpoint},
helm_chart::NatsHelmChartScore,
},
okd::{
crd::route::{RoutePort, RouteSpec, RouteTargetReference, TLSConfig},
route::OKDRouteScore,
@@ -325,21 +325,8 @@ natsBox:
));
debug!("Prepared Helm Chart values : \n{values_yaml:#?}");
let nats = HelmChartScore {
namespace: Some(NonBlankString::from_str(&namespace).unwrap()),
release_name: NonBlankString::from_str(&cluster.name).unwrap(),
chart_name: NonBlankString::from_str("nats/nats").unwrap(),
chart_version: None,
values_overrides: None,
values_yaml,
create_namespace: true,
install_only: false,
repository: Some(HelmRepository::new(
"nats".to_string(),
hurl!("https://nats-io.github.io/k8s/helm/charts/"),
true,
)),
};
let values_yaml = values_yaml.expect("supercluster always builds a values_yaml");
let nats = NatsHelmChartScore::new(cluster.name.clone(), namespace, values_yaml);
nats.interpret(inventory, topology).await
}
}

2
harmony/src/modules/podman/mod.rs
View File

@@ -3,5 +3,5 @@ mod score;
mod topology;
pub use interpret::PodmanV0Interpret;
pub use score::{IotScore, PodmanService, PodmanV0Score};
pub use score::{PodmanService, PodmanV0Score, ReconcileScore};
pub use topology::PodmanTopology;

14
harmony/src/modules/podman/score.rs
View File

@@ -55,7 +55,7 @@ impl PodmanV0Score {
/// log-and-skip the unknown tag.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "type", content = "data")]
pub enum IotScore {
pub enum ReconcileScore {
PodmanV0(PodmanV0Score),
}
@@ -69,16 +69,16 @@ impl<T: Topology + ContainerRuntime> Score<T> for PodmanV0Score {
}
}
impl<T: Topology + ContainerRuntime> Score<T> for IotScore {
impl<T: Topology + ContainerRuntime> Score<T> for ReconcileScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
match self {
IotScore::PodmanV0(score) => score.create_interpret(),
ReconcileScore::PodmanV0(score) => score.create_interpret(),
}
}
fn name(&self) -> String {
match self {
IotScore::PodmanV0(_) => "PodmanV0Score".to_string(),
ReconcileScore::PodmanV0(_) => "PodmanV0Score".to_string(),
}
}
}
@@ -89,7 +89,7 @@ mod tests {
#[test]
fn podman_v0_score_serializes_with_adjacent_tag() {
let score = IotScore::PodmanV0(PodmanV0Score {
let score = ReconcileScore::PodmanV0(PodmanV0Score {
services: vec![PodmanService {
name: "web".to_string(),
image: "nginx:latest".to_string(),
@@ -103,7 +103,7 @@ mod tests {
#[test]
fn podman_v0_score_roundtrip() {
let score = IotScore::PodmanV0(PodmanV0Score {
let score = ReconcileScore::PodmanV0(PodmanV0Score {
services: vec![
PodmanService {
name: "web".to_string(),
@@ -118,7 +118,7 @@ mod tests {
],
});
let serialized = serde_json::to_string(&score).unwrap();
let deserialized: IotScore = serde_json::from_str(&serialized).unwrap();
let deserialized: ReconcileScore = serde_json::from_str(&serialized).unwrap();
assert_eq!(score, deserialized);
}

15
harmony/src/modules/podman/topology.rs
View File

@@ -62,8 +62,21 @@ impl PodmanTopology {
}
async fn ensure_image_present(&self, image: &str) -> Result<(), ExecutorError> {
let opts = PullOpts::builder().reference(image).build();
// Fast path: image already in the local store → no network
// call, no rate-limit exposure. Matches the behaviour a
// Kubernetes `imagePullPolicy: IfNotPresent` would give, and
// it's the right default for a long-lived device agent —
// every podman `pull` against a public registry is rate-
// limited traffic we only want to spend when strictly
// necessary. Upgrades (different `image` string / tag) hit
// this function with a reference that's NOT locally
// present yet and still do the pull below.
let images = self.podman.images();
if images.get(image).exists().await.map_err(to_exec_error)? {
return Ok(());
}
let opts = PullOpts::builder().reference(image).build();
let mut stream = images.pull(&opts);
while let Some(event) = stream.next().await {
let event = event.map_err(to_exec_error)?;

145
iot/iot-agent-v0/src/reconciler.rs
View File

@@ -1,145 +0,0 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use tokio::sync::Mutex;
use harmony::inventory::Inventory;
use harmony::modules::podman::{IotScore, PodmanTopology, PodmanV0Score};
use harmony::score::Score;
/// Cache key → last-seen state, populated by `apply` and consulted by the
/// 30-second periodic tick and the delete path.
struct CachedEntry {
/// Serialized score JSON. Used for string-compare idempotency per
/// ROADMAP §5.5 — cheaper and more deterministic than a hash.
serialized: String,
/// Parsed score. Cached so the periodic reconcile tick and delete
/// handlers don't have to re-parse the JSON.
score: PodmanV0Score,
}
pub struct Reconciler {
topology: Arc<PodmanTopology>,
inventory: Arc<Inventory>,
/// Keyed by NATS KV key (`<device>.<deployment>`). A single entry per
/// KV key — in v0 there is no fan-out from one key to many scores.
state: Mutex<HashMap<String, CachedEntry>>,
}
impl Reconciler {
pub fn new(topology: Arc<PodmanTopology>, inventory: Arc<Inventory>) -> Self {
Self {
topology,
inventory,
state: Mutex::new(HashMap::new()),
}
}
/// Handle a Put event (new or updated score on NATS KV). No-ops if the
/// serialized score is byte-identical to the last-seen value for this
/// key.
pub async fn apply(&self, key: &str, value: &[u8]) -> Result<()> {
let incoming = match serde_json::from_slice::<IotScore>(value) {
Ok(IotScore::PodmanV0(s)) => s,
Err(e) => {
tracing::warn!(key, error = %e, "failed to deserialize score");
return Ok(());
}
};
let serialized = String::from_utf8_lossy(value).into_owned();
{
let state = self.state.lock().await;
if let Some(existing) = state.get(key) {
if existing.serialized == serialized {
tracing::debug!(key, "score unchanged — noop");
return Ok(());
}
}
}
self.run_score(key, &incoming).await?;
let mut state = self.state.lock().await;
state.insert(
key.to_string(),
CachedEntry {
serialized,
score: incoming,
},
);
Ok(())
}
/// Handle a Delete/Purge event. Stops and removes every container
/// referenced by the last cached score for this key. Idempotent: if we
/// never saw a Put for this key (agent restart after delete), logs and
/// returns ok.
pub async fn remove(&self, key: &str) -> Result<()> {
let mut state = self.state.lock().await;
let Some(entry) = state.remove(key) else {
tracing::info!(key, "delete for unknown key — nothing to remove");
return Ok(());
};
drop(state);
use harmony::topology::ContainerRuntime;
for service in &entry.score.services {
if let Err(e) = self.topology.remove_service(&service.name).await {
tracing::warn!(
key,
service = %service.name,
error = %e,
"failed to remove container"
);
} else {
tracing::info!(key, service = %service.name, "removed container");
}
}
Ok(())
}
/// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of
/// event-driven PLEG. Agent polls podman every 30s as ground truth;
/// KV watch events are accelerators." Re-runs each cached score against
/// podman-api; the underlying `ensure_service_running` is idempotent
/// so a converged state produces no log noise.
pub async fn tick(&self) -> Result<()> {
let snapshot: Vec<(String, PodmanV0Score)> = {
let state = self.state.lock().await;
state
.iter()
.map(|(k, v)| (k.clone(), v.score.clone()))
.collect()
};
for (key, score) in snapshot {
if let Err(e) = self.run_score(&key, &score).await {
tracing::warn!(key, error = %e, "periodic reconcile failed");
}
}
Ok(())
}
pub async fn run_periodic(self: Arc<Self>, interval: Duration) {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
if let Err(e) = self.tick().await {
tracing::warn!(error = %e, "reconcile tick error");
}
}
}
async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> {
let interpret = Score::<PodmanTopology>::create_interpret(score);
let outcome = interpret
.execute(&self.inventory, &self.topology)
.await
.map_err(|e| anyhow::anyhow!("PodmanV0Score interpret failed for {key}: {e}"))?;
tracing::info!(key, outcome = ?outcome, "reconciled");
Ok(())
}
}

137
iot/iot-operator-v0/src/controller.rs
View File

@@ -1,137 +0,0 @@
use std::sync::Arc;
use std::time::Duration;
use async_nats::jetstream::kv::Store;
use futures_util::StreamExt;
use harmony_reconciler_contracts::desired_state_key;
use kube::api::{Patch, PatchParams};
use kube::runtime::Controller;
use kube::runtime::controller::Action;
use kube::runtime::finalizer::{Event as FinalizerEvent, finalizer};
use kube::runtime::watcher::Config as WatcherConfig;
use kube::{Api, Client, ResourceExt};
use serde_json::json;
use crate::crd::{Deployment, DeploymentStatus, ScorePayload};
const FINALIZER: &str = "iot.nationtech.io/finalizer";
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("kube api: {0}")]
Kube(#[from] kube::Error),
#[error("nats kv: {0}")]
Kv(String),
#[error("serde: {0}")]
Serde(#[from] serde_json::Error),
#[error("missing namespace on resource")]
MissingNamespace,
#[error("missing target devices")]
MissingTargets,
}
pub struct Context {
pub client: Client,
pub kv: Store,
}
pub async fn run(client: Client, kv: Store) -> anyhow::Result<()> {
let api: Api<Deployment> = Api::all(client.clone());
let ctx = Arc::new(Context { client, kv });
tracing::info!("starting Deployment controller");
Controller::new(api, WatcherConfig::default())
.run(reconcile, error_policy, ctx)
.for_each(|res| async move {
match res {
Ok((obj, _)) => tracing::debug!(?obj, "reconciled"),
Err(e) => tracing::warn!(error = %e, "reconcile error"),
}
})
.await;
Ok(())
}
async fn reconcile(obj: Arc<Deployment>, ctx: Arc<Context>) -> Result<Action, Error> {
let ns = obj.namespace().ok_or(Error::MissingNamespace)?;
let name = obj.name_any();
tracing::info!(%ns, %name, "reconcile");
let api: Api<Deployment> = Api::namespaced(ctx.client.clone(), &ns);
finalizer(&api, FINALIZER, obj, |event| async {
match event {
FinalizerEvent::Apply(d) => apply(d, &api, &ctx.kv).await,
FinalizerEvent::Cleanup(d) => cleanup(d, &ctx.kv).await,
}
})
.await
.map_err(|e| match e {
kube::runtime::finalizer::Error::ApplyFailed(e)
| kube::runtime::finalizer::Error::CleanupFailed(e) => e,
kube::runtime::finalizer::Error::AddFinalizer(e)
| kube::runtime::finalizer::Error::RemoveFinalizer(e) => Error::Kube(e),
kube::runtime::finalizer::Error::UnnamedObject => Error::Kv("unnamed object".into()),
kube::runtime::finalizer::Error::InvalidFinalizer => Error::Kv("invalid finalizer".into()),
})
}
async fn apply(obj: Arc<Deployment>, api: &Api<Deployment>, kv: &Store) -> Result<Action, Error> {
let name = obj.name_any();
if obj.spec.target_devices.is_empty() {
return Err(Error::MissingTargets);
}
let score_json = serialize_score(&obj.spec.score)?;
let already_observed = obj
.status
.as_ref()
.and_then(|s| s.observed_score_string.as_deref())
== Some(score_json.as_str());
if already_observed {
tracing::debug!(%name, "score unchanged; skipping KV write and status patch");
return Ok(Action::requeue(Duration::from_secs(300)));
}
for device_id in &obj.spec.target_devices {
let key = kv_key(device_id, &name);
kv.put(key.clone(), score_json.clone().into_bytes().into())
.await
.map_err(|e| Error::Kv(e.to_string()))?;
tracing::info!(%key, "wrote desired state");
}
let status = json!({
"status": DeploymentStatus {
observed_score_string: Some(score_json),
}
});
api.patch_status(&name, &PatchParams::default(), &Patch::Merge(&status))
.await?;
Ok(Action::requeue(Duration::from_secs(300)))
}
async fn cleanup(obj: Arc<Deployment>, kv: &Store) -> Result<Action, Error> {
let name = obj.name_any();
for device_id in &obj.spec.target_devices {
let key = kv_key(device_id, &name);
kv.delete(&key)
.await
.map_err(|e| Error::Kv(e.to_string()))?;
tracing::info!(%key, "deleted desired state");
}
Ok(Action::await_change())
}
fn serialize_score(score: &ScorePayload) -> Result<String, Error> {
Ok(serde_json::to_string(score)?)
}
fn kv_key(device_id: &str, deployment_name: &str) -> String {
desired_state_key(device_id, deployment_name)
}
fn error_policy(_obj: Arc<Deployment>, err: &Error, _ctx: Arc<Context>) -> Action {
tracing::warn!(error = %err, "requeueing after error");
Action::requeue(Duration::from_secs(30))
}

96
iot/iot-operator-v0/src/install.rs
View File

@@ -1,96 +0,0 @@
//! Install the operator's CRD into a target Kubernetes cluster
//! via a harmony Score — no yaml generation, no kubectl shell-out.
//!
//! The Score side is just [`K8sResourceScore`] over
//! [`Deployment::crd()`]; what this module owns is a thin
//! [`InstallTopology`] that satisfies `K8sclient` by loading the
//! current `KUBECONFIG` directly. We don't use
//! [`K8sAnywhereTopology`] because its `ensure_ready` does a lot of
//! product-level setup (cert-manager, tenant manager, helm probes)
//! that isn't appropriate for a narrow "apply a CRD" action.
use std::sync::Arc;
use anyhow::{Context, Result};
use async_trait::async_trait;
use harmony::inventory::Inventory;
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::score::Score;
use harmony::topology::{K8sclient, PreparationOutcome, Topology};
use harmony_k8s::K8sClient;
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use kube::CustomResourceExt;
use crate::crd::Deployment;
/// Topology that only knows how to hand out a pre-built `K8sClient`.
/// Used by [`install_crds`] so the Score machinery has something
/// that satisfies `K8sclient` without dragging in the full
/// `K8sAnywhereTopology` bootstrap.
///
/// # Architectural smell — do not copy this pattern without reading the roadmap
///
/// Vendoring an ad-hoc `Topology` impl in a module that just wants to
/// apply a CRD is a symptom of a bigger problem: the existing
/// opinionated topologies (`K8sAnywhereTopology`, `HAClusterTopology`)
/// have accumulated product-level side effects in their `ensure_ready`
/// — cert-manager install, tenant manager setup, helm probes — that
/// make them unfit for narrow actions. The correct long-term fix is a
/// minimal reusable `K8sBareTopology` in harmony that carries a
/// `K8sClient` and exposes `K8sclient` with a noop `ensure_ready`, so
/// every narrow Score isn't tempted to vendor its own copy.
///
/// See `ROADMAP/12-code-review-april-2026.md` §12.6 "Topology
/// proliferation". The explicit smoke test for "that roadmap item is
/// done" is: this file can delete `InstallTopology` and replace
/// `topology` construction with a one-liner against the shared type.
struct InstallTopology {
client: Arc<K8sClient>,
}
#[async_trait]
impl Topology for InstallTopology {
fn name(&self) -> &str {
"iot-operator-install"
}
async fn ensure_ready(
&self,
) -> Result<PreparationOutcome, harmony::topology::PreparationError> {
Ok(PreparationOutcome::Noop)
}
}
#[async_trait]
impl K8sclient for InstallTopology {
async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
Ok(self.client.clone())
}
}
/// Apply the operator's CRDs to whatever cluster `KUBECONFIG` points
/// at. Returns once the apply call completes — does **not** wait for
/// the apiserver to mark the CRD `Established`; the caller does that
/// (e.g. with `kubectl wait --for=condition=Established`) if it
/// cares.
pub async fn install_crds() -> Result<()> {
let kube_client = kube::Client::try_default()
.await
.context("building kube client from KUBECONFIG")?;
let topology = InstallTopology {
client: Arc::new(K8sClient::new(kube_client)),
};
let inventory = Inventory::empty();
let crd: CustomResourceDefinition = Deployment::crd();
let score = K8sResourceScore::<CustomResourceDefinition>::single(crd, None);
let interpret = Score::<InstallTopology>::create_interpret(&score);
let outcome = interpret
.execute(&inventory, &topology)
.await
.map_err(|e| anyhow::anyhow!("install CRD: {e}"))
.context("executing K8sResourceScore for Deployment CRD")?;
tracing::info!(?outcome, "CRD installed");
Ok(())
}

73
iot/iot-operator-v0/src/main.rs
View File

@@ -1,73 +0,0 @@
mod controller;
mod crd;
mod install;
use anyhow::Result;
use async_nats::jetstream;
use clap::{Parser, Subcommand};
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
use kube::Client;
#[derive(Parser)]
#[command(
name = "iot-operator-v0",
about = "IoT operator — Deployment CRD → NATS KV"
)]
struct Cli {
#[command(subcommand)]
command: Option<Command>,
#[arg(
long,
env = "NATS_URL",
default_value = "nats://localhost:4222",
global = true
)]
nats_url: String,
#[arg(
long,
env = "KV_BUCKET",
default_value = BUCKET_DESIRED_STATE,
global = true
)]
kv_bucket: String,
}
#[derive(Subcommand)]
enum Command {
/// Run the controller (default when no subcommand is given).
Run,
/// Apply the operator's CRD to the cluster `KUBECONFIG` points
/// at. Uses harmony's typed k8s client — no yaml, no kubectl.
Install,
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();
let cli = Cli::parse();
match cli.command.unwrap_or(Command::Run) {
Command::Install => install::install_crds().await,
Command::Run => run(&cli.nats_url, &cli.kv_bucket).await,
}
}
async fn run(nats_url: &str, bucket: &str) -> Result<()> {
let nats = async_nats::connect(nats_url).await?;
tracing::info!(url = %nats_url, "connected to NATS");
let js = jetstream::new(nats);
let kv = js
.create_key_value(jetstream::kv::Config {
bucket: bucket.to_string(),
..Default::default()
})
.await?;
tracing::info!(bucket = %bucket, "KV bucket ready");
let client = Client::try_default().await?;
controller::run(client, kv).await
}