From a51ca0af27798049191a99efd2f0aace348adfc0 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 18 Jul 2026 11:48:37 -0400 Subject: [PATCH 01/47] feat: add auth identity access skeleton --- Cargo.lock | 31 +++++ Cargo.toml | 2 + harmony_auth/Cargo.toml | 13 +++ harmony_auth/src/lib.rs | 64 +++++++++++ harmony_auth_ui/Cargo.toml | 21 ++++ harmony_auth_ui/src/a11y.css | 40 +++++++ harmony_auth_ui/src/app.css | 1 + harmony_auth_ui/src/main.rs | 127 +++++++++++++++++++++ harmony_auth_ui/src/mock.rs | 104 +++++++++++++++++ harmony_auth_ui/src/views.rs | 212 +++++++++++++++++++++++++++++++++++ 10 files changed, 615 insertions(+) create mode 100644 harmony_auth/Cargo.toml create mode 100644 harmony_auth/src/lib.rs create mode 100644 harmony_auth_ui/Cargo.toml create mode 100644 harmony_auth_ui/src/a11y.css create mode 100644 harmony_auth_ui/src/app.css create mode 100644 harmony_auth_ui/src/main.rs create mode 100644 harmony_auth_ui/src/mock.rs create mode 100644 harmony_auth_ui/src/views.rs diff --git a/Cargo.lock b/Cargo.lock index 24834db1..b87273e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4317,6 +4317,36 @@ dependencies = [ "url", ] +[[package]] +name = "harmony_auth" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "serde", + "thiserror 2.0.18", + "uuid", +] + +[[package]] +name = "harmony_auth_ui" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "chrono", + "clap", + "harmony_auth", + "maud", + "serde", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "harmony_cli" version = "0.1.0" @@ -9564,6 +9594,7 @@ dependencies = [ "getrandom 0.4.2", "js-sys", "rand 0.10.1", + "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 65af54da..f4021894 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,8 @@ members = [ "fleet/harmony-fleet-deploy", "fleet/harmony-fleet-e2e", "harmony-reconciler-contracts", + "harmony_auth", + "harmony_auth_ui", "examples/fleet_server_install", "nats/jwt", "nats/callout", diff --git a/harmony_auth/Cargo.toml b/harmony_auth/Cargo.toml new file mode 100644 index 00000000..3c66a450 --- /dev/null +++ b/harmony_auth/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "harmony_auth" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +async-trait.workspace = true +chrono = { workspace = true, features = ["serde"] } +serde.workspace = true +thiserror.workspace = true +uuid = { workspace = true, features = ["serde"] } diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs new file mode 100644 index 00000000..a2778cb3 --- /dev/null +++ b/harmony_auth/src/lib.rs @@ -0,0 +1,64 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityKind { + Human, + Machine, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Identity { + pub subject_id: String, + pub kind: IdentityKind, + pub display_name: String, + pub login_name: String, + pub email: Option, + pub active: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Selector { + Exact, + Subtree, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccessLevel { + ReadOnly, + ReadWrite, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Grant { + pub id: Uuid, + pub principal_subject_id: String, + pub principal_kind: IdentityKind, + pub mount: String, + pub path: String, + pub selector: Selector, + pub access: AccessLevel, + pub created_by: String, + pub created_at: DateTime, +} + +#[derive(Debug, Error)] +pub enum AuthError { + #[error("identity not found")] + IdentityNotFound, + #[error("authorization backend failed: {0}")] + Backend(String), +} + +#[async_trait] +pub trait AuthService: Send + Sync { + async fn identities(&self, search: Option<&str>) -> Result, AuthError>; + async fn identity(&self, subject_id: &str) -> Result; + async fn grants_for(&self, subject_id: &str) -> Result, AuthError>; +} diff --git a/harmony_auth_ui/Cargo.toml b/harmony_auth_ui/Cargo.toml new file mode 100644 index 00000000..3267057b --- /dev/null +++ b/harmony_auth_ui/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "harmony_auth_ui" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +harmony_auth = { path = "../harmony_auth" } +anyhow.workspace = true +async-trait.workspace = true +axum = "0.8" +chrono = { workspace = true, features = ["serde"] } +clap.workspace = true +maud = { version = "0.27", features = ["axum"] } +serde.workspace = true +tokio.workspace = true +tower-http = { version = "0.6", features = ["set-header"] } +tracing.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter"] } +uuid = { workspace = true, features = ["serde"] } diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css new file mode 100644 index 00000000..a0840b15 --- /dev/null +++ b/harmony_auth_ui/src/a11y.css @@ -0,0 +1,40 @@ + +/* Overrides kept readable while the base stylesheet remains minified. */ +body { + --muted: #56615d; +} + +.nav-disabled { + color: #9ca8b5; + opacity: 1; +} + +.identity-email { + margin-top: 2px; +} + +.metric small { + color: #56615d; +} + +.mobile-meta { + display: none; +} + +.detail-grid > *, +.fact dd, +.path code { + min-width: 0; + overflow-wrap: anywhere; +} + +.facts dl { + margin: 0; +} + +@media (max-width: 760px) { + .mobile-meta { + display: block; + color: #56615d; + } +} diff --git a/harmony_auth_ui/src/app.css b/harmony_auth_ui/src/app.css new file mode 100644 index 00000000..5c1321f9 --- /dev/null +++ b/harmony_auth_ui/src/app.css @@ -0,0 +1 @@ +:root{--ink:#17201d;--muted:#69736f;--paper:#f4f1e9;--surface:#fffdf8;--line:#d9d6cc;--green:#0c6b4f;--lime:#cde86a;--navy:#14243b;--red:#a33b32}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:15px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif}.topbar{height:68px;padding:0 max(24px,calc((100vw - 1180px)/2));display:flex;align-items:center;gap:48px;background:var(--navy);color:#fff}.brand{display:flex;align-items:center;gap:9px;color:#fff;text-decoration:none;font-weight:750;font-size:17px;letter-spacing:-.02em}.brand-mark{display:grid;place-items:center;width:29px;height:29px;border:1px solid #a8bdd5;border-radius:50%;font-family:Georgia,serif}.brand em{font-style:normal;font-size:9px;letter-spacing:.18em;color:var(--lime);align-self:flex-start;margin-top:10px}.topbar nav{display:flex;height:100%;align-items:center;gap:30px}.topbar nav a,.nav-disabled{height:100%;display:flex;align-items:center;color:#b8c4d1;text-decoration:none;font-size:13px;border-bottom:2px solid transparent}.topbar nav a:hover{color:#fff}.topbar nav a[aria-current=page]{color:#fff;border-color:var(--lime)}.nav-disabled{opacity:.42;cursor:not-allowed}.environment{margin-left:auto;padding:6px 10px;border:1px solid #415269;border-radius:99px;color:#c5cfda;font-size:11px}.environment span{display:inline-block;width:6px;height:6px;margin-right:7px;border-radius:50%;background:var(--lime)}main{width:min(1180px,calc(100% - 48px));margin:0 auto;padding:58px 0 90px}.hero{display:grid;grid-template-columns:1fr auto;align-items:end;gap:40px;padding:26px 0 64px}.eyebrow{margin:0 0 12px;color:var(--green);font-size:11px;font-weight:800;letter-spacing:.15em}.hero h1,.page-heading h1,.identity-header h1{max-width:780px;margin:0;font:600 clamp(38px,5vw,66px)/1.02 Georgia,serif;letter-spacing:-.045em}.lede{max-width:700px;margin:22px 0 0;color:var(--muted);font-size:18px}.button{display:inline-flex;justify-content:center;align-items:center;min-height:42px;padding:0 18px;border:1px solid #bcb9b0;border-radius:3px;background:var(--surface);color:var(--ink);font:700 12px inherit;text-decoration:none;cursor:pointer}.button.primary{background:var(--lime);border-color:#b4ce55;color:#17201d}.metrics{display:grid;grid-template-columns:repeat(4,1fr);border:1px solid var(--line);background:var(--surface)}.metric{min-height:142px;padding:24px;border-right:1px solid var(--line);display:flex;flex-direction:column}.metric:last-child{border:0}.metric>span{color:var(--muted);font-size:12px}.metric strong{margin:8px 0 4px;font:600 34px Georgia,serif}.metric small{margin-top:auto;color:#818984}.metric.healthy strong{font:700 18px system-ui}.signal{width:8px;height:8px;border-radius:50%;background:var(--green);display:inline-block;margin-right:6px}.panel{border:1px solid var(--line);background:var(--surface);padding:30px}.panel.split{display:grid;grid-template-columns:1fr 1fr;gap:80px;margin-top:28px;padding:45px}.panel h2{margin:0 0 8px;font:600 26px Georgia,serif}.panel p{color:var(--muted)}.text-link,.back{color:var(--green);font-weight:700;text-decoration:none}.rule-card{display:grid;grid-template-columns:32px 1fr;gap:10px 14px}.rule-card p{margin:0 0 12px}.rule-number{font:700 11px monospace;color:var(--green);padding-top:4px}.page-heading{max-width:760px;margin-bottom:38px}.page-heading h1{font-size:52px}.page-heading>p:last-child{color:var(--muted);font-size:17px}.search{max-width:660px}.search label{display:block;margin-bottom:8px;font-size:12px;font-weight:750}.search-row{display:flex;gap:9px}.search input{width:100%;height:44px;border:1px solid #bcb9b0;background:#fff;padding:0 13px;font:inherit;border-radius:3px}.search input:focus{outline:3px solid #cde86a88;border-color:var(--green)}.list-summary{margin:30px 0 10px;color:var(--muted);font-size:12px}.identity-list{border-top:1px solid var(--line)}.identity-row{display:grid;grid-template-columns:42px minmax(180px,1fr) 90px 100px 24px;align-items:center;gap:18px;padding:17px 8px;border-bottom:1px solid var(--line);color:var(--ink);text-decoration:none}.identity-row:hover{background:#f6f7ef}.avatar{display:grid;place-items:center;width:40px;height:40px;border-radius:50%;background:#dce5dc;color:#244c3d;font-size:11px;font-weight:800}.machine-avatar{border-radius:8px;background:#dce3ed;color:#294866}.identity-main{display:flex;flex-direction:column}.identity-main span{color:var(--muted);font-size:12px}.kind,.status{font-size:11px}.kind{padding:4px 8px;width:max-content;border:1px solid var(--line);border-radius:99px}.status:before{content:"";display:inline-block;width:6px;height:6px;border-radius:50%;margin-right:7px}.status.active{color:var(--green)}.status.active:before{background:var(--green)}.status.suspended{color:var(--red)}.status.suspended:before{background:var(--red)}.arrow{font-size:18px;color:var(--green)}.empty{padding:50px;text-align:center;color:var(--muted)}.empty strong{color:var(--ink)}.empty.compact{padding:35px}.back{display:inline-block;margin-bottom:32px}.identity-header{display:flex;align-items:center;gap:20px;margin-bottom:36px}.identity-header .avatar{width:64px;height:64px;font-size:16px}.identity-header h1{font-size:44px}.identity-header p{margin:5px 0 0;color:var(--muted)}.title-line{display:flex;align-items:center;gap:18px}.detail-grid{display:grid;grid-template-columns:minmax(260px,.7fr) minmax(420px,1.3fr);gap:24px}.facts h2,.access h2{font-size:22px}.fact{display:grid;grid-template-columns:130px 1fr;padding:15px 0;border-bottom:1px solid var(--line)}.fact:last-child{border:0}.fact dt{color:var(--muted);font-size:12px}.fact dd{margin:0;overflow-wrap:anywhere;font-family:ui-monospace,monospace;font-size:12px}.section-title{display:flex;justify-content:space-between;align-items:start;margin-bottom:20px}.section-title p{margin:0}.count{display:grid;place-items:center;width:34px;height:34px;background:var(--navy);color:#fff;border-radius:50%;font-weight:700}.grant{padding:18px 0;border-top:1px solid var(--line)}.path code{font-size:13px;font-weight:700}.grant-meta{display:flex;gap:9px;margin-top:10px;color:var(--muted);font-size:11px}.grant-meta span{padding:4px 8px;background:#f0eee7;border-radius:2px}.grant-meta .access-level{background:#e1ebd4;color:#315a2e}footer{display:flex;justify-content:space-between;width:min(1180px,calc(100% - 48px));margin:0 auto;padding:26px 0;border-top:1px solid var(--line);color:var(--muted);font-size:11px}@media(max-width:760px){.topbar{height:auto;min-height:64px;padding:13px 20px;flex-wrap:wrap;gap:8px 24px}.topbar nav{order:3;width:100%;height:38px;gap:22px;overflow-x:auto}.environment{margin-left:auto}.hero{grid-template-columns:1fr;padding-top:5px}.hero .button{justify-self:start}.metrics{grid-template-columns:1fr 1fr}.metric:nth-child(2){border-right:0}.metric:nth-child(-n+2){border-bottom:1px solid var(--line)}.panel.split,.detail-grid{grid-template-columns:1fr;gap:30px}.identity-row{grid-template-columns:42px 1fr 24px}.identity-row .kind,.identity-row .status{display:none}main{width:min(100% - 28px,1180px);padding-top:36px}.panel{padding:20px}.page-heading h1{font-size:42px}.title-line{align-items:start;flex-direction:column;gap:8px}.identity-header h1{font-size:35px}.search-row{align-items:stretch}.search-row .button{padding:0 12px}footer{width:calc(100% - 28px)}} diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs new file mode 100644 index 00000000..868f6613 --- /dev/null +++ b/harmony_auth_ui/src/main.rs @@ -0,0 +1,127 @@ +mod mock; +mod views; + +use std::{net::SocketAddr, sync::Arc}; + +use anyhow::Result; +use axum::{ + Router, + extract::{Path, Query, State}, + http::{HeaderName, HeaderValue}, + response::{IntoResponse, Response}, + routing::get, +}; +use clap::Parser; +use harmony_auth::{AuthError, AuthService}; +use serde::Deserialize; +use tower_http::set_header::SetResponseHeaderLayer; + +#[derive(Parser)] +struct Args { + #[arg(long, default_value = "127.0.0.1:18081")] + addr: SocketAddr, +} + +#[derive(Clone)] +struct AppState { + auth: Arc, +} + +#[derive(Default, Deserialize)] +struct Search { + q: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + let args = Args::parse(); + let state = AppState { + auth: Arc::new(mock::MockAuth::new()), + }; + let listener = tokio::net::TcpListener::bind(args.addr).await?; + tracing::info!(address = %args.addr, "Harmony Auth UI listening"); + axum::serve(listener, router(state)).await?; + Ok(()) +} + +fn router(state: AppState) -> Router { + Router::new() + .route("/", get(overview)) + .route("/identities", get(identities)) + .route("/identities/{subject_id}", get(identity)) + .route("/static/app.css", get(css)) + .route("/static/htmx.min.js", get(htmx)) + .layer(SetResponseHeaderLayer::overriding( + HeaderName::from_static("content-security-policy"), + HeaderValue::from_static("default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; object-src 'none'"), + )) + .layer(SetResponseHeaderLayer::overriding( + HeaderName::from_static("x-content-type-options"), + HeaderValue::from_static("nosniff"), + )) + .with_state(state) +} + +async fn overview(State(state): State) -> Result { + let identities = state.auth.identities(None).await?; + let mut grants = 0; + for identity in &identities { + grants += state.auth.grants_for(&identity.subject_id).await?.len(); + } + Ok(views::overview(&identities, grants)) +} + +async fn identities( + State(state): State, + Query(search): Query, +) -> Result { + let identities = state.auth.identities(search.q.as_deref()).await?; + Ok(views::identities(&identities, search.q.as_deref())) +} + +async fn identity( + State(state): State, + Path(subject_id): Path, +) -> Result { + let identity = state.auth.identity(&subject_id).await?; + let grants = state.auth.grants_for(&subject_id).await?; + Ok(views::identity(&identity, &grants)) +} + +async fn css() -> impl IntoResponse { + ( + [(axum::http::header::CONTENT_TYPE, "text/css; charset=utf-8")], + format!("{}{}", include_str!("app.css"), include_str!("a11y.css")), + ) +} + +async fn htmx() -> impl IntoResponse { + ( + [( + axum::http::header::CONTENT_TYPE, + "text/javascript; charset=utf-8", + )], + include_bytes!("../../fleet/harmony-fleet-operator/vendor/htmx.min.js").as_slice(), + ) +} + +struct AppError(AuthError); + +impl From for AppError { + fn from(value: AuthError) -> Self { + Self(value) + } +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + let status = match self.0 { + AuthError::IdentityNotFound => axum::http::StatusCode::NOT_FOUND, + AuthError::Backend(_) => axum::http::StatusCode::BAD_GATEWAY, + }; + (status, views::error(status)).into_response() + } +} diff --git a/harmony_auth_ui/src/mock.rs b/harmony_auth_ui/src/mock.rs new file mode 100644 index 00000000..ac5b365d --- /dev/null +++ b/harmony_auth_ui/src/mock.rs @@ -0,0 +1,104 @@ +use async_trait::async_trait; +use chrono::{TimeZone, Utc}; +use harmony_auth::{AccessLevel, AuthError, AuthService, Grant, Identity, IdentityKind, Selector}; +use uuid::Uuid; + +pub struct MockAuth { + identities: Vec, + grants: Vec, +} + +impl MockAuth { + pub fn new() -> Self { + let identities = vec![ + Identity { + subject_id: "218904384720891003".into(), + kind: IdentityKind::Human, + display_name: "Maya Chen".into(), + login_name: "maya.chen".into(), + email: Some("maya@northstar.example".into()), + active: true, + }, + Identity { + subject_id: "218904384720891017".into(), + kind: IdentityKind::Machine, + display_name: "Northstar deployer".into(), + login_name: "northstar-deployer".into(), + email: None, + active: true, + }, + Identity { + subject_id: "218904384720891029".into(), + kind: IdentityKind::Human, + display_name: "Sam Rivera".into(), + login_name: "sam.rivera".into(), + email: Some("sam@harbor.example".into()), + active: false, + }, + ]; + let created_at = Utc.with_ymd_and_hms(2026, 7, 18, 14, 32, 0).unwrap(); + let grants = vec![ + Grant { + id: Uuid::parse_str("01981f2c-8950-7ad2-bd0c-aed3d23621a1").unwrap(), + principal_subject_id: identities[0].subject_id.clone(), + principal_kind: IdentityKind::Human, + mount: "secret".into(), + path: "tenants/northstar/production".into(), + selector: Selector::Subtree, + access: AccessLevel::ReadOnly, + created_by: "177192654770291002".into(), + created_at, + }, + Grant { + id: Uuid::parse_str("01981f2d-42bb-7b03-9243-1e590e25989f").unwrap(), + principal_subject_id: identities[1].subject_id.clone(), + principal_kind: IdentityKind::Machine, + mount: "secret".into(), + path: "tenants/northstar/deploy".into(), + selector: Selector::Subtree, + access: AccessLevel::ReadWrite, + created_by: "177192654770291002".into(), + created_at, + }, + ]; + Self { identities, grants } + } +} + +#[async_trait] +impl AuthService for MockAuth { + async fn identities(&self, search: Option<&str>) -> Result, AuthError> { + let search = search.unwrap_or("").trim().to_ascii_lowercase(); + Ok(self + .identities + .iter() + .filter(|identity| { + search.is_empty() + || identity.display_name.to_ascii_lowercase().contains(&search) + || identity.login_name.to_ascii_lowercase().contains(&search) + || identity + .email + .as_deref() + .is_some_and(|email| email.to_ascii_lowercase().contains(&search)) + }) + .cloned() + .collect()) + } + + async fn identity(&self, subject_id: &str) -> Result { + self.identities + .iter() + .find(|identity| identity.subject_id == subject_id) + .cloned() + .ok_or(AuthError::IdentityNotFound) + } + + async fn grants_for(&self, subject_id: &str) -> Result, AuthError> { + Ok(self + .grants + .iter() + .filter(|grant| grant.principal_subject_id == subject_id) + .cloned() + .collect()) + } +} diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs new file mode 100644 index 00000000..fc339ad1 --- /dev/null +++ b/harmony_auth_ui/src/views.rs @@ -0,0 +1,212 @@ +use axum::http::StatusCode; +use harmony_auth::{AccessLevel, Grant, Identity, IdentityKind, Selector}; +use maud::{DOCTYPE, Markup, html}; + +pub fn overview(identities: &[Identity], grants: usize) -> Markup { + let active = identities.iter().filter(|identity| identity.active).count(); + layout( + "Overview", + "/", + html! { + section class="hero" { + div { + p class="eyebrow" { "AUTHORIZATION CONTROL PLANE" } + h1 { "Know exactly who can reach every secret." } + p class="lede" { "Harmony translates direct access intent into enforceable OpenBao policies without copying identities or secret values." } + } + a class="button primary" href="/identities" { "Review identities" } + } + section class="metrics" aria-label="System summary" { + (metric("Identities", identities.len(), "Discovered from Zitadel")) + (metric("Active", active, "Eligible for new grants")) + (metric("Direct grants", grants, "Canonical access intent")) + article class="metric healthy" { + span class="signal" {} span { "Enforcement" } + strong { "In sync" } + small { "No policy drift detected" } + } + } + section class="panel split" { + div { + p class="eyebrow" { "QUICK ANSWERS" } + h2 { "Start with an identity" } + p { "Find a person or machine, then inspect every secret path they can access and why." } + a class="text-link" href="/identities" { "Browse all identities" } + } + div class="rule-card" { + span class="rule-number" { "01" } + p { "Zitadel subject IDs remain the stable identity key." } + span class="rule-number" { "02" } + p { "Secret values stay hidden. This console manages access, not data." } + } + } + }, + ) +} + +pub fn identities(identities: &[Identity], search: Option<&str>) -> Markup { + layout( + "Identities", + "/identities", + html! { + (page_heading("Identities", "People and machines discovered from Zitadel. Email and names are display data; subject ID is authoritative.")) + section class="panel" { + form class="search" action="/identities" method="get" { + label for="identity-search" { "Search identities" } + div class="search-row" { + input id="identity-search" name="q" type="search" value=[search] placeholder="Name, login, or email"; + button class="button" type="submit" { "Search" } + } + } + div class="list-summary" { (identities.len()) " matching identities" } + div class="identity-list" { + @for identity in identities { + a class="identity-row" href=(format!("/identities/{}", identity.subject_id)) { + (avatar(identity)) + div class="identity-main" { + strong { (&identity.display_name) } + span { (&identity.login_name) } + @if let Some(email) = identity.email.as_deref() { + span class="identity-email" { (email) } + } + span class="mobile-meta" { + (if identity.kind == IdentityKind::Human { "Human" } else { "Machine" }) + " · " + (if identity.active { "Active" } else { "Suspended" }) + } + } + span class=(if identity.kind == IdentityKind::Human { "kind human" } else { "kind machine" }) { + (if identity.kind == IdentityKind::Human { "Human" } else { "Machine" }) + } + span class=(if identity.active { "status active" } else { "status suspended" }) { + (if identity.active { "Active" } else { "Suspended" }) + } + span class="arrow" aria-hidden="true" { "→" } + } + } + @if identities.is_empty() { + div class="empty" { strong { "No identities found" } p { "Try a name, login, or email fragment." } } + } + } + } + }, + ) +} + +pub fn identity(identity: &Identity, grants: &[Grant]) -> Markup { + layout( + &identity.display_name, + "/identities", + html! { + a class="back" href="/identities" { "← All identities" } + section class="identity-header" { + (avatar(identity)) + div { + div class="title-line" { + h1 { (&identity.display_name) } + span class=(if identity.active { "status active" } else { "status suspended" }) { + (if identity.active { "Active" } else { "Suspended" }) + } + } + p { (&identity.login_name) } + } + } + section class="detail-grid" { + article class="panel facts" { + h2 { "Identity" } + dl { + (fact("Type", if identity.kind == IdentityKind::Human { "Human" } else { "Machine" })) + (fact("Email", identity.email.as_deref().unwrap_or("Not applicable"))) + (fact("Zitadel subject ID", &identity.subject_id)) + } + } + article class="panel access" { + div class="section-title" { + div { h2 { "Effective access" } p { "Union of active direct grants" } } + span class="count" { (grants.len()) } + } + @for grant in grants { + div class="grant" { + div class="path" { code { (&grant.mount) "/" (&grant.path) } } + div class="grant-meta" { + span class="access-level" { (if grant.access == AccessLevel::ReadOnly { "Read only" } else { "Read + write" }) } + span { (if grant.selector == Selector::Exact { "Exact secret" } else { "Entire subtree" }) } + } + } + } + @if grants.is_empty() { + div class="empty compact" { strong { "No secret access" } p { "This identity has no active direct grants." } } + } + } + } + }, + ) +} + +pub fn error(status: StatusCode) -> Markup { + layout( + "Request failed", + "", + html! { section class="panel empty" { h1 { (status.as_u16()) } p { "The requested authorization data could not be loaded." } a class="button" href="/" { "Return to overview" } } }, + ) +} + +fn layout(title: &str, current: &str, content: Markup) -> Markup { + html! { + (DOCTYPE) + html lang="en" { + head { + meta charset="utf-8"; + meta name="viewport" content="width=device-width, initial-scale=1"; + title { (title) " · Harmony Auth" } + link rel="stylesheet" href="/static/app.css"; + script src="/static/htmx.min.js" defer {} + } + body { + header class="topbar" { + a class="brand" href="/" { span class="brand-mark" { "H" } span { "Harmony" } em { "AUTH" } } + nav aria-label="Primary" { + (nav_link("/", "Overview", current)) + (nav_link("/identities", "Identities", current)) + span class="nav-disabled" title="Coming in the next delivery slice" { "Secrets" } + span class="nav-disabled" title="Coming in the next delivery slice" { "Access grants" } + } + div class="environment" { span {} "Development" } + } + main { (content) } + footer { "Harmony Auth" span { "Authorization intent, made inspectable." } } + } + } + } +} + +fn nav_link(href: &str, label: &str, current: &str) -> Markup { + let active = if href == "/" { + current == href + } else { + current.starts_with(href) + }; + html! { a href=(href) aria-current=[active.then_some("page")] { (label) } } +} + +fn metric(label: &str, value: usize, note: &str) -> Markup { + html! { article class="metric" { span { (label) } strong { (value) } small { (note) } } } +} + +fn page_heading(title: &str, description: &str) -> Markup { + html! { section class="page-heading" { p class="eyebrow" { "ZITADEL DIRECTORY" } h1 { (title) } p { (description) } } } +} + +fn avatar(identity: &Identity) -> Markup { + let initials: String = identity + .display_name + .split_whitespace() + .filter_map(|part| part.chars().next()) + .take(2) + .collect(); + html! { span class=(if identity.kind == IdentityKind::Human { "avatar" } else { "avatar machine-avatar" }) { (initials.to_uppercase()) } } +} + +fn fact(label: &str, value: &str) -> Markup { + html! { div class="fact" { dt { (label) } dd { (value) } } } +} -- 2.39.5 From 61fa1c2f991072efe48364380a62bcb8669e8a3b Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 18 Jul 2026 11:56:01 -0400 Subject: [PATCH 02/47] feat: inspect generated auth policies --- harmony_auth/src/lib.rs | 116 +++++++++++++++++++++++++++++++++++ harmony_auth_ui/src/a11y.css | 40 ++++++++++++ harmony_auth_ui/src/main.rs | 3 +- harmony_auth_ui/src/views.rs | 14 ++++- 4 files changed, 170 insertions(+), 3 deletions(-) diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index a2778cb3..1b9ebdf5 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use thiserror::Error; use uuid::Uuid; @@ -62,3 +63,118 @@ pub trait AuthService: Send + Sync { async fn identity(&self, subject_id: &str) -> Result; async fn grants_for(&self, subject_id: &str) -> Result, AuthError>; } + +pub fn policy_name(subject_id: &str) -> String { + format!("harmony-identity-{subject_id}") +} + +pub fn render_policy(grants: &[Grant]) -> String { + let mut data_paths = BTreeMap::new(); + let mut metadata_paths = BTreeMap::new(); + for grant in grants { + let path = grant.path.trim_matches('/'); + let suffixes: &[&str] = match grant.selector { + Selector::Exact => &[""], + Selector::Subtree => &["", "/*"], + }; + for suffix in suffixes { + data_paths + .entry(format!("{}/data/{}{}", grant.mount, path, suffix)) + .and_modify(|read_write| *read_write |= grant.access == AccessLevel::ReadWrite) + .or_insert(grant.access == AccessLevel::ReadWrite); + metadata_paths + .entry(format!("{}/metadata/{}{}", grant.mount, path, suffix)) + .and_modify(|list| *list |= grant.selector == Selector::Subtree) + .or_insert(grant.selector == Selector::Subtree); + } + } + + data_paths + .into_iter() + .map(|(path, read_write)| { + let capabilities = if read_write { + "[\"create\", \"delete\", \"patch\", \"read\", \"update\"]" + } else { + "[\"read\"]" + }; + format!("path \"{path}\" {{ capabilities = {capabilities} }}") + }) + .chain(metadata_paths.into_iter().map(|(path, list)| { + let capabilities = if list { + "[\"list\", \"read\"]" + } else { + "[\"read\"]" + }; + format!("path \"{path}\" {{ capabilities = {capabilities} }}") + })) + .collect::>() + .join("\n") +} + +#[cfg(test)] +mod tests { + use chrono::Utc; + use uuid::Uuid; + + use super::*; + + fn grant(path: &str, selector: Selector, access: AccessLevel) -> Grant { + Grant { + id: Uuid::nil(), + principal_subject_id: "218904384720891003".into(), + principal_kind: IdentityKind::Human, + mount: "secret".into(), + path: path.into(), + selector, + access, + created_by: "admin".into(), + created_at: Utc::now(), + } + } + + #[test] + fn policy_is_stable_and_deduplicated() { + let first = grant( + "/tenants/acme/database/", + Selector::Exact, + AccessLevel::ReadOnly, + ); + let second = grant( + "tenants/acme/deploy", + Selector::Subtree, + AccessLevel::ReadWrite, + ); + + let expected = r#"path "secret/data/tenants/acme/database" { capabilities = ["read"] } +path "secret/data/tenants/acme/deploy" { capabilities = ["create", "delete", "patch", "read", "update"] } +path "secret/data/tenants/acme/deploy/*" { capabilities = ["create", "delete", "patch", "read", "update"] } +path "secret/metadata/tenants/acme/database" { capabilities = ["read"] } +path "secret/metadata/tenants/acme/deploy" { capabilities = ["list", "read"] } +path "secret/metadata/tenants/acme/deploy/*" { capabilities = ["list", "read"] }"#; + + assert_eq!( + render_policy(&[second.clone(), first.clone(), first]), + expected + ); + assert_eq!( + render_policy(&[second]), + render_policy(&[grant( + "tenants/acme/deploy", + Selector::Subtree, + AccessLevel::ReadWrite, + )]) + ); + } + + #[test] + fn strongest_overlapping_grant_wins() { + let path = "tenants/acme/database"; + let policy = render_policy(&[ + grant(path, Selector::Exact, AccessLevel::ReadOnly), + grant(path, Selector::Exact, AccessLevel::ReadWrite), + ]); + + assert_eq!(policy.matches(&format!("secret/data/{path}")).count(), 1); + assert!(policy.contains("[\"create\", \"delete\", \"patch\", \"read\", \"update\"]")); + } +} diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css index a0840b15..db8aa8b1 100644 --- a/harmony_auth_ui/src/a11y.css +++ b/harmony_auth_ui/src/a11y.css @@ -32,6 +32,46 @@ body { margin: 0; } +.policy { + margin-top: 20px; + border-top: 1px solid var(--line); + padding-top: 20px; +} + +.policy summary { + cursor: pointer; + color: var(--green); + font-size: 12px; + font-weight: 750; +} + +.policy-heading { + display: flex; + justify-content: space-between; + gap: 16px; + margin: 18px 0 8px; + color: var(--muted); + font-size: 11px; +} + +.policy-heading code { + overflow-wrap: anywhere; + text-align: right; +} + +.policy pre { + max-width: 100%; + margin: 0; + padding: 16px; + overflow-x: auto; + border-radius: 3px; + background: var(--navy); + color: #e8eee9; + font-size: 11px; + white-space: pre; + overflow-wrap: normal; +} + @media (max-width: 760px) { .mobile-meta { display: block; diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index 868f6613..c495ff27 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -88,7 +88,8 @@ async fn identity( ) -> Result { let identity = state.auth.identity(&subject_id).await?; let grants = state.auth.grants_for(&subject_id).await?; - Ok(views::identity(&identity, &grants)) + let policy = harmony_auth::render_policy(&grants); + Ok(views::identity(&identity, &grants, &policy)) } async fn css() -> impl IntoResponse { diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index fc339ad1..c08ba923 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -93,7 +93,7 @@ pub fn identities(identities: &[Identity], search: Option<&str>) -> Markup { ) } -pub fn identity(identity: &Identity, grants: &[Grant]) -> Markup { +pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { layout( &identity.display_name, "/identities", @@ -129,7 +129,7 @@ pub fn identity(identity: &Identity, grants: &[Grant]) -> Markup { div class="grant" { div class="path" { code { (&grant.mount) "/" (&grant.path) } } div class="grant-meta" { - span class="access-level" { (if grant.access == AccessLevel::ReadOnly { "Read only" } else { "Read + write" }) } + span class="access-level" { (if grant.access == AccessLevel::ReadOnly { "Read only" } else { "Read, write + delete" }) } span { (if grant.selector == Selector::Exact { "Exact secret" } else { "Entire subtree" }) } } } @@ -137,6 +137,16 @@ pub fn identity(identity: &Identity, grants: &[Grant]) -> Markup { @if grants.is_empty() { div class="empty compact" { strong { "No secret access" } p { "This identity has no active direct grants." } } } + @if !policy.is_empty() { + details class="policy" { + summary { "Generated OpenBao policy" } + div class="policy-heading" { + span { "Policy name" } + code { (harmony_auth::policy_name(&identity.subject_id)) } + } + pre { code { (policy) } } + } + } } } }, -- 2.39.5 From bc51e94184b2fbd57d6c9637eb62e38f59462844 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 18 Jul 2026 12:12:26 -0400 Subject: [PATCH 03/47] feat: add reviewed direct grant workflow --- harmony_auth/src/lib.rs | 110 +++++++++++++++++++++++++++++++++++ harmony_auth_ui/src/a11y.css | 66 +++++++++++++++++++++ harmony_auth_ui/src/main.rs | 43 ++++++++++++-- harmony_auth_ui/src/mock.rs | 41 ++++++++++++- harmony_auth_ui/src/views.rs | 65 +++++++++++++++++++-- 5 files changed, 313 insertions(+), 12 deletions(-) diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index 1b9ebdf5..985be2b9 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -49,10 +49,27 @@ pub struct Grant { pub created_at: DateTime, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct GrantRequest { + pub principal_subject_id: String, + pub mount: String, + pub path: String, + pub selector: Selector, + pub access: AccessLevel, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GrantPlan { + pub request: GrantRequest, + pub resulting_policy: String, +} + #[derive(Debug, Error)] pub enum AuthError { #[error("identity not found")] IdentityNotFound, + #[error("invalid grant: {0}")] + InvalidGrant(String), #[error("authorization backend failed: {0}")] Backend(String), } @@ -62,6 +79,75 @@ pub trait AuthService: Send + Sync { async fn identities(&self, search: Option<&str>) -> Result, AuthError>; async fn identity(&self, subject_id: &str) -> Result; async fn grants_for(&self, subject_id: &str) -> Result, AuthError>; + async fn apply_grant(&self, plan: GrantPlan, created_by: &str) -> Result; +} + +pub fn plan_grant( + identity: &Identity, + existing: &[Grant], + mut request: GrantRequest, +) -> Result { + if !identity.active { + return Err(AuthError::InvalidGrant("identity is suspended".into())); + } + if request.principal_subject_id != identity.subject_id { + return Err(AuthError::InvalidGrant( + "principal does not match identity".into(), + )); + } + request.mount = request.mount.trim().trim_matches('/').to_string(); + request.path = request.path.trim().trim_matches('/').to_string(); + if request.mount.is_empty() || request.mount.contains('/') { + return Err(AuthError::InvalidGrant( + "mount must be one non-empty path segment".into(), + )); + } + if !request.mount.chars().all(valid_path_character) { + return Err(AuthError::InvalidGrant( + "mount may contain only letters, numbers, dot, dash, and underscore".into(), + )); + } + if request.path.is_empty() { + return Err(AuthError::InvalidGrant("path must not be empty".into())); + } + if request + .path + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(AuthError::InvalidGrant( + "path must not contain empty, dot, or parent segments".into(), + )); + } + if !request + .path + .chars() + .all(|character| character == '/' || valid_path_character(character)) + { + return Err(AuthError::InvalidGrant( + "path segments may contain only letters, numbers, dot, dash, and underscore".into(), + )); + } + let mut grants = existing.to_vec(); + grants.push(Grant { + id: Uuid::nil(), + principal_subject_id: identity.subject_id.clone(), + principal_kind: identity.kind.clone(), + mount: request.mount.clone(), + path: request.path.clone(), + selector: request.selector, + access: request.access, + created_by: String::new(), + created_at: Utc::now(), + }); + Ok(GrantPlan { + request, + resulting_policy: render_policy(&grants), + }) +} + +fn valid_path_character(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') } pub fn policy_name(subject_id: &str) -> String { @@ -177,4 +263,28 @@ path "secret/metadata/tenants/acme/deploy/*" { capabilities = ["list", "read"] } assert_eq!(policy.matches(&format!("secret/data/{path}")).count(), 1); assert!(policy.contains("[\"create\", \"delete\", \"patch\", \"read\", \"update\"]")); } + + #[test] + fn grant_plan_rejects_policy_syntax() { + let identity = Identity { + subject_id: "subject".into(), + kind: IdentityKind::Human, + display_name: "User".into(), + login_name: "user".into(), + email: None, + active: true, + }; + let request = GrantRequest { + principal_subject_id: identity.subject_id.clone(), + mount: "secret".into(), + path: "safe/\" } path \"*\" { capabilities = [\"sudo\"] }".into(), + selector: Selector::Exact, + access: AccessLevel::ReadOnly, + }; + + assert!(matches!( + plan_grant(&identity, &[], request), + Err(AuthError::InvalidGrant(_)) + )); + } } diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css index db8aa8b1..49d991fc 100644 --- a/harmony_auth_ui/src/a11y.css +++ b/harmony_auth_ui/src/a11y.css @@ -72,6 +72,72 @@ body { overflow-wrap: normal; } +.grant-form { + margin-top: 24px; + border-top: 1px solid var(--line); + padding-top: 20px; +} + +.grant-form summary { + cursor: pointer; + color: var(--green); + font-weight: 750; +} + +.grant-form form { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; + margin-top: 20px; +} + +.grant-form label { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 11px; +} + +.grant-form input, +.grant-form select { + min-width: 0; + height: 40px; + border: 1px solid #bcb9b0; + border-radius: 3px; + background: #fff; + padding: 0 10px; + color: var(--ink); + font: 13px inherit; +} + +.grant-form .button { + justify-self: start; +} + +.review-policy { + max-width: 100%; + margin: 20px 0; + padding: 16px; + overflow-x: auto; + background: var(--navy); + color: #e8eee9; + font-size: 11px; +} + +.danger-note { + border-left: 3px solid var(--red); + padding: 12px 14px; + background: #f7e8e5; + color: #742a24 !important; + font-size: 12px; +} + +@media (max-width: 760px) { + .grant-form form { + grid-template-columns: 1fr; + } +} + @media (max-width: 760px) { .mobile-meta { display: block; diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index c495ff27..6226be54 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -6,13 +6,13 @@ use std::{net::SocketAddr, sync::Arc}; use anyhow::Result; use axum::{ Router, - extract::{Path, Query, State}, + extract::{Form, Path, Query, State}, http::{HeaderName, HeaderValue}, - response::{IntoResponse, Response}, - routing::get, + response::{IntoResponse, Redirect, Response}, + routing::{get, post}, }; use clap::Parser; -use harmony_auth::{AuthError, AuthService}; +use harmony_auth::{AuthError, AuthService, GrantRequest}; use serde::Deserialize; use tower_http::set_header::SetResponseHeaderLayer; @@ -52,6 +52,8 @@ fn router(state: AppState) -> Router { .route("/", get(overview)) .route("/identities", get(identities)) .route("/identities/{subject_id}", get(identity)) + .route("/grants/plan", post(grant_plan)) + .route("/grants", post(apply_grant)) .route("/static/app.css", get(css)) .route("/static/htmx.min.js", get(htmx)) .layer(SetResponseHeaderLayer::overriding( @@ -65,6 +67,30 @@ fn router(state: AppState) -> Router { .with_state(state) } +async fn grant_plan( + State(state): State, + Form(request): Form, +) -> Result { + let identity = state.auth.identity(&request.principal_subject_id).await?; + let grants = state.auth.grants_for(&identity.subject_id).await?; + let plan = harmony_auth::plan_grant(&identity, &grants, request)?; + Ok(views::grant_review(&identity, &plan)) +} + +async fn apply_grant( + State(state): State, + Form(request): Form, +) -> Result { + let identity = state.auth.identity(&request.principal_subject_id).await?; + let grants = state.auth.grants_for(&identity.subject_id).await?; + let plan = harmony_auth::plan_grant(&identity, &grants, request)?; + state.auth.apply_grant(plan, "development-admin").await?; + Ok(Redirect::to(&format!( + "/identities/{}", + identity.subject_id + ))) +} + async fn overview(State(state): State) -> Result { let identities = state.auth.identities(None).await?; let mut grants = 0; @@ -119,10 +145,15 @@ impl From for AppError { impl IntoResponse for AppError { fn into_response(self) -> Response { - let status = match self.0 { + let status = match &self.0 { AuthError::IdentityNotFound => axum::http::StatusCode::NOT_FOUND, + AuthError::InvalidGrant(_) => axum::http::StatusCode::UNPROCESSABLE_ENTITY, AuthError::Backend(_) => axum::http::StatusCode::BAD_GATEWAY, }; - (status, views::error(status)).into_response() + let detail = match &self.0 { + AuthError::InvalidGrant(message) => Some(message.as_str()), + _ => None, + }; + (status, views::error(status, detail)).into_response() } } diff --git a/harmony_auth_ui/src/mock.rs b/harmony_auth_ui/src/mock.rs index ac5b365d..918a661f 100644 --- a/harmony_auth_ui/src/mock.rs +++ b/harmony_auth_ui/src/mock.rs @@ -1,11 +1,12 @@ use async_trait::async_trait; use chrono::{TimeZone, Utc}; use harmony_auth::{AccessLevel, AuthError, AuthService, Grant, Identity, IdentityKind, Selector}; +use tokio::sync::RwLock; use uuid::Uuid; pub struct MockAuth { identities: Vec, - grants: Vec, + grants: RwLock>, } impl MockAuth { @@ -61,7 +62,10 @@ impl MockAuth { created_at, }, ]; - Self { identities, grants } + Self { + identities, + grants: RwLock::new(grants), + } } } @@ -96,9 +100,42 @@ impl AuthService for MockAuth { async fn grants_for(&self, subject_id: &str) -> Result, AuthError> { Ok(self .grants + .read() + .await .iter() .filter(|grant| grant.principal_subject_id == subject_id) .cloned() .collect()) } + + async fn apply_grant( + &self, + plan: harmony_auth::GrantPlan, + created_by: &str, + ) -> Result { + let identity = self.identity(&plan.request.principal_subject_id).await?; + let mut grants = self.grants.write().await; + if let Some(grant) = grants.iter().find(|grant| { + grant.principal_subject_id == plan.request.principal_subject_id + && grant.mount == plan.request.mount + && grant.path == plan.request.path + && grant.selector == plan.request.selector + && grant.access == plan.request.access + }) { + return Ok(grant.clone()); + } + let grant = Grant { + id: Uuid::new_v4(), + principal_subject_id: identity.subject_id, + principal_kind: identity.kind, + mount: plan.request.mount, + path: plan.request.path, + selector: plan.request.selector, + access: plan.request.access, + created_by: created_by.into(), + created_at: Utc::now(), + }; + grants.push(grant.clone()); + Ok(grant) + } } diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index c08ba923..975fc65f 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -1,5 +1,5 @@ use axum::http::StatusCode; -use harmony_auth::{AccessLevel, Grant, Identity, IdentityKind, Selector}; +use harmony_auth::{AccessLevel, Grant, GrantPlan, Identity, IdentityKind, Selector}; use maud::{DOCTYPE, Markup, html}; pub fn overview(identities: &[Identity], grants: usize) -> Markup { @@ -144,7 +144,20 @@ pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { span { "Policy name" } code { (harmony_auth::policy_name(&identity.subject_id)) } } - pre { code { (policy) } } + pre tabindex="0" { code { (policy) } } + } + } + @if identity.active { + details class="grant-form" { + summary { "Grant direct access" } + form action="/grants/plan" method="post" { + input type="hidden" name="principal_subject_id" value=(&identity.subject_id); + label { "Secret mount" input name="mount" value="secret" required; } + label { "Logical path" input name="path" placeholder="tenants/acme/database" required; } + label { "Scope" select name="selector" { option value="exact" { "Exact secret" } option value="subtree" { "Entire subtree" } } } + label { "Access" select name="access" { option value="read_only" { "Read only" } option value="read_write" { "Read, write + delete" } } } + button class="button primary" type="submit" { "Review grant" } + } } } } @@ -153,11 +166,55 @@ pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { ) } -pub fn error(status: StatusCode) -> Markup { +pub fn grant_review(identity: &Identity, plan: &GrantPlan) -> Markup { + layout( + "Review grant", + "/identities", + html! { + a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } + (page_heading("Review direct grant", "Confirm the authorization intent and generated enforcement change before applying it.")) + section class="detail-grid" { + article class="panel facts" { + h2 { "Authorization intent" } + dl { + (fact("Identity", &identity.display_name)) + (fact("Secret", &format!("{}/{}", plan.request.mount, plan.request.path))) + (fact("Scope", if plan.request.selector == Selector::Exact { "Exact secret" } else { "Entire subtree" })) + (fact("Access", if plan.request.access == AccessLevel::ReadOnly { "Read only" } else { "Read, write + delete" })) + } + } + article class="panel access" { + h2 { "Generated policy after apply" } + pre class="review-policy" tabindex="0" { code { (&plan.resulting_policy) } } + @if plan.request.access == AccessLevel::ReadWrite { + p class="danger-note" { + strong { "Destructive access." } + @if plan.request.selector == Selector::Subtree { + " This identity can create, change, and delete this secret and every secret below it." + } @else { + " This identity can create, change, and delete this secret." + } + } + } + form action="/grants" method="post" { + input type="hidden" name="principal_subject_id" value=(&plan.request.principal_subject_id); + input type="hidden" name="mount" value=(&plan.request.mount); + input type="hidden" name="path" value=(&plan.request.path); + input type="hidden" name="selector" value=(if plan.request.selector == Selector::Exact { "exact" } else { "subtree" }); + input type="hidden" name="access" value=(if plan.request.access == AccessLevel::ReadOnly { "read_only" } else { "read_write" }); + button class="button primary" type="submit" { "Apply direct grant" } + } + } + } + }, + ) +} + +pub fn error(status: StatusCode, detail: Option<&str>) -> Markup { layout( "Request failed", "", - html! { section class="panel empty" { h1 { (status.as_u16()) } p { "The requested authorization data could not be loaded." } a class="button" href="/" { "Return to overview" } } }, + html! { section class="panel empty" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { "Grant could not be reviewed: " (detail) "." } } @else { p { "The requested authorization data could not be loaded." } } a class="button" href="/" { "Return to overview" } } }, ) } -- 2.39.5 From 5136d21a91e7af9fa3c9e8fca618d1d4121f9741 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 19 Jul 2026 11:08:50 -0400 Subject: [PATCH 04/47] feat: connect browser-managed auth profiles --- Cargo.lock | 3 + harmony_auth_ui/Cargo.toml | 3 + harmony_auth_ui/src/a11y.css | 180 ++++++++++++++++ harmony_auth_ui/src/backend.rs | 275 +++++++++++++++++++++++++ harmony_auth_ui/src/main.rs | 353 +++++++++++++++++++++++++++++--- harmony_auth_ui/src/mock.rs | 141 ------------- harmony_auth_ui/src/profiles.js | 163 +++++++++++++++ harmony_auth_ui/src/views.rs | 117 +++++++++-- 8 files changed, 1052 insertions(+), 183 deletions(-) create mode 100644 harmony_auth_ui/src/backend.rs delete mode 100644 harmony_auth_ui/src/mock.rs create mode 100644 harmony_auth_ui/src/profiles.js diff --git a/Cargo.lock b/Cargo.lock index b87273e5..0efd5432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4335,11 +4335,14 @@ dependencies = [ "anyhow", "async-trait", "axum", + "axum-extra", "chrono", "clap", "harmony_auth", "maud", + "reqwest 0.12.28", "serde", + "serde_json", "tokio", "tower-http", "tracing", diff --git a/harmony_auth_ui/Cargo.toml b/harmony_auth_ui/Cargo.toml index 3267057b..d9da85ca 100644 --- a/harmony_auth_ui/Cargo.toml +++ b/harmony_auth_ui/Cargo.toml @@ -10,10 +10,13 @@ harmony_auth = { path = "../harmony_auth" } anyhow.workspace = true async-trait.workspace = true axum = "0.8" +axum-extra = { version = "0.10", features = ["cookie"] } chrono = { workspace = true, features = ["serde"] } clap.workspace = true maud = { version = "0.27", features = ["axum"] } +reqwest.workspace = true serde.workspace = true +serde_json.workspace = true tokio.workspace = true tower-http = { version = "0.6", features = ["set-header"] } tracing.workspace = true diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css index 49d991fc..59d6376c 100644 --- a/harmony_auth_ui/src/a11y.css +++ b/harmony_auth_ui/src/a11y.css @@ -138,6 +138,186 @@ body { } } +.profile-switch { + display: flex; + align-items: center; + gap: 8px; + margin-left: auto; + padding: 7px 10px; + border: 1px solid #415269; + border-radius: 99px; + color: #eef3f6; + text-decoration: none; + font-size: 12px; +} + +.profile-switch small { + color: #aebac6; +} + +.profile-hero { + padding-bottom: 42px; +} + +.profile-workspace { + display: grid; + grid-template-columns: minmax(0, 1.4fr) minmax(300px, .6fr); + gap: 24px; + align-items: start; +} + +.profile-list { + display: grid; + gap: 14px; +} + +.profile-card { + display: grid; + grid-template-columns: minmax(180px, .7fr) minmax(280px, 1.3fr) auto; + gap: 24px; + align-items: center; + padding: 24px; + border: 1px solid var(--line); + background: var(--surface); +} + +.profile-card h2 { + margin: 6px 0 0; + font: 600 22px Georgia, serif; +} + +.connection-state { + color: var(--muted); + font-size: 10px; + font-weight: 800; + letter-spacing: .08em; + text-transform: uppercase; +} + +.connection-state::before { + content: ""; + display: inline-block; + width: 7px; + height: 7px; + margin-right: 7px; + border-radius: 50%; + background: #929b96; +} + +.connection-state.connected { + color: var(--green); +} + +.connection-state.connected::before { + background: var(--green); +} + +.profile-card dl { + display: grid; + grid-template-columns: 65px minmax(0, 1fr); + gap: 5px 12px; + margin: 0; + font-size: 11px; +} + +.profile-card dt { + color: var(--muted); +} + +.profile-card dd { + margin: 0; + overflow-wrap: anywhere; + font-family: ui-monospace, monospace; +} + +.profile-actions, +.form-actions { + display: flex; + gap: 8px; + flex-wrap: wrap; +} + +.quiet-danger { + color: var(--red); +} + +.profile-editor, +.connect-dialog form { + display: grid; + gap: 16px; +} + +.profile-editor[hidden] { + display: none; +} + +.profile-editor label, +.connect-dialog label { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 11px; + font-weight: 700; +} + +.profile-editor input, +.connect-dialog input { + width: 100%; + height: 42px; + border: 1px solid #bcb9b0; + border-radius: 3px; + padding: 0 11px; + font: 13px inherit; +} + +.connect-dialog { + width: min(540px, calc(100% - 28px)); + border: 1px solid var(--line); + border-radius: 4px; + padding: 30px; + background: var(--surface); + color: var(--ink); +} + +.connect-dialog::backdrop { + background: rgba(10, 20, 31, .66); +} + +.connect-dialog h2 { + margin: 0; + font: 600 30px Georgia, serif; +} + +.connect-dialog p, +.connect-dialog small { + margin: 0; + color: var(--muted); +} + +.root-warning { + border-left: 3px solid #b88122; + padding-left: 10px; +} + +.connection-error { + margin-bottom: 24px; + border-left: 3px solid var(--red); + padding: 14px 18px; + background: #f7e8e5; + color: #742a24; +} + +.connection-error p { + margin: 3px 0 0; +} + +@media (max-width: 900px) { + .profile-workspace, + .profile-card { + grid-template-columns: 1fr; + } +} + @media (max-width: 760px) { .mobile-meta { display: block; diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth_ui/src/backend.rs new file mode 100644 index 00000000..15b016e7 --- /dev/null +++ b/harmony_auth_ui/src/backend.rs @@ -0,0 +1,275 @@ +use async_trait::async_trait; +use harmony_auth::{AuthError, AuthService, Grant, GrantPlan, Identity, IdentityKind}; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; + +pub struct BackendAuth { + client: Client, + zitadel_url: String, + zitadel_pat: String, + openbao_url: String, + openbao_token: String, +} + +impl BackendAuth { + pub fn new( + client: Client, + zitadel_url: String, + zitadel_pat: String, + openbao_url: String, + openbao_token: String, + ) -> Self { + Self { + client, + zitadel_url: zitadel_url.trim_end_matches('/').into(), + zitadel_pat, + openbao_url: openbao_url.trim_end_matches('/').into(), + openbao_token, + } + } + + pub async fn validate(&self) -> Result<(), String> { + let zitadel = self + .client + .get(format!("{}/management/v1/orgs/me", self.zitadel_url)) + .bearer_auth(&self.zitadel_pat) + .send() + .await + .map_err(|error| format!("Zitadel could not be reached: {error}"))?; + if !zitadel.status().is_success() { + return Err(format!( + "Zitadel rejected the service-account PAT ({})", + zitadel.status() + )); + } + + let openbao = self + .client + .get(format!("{}/v1/auth/token/lookup-self", self.openbao_url)) + .header("X-Vault-Token", &self.openbao_token) + .send() + .await + .map_err(|error| format!("OpenBao could not be reached: {error}"))?; + if !openbao.status().is_success() { + return Err(format!("OpenBao rejected the token ({})", openbao.status())); + } + Ok(()) + } + + async fn all_identities(&self) -> Result, AuthError> { + let response = self + .client + .post(format!("{}/management/v1/users/_search", self.zitadel_url)) + .bearer_auth(&self.zitadel_pat) + .json(&json!({ "query": { "offset": "0", "limit": 1000 } })) + .send() + .await + .map_err(backend)? + .error_for_status() + .map_err(backend)?; + let body: Value = response.json().await.map_err(backend)?; + Ok(body["result"] + .as_array() + .into_iter() + .flatten() + .filter_map(parse_identity) + .collect()) + } + + async fn request_openbao(&self, path: &str) -> Result { + self.client + .get(format!("{}/v1/{path}", self.openbao_url)) + .header("X-Vault-Token", &self.openbao_token) + .send() + .await + .map_err(backend) + } +} + +#[async_trait] +impl AuthService for BackendAuth { + async fn identities(&self, search: Option<&str>) -> Result, AuthError> { + let search = search.unwrap_or("").trim().to_ascii_lowercase(); + Ok(self + .all_identities() + .await? + .into_iter() + .filter(|identity| { + search.is_empty() + || identity.display_name.to_ascii_lowercase().contains(&search) + || identity.login_name.to_ascii_lowercase().contains(&search) + || identity + .email + .as_deref() + .is_some_and(|email| email.to_ascii_lowercase().contains(&search)) + }) + .collect()) + } + + async fn identity(&self, subject_id: &str) -> Result { + self.all_identities() + .await? + .into_iter() + .find(|identity| identity.subject_id == subject_id) + .ok_or(AuthError::IdentityNotFound) + } + + async fn grants_for(&self, subject_id: &str) -> Result, AuthError> { + let response = self + .request_openbao(&format!( + "harmony_auth/metadata/grants/by-principal/{subject_id}?list=true" + )) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(vec![]); + } + let body: Value = response + .error_for_status() + .map_err(backend)? + .json() + .await + .map_err(backend)?; + let mut grants = Vec::new(); + for id in body["data"]["keys"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + let response = self + .request_openbao(&format!("harmony_auth/data/grants/by-id/{id}")) + .await? + .error_for_status() + .map_err(backend)?; + let body: Value = response.json().await.map_err(backend)?; + grants.push(serde_json::from_value(body["data"]["data"].clone()).map_err(backend)?); + } + Ok(grants) + } + + async fn apply_grant(&self, _plan: GrantPlan, _created_by: &str) -> Result { + Err(AuthError::Backend( + "grant mutations require JWT role configuration and are not enabled yet".into(), + )) + } +} + +fn parse_identity(value: &Value) -> Option { + let human = value.get("human"); + let machine = value.get("machine"); + let display_name = human + .and_then(|human| human.pointer("/profile/displayName")) + .or_else(|| machine.and_then(|machine| machine.get("name"))) + .and_then(Value::as_str) + .or_else(|| value.get("preferredLoginName").and_then(Value::as_str))? + .to_string(); + Some(Identity { + subject_id: value.get("id")?.as_str()?.into(), + kind: if human.is_some() { + IdentityKind::Human + } else { + IdentityKind::Machine + }, + display_name, + login_name: value + .get("preferredLoginName") + .or_else(|| value.get("userName")) + .and_then(Value::as_str) + .unwrap_or_default() + .into(), + email: human + .and_then(|human| human.pointer("/email/email")) + .and_then(Value::as_str) + .map(str::to_string), + active: value.get("state").is_none_or(|state| { + state.as_i64() == Some(1) || state.as_str() == Some("USER_STATE_ACTIVE") + }), + }) +} + +fn backend(error: impl std::fmt::Display) -> AuthError { + AuthError::Backend(error.to_string()) +} + +#[cfg(test)] +mod tests { + use axum::{ + Json, Router, + http::{HeaderMap, StatusCode}, + routing::{get, post}, + }; + use serde_json::json; + + use super::*; + + #[tokio::test] + async fn validates_credentials_and_maps_zitadel_identities() { + async fn zitadel(headers: HeaderMap) -> StatusCode { + if headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + == Some("Bearer zitadel-pat") + { + StatusCode::OK + } else { + StatusCode::UNAUTHORIZED + } + } + async fn openbao(headers: HeaderMap) -> StatusCode { + if headers + .get("x-vault-token") + .and_then(|value| value.to_str().ok()) + == Some("openbao-token") + { + StatusCode::OK + } else { + StatusCode::FORBIDDEN + } + } + async fn users() -> Json { + Json(json!({ + "result": [ + { + "id": "human-subject", + "state": 1, + "userName": "maya", + "preferredLoginName": "maya@example.com", + "human": { + "profile": { "displayName": "Maya Chen" }, + "email": { "email": "maya@example.com" } + } + }, + { + "id": "machine-subject", + "state": "USER_STATE_ACTIVE", + "userName": "deployer", + "machine": { "name": "Production deployer" } + } + ] + })) + } + + let app = Router::new() + .route("/management/v1/orgs/me", get(zitadel)) + .route("/management/v1/users/_search", post(users)) + .route("/v1/auth/token/lookup-self", get(openbao)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let backend = BackendAuth::new( + Client::new(), + base_url.clone(), + "zitadel-pat".into(), + base_url, + "openbao-token".into(), + ); + + backend.validate().await.unwrap(); + let identities = backend.identities(None).await.unwrap(); + assert_eq!(identities.len(), 2); + assert_eq!(identities[0].display_name, "Maya Chen"); + assert_eq!(identities[0].email.as_deref(), Some("maya@example.com")); + assert_eq!(identities[1].kind, IdentityKind::Machine); + } +} diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index 6226be54..d51ac4bc 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -1,19 +1,23 @@ -mod mock; +mod backend; mod views; -use std::{net::SocketAddr, sync::Arc}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; use anyhow::Result; use axum::{ - Router, + Json, Router, + body::Body, extract::{Form, Path, Query, State}, - http::{HeaderName, HeaderValue}, + http::{HeaderName, HeaderValue, Method, Request, StatusCode, header}, + middleware::{self, Next}, response::{IntoResponse, Redirect, Response}, routing::{get, post}, }; +use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite}; use clap::Parser; use harmony_auth::{AuthError, AuthService, GrantRequest}; use serde::Deserialize; +use tokio::sync::RwLock; use tower_http::set_header::SetResponseHeaderLayer; #[derive(Parser)] @@ -24,6 +28,14 @@ struct Args { #[derive(Clone)] struct AppState { + client: reqwest::Client, + sessions: Arc>>>, + connection_errors: Arc>>, +} + +#[derive(Clone)] +struct ConnectedProfile { + name: String, auth: Arc, } @@ -32,6 +44,29 @@ struct Search { q: Option, } +#[derive(Default, Deserialize)] +struct ProfileQuery { + connection_error: Option, +} + +#[derive(Deserialize)] +struct ConnectForm { + profile_id: String, + name: String, + zitadel_url: String, + zitadel_pat: String, + openbao_url: String, + openbao_token: String, +} + +#[derive(Deserialize)] +struct ProfileSelection { + profile_id: String, +} + +const SESSION_COOKIE: &str = "harmony_auth_session"; +const PROFILE_COOKIE: &str = "harmony_auth_profile"; + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -39,7 +74,12 @@ async fn main() -> Result<()> { .init(); let args = Args::parse(); let state = AppState { - auth: Arc::new(mock::MockAuth::new()), + client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(10)) + .build()?, + sessions: Arc::default(), + connection_errors: Arc::default(), }; let listener = tokio::net::TcpListener::bind(args.addr).await?; tracing::info!(address = %args.addr, "Harmony Auth UI listening"); @@ -49,13 +89,20 @@ async fn main() -> Result<()> { fn router(state: AppState) -> Router { Router::new() - .route("/", get(overview)) + .route("/", get(profiles)) + .route("/profiles/connect", post(connect_profile)) + .route("/profiles/switch", post(switch_profile)) + .route("/profiles/disconnect", post(disconnect_profile)) + .route("/profiles/connected", get(connected_profiles)) + .route("/dashboard", get(overview)) .route("/identities", get(identities)) .route("/identities/{subject_id}", get(identity)) .route("/grants/plan", post(grant_plan)) .route("/grants", post(apply_grant)) .route("/static/app.css", get(css)) .route("/static/htmx.min.js", get(htmx)) + .route("/static/profiles.js", get(profiles_js)) + .route("/favicon.ico", get(|| async { StatusCode::NO_CONTENT })) .layer(SetResponseHeaderLayer::overriding( HeaderName::from_static("content-security-policy"), HeaderValue::from_static("default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; object-src 'none'"), @@ -64,58 +111,289 @@ fn router(state: AppState) -> Router { HeaderName::from_static("x-content-type-options"), HeaderValue::from_static("nosniff"), )) + .layer(middleware::from_fn(request_security)) .with_state(state) } +async fn request_security(req: Request, next: Next) -> Response { + if req.method() == Method::POST && !same_origin(&req) { + return (StatusCode::FORBIDDEN, "origin check failed").into_response(); + } + let mut response = next.run(req).await; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response.headers_mut().insert( + header::REFERRER_POLICY, + HeaderValue::from_static("same-origin"), + ); + response.headers_mut().insert( + "permissions-policy", + HeaderValue::from_static("geolocation=(), microphone=(), camera=()"), + ); + response +} + +fn same_origin(req: &Request) -> bool { + let Some(host) = req + .headers() + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + else { + return false; + }; + req.headers() + .get(header::ORIGIN) + .or_else(|| req.headers().get(header::REFERER)) + .and_then(|value| value.to_str().ok()) + .and_then(|value| reqwest::Url::parse(value).ok()) + .and_then(|url| { + url.host_str().map(|hostname| match url.port() { + Some(port) => format!("{hostname}:{port}"), + None => hostname.to_string(), + }) + }) + .is_some_and(|origin| origin.eq_ignore_ascii_case(host)) +} + +async fn profiles( + State(state): State, + jar: CookieJar, + Query(query): Query, +) -> maud::Markup { + let error = if query.connection_error.is_some() { + if let Some(session_id) = jar.get(SESSION_COOKIE) { + state + .connection_errors + .write() + .await + .remove(session_id.value()) + } else { + None + } + } else { + None + }; + views::profiles(error.as_deref()) +} + +async fn connected_profiles(State(state): State, jar: CookieJar) -> Json> { + let Some(session_id) = jar.get(SESSION_COOKIE) else { + return Json(vec![]); + }; + Json( + state + .sessions + .read() + .await + .get(session_id.value()) + .map(|profiles| profiles.keys().cloned().collect()) + .unwrap_or_default(), + ) +} + +async fn connect_profile( + State(state): State, + jar: CookieJar, + Form(form): Form, +) -> (CookieJar, Redirect) { + let session_id = jar + .get(SESSION_COOKIE) + .map(|cookie| cookie.value().to_string()) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let jar = jar.add(session_cookie(SESSION_COOKIE, session_id.clone())); + if form.profile_id.trim().is_empty() || form.name.trim().is_empty() { + state + .connection_errors + .write() + .await + .insert(session_id, "Profile name is required".into()); + return (jar, Redirect::to("/?connection_error=1")); + } + let backend = Arc::new(backend::BackendAuth::new( + state.client.clone(), + form.zitadel_url, + form.zitadel_pat, + form.openbao_url, + form.openbao_token, + )); + if let Err(error) = backend.validate().await { + state + .connection_errors + .write() + .await + .insert(session_id, error); + return (jar, Redirect::to("/?connection_error=1")); + } + state + .sessions + .write() + .await + .entry(session_id.clone()) + .or_default() + .insert( + form.profile_id.clone(), + ConnectedProfile { + name: form.name, + auth: backend, + }, + ); + ( + jar.add(session_cookie(PROFILE_COOKIE, form.profile_id)), + Redirect::to("/dashboard"), + ) +} + +async fn switch_profile( + State(state): State, + jar: CookieJar, + Form(selection): Form, +) -> Result<(CookieJar, Redirect), AppError> { + let session_id = jar + .get(SESSION_COOKIE) + .ok_or(AppError::Disconnected)? + .value(); + let sessions = state.sessions.read().await; + if !sessions + .get(session_id) + .is_some_and(|profiles| profiles.contains_key(&selection.profile_id)) + { + return Err(AppError::Disconnected); + } + drop(sessions); + Ok(( + jar.add(session_cookie(PROFILE_COOKIE, selection.profile_id)), + Redirect::to("/dashboard"), + )) +} + +async fn disconnect_profile( + State(state): State, + jar: CookieJar, + Form(selection): Form, +) -> (CookieJar, Redirect) { + let mut empty = false; + if let Some(session_id) = jar.get(SESSION_COOKIE) + && let Some(profiles) = state.sessions.write().await.get_mut(session_id.value()) + { + profiles.remove(&selection.profile_id); + empty = profiles.is_empty(); + } + let jar = if jar + .get(PROFILE_COOKIE) + .is_some_and(|profile| profile.value() == selection.profile_id) + { + jar.remove(removal_cookie(PROFILE_COOKIE)) + } else { + jar + }; + let jar = if empty { + jar.remove(removal_cookie(SESSION_COOKIE)) + } else { + jar + }; + (jar, Redirect::to("/")) +} + +fn session_cookie(name: &'static str, value: String) -> Cookie<'static> { + Cookie::build((name, value)) + .http_only(true) + .same_site(SameSite::Lax) + .path("/") + .build() +} + +fn removal_cookie(name: &'static str) -> Cookie<'static> { + Cookie::build(name).path("/").build() +} + +async fn active_profile(state: &AppState, jar: &CookieJar) -> Result { + let session_id = jar + .get(SESSION_COOKIE) + .ok_or(AppError::Disconnected)? + .value(); + let profile_id = jar + .get(PROFILE_COOKIE) + .ok_or(AppError::Disconnected)? + .value(); + state + .sessions + .read() + .await + .get(session_id) + .and_then(|profiles| profiles.get(profile_id)) + .cloned() + .ok_or(AppError::Disconnected) +} + async fn grant_plan( State(state): State, + jar: CookieJar, Form(request): Form, ) -> Result { - let identity = state.auth.identity(&request.principal_subject_id).await?; - let grants = state.auth.grants_for(&identity.subject_id).await?; + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&request.principal_subject_id).await?; + let grants = profile.auth.grants_for(&identity.subject_id).await?; let plan = harmony_auth::plan_grant(&identity, &grants, request)?; - Ok(views::grant_review(&identity, &plan)) + Ok(views::grant_review(&profile.view(), &identity, &plan)) } async fn apply_grant( State(state): State, + jar: CookieJar, Form(request): Form, ) -> Result { - let identity = state.auth.identity(&request.principal_subject_id).await?; - let grants = state.auth.grants_for(&identity.subject_id).await?; + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&request.principal_subject_id).await?; + let grants = profile.auth.grants_for(&identity.subject_id).await?; let plan = harmony_auth::plan_grant(&identity, &grants, request)?; - state.auth.apply_grant(plan, "development-admin").await?; + profile.auth.apply_grant(plan, "profile-operator").await?; Ok(Redirect::to(&format!( "/identities/{}", identity.subject_id ))) } -async fn overview(State(state): State) -> Result { - let identities = state.auth.identities(None).await?; +async fn overview(State(state): State, jar: CookieJar) -> Result { + let profile = active_profile(&state, &jar).await?; + let identities = profile.auth.identities(None).await?; let mut grants = 0; for identity in &identities { - grants += state.auth.grants_for(&identity.subject_id).await?.len(); + grants += profile.auth.grants_for(&identity.subject_id).await?.len(); } - Ok(views::overview(&identities, grants)) + Ok(views::overview(&profile.view(), &identities, grants)) } async fn identities( State(state): State, + jar: CookieJar, Query(search): Query, ) -> Result { - let identities = state.auth.identities(search.q.as_deref()).await?; - Ok(views::identities(&identities, search.q.as_deref())) + let profile = active_profile(&state, &jar).await?; + let identities = profile.auth.identities(search.q.as_deref()).await?; + Ok(views::identities( + &profile.view(), + &identities, + search.q.as_deref(), + )) } async fn identity( State(state): State, + jar: CookieJar, Path(subject_id): Path, ) -> Result { - let identity = state.auth.identity(&subject_id).await?; - let grants = state.auth.grants_for(&subject_id).await?; + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&subject_id).await?; + let grants = profile.auth.grants_for(&subject_id).await?; let policy = harmony_auth::render_policy(&grants); - Ok(views::identity(&identity, &grants, &policy)) + Ok(views::identity( + &profile.view(), + &identity, + &grants, + &policy, + false, + )) } async fn css() -> impl IntoResponse { @@ -135,25 +413,50 @@ async fn htmx() -> impl IntoResponse { ) } -struct AppError(AuthError); +async fn profiles_js() -> impl IntoResponse { + ( + [( + axum::http::header::CONTENT_TYPE, + "text/javascript; charset=utf-8", + )], + include_str!("profiles.js"), + ) +} + +enum AppError { + Auth(AuthError), + Disconnected, +} impl From for AppError { fn from(value: AuthError) -> Self { - Self(value) + Self::Auth(value) } } impl IntoResponse for AppError { fn into_response(self) -> Response { - let status = match &self.0 { + if matches!(self, Self::Disconnected) { + return Redirect::to("/").into_response(); + } + let Self::Auth(error) = self else { + unreachable!() + }; + let status = match &error { AuthError::IdentityNotFound => axum::http::StatusCode::NOT_FOUND, AuthError::InvalidGrant(_) => axum::http::StatusCode::UNPROCESSABLE_ENTITY, AuthError::Backend(_) => axum::http::StatusCode::BAD_GATEWAY, }; - let detail = match &self.0 { + let detail = match &error { AuthError::InvalidGrant(message) => Some(message.as_str()), _ => None, }; (status, views::error(status, detail)).into_response() } } + +impl ConnectedProfile { + fn view(&self) -> views::Profile<'_> { + views::Profile { name: &self.name } + } +} diff --git a/harmony_auth_ui/src/mock.rs b/harmony_auth_ui/src/mock.rs deleted file mode 100644 index 918a661f..00000000 --- a/harmony_auth_ui/src/mock.rs +++ /dev/null @@ -1,141 +0,0 @@ -use async_trait::async_trait; -use chrono::{TimeZone, Utc}; -use harmony_auth::{AccessLevel, AuthError, AuthService, Grant, Identity, IdentityKind, Selector}; -use tokio::sync::RwLock; -use uuid::Uuid; - -pub struct MockAuth { - identities: Vec, - grants: RwLock>, -} - -impl MockAuth { - pub fn new() -> Self { - let identities = vec![ - Identity { - subject_id: "218904384720891003".into(), - kind: IdentityKind::Human, - display_name: "Maya Chen".into(), - login_name: "maya.chen".into(), - email: Some("maya@northstar.example".into()), - active: true, - }, - Identity { - subject_id: "218904384720891017".into(), - kind: IdentityKind::Machine, - display_name: "Northstar deployer".into(), - login_name: "northstar-deployer".into(), - email: None, - active: true, - }, - Identity { - subject_id: "218904384720891029".into(), - kind: IdentityKind::Human, - display_name: "Sam Rivera".into(), - login_name: "sam.rivera".into(), - email: Some("sam@harbor.example".into()), - active: false, - }, - ]; - let created_at = Utc.with_ymd_and_hms(2026, 7, 18, 14, 32, 0).unwrap(); - let grants = vec![ - Grant { - id: Uuid::parse_str("01981f2c-8950-7ad2-bd0c-aed3d23621a1").unwrap(), - principal_subject_id: identities[0].subject_id.clone(), - principal_kind: IdentityKind::Human, - mount: "secret".into(), - path: "tenants/northstar/production".into(), - selector: Selector::Subtree, - access: AccessLevel::ReadOnly, - created_by: "177192654770291002".into(), - created_at, - }, - Grant { - id: Uuid::parse_str("01981f2d-42bb-7b03-9243-1e590e25989f").unwrap(), - principal_subject_id: identities[1].subject_id.clone(), - principal_kind: IdentityKind::Machine, - mount: "secret".into(), - path: "tenants/northstar/deploy".into(), - selector: Selector::Subtree, - access: AccessLevel::ReadWrite, - created_by: "177192654770291002".into(), - created_at, - }, - ]; - Self { - identities, - grants: RwLock::new(grants), - } - } -} - -#[async_trait] -impl AuthService for MockAuth { - async fn identities(&self, search: Option<&str>) -> Result, AuthError> { - let search = search.unwrap_or("").trim().to_ascii_lowercase(); - Ok(self - .identities - .iter() - .filter(|identity| { - search.is_empty() - || identity.display_name.to_ascii_lowercase().contains(&search) - || identity.login_name.to_ascii_lowercase().contains(&search) - || identity - .email - .as_deref() - .is_some_and(|email| email.to_ascii_lowercase().contains(&search)) - }) - .cloned() - .collect()) - } - - async fn identity(&self, subject_id: &str) -> Result { - self.identities - .iter() - .find(|identity| identity.subject_id == subject_id) - .cloned() - .ok_or(AuthError::IdentityNotFound) - } - - async fn grants_for(&self, subject_id: &str) -> Result, AuthError> { - Ok(self - .grants - .read() - .await - .iter() - .filter(|grant| grant.principal_subject_id == subject_id) - .cloned() - .collect()) - } - - async fn apply_grant( - &self, - plan: harmony_auth::GrantPlan, - created_by: &str, - ) -> Result { - let identity = self.identity(&plan.request.principal_subject_id).await?; - let mut grants = self.grants.write().await; - if let Some(grant) = grants.iter().find(|grant| { - grant.principal_subject_id == plan.request.principal_subject_id - && grant.mount == plan.request.mount - && grant.path == plan.request.path - && grant.selector == plan.request.selector - && grant.access == plan.request.access - }) { - return Ok(grant.clone()); - } - let grant = Grant { - id: Uuid::new_v4(), - principal_subject_id: identity.subject_id, - principal_kind: identity.kind, - mount: plan.request.mount, - path: plan.request.path, - selector: plan.request.selector, - access: plan.request.access, - created_by: created_by.into(), - created_at: Utc::now(), - }; - grants.push(grant.clone()); - Ok(grant) - } -} diff --git a/harmony_auth_ui/src/profiles.js b/harmony_auth_ui/src/profiles.js new file mode 100644 index 00000000..f0cbcad4 --- /dev/null +++ b/harmony_auth_ui/src/profiles.js @@ -0,0 +1,163 @@ +const storageKey = 'harmony.auth.profiles'; +const list = document.querySelector('#profile-list'); +const editor = document.querySelector('#profile-editor'); +const dialog = document.querySelector('#connect-dialog'); +let focusAfterDialog; + +function profiles() { + try { + const value = JSON.parse(localStorage.getItem(storageKey) || '[]'); + return Array.isArray(value) ? value : []; + } catch (_) { + return []; + } +} + +function save(items) { + localStorage.setItem(storageKey, JSON.stringify(items)); +} + +function submit(path, profileId) { + const form = document.createElement('form'); + form.method = 'post'; + form.action = path; + const input = document.createElement('input'); + input.type = 'hidden'; + input.name = 'profile_id'; + input.value = profileId; + form.append(input); + document.body.append(form); + form.submit(); +} + +function connect(profile, trigger) { + focusAfterDialog = trigger; + const form = dialog.querySelector('form'); + for (const field of ['profile_id', 'name', 'zitadel_url', 'openbao_url']) { + form.elements[field].value = profile[field === 'profile_id' ? 'id' : field]; + } + form.elements.zitadel_pat.value = ''; + form.elements.openbao_token.value = ''; + document.querySelector('#connect-title').textContent = `Connect ${profile.name}`; + dialog.showModal(); + form.elements.zitadel_pat.focus(); +} + +async function render() { + const connected = new Set(await fetch('/profiles/connected').then(response => response.json())); + const items = profiles(); + list.replaceChildren(); + if (!items.length) { + const empty = document.createElement('div'); + empty.className = 'panel empty profile-empty'; + const title = document.createElement('strong'); + title.textContent = 'No profiles yet'; + const detail = document.createElement('p'); + detail.textContent = 'Add the first Zitadel and OpenBao pair to begin.'; + empty.append(title, detail); + list.append(empty); + return; + } + for (const profile of items) { + const card = document.createElement('article'); + card.className = 'profile-card'; + const heading = document.createElement('div'); + const status = document.createElement('span'); + status.className = connected.has(profile.id) ? 'connection-state connected' : 'connection-state'; + status.textContent = connected.has(profile.id) ? 'Connected' : 'Disconnected'; + const name = document.createElement('h2'); + name.textContent = profile.name; + heading.append(status, name); + const endpoints = document.createElement('dl'); + for (const [label, value] of [['Zitadel', profile.zitadel_url], ['OpenBao', profile.openbao_url]]) { + const term = document.createElement('dt'); + term.textContent = label; + const description = document.createElement('dd'); + description.textContent = value; + endpoints.append(term, description); + } + const actions = document.createElement('div'); + actions.className = 'profile-actions'; + const primary = document.createElement('button'); + primary.className = 'button primary'; + primary.textContent = connected.has(profile.id) ? 'Open' : 'Connect'; + primary.addEventListener('click', () => connected.has(profile.id) + ? submit('/profiles/switch', profile.id) + : connect(profile, primary)); + const edit = document.createElement('button'); + edit.className = 'button'; + edit.textContent = 'Edit'; + edit.addEventListener('click', () => editProfile(profile)); + const remove = document.createElement('button'); + remove.className = 'button quiet-danger'; + remove.textContent = 'Delete'; + remove.addEventListener('click', async () => { + if (!confirm(`Delete ${profile.name}? Credentials held by this server will also be forgotten.`)) return; + save(profiles().filter(item => item.id !== profile.id)); + await fetch('/profiles/disconnect', { + method: 'POST', + headers: {'content-type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({profile_id: profile.id}), + }); + render(); + }); + actions.append(primary, edit, remove); + card.append(heading, endpoints, actions); + list.append(card); + } +} + +function editProfile(profile) { + editor.hidden = false; + editor.dataset.profileId = profile.id; + editor.elements.name.value = profile.name; + editor.elements.zitadel_url.value = profile.zitadel_url; + editor.elements.openbao_url.value = profile.openbao_url; + editor.elements.name.focus(); +} + +document.querySelector('#show-profile-form').addEventListener('click', () => { + editor.reset(); + editor.dataset.profileId = ''; + editor.hidden = false; + editor.elements.name.focus(); +}); + +document.querySelector('#cancel-profile').addEventListener('click', () => { + editor.hidden = true; +}); + +editor.addEventListener('submit', async event => { + event.preventDefault(); + const data = new FormData(editor); + const profile = { + id: editor.dataset.profileId || crypto.randomUUID(), + name: data.get('name').trim(), + zitadel_url: data.get('zitadel_url').replace(/\/+$/, ''), + openbao_url: data.get('openbao_url').replace(/\/+$/, ''), + }; + const items = profiles(); + const existing = items.findIndex(item => item.id === profile.id); + if (existing >= 0) { + await fetch('/profiles/disconnect', { + method: 'POST', + headers: {'content-type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({profile_id: profile.id}), + }); + items[existing] = profile; + } else { + items.push(profile); + } + save(items); + editor.hidden = true; + render(); + connect(profile, document.querySelector('#show-profile-form')); +}); + +document.querySelector('#cancel-connect').addEventListener('click', () => dialog.close()); +dialog.addEventListener('click', event => { + if (event.target === dialog) dialog.close(); +}); +dialog.addEventListener('close', () => focusAfterDialog?.focus()); + +render(); diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index 975fc65f..bcd3d0ec 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -2,17 +2,84 @@ use axum::http::StatusCode; use harmony_auth::{AccessLevel, Grant, GrantPlan, Identity, IdentityKind, Selector}; use maud::{DOCTYPE, Markup, html}; -pub fn overview(identities: &[Identity], grants: usize) -> Markup { +pub struct Profile<'a> { + pub name: &'a str, +} + +pub fn profiles(connection_error: Option<&str>) -> Markup { + layout( + "Profiles", + "", + None, + html! { + section class="hero profile-hero" { + div { + p class="eyebrow" { "INFRASTRUCTURE PROFILES" } + h1 { "Choose where you want to work." } + p class="lede" { "Profiles remember backend locations in this browser. Credentials stay in server memory only while connected." } + } + button class="button primary" id="show-profile-form" type="button" { "Add profile" } + } + @if let Some(error) = connection_error { + div class="connection-error" role="alert" { strong { "Connection failed" } p { (error) } } + } + section class="profile-workspace" { + div id="profile-list" class="profile-list" aria-live="polite" {} + form id="profile-editor" class="panel profile-editor" hidden { + div class="section-title" { div { h2 { "New profile" } p { "Only this non-secret metadata is saved in your browser." } } } + label { "Profile name" input name="name" required placeholder="NationTech production"; } + label { "Zitadel URL" input name="zitadel_url" type="url" required placeholder="https://sso.example.com"; } + label { "OpenBao URL" input name="openbao_url" type="url" required placeholder="https://secrets.example.com"; } + div class="form-actions" { + button class="button primary" type="submit" { "Save and connect" } + button class="button" id="cancel-profile" type="button" { "Cancel" } + } + } + } + dialog id="connect-dialog" class="connect-dialog" { + form action="/profiles/connect" method="post" autocomplete="off" { + input type="hidden" name="profile_id"; + input type="hidden" name="name"; + input type="hidden" name="zitadel_url"; + input type="hidden" name="openbao_url"; + p class="eyebrow" { "CONNECT BACKENDS" } + h2 id="connect-title" { "Connect profile" } + p { "Credentials are held in this server process and disappear when it restarts." } + label { "Zitadel service-account PAT" input name="zitadel_pat" type="password" required autocomplete="off"; } + small { "PATs belong to Zitadel service accounts. The account needs the administrator permissions you intend to use." } + label { "OpenBao token" input name="openbao_token" type="password" required autocomplete="off"; } + small class="root-warning" { "The initial root token works for bootstrap, but OpenBao recommends replacing it with a limited administrator token." } + details { + summary { "How do I create a Zitadel PAT?" } + ol { + li { "Sign in to the Zitadel Console with your administrator account." } + li { "Create a service account and grant its required administrator role." } + li { "Create an expiring Personal Access Token on that service account." } + } + } + div class="form-actions" { + button class="button primary" type="submit" { "Validate and connect" } + button class="button" id="cancel-connect" type="button" { "Cancel" } + } + } + } + script src="/static/profiles.js" defer {} + }, + ) +} + +pub fn overview(profile: &Profile<'_>, identities: &[Identity], grants: usize) -> Markup { let active = identities.iter().filter(|identity| identity.active).count(); layout( "Overview", - "/", + "/dashboard", + Some(profile), html! { section class="hero" { div { p class="eyebrow" { "AUTHORIZATION CONTROL PLANE" } - h1 { "Know exactly who can reach every secret." } - p class="lede" { "Harmony translates direct access intent into enforceable OpenBao policies without copying identities or secret values." } + h1 { "Understand direct secret access, identity by identity." } + p class="lede" { "Harmony makes direct access intent inspectable without copying identities or secret values." } } a class="button primary" href="/identities" { "Review identities" } } @@ -21,9 +88,9 @@ pub fn overview(identities: &[Identity], grants: usize) -> Markup { (metric("Active", active, "Eligible for new grants")) (metric("Direct grants", grants, "Canonical access intent")) article class="metric healthy" { - span class="signal" {} span { "Enforcement" } - strong { "In sync" } - small { "No policy drift detected" } + span { "Current mode" } + strong { "Inspect" } + small { "Mutations await JWT role configuration" } } } section class="panel split" { @@ -44,10 +111,11 @@ pub fn overview(identities: &[Identity], grants: usize) -> Markup { ) } -pub fn identities(identities: &[Identity], search: Option<&str>) -> Markup { +pub fn identities(profile: &Profile<'_>, identities: &[Identity], search: Option<&str>) -> Markup { layout( "Identities", "/identities", + Some(profile), html! { (page_heading("Identities", "People and machines discovered from Zitadel. Email and names are display data; subject ID is authoritative.")) section class="panel" { @@ -93,10 +161,17 @@ pub fn identities(identities: &[Identity], search: Option<&str>) -> Markup { ) } -pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { +pub fn identity( + profile: &Profile<'_>, + identity: &Identity, + grants: &[Grant], + policy: &str, + can_manage: bool, +) -> Markup { layout( &identity.display_name, "/identities", + Some(profile), html! { a class="back" href="/identities" { "← All identities" } section class="identity-header" { @@ -147,7 +222,7 @@ pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { pre tabindex="0" { code { (policy) } } } } - @if identity.active { + @if identity.active && can_manage { details class="grant-form" { summary { "Grant direct access" } form action="/grants/plan" method="post" { @@ -166,10 +241,11 @@ pub fn identity(identity: &Identity, grants: &[Grant], policy: &str) -> Markup { ) } -pub fn grant_review(identity: &Identity, plan: &GrantPlan) -> Markup { +pub fn grant_review(profile: &Profile<'_>, identity: &Identity, plan: &GrantPlan) -> Markup { layout( "Review grant", "/identities", + Some(profile), html! { a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } (page_heading("Review direct grant", "Confirm the authorization intent and generated enforcement change before applying it.")) @@ -214,11 +290,12 @@ pub fn error(status: StatusCode, detail: Option<&str>) -> Markup { layout( "Request failed", "", + None, html! { section class="panel empty" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { "Grant could not be reviewed: " (detail) "." } } @else { p { "The requested authorization data could not be loaded." } } a class="button" href="/" { "Return to overview" } } }, ) } -fn layout(title: &str, current: &str, content: Markup) -> Markup { +fn layout(title: &str, current: &str, profile: Option<&Profile<'_>>, content: Markup) -> Markup { html! { (DOCTYPE) html lang="en" { @@ -233,12 +310,18 @@ fn layout(title: &str, current: &str, content: Markup) -> Markup { header class="topbar" { a class="brand" href="/" { span class="brand-mark" { "H" } span { "Harmony" } em { "AUTH" } } nav aria-label="Primary" { - (nav_link("/", "Overview", current)) - (nav_link("/identities", "Identities", current)) - span class="nav-disabled" title="Coming in the next delivery slice" { "Secrets" } - span class="nav-disabled" title="Coming in the next delivery slice" { "Access grants" } + @if profile.is_some() { + (nav_link("/dashboard", "Overview", current)) + (nav_link("/identities", "Identities", current)) + } + } + @if let Some(profile) = profile { + a class="profile-switch" href="/" title="Switch or manage profiles" { + span class="signal" {} + span { (profile.name) } + small { "Switch" } + } } - div class="environment" { span {} "Development" } } main { (content) } footer { "Harmony Auth" span { "Authorization intent, made inspectable." } } -- 2.39.5 From 0e232b3b406fc3ea89d59f80cac330d0519709f7 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 19 Jul 2026 18:33:32 -0400 Subject: [PATCH 05/47] feat: refresh profile credentials independently --- harmony_auth_ui/src/a11y.css | 17 ++++++++++ harmony_auth_ui/src/backend.rs | 14 ++++++++ harmony_auth_ui/src/main.rs | 58 +++++++++++++++++++++++++-------- harmony_auth_ui/src/profiles.js | 23 +++++++++++-- harmony_auth_ui/src/views.rs | 8 ++++- 5 files changed, 103 insertions(+), 17 deletions(-) diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css index 59d6376c..74ee65d6 100644 --- a/harmony_auth_ui/src/a11y.css +++ b/harmony_auth_ui/src/a11y.css @@ -299,6 +299,23 @@ body { padding-left: 10px; } +.token-help pre { + width: 100%; + min-width: 0; + max-width: 100%; + overflow-x: auto; + padding: 12px; + background: var(--navy); + color: #e8eee9; + font-size: 10px; +} + +.connect-dialog form, +.connect-dialog form > *, +.token-help { + min-width: 0; +} + .connection-error { margin-bottom: 24px; border-left: 3px solid var(--red); diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth_ui/src/backend.rs index 15b016e7..83baf15a 100644 --- a/harmony_auth_ui/src/backend.rs +++ b/harmony_auth_ui/src/backend.rs @@ -56,6 +56,20 @@ impl BackendAuth { Ok(()) } + pub fn with_credentials(&self, zitadel_pat: String, openbao_token: String) -> Self { + Self::new( + self.client.clone(), + self.zitadel_url.clone(), + zitadel_pat, + self.openbao_url.clone(), + openbao_token, + ) + } + + pub fn credentials(&self) -> (&str, &str) { + (&self.zitadel_pat, &self.openbao_token) + } + async fn all_identities(&self) -> Result, AuthError> { let response = self .client diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index d51ac4bc..2a113dea 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -36,7 +36,7 @@ struct AppState { #[derive(Clone)] struct ConnectedProfile { name: String, - auth: Arc, + auth: Arc, } #[derive(Default, Deserialize)] @@ -210,13 +210,43 @@ async fn connect_profile( .insert(session_id, "Profile name is required".into()); return (jar, Redirect::to("/?connection_error=1")); } - let backend = Arc::new(backend::BackendAuth::new( - state.client.clone(), - form.zitadel_url, - form.zitadel_pat, - form.openbao_url, - form.openbao_token, - )); + let existing = state + .sessions + .read() + .await + .get(&session_id) + .and_then(|profiles| profiles.get(&form.profile_id)) + .cloned(); + let backend = if let Some(existing) = &existing { + let (old_zitadel, old_openbao) = existing.auth.credentials(); + Arc::new(existing.auth.with_credentials( + if form.zitadel_pat.is_empty() { + old_zitadel.into() + } else { + form.zitadel_pat + }, + if form.openbao_token.is_empty() { + old_openbao.into() + } else { + form.openbao_token + }, + )) + } else { + if form.zitadel_pat.is_empty() || form.openbao_token.is_empty() { + state.connection_errors.write().await.insert( + session_id, + "Both credentials are required for a new connection".into(), + ); + return (jar, Redirect::to("/?connection_error=1")); + } + Arc::new(backend::BackendAuth::new( + state.client.clone(), + form.zitadel_url, + form.zitadel_pat, + form.openbao_url, + form.openbao_token, + )) + }; if let Err(error) = backend.validate().await { state .connection_errors @@ -332,8 +362,9 @@ async fn grant_plan( Form(request): Form, ) -> Result { let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&request.principal_subject_id).await?; - let grants = profile.auth.grants_for(&identity.subject_id).await?; + let auth: &dyn AuthService = profile.auth.as_ref(); + let identity = auth.identity(&request.principal_subject_id).await?; + let grants = auth.grants_for(&identity.subject_id).await?; let plan = harmony_auth::plan_grant(&identity, &grants, request)?; Ok(views::grant_review(&profile.view(), &identity, &plan)) } @@ -344,10 +375,11 @@ async fn apply_grant( Form(request): Form, ) -> Result { let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&request.principal_subject_id).await?; - let grants = profile.auth.grants_for(&identity.subject_id).await?; + let auth: &dyn AuthService = profile.auth.as_ref(); + let identity = auth.identity(&request.principal_subject_id).await?; + let grants = auth.grants_for(&identity.subject_id).await?; let plan = harmony_auth::plan_grant(&identity, &grants, request)?; - profile.auth.apply_grant(plan, "profile-operator").await?; + auth.apply_grant(plan, "profile-operator").await?; Ok(Redirect::to(&format!( "/identities/{}", identity.subject_id diff --git a/harmony_auth_ui/src/profiles.js b/harmony_auth_ui/src/profiles.js index f0cbcad4..ce7da9f8 100644 --- a/harmony_auth_ui/src/profiles.js +++ b/harmony_auth_ui/src/profiles.js @@ -30,7 +30,7 @@ function submit(path, profileId) { form.submit(); } -function connect(profile, trigger) { +function connect(profile, trigger, connected = false) { focusAfterDialog = trigger; const form = dialog.querySelector('form'); for (const field of ['profile_id', 'name', 'zitadel_url', 'openbao_url']) { @@ -38,7 +38,16 @@ function connect(profile, trigger) { } form.elements.zitadel_pat.value = ''; form.elements.openbao_token.value = ''; - document.querySelector('#connect-title').textContent = `Connect ${profile.name}`; + document.querySelector('#connect-title').textContent = connected + ? `Update credentials for ${profile.name}` + : `Connect ${profile.name}`; + document.querySelector('#credential-instruction').textContent = connected + ? 'Leave either field blank to keep its current credential. Submit both blank to revalidate the current connection.' + : 'Enter both credentials for the initial connection.'; + for (const field of [form.elements.zitadel_pat, form.elements.openbao_token]) { + field.required = !connected; + field.placeholder = connected ? 'Leave blank to keep current credential' : ''; + } dialog.showModal(); form.elements.zitadel_pat.focus(); } @@ -84,6 +93,14 @@ async function render() { primary.addEventListener('click', () => connected.has(profile.id) ? submit('/profiles/switch', profile.id) : connect(profile, primary)); + actions.append(primary); + if (connected.has(profile.id)) { + const credentials = document.createElement('button'); + credentials.className = 'button'; + credentials.textContent = 'Credentials'; + credentials.addEventListener('click', () => connect(profile, credentials, true)); + actions.append(credentials); + } const edit = document.createElement('button'); edit.className = 'button'; edit.textContent = 'Edit'; @@ -101,7 +118,7 @@ async function render() { }); render(); }); - actions.append(primary, edit, remove); + actions.append(edit, remove); card.append(heading, endpoints, actions); list.append(card); } diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index bcd3d0ec..780f2b13 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -45,10 +45,16 @@ pub fn profiles(connection_error: Option<&str>) -> Markup { p class="eyebrow" { "CONNECT BACKENDS" } h2 id="connect-title" { "Connect profile" } p { "Credentials are held in this server process and disappear when it restarts." } + p id="credential-instruction" {} label { "Zitadel service-account PAT" input name="zitadel_pat" type="password" required autocomplete="off"; } small { "PATs belong to Zitadel service accounts. The account needs the administrator permissions you intend to use." } label { "OpenBao token" input name="openbao_token" type="password" required autocomplete="off"; } - small class="root-warning" { "The initial root token works for bootstrap, but OpenBao recommends replacing it with a limited administrator token." } + small class="root-warning" { "The initial root token works for bootstrap. Prefer a temporary, non-renewable root-policy token for this session." } + details class="token-help" { + summary { "Create a temporary OpenBao administrator token" } + p { "Run this while authenticated as root, then give the resulting one-hour token to Harmony:" } + pre tabindex="0" { code { "bao token create -policy=root -ttl=1h -explicit-max-ttl=1h -renewable=false -display-name=temporary-admin" } } + } details { summary { "How do I create a Zitadel PAT?" } ol { -- 2.39.5 From 6c7fd2a659eaf590c9cd67758422e746ed697967 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 19 Jul 2026 18:52:10 -0400 Subject: [PATCH 06/47] feat: inspect and manage identity jwt roles --- harmony_auth/src/lib.rs | 116 +++++++++++++++++ harmony_auth_ui/src/a11y.css | 21 ++++ harmony_auth_ui/src/backend.rs | 223 +++++++++++++++++++++++++++++++-- harmony_auth_ui/src/main.rs | 71 ++++++++++- harmony_auth_ui/src/views.rs | 95 +++++++++++++- 5 files changed, 512 insertions(+), 14 deletions(-) diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index 985be2b9..815505a6 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -64,6 +64,24 @@ pub struct GrantPlan { pub resulting_policy: String, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct JwtRole { + pub name: String, + pub bound_subject: String, + pub bound_audiences: Vec, + pub token_policies: Vec, + pub role_type: String, + pub user_claim: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct JwtRolePlan { + pub subject_id: String, + pub bound_audiences: Vec, + pub token_policies: Vec, + pub creates_role: bool, +} + #[derive(Debug, Error)] pub enum AuthError { #[error("identity not found")] @@ -80,6 +98,67 @@ pub trait AuthService: Send + Sync { async fn identity(&self, subject_id: &str) -> Result; async fn grants_for(&self, subject_id: &str) -> Result, AuthError>; async fn apply_grant(&self, plan: GrantPlan, created_by: &str) -> Result; + async fn jwt_role(&self, subject_id: &str) -> Result, AuthError>; + async fn acl_policies(&self) -> Result, AuthError>; + async fn apply_jwt_role(&self, plan: JwtRolePlan) -> Result; +} + +pub fn plan_jwt_role( + identity: &Identity, + current: Option<&JwtRole>, + audiences: &str, + policies: &str, +) -> Result { + if !identity.active { + return Err(AuthError::InvalidGrant("identity is suspended".into())); + } + if let Some(current) = current { + if current.role_type != "jwt" { + return Err(AuthError::InvalidGrant( + "the subject-named role exists but is not a JWT role".into(), + )); + } + if current.bound_subject != identity.subject_id { + return Err(AuthError::InvalidGrant( + "the subject-named role is bound to a different identity".into(), + )); + } + } + let mut bound_audiences = comma_values(audiences); + let mut token_policies = comma_values(policies); + bound_audiences.sort_unstable(); + bound_audiences.dedup(); + token_policies.sort_unstable(); + token_policies.dedup(); + if token_policies.is_empty() { + return Err(AuthError::InvalidGrant( + "at least one ACL policy is required".into(), + )); + } + if token_policies.iter().any(|policy| { + !policy.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | '/' | ':') + }) + }) { + return Err(AuthError::InvalidGrant( + "policy names contain unsupported characters".into(), + )); + } + Ok(JwtRolePlan { + subject_id: identity.subject_id.clone(), + bound_audiences, + token_policies, + creates_role: current.is_none(), + }) +} + +fn comma_values(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect() } pub fn plan_grant( @@ -287,4 +366,41 @@ path "secret/metadata/tenants/acme/deploy/*" { capabilities = ["list", "read"] } Err(AuthError::InvalidGrant(_)) )); } + + #[test] + fn jwt_role_plan_normalizes_policies_and_audiences() { + let identity = Identity { + subject_id: "subject".into(), + kind: IdentityKind::Human, + display_name: "User".into(), + login_name: "user".into(), + email: None, + active: true, + }; + + let plan = plan_jwt_role( + &identity, + None, + "audience-b, audience-a, audience-a", + "tenant/viewer, tenant/viewer, deploy.prod", + ) + .unwrap(); + + assert!(plan.creates_role); + assert_eq!(plan.bound_audiences, ["audience-a", "audience-b"]); + assert_eq!(plan.token_policies, ["deploy.prod", "tenant/viewer"]); + + let collision = JwtRole { + name: "subject".into(), + bound_subject: "someone-else".into(), + bound_audiences: vec![], + token_policies: vec![], + role_type: "jwt".into(), + user_claim: "sub".into(), + }; + assert!(matches!( + plan_jwt_role(&identity, Some(&collision), "", "tenant/viewer"), + Err(AuthError::InvalidGrant(_)) + )); + } } diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css index 74ee65d6..9353830c 100644 --- a/harmony_auth_ui/src/a11y.css +++ b/harmony_auth_ui/src/a11y.css @@ -132,6 +132,27 @@ body { font-size: 12px; } +.role-panel { + margin-top: 24px; +} + +.role-facts { + margin: 0; +} + +.role-form form { + grid-template-columns: 1fr 1fr; +} + +.available-policies, +.field-help { + grid-column: 1 / -1; + margin: 0; + overflow-wrap: anywhere; + color: var(--muted); + font-size: 11px; +} + @media (max-width: 760px) { .grant-form form { grid-template-columns: 1fr; diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth_ui/src/backend.rs index 83baf15a..890b9bb9 100644 --- a/harmony_auth_ui/src/backend.rs +++ b/harmony_auth_ui/src/backend.rs @@ -1,7 +1,9 @@ use async_trait::async_trait; -use harmony_auth::{AuthError, AuthService, Grant, GrantPlan, Identity, IdentityKind}; -use reqwest::{Client, StatusCode}; -use serde_json::{Value, json}; +use harmony_auth::{ + AuthError, AuthService, Grant, GrantPlan, Identity, IdentityKind, JwtRole, JwtRolePlan, +}; +use reqwest::{Client, Method, StatusCode}; +use serde_json::{Map, Value, json}; pub struct BackendAuth { client: Client, @@ -91,12 +93,39 @@ impl BackendAuth { } async fn request_openbao(&self, path: &str) -> Result { - self.client - .get(format!("{}/v1/{path}", self.openbao_url)) - .header("X-Vault-Token", &self.openbao_token) - .send() + self.openbao(Method::GET, path, None).await + } + + async fn openbao( + &self, + method: Method, + path: &str, + body: Option, + ) -> Result { + let mut request = self + .client + .request(method, format!("{}/v1/{path}", self.openbao_url)) + .header("X-Vault-Token", &self.openbao_token); + if let Some(body) = body { + request = request.json(&body); + } + request.send().await.map_err(backend) + } + + async fn role_value(&self, subject_id: &str) -> Result, AuthError> { + let response = self + .request_openbao(&format!("auth/jwt/role/{subject_id}")) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + let body: Value = response + .error_for_status() + .map_err(backend)? + .json() .await - .map_err(backend) + .map_err(backend)?; + Ok(Some(body["data"].clone())) } } @@ -166,6 +195,117 @@ impl AuthService for BackendAuth { "grant mutations require JWT role configuration and are not enabled yet".into(), )) } + + async fn jwt_role(&self, subject_id: &str) -> Result, AuthError> { + Ok(self + .role_value(subject_id) + .await? + .map(|value| parse_role(subject_id, &value))) + } + + async fn acl_policies(&self) -> Result, AuthError> { + let body: Value = self + .request_openbao("sys/policies/acl?list=true") + .await? + .error_for_status() + .map_err(backend)? + .json() + .await + .map_err(backend)?; + Ok(strings(&body["data"]["keys"])) + } + + async fn apply_jwt_role(&self, plan: JwtRolePlan) -> Result { + let current = self.role_value(&plan.subject_id).await?; + let mut body = Map::new(); + if let Some(current) = current.as_ref().and_then(Value::as_object) { + for key in ROLE_FIELDS { + if let Some(value) = current.get(*key) { + body.insert((*key).into(), value.clone()); + } + } + } + body.insert("role_type".into(), Value::String("jwt".into())); + body.insert("user_claim".into(), Value::String("sub".into())); + body.insert( + "bound_subject".into(), + Value::String(plan.subject_id.clone()), + ); + body.insert("bound_audiences".into(), json!(plan.bound_audiences)); + body.insert("token_policies".into(), json!(plan.token_policies)); + body.remove("policies"); + self.openbao( + Method::POST, + &format!("auth/jwt/role/{}", plan.subject_id), + Some(Value::Object(body)), + ) + .await? + .error_for_status() + .map_err(backend)?; + self.jwt_role(&plan.subject_id) + .await? + .ok_or_else(|| AuthError::Backend("OpenBao did not return the saved JWT role".into())) + } +} + +const ROLE_FIELDS: &[&str] = &[ + "allowed_redirect_uris", + "bound_audiences", + "bound_claims", + "bound_claims_type", + "bound_subject", + "callback_mode", + "claim_mappings", + "clock_skew_leeway", + "expiration_leeway", + "groups_claim", + "max_age", + "not_before_leeway", + "oauth2_metadata", + "oidc_disable_confirmation", + "oidc_scopes", + "poll_interval", + "role_type", + "token_bound_cidrs", + "token_explicit_max_ttl", + "token_max_ttl", + "token_no_default_policy", + "token_num_uses", + "token_period", + "token_policies", + "token_policies_template_claims", + "token_strictly_bind_ip", + "token_ttl", + "token_type", + "user_claim", + "user_claim_json_pointer", + "verbose_oidc_logging", +]; + +fn parse_role(name: &str, value: &Value) -> JwtRole { + let token_policies = strings(&value["token_policies"]); + JwtRole { + name: name.into(), + bound_subject: value["bound_subject"].as_str().unwrap_or_default().into(), + bound_audiences: strings(&value["bound_audiences"]), + token_policies: if token_policies.is_empty() { + strings(&value["policies"]) + } else { + token_policies + }, + role_type: value["role_type"].as_str().unwrap_or("jwt").into(), + user_claim: value["user_claim"].as_str().unwrap_or_default().into(), + } +} + +fn strings(value: &Value) -> Vec { + value + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() } fn parse_identity(value: &Value) -> Option { @@ -207,8 +347,11 @@ fn backend(error: impl std::fmt::Display) -> AuthError { #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + use axum::{ Json, Router, + extract::State, http::{HeaderMap, StatusCode}, routing::{get, post}, }; @@ -286,4 +429,68 @@ mod tests { assert_eq!(identities[0].email.as_deref(), Some("maya@example.com")); assert_eq!(identities[1].kind, IdentityKind::Machine); } + + #[tokio::test] + async fn role_update_preserves_existing_login_settings() { + async fn read_role(State(role): State>>) -> Json { + Json(json!({ "data": role.lock().unwrap().clone() })) + } + async fn write_role( + State(role): State>>, + Json(body): Json, + ) -> StatusCode { + *role.lock().unwrap() = body; + StatusCode::NO_CONTENT + } + + let role = Arc::new(Mutex::new(json!({ + "role_type": "jwt", + "user_claim": "sub", + "bound_subject": "subject", + "bound_audiences": ["old-audience"], + "token_policies": ["old-policy"], + "groups_claim": "groups", + "bound_claims": { "department": "platform" }, + "claim_mappings": { "email": "email" }, + "clock_skew_leeway": 30, + "token_bound_cidrs": ["10.0.0.0/8"], + "token_explicit_max_ttl": 900, + "token_ttl": 300, + "token_type": "batch" + }))); + let app = Router::new() + .route("/v1/auth/jwt/role/subject", get(read_role).post(write_role)) + .with_state(role.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let backend = BackendAuth::new( + Client::new(), + "http://unused".into(), + "unused".into(), + base_url, + "openbao-token".into(), + ); + + let saved = backend + .apply_jwt_role(JwtRolePlan { + subject_id: "subject".into(), + bound_audiences: vec!["new-audience".into()], + token_policies: vec!["new-policy".into()], + creates_role: false, + }) + .await + .unwrap(); + + assert_eq!(saved.token_policies, ["new-policy"]); + let written = role.lock().unwrap(); + assert_eq!(written["groups_claim"], "groups"); + assert_eq!(written["token_ttl"], 300); + assert_eq!(written["token_type"], "batch"); + assert_eq!(written["bound_claims"]["department"], "platform"); + assert_eq!(written["claim_mappings"]["email"], "email"); + assert_eq!(written["clock_skew_leeway"], 30); + assert_eq!(written["token_bound_cidrs"][0], "10.0.0.0/8"); + assert_eq!(written["token_explicit_max_ttl"], 900); + } } diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index 2a113dea..ca40e1b8 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -15,7 +15,7 @@ use axum::{ }; use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite}; use clap::Parser; -use harmony_auth::{AuthError, AuthService, GrantRequest}; +use harmony_auth::{AuthError, AuthService, GrantRequest, Identity, JwtRolePlan}; use serde::Deserialize; use tokio::sync::RwLock; use tower_http::set_header::SetResponseHeaderLayer; @@ -64,6 +64,12 @@ struct ProfileSelection { profile_id: String, } +#[derive(Deserialize)] +struct JwtRoleForm { + audiences: String, + policies: String, +} + const SESSION_COOKIE: &str = "harmony_auth_session"; const PROFILE_COOKIE: &str = "harmony_auth_profile"; @@ -99,6 +105,11 @@ fn router(state: AppState) -> Router { .route("/identities/{subject_id}", get(identity)) .route("/grants/plan", post(grant_plan)) .route("/grants", post(apply_grant)) + .route( + "/identities/{subject_id}/roles/review", + get(role_review), + ) + .route("/identities/{subject_id}/roles", post(apply_role)) .route("/static/app.css", get(css)) .route("/static/htmx.min.js", get(htmx)) .route("/static/profiles.js", get(profiles_js)) @@ -386,6 +397,58 @@ async fn apply_grant( ))) } +async fn role_review( + State(state): State, + jar: CookieJar, + Path(subject_id): Path, + Query(form): Query, +) -> Result { + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&subject_id).await?; + let plan = requested_role_plan(&profile.auth, &identity, &form).await?; + Ok(views::role_review(&profile.view(), &identity, &plan)) +} + +async fn apply_role( + State(state): State, + jar: CookieJar, + Path(subject_id): Path, + Form(form): Form, +) -> Result { + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&subject_id).await?; + let plan = requested_role_plan(&profile.auth, &identity, &form).await?; + profile.auth.apply_jwt_role(plan).await?; + Ok(Redirect::to(&format!( + "/identities/{}", + identity.subject_id + ))) +} + +async fn requested_role_plan( + auth: &backend::BackendAuth, + identity: &Identity, + form: &JwtRoleForm, +) -> Result { + let current = auth.jwt_role(&identity.subject_id).await?; + let plan = + harmony_auth::plan_jwt_role(identity, current.as_ref(), &form.audiences, &form.policies)?; + let available = auth.acl_policies().await?; + let missing = plan + .token_policies + .iter() + .filter(|policy| !available.contains(policy)) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(AuthError::InvalidGrant(format!( + "OpenBao ACL policies do not exist: {}", + missing.join(", ") + ))); + } + Ok(plan) +} + async fn overview(State(state): State, jar: CookieJar) -> Result { let profile = active_profile(&state, &jar).await?; let identities = profile.auth.identities(None).await?; @@ -418,13 +481,17 @@ async fn identity( let profile = active_profile(&state, &jar).await?; let identity = profile.auth.identity(&subject_id).await?; let grants = profile.auth.grants_for(&subject_id).await?; + let role = profile.auth.jwt_role(&subject_id).await?; + let policies = profile.auth.acl_policies().await?; let policy = harmony_auth::render_policy(&grants); Ok(views::identity( &profile.view(), &identity, &grants, + role.as_ref(), + &policies, &policy, - false, + true, )) } diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index 780f2b13..ccea4f9f 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -1,5 +1,7 @@ use axum::http::StatusCode; -use harmony_auth::{AccessLevel, Grant, GrantPlan, Identity, IdentityKind, Selector}; +use harmony_auth::{ + AccessLevel, Grant, GrantPlan, Identity, IdentityKind, JwtRole, JwtRolePlan, Selector, +}; use maud::{DOCTYPE, Markup, html}; pub struct Profile<'a> { @@ -171,6 +173,8 @@ pub fn identity( profile: &Profile<'_>, identity: &Identity, grants: &[Grant], + role: Option<&JwtRole>, + available_policies: &[String], policy: &str, can_manage: bool, ) -> Markup { @@ -203,7 +207,7 @@ pub fn identity( } article class="panel access" { div class="section-title" { - div { h2 { "Effective access" } p { "Union of active direct grants" } } + div { h2 { "Harmony direct grants" } p { "Canonical authorization intent" } } span class="count" { (grants.len()) } } @for grant in grants { @@ -216,7 +220,7 @@ pub fn identity( } } @if grants.is_empty() { - div class="empty compact" { strong { "No secret access" } p { "This identity has no active direct grants." } } + div class="empty compact" { strong { "No Harmony direct grants" } p { "This does not mean the identity has no access. Inspect its live JWT role below." } } } @if !policy.is_empty() { details class="policy" { @@ -243,6 +247,81 @@ pub fn identity( } } } + section class="panel role-panel" { + div class="section-title" { + div { h2 { "OpenBao JWT role" } p { "JWT login configuration at auth/jwt/role/" (identity.subject_id) } } + @if role.is_some() { span class="status active" { "Configured" } } + } + @if let Some(role) = role { + dl class="role-facts" { + (fact("Role name", &role.name)) + (fact("Bound subject", &role.bound_subject)) + (fact("Role type", &role.role_type)) + (fact("User claim", &role.user_claim)) + (fact("Audiences", &display_values(&role.bound_audiences))) + (fact("Token policies", &display_values(&role.token_policies))) + } + p class="field-help" { "Token policies are attached on the next JWT login. Existing OpenBao tokens are unchanged." } + } @else { + div class="empty compact" { strong { "No subject JWT role" } p { "Create one to bind this Zitadel subject to OpenBao ACL policies." } } + } + @if can_manage { + details class="grant-form role-form" { + summary { (if role.is_some() { "Edit JWT role" } else { "Create JWT role" }) } + form action=(format!("/identities/{}/roles/review", identity.subject_id)) method="get" { + label { "Bound audiences" input name="audiences" value=(role.map(|role| role.bound_audiences.join(", ")).unwrap_or_default()) placeholder="Zitadel project resource ID"; } + label { "ACL policies" input name="policies" value=(role.map(|role| role.token_policies.join(", ")).unwrap_or_default()) required placeholder="policy-one, policy-two"; } + @if !available_policies.is_empty() { + p class="available-policies" { "Available: " (available_policies.join(", ")) } + } + p class="field-help" { "Harmony fixes role type to jwt, user claim to sub, and bound subject to this identity. Supported existing TTL, claim, CIDR, and token settings are preserved." } + button class="button primary" type="submit" { "Review role" } + } + } + } + } + }, + ) +} + +pub fn role_review(profile: &Profile<'_>, identity: &Identity, plan: &JwtRolePlan) -> Markup { + layout( + "Review JWT role", + "/identities", + Some(profile), + html! { + a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } + (page_heading("Review JWT role", "Confirm the live OpenBao login binding before applying it.")) + section class="detail-grid" { + article class="panel facts" { + h2 { (if plan.creates_role { "Create role" } else { "Update role" }) } + dl { + (fact("Identity", &identity.display_name)) + (fact("Role name", &plan.subject_id)) + (fact("Bound subject", &plan.subject_id)) + (fact("Role type", "jwt")) + (fact("User claim", "sub")) + } + } + article class="panel access" { + h2 { "Token authorization" } + dl { + (fact("Bound audiences", &display_values(&plan.bound_audiences))) + (fact("ACL policies", &display_values(&plan.token_policies))) + } + @if plan.bound_audiences.is_empty() { + p class="danger-note" { strong { "No audience constraint." } " JWTs carrying an aud claim may be rejected by OpenBao. Add the Zitadel project resource ID unless these tokens have no audience." } + } + @if plan.token_policies.iter().any(|policy| policy == "root") { + p class="danger-note" { strong { "Unrestricted OpenBao access." } " The root ACL policy allows this identity to perform any operation." } + } + form action=(format!("/identities/{}/roles", plan.subject_id)) method="post" { + input type="hidden" name="audiences" value=(plan.bound_audiences.join(",")); + input type="hidden" name="policies" value=(plan.token_policies.join(",")); + button class="button primary" type="submit" { (if plan.creates_role { "Create JWT role" } else { "Apply role changes" }) } + } + } + } }, ) } @@ -297,7 +376,7 @@ pub fn error(status: StatusCode, detail: Option<&str>) -> Markup { "Request failed", "", None, - html! { section class="panel empty" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { "Grant could not be reviewed: " (detail) "." } } @else { p { "The requested authorization data could not be loaded." } } a class="button" href="/" { "Return to overview" } } }, + html! { section class="panel empty" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { "Change could not be reviewed: " (detail) "." } } @else { p { "The requested authorization data could not be loaded." } } a class="button" href="/" { "Return to profiles" } } }, ) } @@ -366,3 +445,11 @@ fn avatar(identity: &Identity) -> Markup { fn fact(label: &str, value: &str) -> Markup { html! { div class="fact" { dt { (label) } dd { (value) } } } } + +fn display_values(values: &[String]) -> String { + if values.is_empty() { + "None".into() + } else { + values.join(", ") + } +} -- 2.39.5 From 015c262275908ba42ec524090f23ee395da35bfc Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 19 Jul 2026 21:15:15 -0400 Subject: [PATCH 07/47] feat: harmony auth ui redesign around harmony specific permission model --- Cargo.lock | 1 + harmony_auth/Cargo.toml | 1 + harmony_auth/src/lib.rs | 525 +++++++++-------------- harmony_auth_ui/src/a11y.css | 364 ---------------- harmony_auth_ui/src/app.css | 140 +++++- harmony_auth_ui/src/backend.rs | 750 ++++++++++++++++++++------------- harmony_auth_ui/src/main.rs | 418 +++++++++--------- harmony_auth_ui/src/views.rs | 581 +++++++++++-------------- 8 files changed, 1285 insertions(+), 1495 deletions(-) delete mode 100644 harmony_auth_ui/src/a11y.css diff --git a/Cargo.lock b/Cargo.lock index 0efd5432..f10435a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4324,6 +4324,7 @@ dependencies = [ "async-trait", "chrono", "serde", + "sha2 0.10.9", "thiserror 2.0.18", "uuid", ] diff --git a/harmony_auth/Cargo.toml b/harmony_auth/Cargo.toml index 3c66a450..70e7a46e 100644 --- a/harmony_auth/Cargo.toml +++ b/harmony_auth/Cargo.toml @@ -9,5 +9,6 @@ license.workspace = true async-trait.workspace = true chrono = { workspace = true, features = ["serde"] } serde.workspace = true +sha2.workspace = true thiserror.workspace = true uuid = { workspace = true, features = ["serde"] } diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index 815505a6..a0454e62 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use sha2::{Digest, Sha256}; use thiserror::Error; use uuid::Uuid; @@ -9,7 +9,7 @@ use uuid::Uuid; #[serde(rename_all = "snake_case")] pub enum IdentityKind { Human, - Machine, + Service, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -24,70 +24,132 @@ pub struct Identity { #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub enum Selector { - Exact, - Subtree, +pub enum Permission { + TenantAdmin, + CdDeployer, + ReadOnly, } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccessLevel { - ReadOnly, - ReadWrite, +impl Permission { + pub fn label(self) -> &'static str { + match self { + Self::TenantAdmin => "Tenant Admin", + Self::CdDeployer => "CD Deployer", + Self::ReadOnly => "Read-only", + } + } + + pub fn slug(self) -> &'static str { + match self { + Self::TenantAdmin => "admin", + Self::CdDeployer => "cd", + Self::ReadOnly => "viewer", + } + } + + pub fn description(self) -> &'static str { + match self { + Self::TenantAdmin => "Read, create, change, and delete secrets", + Self::CdDeployer => "Read deployment secrets", + Self::ReadOnly => "Read secrets", + } + } } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct Grant { +pub struct Scope { + pub tenant: String, + pub project: Option, +} + +impl Scope { + pub fn new(tenant: &str, project: Option<&str>) -> Result { + let tenant = tenant.trim().to_ascii_lowercase(); + let project = project + .map(str::trim) + .filter(|project| !project.is_empty()) + .map(str::to_ascii_lowercase); + if !valid_slug(&tenant) || project.as_deref().is_some_and(|value| !valid_slug(value)) { + return Err(AuthError::Invalid( + "tenant and project may contain lowercase letters, numbers, and dashes".into(), + )); + } + Ok(Self { tenant, project }) + } + + pub fn path(&self) -> String { + self.project.as_ref().map_or_else( + || self.tenant.clone(), + |project| format!("{}/{project}", self.tenant), + ) + } + + pub fn label(&self) -> String { + self.path() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Assignment { pub id: Uuid, - pub principal_subject_id: String, - pub principal_kind: IdentityKind, - pub mount: String, - pub path: String, - pub selector: Selector, - pub access: AccessLevel, - pub created_by: String, + pub subject_id: String, + pub permission: Permission, + pub scope: Scope, + pub policy_name: String, pub created_at: DateTime, } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct GrantRequest { - pub principal_subject_id: String, - pub mount: String, - pub path: String, - pub selector: Selector, - pub access: AccessLevel, -} - #[derive(Clone, Debug, Eq, PartialEq)] -pub struct GrantPlan { - pub request: GrantRequest, - pub resulting_policy: String, +pub struct ImportedAccess { + pub role_name: String, + pub policy_name: String, + pub secret_paths: Vec, + pub effect: String, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct JwtRole { pub name: String, - pub bound_subject: String, - pub bound_audiences: Vec, - pub token_policies: Vec, - pub role_type: String, - pub user_claim: String, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct JwtRolePlan { pub subject_id: String, pub bound_audiences: Vec, pub token_policies: Vec, - pub creates_role: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct IdentityAccess { + pub assignments: Vec, + pub imported: Vec, + pub roles: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TenantSummary { + pub scope: Scope, + pub humans: usize, + pub services: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct AssignmentRequest { + pub subject_id: String, + pub permission: Permission, + pub tenant: String, + pub project: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AssignmentPlan { + pub assignment: Assignment, + pub summary: String, + pub policy: String, } #[derive(Debug, Error)] pub enum AuthError { #[error("identity not found")] IdentityNotFound, - #[error("invalid grant: {0}")] - InvalidGrant(String), + #[error("invalid request: {0}")] + Invalid(String), #[error("authorization backend failed: {0}")] Backend(String), } @@ -96,311 +158,144 @@ pub enum AuthError { pub trait AuthService: Send + Sync { async fn identities(&self, search: Option<&str>) -> Result, AuthError>; async fn identity(&self, subject_id: &str) -> Result; - async fn grants_for(&self, subject_id: &str) -> Result, AuthError>; - async fn apply_grant(&self, plan: GrantPlan, created_by: &str) -> Result; - async fn jwt_role(&self, subject_id: &str) -> Result, AuthError>; - async fn acl_policies(&self) -> Result, AuthError>; - async fn apply_jwt_role(&self, plan: JwtRolePlan) -> Result; + async fn access(&self, subject_id: &str) -> Result; + async fn access_for(&self, identities: &[Identity]) -> Result, AuthError>; + async fn tenants(&self) -> Result, AuthError>; + async fn plan_assignment( + &self, + request: AssignmentRequest, + ) -> Result; + async fn apply_assignment(&self, plan: AssignmentPlan) -> Result; + async fn remove_assignment( + &self, + subject_id: &str, + assignment_id: Uuid, + ) -> Result<(), AuthError>; } -pub fn plan_jwt_role( +pub fn plan_assignment( identity: &Identity, - current: Option<&JwtRole>, - audiences: &str, - policies: &str, -) -> Result { + request: AssignmentRequest, + mount: &str, +) -> Result { if !identity.active { - return Err(AuthError::InvalidGrant("identity is suspended".into())); + return Err(AuthError::Invalid("identity is suspended".into())); } - if let Some(current) = current { - if current.role_type != "jwt" { - return Err(AuthError::InvalidGrant( - "the subject-named role exists but is not a JWT role".into(), - )); - } - if current.bound_subject != identity.subject_id { - return Err(AuthError::InvalidGrant( - "the subject-named role is bound to a different identity".into(), - )); - } + if identity.subject_id != request.subject_id { + return Err(AuthError::Invalid("identity does not match request".into())); } - let mut bound_audiences = comma_values(audiences); - let mut token_policies = comma_values(policies); - bound_audiences.sort_unstable(); - bound_audiences.dedup(); - token_policies.sort_unstable(); - token_policies.dedup(); - if token_policies.is_empty() { - return Err(AuthError::InvalidGrant( - "at least one ACL policy is required".into(), + if request.permission == Permission::CdDeployer && identity.kind != IdentityKind::Service { + return Err(AuthError::Invalid( + "CD Deployer is intended for a service account".into(), )); } - if token_policies.iter().any(|policy| { - !policy.chars().all(|character| { - character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | '/' | ':') - }) - }) { - return Err(AuthError::InvalidGrant( - "policy names contain unsupported characters".into(), - )); + if request.permission == Permission::CdDeployer && request.project.trim().is_empty() { + return Err(AuthError::Invalid("CD Deployer requires a project".into())); } - Ok(JwtRolePlan { + let scope = Scope::new(&request.tenant, Some(&request.project))?; + let hash = Sha256::digest(scope.path().as_bytes()); + let policy_name = format!("harmony-{}-{}", request.permission.slug(), hex(&hash[..6])); + let assignment = Assignment { + id: Uuid::new_v4(), subject_id: identity.subject_id.clone(), - bound_audiences, - token_policies, - creates_role: current.is_none(), - }) -} - -fn comma_values(value: &str) -> Vec { - value - .split(',') - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .collect() -} - -pub fn plan_grant( - identity: &Identity, - existing: &[Grant], - mut request: GrantRequest, -) -> Result { - if !identity.active { - return Err(AuthError::InvalidGrant("identity is suspended".into())); - } - if request.principal_subject_id != identity.subject_id { - return Err(AuthError::InvalidGrant( - "principal does not match identity".into(), - )); - } - request.mount = request.mount.trim().trim_matches('/').to_string(); - request.path = request.path.trim().trim_matches('/').to_string(); - if request.mount.is_empty() || request.mount.contains('/') { - return Err(AuthError::InvalidGrant( - "mount must be one non-empty path segment".into(), - )); - } - if !request.mount.chars().all(valid_path_character) { - return Err(AuthError::InvalidGrant( - "mount may contain only letters, numbers, dot, dash, and underscore".into(), - )); - } - if request.path.is_empty() { - return Err(AuthError::InvalidGrant("path must not be empty".into())); - } - if request - .path - .split('/') - .any(|segment| segment.is_empty() || segment == "." || segment == "..") - { - return Err(AuthError::InvalidGrant( - "path must not contain empty, dot, or parent segments".into(), - )); - } - if !request - .path - .chars() - .all(|character| character == '/' || valid_path_character(character)) - { - return Err(AuthError::InvalidGrant( - "path segments may contain only letters, numbers, dot, dash, and underscore".into(), - )); - } - let mut grants = existing.to_vec(); - grants.push(Grant { - id: Uuid::nil(), - principal_subject_id: identity.subject_id.clone(), - principal_kind: identity.kind.clone(), - mount: request.mount.clone(), - path: request.path.clone(), - selector: request.selector, - access: request.access, - created_by: String::new(), + permission: request.permission, + scope, + policy_name, created_at: Utc::now(), - }); - Ok(GrantPlan { - request, - resulting_policy: render_policy(&grants), + }; + Ok(AssignmentPlan { + summary: format!( + "Assign {} as {} for {}.", + identity.display_name, + request.permission.label(), + assignment.scope.label() + ), + policy: render_policy(mount, &assignment.scope, request.permission), + assignment, }) } -fn valid_path_character(character: char) -> bool { - character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') +pub fn render_policy(mount: &str, scope: &Scope, permission: Permission) -> String { + let path = scope.path(); + let data = match permission { + Permission::TenantAdmin => "[\"create\", \"delete\", \"patch\", \"read\", \"update\"]", + Permission::CdDeployer | Permission::ReadOnly => "[\"read\"]", + }; + format!( + "path \"{mount}/data/{path}/*\" {{ capabilities = {data} }}\n\ + path \"{mount}/metadata/{path}/*\" {{ capabilities = [\"list\", \"read\"] }}" + ) } -pub fn policy_name(subject_id: &str) -> String { - format!("harmony-identity-{subject_id}") -} - -pub fn render_policy(grants: &[Grant]) -> String { - let mut data_paths = BTreeMap::new(); - let mut metadata_paths = BTreeMap::new(); - for grant in grants { - let path = grant.path.trim_matches('/'); - let suffixes: &[&str] = match grant.selector { - Selector::Exact => &[""], - Selector::Subtree => &["", "/*"], - }; - for suffix in suffixes { - data_paths - .entry(format!("{}/data/{}{}", grant.mount, path, suffix)) - .and_modify(|read_write| *read_write |= grant.access == AccessLevel::ReadWrite) - .or_insert(grant.access == AccessLevel::ReadWrite); - metadata_paths - .entry(format!("{}/metadata/{}{}", grant.mount, path, suffix)) - .and_modify(|list| *list |= grant.selector == Selector::Subtree) - .or_insert(grant.selector == Selector::Subtree); - } - } - - data_paths - .into_iter() - .map(|(path, read_write)| { - let capabilities = if read_write { - "[\"create\", \"delete\", \"patch\", \"read\", \"update\"]" - } else { - "[\"read\"]" - }; - format!("path \"{path}\" {{ capabilities = {capabilities} }}") +fn valid_slug(value: &str) -> bool { + !value.is_empty() + && value.len() <= 63 + && value.chars().all(|character| { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' }) - .chain(metadata_paths.into_iter().map(|(path, list)| { - let capabilities = if list { - "[\"list\", \"read\"]" - } else { - "[\"read\"]" - }; - format!("path \"{path}\" {{ capabilities = {capabilities} }}") - })) - .collect::>() - .join("\n") + && !value.starts_with('-') + && !value.ends_with('-') +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() } #[cfg(test)] mod tests { - use chrono::Utc; - use uuid::Uuid; - use super::*; - fn grant(path: &str, selector: Selector, access: AccessLevel) -> Grant { - Grant { - id: Uuid::nil(), - principal_subject_id: "218904384720891003".into(), - principal_kind: IdentityKind::Human, - mount: "secret".into(), - path: path.into(), - selector, - access, - created_by: "admin".into(), - created_at: Utc::now(), + fn service() -> Identity { + Identity { + subject_id: "123".into(), + kind: IdentityKind::Service, + display_name: "Folk CD".into(), + login_name: "folk-cd".into(), + email: None, + active: true, } } #[test] - fn policy_is_stable_and_deduplicated() { - let first = grant( - "/tenants/acme/database/", - Selector::Exact, - AccessLevel::ReadOnly, - ); - let second = grant( - "tenants/acme/deploy", - Selector::Subtree, - AccessLevel::ReadWrite, - ); - - let expected = r#"path "secret/data/tenants/acme/database" { capabilities = ["read"] } -path "secret/data/tenants/acme/deploy" { capabilities = ["create", "delete", "patch", "read", "update"] } -path "secret/data/tenants/acme/deploy/*" { capabilities = ["create", "delete", "patch", "read", "update"] } -path "secret/metadata/tenants/acme/database" { capabilities = ["read"] } -path "secret/metadata/tenants/acme/deploy" { capabilities = ["list", "read"] } -path "secret/metadata/tenants/acme/deploy/*" { capabilities = ["list", "read"] }"#; - - assert_eq!( - render_policy(&[second.clone(), first.clone(), first]), - expected - ); - assert_eq!( - render_policy(&[second]), - render_policy(&[grant( - "tenants/acme/deploy", - Selector::Subtree, - AccessLevel::ReadWrite, - )]) - ); - } - - #[test] - fn strongest_overlapping_grant_wins() { - let path = "tenants/acme/database"; - let policy = render_policy(&[ - grant(path, Selector::Exact, AccessLevel::ReadOnly), - grant(path, Selector::Exact, AccessLevel::ReadWrite), - ]); - - assert_eq!(policy.matches(&format!("secret/data/{path}")).count(), 1); - assert!(policy.contains("[\"create\", \"delete\", \"patch\", \"read\", \"update\"]")); - } - - #[test] - fn grant_plan_rejects_policy_syntax() { - let identity = Identity { - subject_id: "subject".into(), - kind: IdentityKind::Human, - display_name: "User".into(), - login_name: "user".into(), - email: None, - active: true, - }; - let request = GrantRequest { - principal_subject_id: identity.subject_id.clone(), - mount: "secret".into(), - path: "safe/\" } path \"*\" { capabilities = [\"sudo\"] }".into(), - selector: Selector::Exact, - access: AccessLevel::ReadOnly, - }; - - assert!(matches!( - plan_grant(&identity, &[], request), - Err(AuthError::InvalidGrant(_)) - )); - } - - #[test] - fn jwt_role_plan_normalizes_policies_and_audiences() { - let identity = Identity { - subject_id: "subject".into(), - kind: IdentityKind::Human, - display_name: "User".into(), - login_name: "user".into(), - email: None, - active: true, - }; - - let plan = plan_jwt_role( - &identity, - None, - "audience-b, audience-a, audience-a", - "tenant/viewer, tenant/viewer, deploy.prod", + fn plans_harmony_permission_in_business_terms() { + let plan = plan_assignment( + &service(), + AssignmentRequest { + subject_id: "123".into(), + permission: Permission::CdDeployer, + tenant: "Devsights".into(), + project: "folk-timesheet".into(), + }, + "secret", ) .unwrap(); - assert!(plan.creates_role); - assert_eq!(plan.bound_audiences, ["audience-a", "audience-b"]); - assert_eq!(plan.token_policies, ["deploy.prod", "tenant/viewer"]); + assert_eq!(plan.assignment.scope.path(), "devsights/folk-timesheet"); + assert!(plan.assignment.policy_name.starts_with("harmony-cd-")); + assert!(plan.summary.contains("Folk CD as CD Deployer")); + assert!( + plan.policy + .contains("secret/data/devsights/folk-timesheet/*") + ); + assert!(!plan.policy.contains("create")); + } - let collision = JwtRole { - name: "subject".into(), - bound_subject: "someone-else".into(), - bound_audiences: vec![], - token_policies: vec![], - role_type: "jwt".into(), - user_claim: "sub".into(), - }; + #[test] + fn rejects_cd_permission_for_human() { + let mut human = service(); + human.kind = IdentityKind::Human; assert!(matches!( - plan_jwt_role(&identity, Some(&collision), "", "tenant/viewer"), - Err(AuthError::InvalidGrant(_)) + plan_assignment( + &human, + AssignmentRequest { + subject_id: "123".into(), + permission: Permission::CdDeployer, + tenant: "devsights".into(), + project: "folk".into(), + }, + "secret" + ), + Err(AuthError::Invalid(_)) )); } } diff --git a/harmony_auth_ui/src/a11y.css b/harmony_auth_ui/src/a11y.css deleted file mode 100644 index 9353830c..00000000 --- a/harmony_auth_ui/src/a11y.css +++ /dev/null @@ -1,364 +0,0 @@ - -/* Overrides kept readable while the base stylesheet remains minified. */ -body { - --muted: #56615d; -} - -.nav-disabled { - color: #9ca8b5; - opacity: 1; -} - -.identity-email { - margin-top: 2px; -} - -.metric small { - color: #56615d; -} - -.mobile-meta { - display: none; -} - -.detail-grid > *, -.fact dd, -.path code { - min-width: 0; - overflow-wrap: anywhere; -} - -.facts dl { - margin: 0; -} - -.policy { - margin-top: 20px; - border-top: 1px solid var(--line); - padding-top: 20px; -} - -.policy summary { - cursor: pointer; - color: var(--green); - font-size: 12px; - font-weight: 750; -} - -.policy-heading { - display: flex; - justify-content: space-between; - gap: 16px; - margin: 18px 0 8px; - color: var(--muted); - font-size: 11px; -} - -.policy-heading code { - overflow-wrap: anywhere; - text-align: right; -} - -.policy pre { - max-width: 100%; - margin: 0; - padding: 16px; - overflow-x: auto; - border-radius: 3px; - background: var(--navy); - color: #e8eee9; - font-size: 11px; - white-space: pre; - overflow-wrap: normal; -} - -.grant-form { - margin-top: 24px; - border-top: 1px solid var(--line); - padding-top: 20px; -} - -.grant-form summary { - cursor: pointer; - color: var(--green); - font-weight: 750; -} - -.grant-form form { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 16px; - margin-top: 20px; -} - -.grant-form label { - display: grid; - gap: 6px; - color: var(--muted); - font-size: 11px; -} - -.grant-form input, -.grant-form select { - min-width: 0; - height: 40px; - border: 1px solid #bcb9b0; - border-radius: 3px; - background: #fff; - padding: 0 10px; - color: var(--ink); - font: 13px inherit; -} - -.grant-form .button { - justify-self: start; -} - -.review-policy { - max-width: 100%; - margin: 20px 0; - padding: 16px; - overflow-x: auto; - background: var(--navy); - color: #e8eee9; - font-size: 11px; -} - -.danger-note { - border-left: 3px solid var(--red); - padding: 12px 14px; - background: #f7e8e5; - color: #742a24 !important; - font-size: 12px; -} - -.role-panel { - margin-top: 24px; -} - -.role-facts { - margin: 0; -} - -.role-form form { - grid-template-columns: 1fr 1fr; -} - -.available-policies, -.field-help { - grid-column: 1 / -1; - margin: 0; - overflow-wrap: anywhere; - color: var(--muted); - font-size: 11px; -} - -@media (max-width: 760px) { - .grant-form form { - grid-template-columns: 1fr; - } -} - -.profile-switch { - display: flex; - align-items: center; - gap: 8px; - margin-left: auto; - padding: 7px 10px; - border: 1px solid #415269; - border-radius: 99px; - color: #eef3f6; - text-decoration: none; - font-size: 12px; -} - -.profile-switch small { - color: #aebac6; -} - -.profile-hero { - padding-bottom: 42px; -} - -.profile-workspace { - display: grid; - grid-template-columns: minmax(0, 1.4fr) minmax(300px, .6fr); - gap: 24px; - align-items: start; -} - -.profile-list { - display: grid; - gap: 14px; -} - -.profile-card { - display: grid; - grid-template-columns: minmax(180px, .7fr) minmax(280px, 1.3fr) auto; - gap: 24px; - align-items: center; - padding: 24px; - border: 1px solid var(--line); - background: var(--surface); -} - -.profile-card h2 { - margin: 6px 0 0; - font: 600 22px Georgia, serif; -} - -.connection-state { - color: var(--muted); - font-size: 10px; - font-weight: 800; - letter-spacing: .08em; - text-transform: uppercase; -} - -.connection-state::before { - content: ""; - display: inline-block; - width: 7px; - height: 7px; - margin-right: 7px; - border-radius: 50%; - background: #929b96; -} - -.connection-state.connected { - color: var(--green); -} - -.connection-state.connected::before { - background: var(--green); -} - -.profile-card dl { - display: grid; - grid-template-columns: 65px minmax(0, 1fr); - gap: 5px 12px; - margin: 0; - font-size: 11px; -} - -.profile-card dt { - color: var(--muted); -} - -.profile-card dd { - margin: 0; - overflow-wrap: anywhere; - font-family: ui-monospace, monospace; -} - -.profile-actions, -.form-actions { - display: flex; - gap: 8px; - flex-wrap: wrap; -} - -.quiet-danger { - color: var(--red); -} - -.profile-editor, -.connect-dialog form { - display: grid; - gap: 16px; -} - -.profile-editor[hidden] { - display: none; -} - -.profile-editor label, -.connect-dialog label { - display: grid; - gap: 6px; - color: var(--muted); - font-size: 11px; - font-weight: 700; -} - -.profile-editor input, -.connect-dialog input { - width: 100%; - height: 42px; - border: 1px solid #bcb9b0; - border-radius: 3px; - padding: 0 11px; - font: 13px inherit; -} - -.connect-dialog { - width: min(540px, calc(100% - 28px)); - border: 1px solid var(--line); - border-radius: 4px; - padding: 30px; - background: var(--surface); - color: var(--ink); -} - -.connect-dialog::backdrop { - background: rgba(10, 20, 31, .66); -} - -.connect-dialog h2 { - margin: 0; - font: 600 30px Georgia, serif; -} - -.connect-dialog p, -.connect-dialog small { - margin: 0; - color: var(--muted); -} - -.root-warning { - border-left: 3px solid #b88122; - padding-left: 10px; -} - -.token-help pre { - width: 100%; - min-width: 0; - max-width: 100%; - overflow-x: auto; - padding: 12px; - background: var(--navy); - color: #e8eee9; - font-size: 10px; -} - -.connect-dialog form, -.connect-dialog form > *, -.token-help { - min-width: 0; -} - -.connection-error { - margin-bottom: 24px; - border-left: 3px solid var(--red); - padding: 14px 18px; - background: #f7e8e5; - color: #742a24; -} - -.connection-error p { - margin: 3px 0 0; -} - -@media (max-width: 900px) { - .profile-workspace, - .profile-card { - grid-template-columns: 1fr; - } -} - -@media (max-width: 760px) { - .mobile-meta { - display: block; - color: #56615d; - } -} diff --git a/harmony_auth_ui/src/app.css b/harmony_auth_ui/src/app.css index 5c1321f9..14a9a59f 100644 --- a/harmony_auth_ui/src/app.css +++ b/harmony_auth_ui/src/app.css @@ -1 +1,139 @@ -:root{--ink:#17201d;--muted:#69736f;--paper:#f4f1e9;--surface:#fffdf8;--line:#d9d6cc;--green:#0c6b4f;--lime:#cde86a;--navy:#14243b;--red:#a33b32}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font:15px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif}.topbar{height:68px;padding:0 max(24px,calc((100vw - 1180px)/2));display:flex;align-items:center;gap:48px;background:var(--navy);color:#fff}.brand{display:flex;align-items:center;gap:9px;color:#fff;text-decoration:none;font-weight:750;font-size:17px;letter-spacing:-.02em}.brand-mark{display:grid;place-items:center;width:29px;height:29px;border:1px solid #a8bdd5;border-radius:50%;font-family:Georgia,serif}.brand em{font-style:normal;font-size:9px;letter-spacing:.18em;color:var(--lime);align-self:flex-start;margin-top:10px}.topbar nav{display:flex;height:100%;align-items:center;gap:30px}.topbar nav a,.nav-disabled{height:100%;display:flex;align-items:center;color:#b8c4d1;text-decoration:none;font-size:13px;border-bottom:2px solid transparent}.topbar nav a:hover{color:#fff}.topbar nav a[aria-current=page]{color:#fff;border-color:var(--lime)}.nav-disabled{opacity:.42;cursor:not-allowed}.environment{margin-left:auto;padding:6px 10px;border:1px solid #415269;border-radius:99px;color:#c5cfda;font-size:11px}.environment span{display:inline-block;width:6px;height:6px;margin-right:7px;border-radius:50%;background:var(--lime)}main{width:min(1180px,calc(100% - 48px));margin:0 auto;padding:58px 0 90px}.hero{display:grid;grid-template-columns:1fr auto;align-items:end;gap:40px;padding:26px 0 64px}.eyebrow{margin:0 0 12px;color:var(--green);font-size:11px;font-weight:800;letter-spacing:.15em}.hero h1,.page-heading h1,.identity-header h1{max-width:780px;margin:0;font:600 clamp(38px,5vw,66px)/1.02 Georgia,serif;letter-spacing:-.045em}.lede{max-width:700px;margin:22px 0 0;color:var(--muted);font-size:18px}.button{display:inline-flex;justify-content:center;align-items:center;min-height:42px;padding:0 18px;border:1px solid #bcb9b0;border-radius:3px;background:var(--surface);color:var(--ink);font:700 12px inherit;text-decoration:none;cursor:pointer}.button.primary{background:var(--lime);border-color:#b4ce55;color:#17201d}.metrics{display:grid;grid-template-columns:repeat(4,1fr);border:1px solid var(--line);background:var(--surface)}.metric{min-height:142px;padding:24px;border-right:1px solid var(--line);display:flex;flex-direction:column}.metric:last-child{border:0}.metric>span{color:var(--muted);font-size:12px}.metric strong{margin:8px 0 4px;font:600 34px Georgia,serif}.metric small{margin-top:auto;color:#818984}.metric.healthy strong{font:700 18px system-ui}.signal{width:8px;height:8px;border-radius:50%;background:var(--green);display:inline-block;margin-right:6px}.panel{border:1px solid var(--line);background:var(--surface);padding:30px}.panel.split{display:grid;grid-template-columns:1fr 1fr;gap:80px;margin-top:28px;padding:45px}.panel h2{margin:0 0 8px;font:600 26px Georgia,serif}.panel p{color:var(--muted)}.text-link,.back{color:var(--green);font-weight:700;text-decoration:none}.rule-card{display:grid;grid-template-columns:32px 1fr;gap:10px 14px}.rule-card p{margin:0 0 12px}.rule-number{font:700 11px monospace;color:var(--green);padding-top:4px}.page-heading{max-width:760px;margin-bottom:38px}.page-heading h1{font-size:52px}.page-heading>p:last-child{color:var(--muted);font-size:17px}.search{max-width:660px}.search label{display:block;margin-bottom:8px;font-size:12px;font-weight:750}.search-row{display:flex;gap:9px}.search input{width:100%;height:44px;border:1px solid #bcb9b0;background:#fff;padding:0 13px;font:inherit;border-radius:3px}.search input:focus{outline:3px solid #cde86a88;border-color:var(--green)}.list-summary{margin:30px 0 10px;color:var(--muted);font-size:12px}.identity-list{border-top:1px solid var(--line)}.identity-row{display:grid;grid-template-columns:42px minmax(180px,1fr) 90px 100px 24px;align-items:center;gap:18px;padding:17px 8px;border-bottom:1px solid var(--line);color:var(--ink);text-decoration:none}.identity-row:hover{background:#f6f7ef}.avatar{display:grid;place-items:center;width:40px;height:40px;border-radius:50%;background:#dce5dc;color:#244c3d;font-size:11px;font-weight:800}.machine-avatar{border-radius:8px;background:#dce3ed;color:#294866}.identity-main{display:flex;flex-direction:column}.identity-main span{color:var(--muted);font-size:12px}.kind,.status{font-size:11px}.kind{padding:4px 8px;width:max-content;border:1px solid var(--line);border-radius:99px}.status:before{content:"";display:inline-block;width:6px;height:6px;border-radius:50%;margin-right:7px}.status.active{color:var(--green)}.status.active:before{background:var(--green)}.status.suspended{color:var(--red)}.status.suspended:before{background:var(--red)}.arrow{font-size:18px;color:var(--green)}.empty{padding:50px;text-align:center;color:var(--muted)}.empty strong{color:var(--ink)}.empty.compact{padding:35px}.back{display:inline-block;margin-bottom:32px}.identity-header{display:flex;align-items:center;gap:20px;margin-bottom:36px}.identity-header .avatar{width:64px;height:64px;font-size:16px}.identity-header h1{font-size:44px}.identity-header p{margin:5px 0 0;color:var(--muted)}.title-line{display:flex;align-items:center;gap:18px}.detail-grid{display:grid;grid-template-columns:minmax(260px,.7fr) minmax(420px,1.3fr);gap:24px}.facts h2,.access h2{font-size:22px}.fact{display:grid;grid-template-columns:130px 1fr;padding:15px 0;border-bottom:1px solid var(--line)}.fact:last-child{border:0}.fact dt{color:var(--muted);font-size:12px}.fact dd{margin:0;overflow-wrap:anywhere;font-family:ui-monospace,monospace;font-size:12px}.section-title{display:flex;justify-content:space-between;align-items:start;margin-bottom:20px}.section-title p{margin:0}.count{display:grid;place-items:center;width:34px;height:34px;background:var(--navy);color:#fff;border-radius:50%;font-weight:700}.grant{padding:18px 0;border-top:1px solid var(--line)}.path code{font-size:13px;font-weight:700}.grant-meta{display:flex;gap:9px;margin-top:10px;color:var(--muted);font-size:11px}.grant-meta span{padding:4px 8px;background:#f0eee7;border-radius:2px}.grant-meta .access-level{background:#e1ebd4;color:#315a2e}footer{display:flex;justify-content:space-between;width:min(1180px,calc(100% - 48px));margin:0 auto;padding:26px 0;border-top:1px solid var(--line);color:var(--muted);font-size:11px}@media(max-width:760px){.topbar{height:auto;min-height:64px;padding:13px 20px;flex-wrap:wrap;gap:8px 24px}.topbar nav{order:3;width:100%;height:38px;gap:22px;overflow-x:auto}.environment{margin-left:auto}.hero{grid-template-columns:1fr;padding-top:5px}.hero .button{justify-self:start}.metrics{grid-template-columns:1fr 1fr}.metric:nth-child(2){border-right:0}.metric:nth-child(-n+2){border-bottom:1px solid var(--line)}.panel.split,.detail-grid{grid-template-columns:1fr;gap:30px}.identity-row{grid-template-columns:42px 1fr 24px}.identity-row .kind,.identity-row .status{display:none}main{width:min(100% - 28px,1180px);padding-top:36px}.panel{padding:20px}.page-heading h1{font-size:42px}.title-line{align-items:start;flex-direction:column;gap:8px}.identity-header h1{font-size:35px}.search-row{align-items:stretch}.search-row .button{padding:0 12px}footer{width:calc(100% - 28px)}} +:root { + --ink: #17201d; + --muted: #5a6560; + --paper: #f4f1e9; + --surface: #fffdf8; + --line: #d7d4ca; + --green: #0c6b4f; + --lime: #cde86a; + --navy: #14243b; + --red: #9c342d; +} + +* { box-sizing: border-box; } +body { margin: 0; background: var(--paper); color: var(--ink); font: 14px/1.45 system-ui, sans-serif; } +a { color: var(--green); } +button, input, select { font: inherit; } +.topbar { height: 60px; padding: 0 max(20px, calc((100vw - 1160px) / 2)); display: flex; align-items: center; gap: 38px; background: var(--navy); color: white; } +.brand { display: flex; align-items: center; gap: 9px; color: white; text-decoration: none; font-weight: 750; } +.brand-mark { display: grid; place-items: center; width: 27px; height: 27px; border: 1px solid #a8bdd5; border-radius: 50%; font-family: Georgia, serif; } +.topbar nav { display: flex; align-self: stretch; gap: 26px; } +.topbar nav a { display: flex; align-items: center; border-bottom: 2px solid transparent; color: #bdc8d3; text-decoration: none; font-size: 12px; } +.topbar nav a[aria-current="page"] { border-color: var(--lime); color: white; } +.profile-switch { margin-left: auto; display: flex; align-items: center; gap: 7px; padding: 6px 10px; border: 1px solid #415269; border-radius: 99px; color: white; text-decoration: none; font-size: 11px; } +.profile-switch small { color: #aebac6; } +.signal { width: 7px; height: 7px; border-radius: 50%; background: var(--lime); } +main { width: min(1160px, calc(100% - 36px)); margin: 0 auto; padding: 34px 0 70px; } +h1, h2, p { margin-top: 0; } +h1 { margin-bottom: 4px; font: 600 25px/1.2 Georgia, serif; } +h2 { margin-bottom: 14px; font: 600 18px/1.25 Georgia, serif; } +.page-head { display: flex; justify-content: space-between; gap: 20px; align-items: center; margin-bottom: 20px; } +.page-head p, .identity-head p { margin: 0; color: var(--muted); } +.card { border: 1px solid var(--line); background: var(--surface); padding: 22px; } +.button { min-height: 36px; padding: 0 13px; display: inline-flex; align-items: center; justify-content: center; border: 1px solid #bdb9ae; border-radius: 3px; background: var(--surface); color: var(--ink); text-decoration: none; cursor: pointer; font-size: 12px; font-weight: 700; } +.button.primary { border-color: #b0ca50; background: var(--lime); } +.button.danger, .quiet-danger { color: var(--red); } +.text-link, .row-link, .back { font-weight: 700; text-decoration: none; font-size: 12px; } +.back { display: inline-block; margin-bottom: 20px; } +.filters { display: flex; gap: 10px; margin-bottom: 14px; padding: 13px; } +.filters input, .filters select, .form input, .form select, .dialog input { height: 38px; min-width: 0; border: 1px solid #bbb8ae; border-radius: 3px; background: white; padding: 0 10px; color: var(--ink); } +.filters .search-field { flex: 1; } +.filters .search-field input { width: 100%; } +.filters label:not(.search-field) input { width: 150px; } +.table-card { padding: 0; overflow-x: auto; } +.table-summary { padding: 13px 17px; border-bottom: 1px solid var(--line); color: var(--muted); font-size: 11px; } +table { width: 100%; border-collapse: collapse; } +th { color: var(--muted); font-size: 10px; text-align: left; text-transform: uppercase; letter-spacing: .06em; } +th, td { padding: 13px 17px; border-bottom: 1px solid var(--line); vertical-align: middle; } +tbody tr:last-child td { border-bottom: 0; } +td strong, td small { display: block; } +td small { margin-top: 2px; color: var(--muted); } +.permission-cell { min-width: 260px; } +.badge { display: inline-flex; margin: 2px 5px 2px 0; padding: 3px 7px; border: 1px solid var(--line); border-radius: 99px; font-size: 10px; } +.badge.permission { border-color: #bdd1a0; background: #edf3e5; color: #33542d; } +.badge.imported { border-color: #d7c49e; background: #f5eedf; color: #6c5022; } +.status { font-size: 11px; } +.status::before { content: ""; display: inline-block; width: 6px; height: 6px; margin-right: 6px; border-radius: 50%; } +.status.active { color: var(--green); } +.status.active::before { background: var(--green); } +.status.inactive { color: var(--red); } +.status.inactive::before { background: var(--red); } +.muted { color: var(--muted); } +.empty { padding: 42px 20px; text-align: center; color: var(--muted); } +.empty.compact { padding: 25px 5px; } +.identity-head { display: flex; align-items: center; gap: 14px; margin-bottom: 20px; } +.identity-head > .status { margin-left: auto; } +.avatar { width: 46px; height: 46px; display: grid; place-items: center; border-radius: 50%; background: #dce5dc; color: #244c3d; font-size: 12px; font-weight: 800; } +.two-column { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; align-items: start; } +.section-head { display: flex; align-items: center; justify-content: space-between; gap: 15px; } +.count { min-width: 26px; height: 26px; display: grid; place-items: center; border-radius: 50%; background: var(--navy); color: white; font-size: 11px; } +.assignment { display: flex; justify-content: space-between; gap: 15px; align-items: center; padding: 14px 0; border-top: 1px solid var(--line); } +.assignment p, .assignment small { margin: 2px 0 0; color: var(--muted); } +.form { display: grid; gap: 13px; } +.form label, .dialog label { display: grid; gap: 5px; color: var(--muted); font-size: 11px; font-weight: 700; } +.assignment-form .button { justify-self: start; } +.imported-section, .advanced { margin-top: 16px; } +.imported-row { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 18px; padding: 11px 0; border-top: 1px solid var(--line); } +.imported-row small { display: block; margin-bottom: 3px; color: var(--muted); } +details summary { cursor: pointer; color: var(--green); font-weight: 700; font-size: 12px; } +.advanced dl, .review-card dl { margin: 15px 0; } +.fact { display: grid; grid-template-columns: 150px minmax(0, 1fr); padding: 9px 0; border-bottom: 1px solid var(--line); } +.fact dt { color: var(--muted); } +.fact dd { margin: 0; overflow-wrap: anywhere; } +.review-card { max-width: 720px; } +.review-card form { margin-top: 18px; } +.alert { display: flex; gap: 12px; padding: 12px 14px; margin-bottom: 15px; border-left: 3px solid; } +.alert.error { border-color: var(--red); background: #f7e8e5; color: #742a24; } +.alert.warning { margin-top: 15px; border-color: #b88122; background: #f7f0df; color: #664915; } +.advanced-inline { margin-top: 20px; border-top: 1px solid var(--line); padding-top: 14px; } +pre { max-width: 100%; overflow-x: auto; padding: 13px; background: var(--navy); color: #e8eee9; font-size: 10px; white-space: pre; } +.creation ol { min-height: 95px; padding-left: 20px; color: var(--muted); } +.profile-layout { display: grid; grid-template-columns: minmax(0, 1.4fr) minmax(300px, .6fr); gap: 16px; align-items: start; } +.profile-list { display: grid; gap: 12px; } +.profile-card { display: grid; grid-template-columns: minmax(160px, .7fr) minmax(260px, 1.3fr) auto; gap: 20px; align-items: center; padding: 20px; border: 1px solid var(--line); background: var(--surface); } +.profile-card h2 { margin: 5px 0 0; } +.connection-state { color: var(--muted); font-size: 9px; font-weight: 800; letter-spacing: .08em; text-transform: uppercase; } +.connection-state::before { content: ""; display: inline-block; width: 6px; height: 6px; margin-right: 6px; border-radius: 50%; background: #929b96; } +.connection-state.connected { color: var(--green); } +.connection-state.connected::before { background: var(--green); } +.profile-card dl { display: grid; grid-template-columns: 55px minmax(0, 1fr); gap: 4px 10px; margin: 0; font-size: 10px; } +.profile-card dt { color: var(--muted); } +.profile-card dd { margin: 0; overflow-wrap: anywhere; font-family: ui-monospace, monospace; } +.profile-actions, .actions { display: flex; gap: 7px; flex-wrap: wrap; } +#profile-editor[hidden] { display: none; } +.dialog { width: min(520px, calc(100% - 28px)); border: 1px solid var(--line); border-radius: 4px; padding: 26px; background: var(--surface); color: var(--ink); } +.dialog::backdrop { background: rgba(10, 20, 31, .66); } +.dialog form, .dialog form > *, .dialog details { min-width: 0; } +.dialog form { display: grid; gap: 14px; } +.dialog pre { width: 100%; min-width: 0; } +.sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; border: 0; } + +@media (max-width: 800px) { + .topbar { height: auto; min-height: 58px; padding: 10px 16px; flex-wrap: wrap; gap: 10px 22px; } + .topbar nav { order: 3; width: 100%; height: 34px; } + .profile-switch { margin-left: auto; } + main { width: min(100% - 24px, 1160px); padding-top: 24px; } + .filters { flex-wrap: wrap; } + .filters .search-field { flex-basis: 100%; } + .filters label:not(.search-field) { flex: 1; } + .filters label:not(.search-field) input, .filters select { width: 100%; } + .two-column, .profile-layout, .profile-card { grid-template-columns: 1fr; } + .imported-row { grid-template-columns: 1fr; gap: 3px; } + .creation ol { min-height: 0; } +} + +@media (max-width: 430px) { + .page-head { align-items: flex-start; } + .page-head .button { flex-shrink: 0; } + .card { padding: 17px; } + .fact { grid-template-columns: 110px minmax(0, 1fr); } + .dialog { padding: 20px; } + .table-card { overflow: visible; } + .table-card table, .table-card tbody, .table-card tr, .table-card td { display: block; width: 100%; } + .table-card thead { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0, 0, 0, 0); } + .table-card tr { padding: 11px 14px; border-bottom: 1px solid var(--line); } + .table-card tbody tr:last-child { border-bottom: 0; } + .table-card td { display: grid; grid-template-columns: 92px minmax(0, 1fr); gap: 10px; padding: 6px 0; border: 0; } + .table-card td::before { content: attr(data-label); color: var(--muted); font-size: 10px; font-weight: 700; text-transform: uppercase; } + .permission-cell { min-width: 0; } +} diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth_ui/src/backend.rs index 890b9bb9..d79d57a6 100644 --- a/harmony_auth_ui/src/backend.rs +++ b/harmony_auth_ui/src/backend.rs @@ -1,9 +1,13 @@ +use std::collections::{BTreeMap, BTreeSet}; + use async_trait::async_trait; use harmony_auth::{ - AuthError, AuthService, Grant, GrantPlan, Identity, IdentityKind, JwtRole, JwtRolePlan, + Assignment, AssignmentPlan, AssignmentRequest, AuthError, AuthService, Identity, + IdentityAccess, IdentityKind, ImportedAccess, JwtRole, Scope, TenantSummary, }; use reqwest::{Client, Method, StatusCode}; use serde_json::{Map, Value, json}; +use uuid::Uuid; pub struct BackendAuth { client: Client, @@ -13,6 +17,11 @@ pub struct BackendAuth { openbao_token: String, } +struct RoleRecord { + role: JwtRole, + raw: Value, +} + impl BackendAuth { pub fn new( client: Client, @@ -30,6 +39,10 @@ impl BackendAuth { } } + pub fn zitadel_url(&self) -> &str { + &self.zitadel_url + } + pub async fn validate(&self) -> Result<(), String> { let zitadel = self .client @@ -44,14 +57,10 @@ impl BackendAuth { zitadel.status() )); } - let openbao = self - .client - .get(format!("{}/v1/auth/token/lookup-self", self.openbao_url)) - .header("X-Vault-Token", &self.openbao_token) - .send() + .openbao(Method::GET, "auth/token/lookup-self", None) .await - .map_err(|error| format!("OpenBao could not be reached: {error}"))?; + .map_err(|error| error.to_string())?; if !openbao.status().is_success() { return Err(format!("OpenBao rejected the token ({})", openbao.status())); } @@ -73,7 +82,7 @@ impl BackendAuth { } async fn all_identities(&self) -> Result, AuthError> { - let response = self + let body: Value = self .client .post(format!("{}/management/v1/users/_search", self.zitadel_url)) .bearer_auth(&self.zitadel_pat) @@ -82,8 +91,10 @@ impl BackendAuth { .await .map_err(backend)? .error_for_status() + .map_err(backend)? + .json() + .await .map_err(backend)?; - let body: Value = response.json().await.map_err(backend)?; Ok(body["result"] .as_array() .into_iter() @@ -92,10 +103,6 @@ impl BackendAuth { .collect()) } - async fn request_openbao(&self, path: &str) -> Result { - self.openbao(Method::GET, path, None).await - } - async fn openbao( &self, method: Method, @@ -112,10 +119,8 @@ impl BackendAuth { request.send().await.map_err(backend) } - async fn role_value(&self, subject_id: &str) -> Result, AuthError> { - let response = self - .request_openbao(&format!("auth/jwt/role/{subject_id}")) - .await?; + async fn json(&self, path: &str) -> Result, AuthError> { + let response = self.openbao(Method::GET, path, None).await?; if response.status() == StatusCode::NOT_FOUND { return Ok(None); } @@ -127,6 +132,207 @@ impl BackendAuth { .map_err(backend)?; Ok(Some(body["data"].clone())) } + + async fn role_names(&self) -> Result, AuthError> { + Ok(self + .json("auth/jwt/role?list=true") + .await? + .map_or_else(Vec::new, |data| strings(&data["keys"]))) + } + + async fn roles(&self) -> Result, AuthError> { + let mut roles = Vec::new(); + for name in self.role_names().await? { + if let Some(raw) = self.json(&format!("auth/jwt/role/{name}")).await? { + let subject_id = role_subject(&raw); + if !subject_id.is_empty() { + roles.push(RoleRecord { + role: JwtRole { + name, + subject_id, + bound_audiences: strings(&raw["bound_audiences"]), + token_policies: role_policies(&raw), + }, + raw, + }); + } + } + } + Ok(roles) + } + + async fn matching_roles(&self, subject_id: &str) -> Result, AuthError> { + Ok(self + .roles() + .await? + .into_iter() + .filter(|role| role.role.subject_id == subject_id) + .collect()) + } + + async fn policy(&self, name: &str) -> Result, AuthError> { + Ok(self + .json(&format!("sys/policies/acl/{name}")) + .await? + .and_then(|data| data["policy"].as_str().map(str::to_string))) + } + + async fn assignments(&self, subject_id: &str) -> Result, AuthError> { + let Some(data) = self + .json(&format!( + "harmony_auth/metadata/assignments/{subject_id}?list=true" + )) + .await? + else { + return Ok(vec![]); + }; + let mut assignments = Vec::new(); + for id in strings(&data["keys"]) { + if let Some(data) = self + .json(&format!("harmony_auth/data/assignments/{subject_id}/{id}")) + .await? + { + assignments.push(serde_json::from_value(data["data"].clone()).map_err(backend)?); + } + } + Ok(assignments) + } + + async fn ensure_intent_mount(&self) -> Result<(), AuthError> { + let mounts = self + .json("sys/mounts") + .await? + .ok_or_else(|| AuthError::Backend("OpenBao did not return its mounts".into()))?; + if mounts.get("harmony_auth/").is_none() { + self.openbao( + Method::POST, + "sys/mounts/harmony_auth", + Some(json!({ "type": "kv", "options": { "version": "2" } })), + ) + .await? + .error_for_status() + .map_err(backend)?; + } + Ok(()) + } + + async fn role_for_assignment(&self, subject_id: &str) -> Result { + let mut matching = self.matching_roles(subject_id).await?; + if matching.len() > 1 { + return Err(AuthError::Invalid( + "multiple JWT roles match this identity; resolve the ambiguity in OpenBao".into(), + )); + } + if let Some(role) = matching.pop() { + return Ok(role); + } + let audiences = self + .roles() + .await? + .into_iter() + .flat_map(|role| role.role.bound_audiences) + .collect::>(); + if audiences.len() != 1 { + return Err(AuthError::Invalid( + "cannot infer one JWT audience from existing OpenBao roles".into(), + )); + } + let audience = audiences.into_iter().next().unwrap(); + let raw = json!({ + "role_type": "jwt", + "user_claim": "sub", + "bound_claims": { "sub": subject_id }, + "bound_audiences": [audience], + "token_policies": [], + "token_ttl": "1h", + "token_max_ttl": "2h", + "token_type": "service", + "token_no_default_policy": true + }); + Ok(RoleRecord { + role: JwtRole { + name: format!("harmony-{subject_id}"), + subject_id: subject_id.into(), + bound_audiences: strings(&raw["bound_audiences"]), + token_policies: vec![], + }, + raw, + }) + } + + async fn write_role(&self, role: &RoleRecord, policies: &[String]) -> Result<(), AuthError> { + let mut raw = writable_role(&role.raw); + raw.insert("token_policies".into(), json!(policies)); + raw.remove("policies"); + self.openbao( + Method::POST, + &format!("auth/jwt/role/{}", role.role.name), + Some(Value::Object(raw)), + ) + .await? + .error_for_status() + .map_err(backend)?; + Ok(()) + } + + async fn access_for_identities( + &self, + identities: &[Identity], + ) -> Result, AuthError> { + let roles = self.roles().await?; + let policy_names = roles + .iter() + .flat_map(|role| role.role.token_policies.iter()) + .filter(|policy| policy.as_str() != "default") + .cloned() + .collect::>(); + let mut policy_details = BTreeMap::new(); + for policy in policy_names { + let body = self.policy(&policy).await?.unwrap_or_default(); + policy_details.insert(policy, (secret_paths(&body), policy_effect(&body))); + } + let mut access = Vec::with_capacity(identities.len()); + for identity in identities { + let assignments = self.assignments(&identity.subject_id).await?; + let managed = assignments + .iter() + .map(|assignment| assignment.policy_name.as_str()) + .collect::>(); + let matching = roles + .iter() + .filter(|role| role.role.subject_id == identity.subject_id) + .collect::>(); + let imported = matching + .iter() + .flat_map(|role| { + role.role + .token_policies + .iter() + .filter(|policy| { + policy.as_str() != "default" && !managed.contains(policy.as_str()) + }) + .map(|policy| ImportedAccess { + role_name: role.role.name.clone(), + policy_name: policy.clone(), + secret_paths: policy_details + .get(policy) + .map(|details| details.0.clone()) + .unwrap_or_default(), + effect: policy_details + .get(policy) + .map(|details| details.1.clone()) + .unwrap_or_else(|| "Custom OpenBao access".into()), + }) + }) + .collect(); + access.push(IdentityAccess { + assignments, + imported, + roles: matching.into_iter().map(|role| role.role.clone()).collect(), + }); + } + Ok(access) + } } #[async_trait] @@ -157,95 +363,244 @@ impl AuthService for BackendAuth { .ok_or(AuthError::IdentityNotFound) } - async fn grants_for(&self, subject_id: &str) -> Result, AuthError> { - let response = self - .request_openbao(&format!( - "harmony_auth/metadata/grants/by-principal/{subject_id}?list=true" - )) - .await?; - if response.status() == StatusCode::NOT_FOUND { - return Ok(vec![]); - } - let body: Value = response - .error_for_status() - .map_err(backend)? - .json() - .await - .map_err(backend)?; - let mut grants = Vec::new(); - for id in body["data"]["keys"] - .as_array() - .into_iter() - .flatten() - .filter_map(Value::as_str) - { - let response = self - .request_openbao(&format!("harmony_auth/data/grants/by-id/{id}")) - .await? - .error_for_status() - .map_err(backend)?; - let body: Value = response.json().await.map_err(backend)?; - grants.push(serde_json::from_value(body["data"]["data"].clone()).map_err(backend)?); - } - Ok(grants) - } - - async fn apply_grant(&self, _plan: GrantPlan, _created_by: &str) -> Result { - Err(AuthError::Backend( - "grant mutations require JWT role configuration and are not enabled yet".into(), - )) - } - - async fn jwt_role(&self, subject_id: &str) -> Result, AuthError> { - Ok(self - .role_value(subject_id) + async fn access(&self, subject_id: &str) -> Result { + let identity = self.identity(subject_id).await?; + self.access_for_identities(&[identity]) .await? - .map(|value| parse_role(subject_id, &value))) + .pop() + .ok_or(AuthError::IdentityNotFound) } - async fn acl_policies(&self) -> Result, AuthError> { - let body: Value = self - .request_openbao("sys/policies/acl?list=true") - .await? - .error_for_status() - .map_err(backend)? - .json() - .await - .map_err(backend)?; - Ok(strings(&body["data"]["keys"])) + async fn access_for(&self, identities: &[Identity]) -> Result, AuthError> { + self.access_for_identities(identities).await } - async fn apply_jwt_role(&self, plan: JwtRolePlan) -> Result { - let current = self.role_value(&plan.subject_id).await?; - let mut body = Map::new(); - if let Some(current) = current.as_ref().and_then(Value::as_object) { - for key in ROLE_FIELDS { - if let Some(value) = current.get(*key) { - body.insert((*key).into(), value.clone()); + async fn tenants(&self) -> Result, AuthError> { + let identities = self.all_identities().await?; + let mut tenants: BTreeMap, BTreeSet)> = + BTreeMap::new(); + let access = self.access_for_identities(&identities).await?; + for (identity, access) in identities.into_iter().zip(access) { + for scope in access + .assignments + .iter() + .map(|assignment| assignment.scope.clone()) + .chain(access.imported.iter().flat_map(|access| { + access.secret_paths.iter().filter_map(|path| { + let mut parts = path.split('/'); + Scope::new(parts.next()?, parts.next()).ok() + }) + })) + { + let entry = tenants + .entry(scope.path()) + .or_insert_with(|| (scope, BTreeSet::new(), BTreeSet::new())); + match identity.kind { + IdentityKind::Human => &mut entry.1, + IdentityKind::Service => &mut entry.2, } + .insert(identity.subject_id.clone()); } } - body.insert("role_type".into(), Value::String("jwt".into())); - body.insert("user_claim".into(), Value::String("sub".into())); - body.insert( - "bound_subject".into(), - Value::String(plan.subject_id.clone()), - ); - body.insert("bound_audiences".into(), json!(plan.bound_audiences)); - body.insert("token_policies".into(), json!(plan.token_policies)); - body.remove("policies"); + Ok(tenants + .into_values() + .map(|(scope, humans, services)| TenantSummary { + scope, + humans: humans.len(), + services: services.len(), + }) + .collect()) + } + + async fn plan_assignment( + &self, + request: AssignmentRequest, + ) -> Result { + let identity = self.identity(&request.subject_id).await?; + let mut plan = harmony_auth::plan_assignment(&identity, request, "secret")?; + if let Some(existing) = self + .assignments(&identity.subject_id) + .await? + .into_iter() + .find(|assignment| { + assignment.permission == plan.assignment.permission + && assignment.scope == plan.assignment.scope + }) + { + plan.assignment.id = existing.id; + plan.assignment.created_at = existing.created_at; + } + self.role_for_assignment(&identity.subject_id).await?; + Ok(plan) + } + + async fn apply_assignment(&self, plan: AssignmentPlan) -> Result { + let role = self + .role_for_assignment(&plan.assignment.subject_id) + .await?; + self.ensure_intent_mount().await?; self.openbao( - Method::POST, - &format!("auth/jwt/role/{}", plan.subject_id), - Some(Value::Object(body)), + Method::PUT, + &format!("sys/policies/acl/{}", plan.assignment.policy_name), + Some(json!({ "policy": plan.policy })), ) .await? .error_for_status() .map_err(backend)?; - self.jwt_role(&plan.subject_id) - .await? - .ok_or_else(|| AuthError::Backend("OpenBao did not return the saved JWT role".into())) + let mut policies = role.role.token_policies.clone(); + if !policies.contains(&plan.assignment.policy_name) { + policies.push(plan.assignment.policy_name.clone()); + policies.sort_unstable(); + } + self.write_role(&role, &policies).await?; + self.openbao( + Method::POST, + &format!( + "harmony_auth/data/assignments/{}/{}", + plan.assignment.subject_id, plan.assignment.id + ), + Some(json!({ "data": plan.assignment })), + ) + .await? + .error_for_status() + .map_err(backend)?; + Ok(plan.assignment) } + + async fn remove_assignment( + &self, + subject_id: &str, + assignment_id: Uuid, + ) -> Result<(), AuthError> { + let assignment = self + .assignments(subject_id) + .await? + .into_iter() + .find(|assignment| assignment.id == assignment_id) + .ok_or_else(|| AuthError::Invalid("assignment does not exist".into()))?; + let mut roles = self.matching_roles(subject_id).await?; + if roles.len() != 1 { + return Err(AuthError::Invalid( + "expected exactly one JWT role for this identity".into(), + )); + } + let role = roles.pop().unwrap(); + let policies = role + .role + .token_policies + .iter() + .filter(|policy| *policy != &assignment.policy_name) + .cloned() + .collect::>(); + self.write_role(&role, &policies).await?; + self.openbao( + Method::DELETE, + &format!("harmony_auth/data/assignments/{subject_id}/{assignment_id}"), + None, + ) + .await? + .error_for_status() + .map_err(backend)?; + Ok(()) + } +} + +fn parse_identity(value: &Value) -> Option { + let human = value.get("human"); + let machine = value.get("machine"); + Some(Identity { + subject_id: value.get("id")?.as_str()?.into(), + kind: if human.is_some() { + IdentityKind::Human + } else { + IdentityKind::Service + }, + display_name: human + .and_then(|human| human.pointer("/profile/displayName")) + .or_else(|| machine.and_then(|machine| machine.get("name"))) + .and_then(Value::as_str) + .or_else(|| value.get("preferredLoginName").and_then(Value::as_str))? + .into(), + login_name: value + .get("preferredLoginName") + .or_else(|| value.get("userName")) + .and_then(Value::as_str) + .unwrap_or_default() + .into(), + email: human + .and_then(|human| human.pointer("/email/email")) + .and_then(Value::as_str) + .map(str::to_string), + active: value.get("state").is_none_or(|state| { + state.as_i64() == Some(1) || state.as_str() == Some("USER_STATE_ACTIVE") + }), + }) +} + +fn role_subject(role: &Value) -> String { + role["bound_subject"] + .as_str() + .filter(|subject| !subject.is_empty() && *subject != "n/a") + .or_else(|| role.pointer("/bound_claims/sub").and_then(Value::as_str)) + .unwrap_or_default() + .into() +} + +fn role_policies(role: &Value) -> Vec { + let policies = strings(&role["token_policies"]); + if policies.is_empty() { + strings(&role["policies"]) + } else { + policies + } +} + +fn strings(value: &Value) -> Vec { + value + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() +} + +fn secret_paths(policy: &str) -> Vec { + policy + .lines() + .filter_map(|line| line.split_once("path \"").map(|(_, rest)| rest)) + .filter_map(|rest| rest.split_once('"').map(|(path, _)| path)) + .filter_map(|path| path.split_once("/data/").map(|(_, logical)| logical)) + .map(|path| path.trim_end_matches("/*").to_string()) + .collect::>() + .into_iter() + .collect() +} + +fn policy_effect(policy: &str) -> String { + if policy.contains("\"create\"") + || policy.contains("\"update\"") + || policy.contains("\"patch\"") + || policy.contains("\"delete\"") + { + "Read and manage secrets".into() + } else if policy.contains("\"read\"") { + "Read secrets".into() + } else { + "Custom OpenBao access".into() + } +} + +fn writable_role(role: &Value) -> Map { + let mut writable = Map::new(); + if let Some(role) = role.as_object() { + for key in ROLE_FIELDS { + if let Some(value) = role.get(*key) { + writable.insert((*key).into(), value.clone()); + } + } + } + writable } const ROLE_FIELDS: &[&str] = &[ @@ -282,215 +637,24 @@ const ROLE_FIELDS: &[&str] = &[ "verbose_oidc_logging", ]; -fn parse_role(name: &str, value: &Value) -> JwtRole { - let token_policies = strings(&value["token_policies"]); - JwtRole { - name: name.into(), - bound_subject: value["bound_subject"].as_str().unwrap_or_default().into(), - bound_audiences: strings(&value["bound_audiences"]), - token_policies: if token_policies.is_empty() { - strings(&value["policies"]) - } else { - token_policies - }, - role_type: value["role_type"].as_str().unwrap_or("jwt").into(), - user_claim: value["user_claim"].as_str().unwrap_or_default().into(), - } -} - -fn strings(value: &Value) -> Vec { - value - .as_array() - .into_iter() - .flatten() - .filter_map(Value::as_str) - .map(str::to_string) - .collect() -} - -fn parse_identity(value: &Value) -> Option { - let human = value.get("human"); - let machine = value.get("machine"); - let display_name = human - .and_then(|human| human.pointer("/profile/displayName")) - .or_else(|| machine.and_then(|machine| machine.get("name"))) - .and_then(Value::as_str) - .or_else(|| value.get("preferredLoginName").and_then(Value::as_str))? - .to_string(); - Some(Identity { - subject_id: value.get("id")?.as_str()?.into(), - kind: if human.is_some() { - IdentityKind::Human - } else { - IdentityKind::Machine - }, - display_name, - login_name: value - .get("preferredLoginName") - .or_else(|| value.get("userName")) - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - email: human - .and_then(|human| human.pointer("/email/email")) - .and_then(Value::as_str) - .map(str::to_string), - active: value.get("state").is_none_or(|state| { - state.as_i64() == Some(1) || state.as_str() == Some("USER_STATE_ACTIVE") - }), - }) -} - fn backend(error: impl std::fmt::Display) -> AuthError { AuthError::Backend(error.to_string()) } #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; - - use axum::{ - Json, Router, - extract::State, - http::{HeaderMap, StatusCode}, - routing::{get, post}, - }; - use serde_json::json; - use super::*; - #[tokio::test] - async fn validates_credentials_and_maps_zitadel_identities() { - async fn zitadel(headers: HeaderMap) -> StatusCode { - if headers - .get("authorization") - .and_then(|value| value.to_str().ok()) - == Some("Bearer zitadel-pat") - { - StatusCode::OK - } else { - StatusCode::UNAUTHORIZED - } - } - async fn openbao(headers: HeaderMap) -> StatusCode { - if headers - .get("x-vault-token") - .and_then(|value| value.to_str().ok()) - == Some("openbao-token") - { - StatusCode::OK - } else { - StatusCode::FORBIDDEN - } - } - async fn users() -> Json { - Json(json!({ - "result": [ - { - "id": "human-subject", - "state": 1, - "userName": "maya", - "preferredLoginName": "maya@example.com", - "human": { - "profile": { "displayName": "Maya Chen" }, - "email": { "email": "maya@example.com" } - } - }, - { - "id": "machine-subject", - "state": "USER_STATE_ACTIVE", - "userName": "deployer", - "machine": { "name": "Production deployer" } - } - ] - })) - } - - let app = Router::new() - .route("/management/v1/orgs/me", get(zitadel)) - .route("/management/v1/users/_search", post(users)) - .route("/v1/auth/token/lookup-self", get(openbao)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - - let backend = BackendAuth::new( - Client::new(), - base_url.clone(), - "zitadel-pat".into(), - base_url, - "openbao-token".into(), + #[test] + fn discovers_subject_from_bound_claims_and_policy_paths() { + let role = json!({ "bound_subject": "n/a", "bound_claims": { "sub": "123" } }); + assert_eq!(role_subject(&role), "123"); + assert_eq!( + secret_paths( + "path \"secret/data/devsights/folk/*\" { capabilities = [\"read\"] }\n\ + path \"secret/metadata/devsights/folk/*\" { capabilities = [\"list\"] }" + ), + ["devsights/folk"] ); - - backend.validate().await.unwrap(); - let identities = backend.identities(None).await.unwrap(); - assert_eq!(identities.len(), 2); - assert_eq!(identities[0].display_name, "Maya Chen"); - assert_eq!(identities[0].email.as_deref(), Some("maya@example.com")); - assert_eq!(identities[1].kind, IdentityKind::Machine); - } - - #[tokio::test] - async fn role_update_preserves_existing_login_settings() { - async fn read_role(State(role): State>>) -> Json { - Json(json!({ "data": role.lock().unwrap().clone() })) - } - async fn write_role( - State(role): State>>, - Json(body): Json, - ) -> StatusCode { - *role.lock().unwrap() = body; - StatusCode::NO_CONTENT - } - - let role = Arc::new(Mutex::new(json!({ - "role_type": "jwt", - "user_claim": "sub", - "bound_subject": "subject", - "bound_audiences": ["old-audience"], - "token_policies": ["old-policy"], - "groups_claim": "groups", - "bound_claims": { "department": "platform" }, - "claim_mappings": { "email": "email" }, - "clock_skew_leeway": 30, - "token_bound_cidrs": ["10.0.0.0/8"], - "token_explicit_max_ttl": 900, - "token_ttl": 300, - "token_type": "batch" - }))); - let app = Router::new() - .route("/v1/auth/jwt/role/subject", get(read_role).post(write_role)) - .with_state(role.clone()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let backend = BackendAuth::new( - Client::new(), - "http://unused".into(), - "unused".into(), - base_url, - "openbao-token".into(), - ); - - let saved = backend - .apply_jwt_role(JwtRolePlan { - subject_id: "subject".into(), - bound_audiences: vec!["new-audience".into()], - token_policies: vec!["new-policy".into()], - creates_role: false, - }) - .await - .unwrap(); - - assert_eq!(saved.token_policies, ["new-policy"]); - let written = role.lock().unwrap(); - assert_eq!(written["groups_claim"], "groups"); - assert_eq!(written["token_ttl"], 300); - assert_eq!(written["token_type"], "batch"); - assert_eq!(written["bound_claims"]["department"], "platform"); - assert_eq!(written["claim_mappings"]["email"], "email"); - assert_eq!(written["clock_skew_leeway"], 30); - assert_eq!(written["token_bound_cidrs"][0], "10.0.0.0/8"); - assert_eq!(written["token_explicit_max_ttl"], 900); } } diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index ca40e1b8..4285ac65 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -15,10 +15,11 @@ use axum::{ }; use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite}; use clap::Parser; -use harmony_auth::{AuthError, AuthService, GrantRequest, Identity, JwtRolePlan}; +use harmony_auth::{AssignmentRequest, AuthError, AuthService, IdentityKind, Permission}; use serde::Deserialize; use tokio::sync::RwLock; use tower_http::set_header::SetResponseHeaderLayer; +use uuid::Uuid; #[derive(Parser)] struct Args { @@ -40,8 +41,10 @@ struct ConnectedProfile { } #[derive(Default, Deserialize)] -struct Search { +struct DashboardQuery { q: Option, + kind: Option, + tenant: Option, } #[derive(Default, Deserialize)] @@ -65,9 +68,11 @@ struct ProfileSelection { } #[derive(Deserialize)] -struct JwtRoleForm { - audiences: String, - policies: String, +struct AssignmentForm { + permission: Permission, + tenant: String, + #[serde(default)] + project: String, } const SESSION_COOKIE: &str = "harmony_auth_session"; @@ -100,23 +105,33 @@ fn router(state: AppState) -> Router { .route("/profiles/switch", post(switch_profile)) .route("/profiles/disconnect", post(disconnect_profile)) .route("/profiles/connected", get(connected_profiles)) - .route("/dashboard", get(overview)) - .route("/identities", get(identities)) + .route("/dashboard", get(dashboard)) + .route("/identities", get(dashboard)) + .route("/identities/new", get(new_identity)) .route("/identities/{subject_id}", get(identity)) - .route("/grants/plan", post(grant_plan)) - .route("/grants", post(apply_grant)) + .route("/tenants", get(tenants)) .route( - "/identities/{subject_id}/roles/review", - get(role_review), + "/identities/{subject_id}/assignments/review", + get(assignment_review), + ) + .route( + "/identities/{subject_id}/assignments", + post(apply_assignment), + ) + .route( + "/identities/{subject_id}/assignments/{assignment_id}/remove", + post(remove_assignment), + ) + .route( + "/identities/{subject_id}/assignments/{assignment_id}/remove/review", + get(remove_assignment_review), ) - .route("/identities/{subject_id}/roles", post(apply_role)) .route("/static/app.css", get(css)) - .route("/static/htmx.min.js", get(htmx)) .route("/static/profiles.js", get(profiles_js)) .route("/favicon.ico", get(|| async { StatusCode::NO_CONTENT })) .layer(SetResponseHeaderLayer::overriding( HeaderName::from_static("content-security-policy"), - HeaderValue::from_static("default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; object-src 'none'"), + HeaderValue::from_static("default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; object-src 'none'"), )) .layer(SetResponseHeaderLayer::overriding( HeaderName::from_static("x-content-type-options"), @@ -211,16 +226,8 @@ async fn connect_profile( let session_id = jar .get(SESSION_COOKIE) .map(|cookie| cookie.value().to_string()) - .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + .unwrap_or_else(|| Uuid::new_v4().to_string()); let jar = jar.add(session_cookie(SESSION_COOKIE, session_id.clone())); - if form.profile_id.trim().is_empty() || form.name.trim().is_empty() { - state - .connection_errors - .write() - .await - .insert(session_id, "Profile name is required".into()); - return (jar, Redirect::to("/?connection_error=1")); - } let existing = state .sessions .read() @@ -242,14 +249,15 @@ async fn connect_profile( form.openbao_token }, )) + } else if form.zitadel_pat.is_empty() || form.openbao_token.is_empty() { + return connection_error( + &state, + jar, + session_id, + "Both credentials are required for a new connection".into(), + ) + .await; } else { - if form.zitadel_pat.is_empty() || form.openbao_token.is_empty() { - state.connection_errors.write().await.insert( - session_id, - "Both credentials are required for a new connection".into(), - ); - return (jar, Redirect::to("/?connection_error=1")); - } Arc::new(backend::BackendAuth::new( state.client.clone(), form.zitadel_url, @@ -259,18 +267,13 @@ async fn connect_profile( )) }; if let Err(error) = backend.validate().await { - state - .connection_errors - .write() - .await - .insert(session_id, error); - return (jar, Redirect::to("/?connection_error=1")); + return connection_error(&state, jar, session_id, error).await; } state .sessions .write() .await - .entry(session_id.clone()) + .entry(session_id) .or_default() .insert( form.profile_id.clone(), @@ -279,12 +282,28 @@ async fn connect_profile( auth: backend, }, ); + tracing::info!(profile = %form.profile_id, "connected infrastructure profile"); ( jar.add(session_cookie(PROFILE_COOKIE, form.profile_id)), Redirect::to("/dashboard"), ) } +async fn connection_error( + state: &AppState, + jar: CookieJar, + session_id: String, + error: String, +) -> (CookieJar, Redirect) { + tracing::warn!(%error, "profile connection failed"); + state + .connection_errors + .write() + .await + .insert(session_id, error); + (jar, Redirect::to("/?connection_error=1")) +} + async fn switch_profile( State(state): State, jar: CookieJar, @@ -294,14 +313,15 @@ async fn switch_profile( .get(SESSION_COOKIE) .ok_or(AppError::Disconnected)? .value(); - let sessions = state.sessions.read().await; - if !sessions + if !state + .sessions + .read() + .await .get(session_id) .is_some_and(|profiles| profiles.contains_key(&selection.profile_id)) { return Err(AppError::Disconnected); } - drop(sessions); Ok(( jar.add(session_cookie(PROFILE_COOKIE, selection.profile_id)), Redirect::to("/dashboard"), @@ -336,18 +356,6 @@ async fn disconnect_profile( (jar, Redirect::to("/")) } -fn session_cookie(name: &'static str, value: String) -> Cookie<'static> { - Cookie::build((name, value)) - .http_only(true) - .same_site(SameSite::Lax) - .path("/") - .build() -} - -fn removal_cookie(name: &'static str) -> Cookie<'static> { - Cookie::build(name).path("/").build() -} - async fn active_profile(state: &AppState, jar: &CookieJar) -> Result { let session_id = jar .get(SESSION_COOKIE) @@ -367,109 +375,46 @@ async fn active_profile(state: &AppState, jar: &CookieJar) -> Result, jar: CookieJar, - Form(request): Form, + Query(query): Query, ) -> Result { let profile = active_profile(&state, &jar).await?; - let auth: &dyn AuthService = profile.auth.as_ref(); - let identity = auth.identity(&request.principal_subject_id).await?; - let grants = auth.grants_for(&identity.subject_id).await?; - let plan = harmony_auth::plan_grant(&identity, &grants, request)?; - Ok(views::grant_review(&profile.view(), &identity, &plan)) -} - -async fn apply_grant( - State(state): State, - jar: CookieJar, - Form(request): Form, -) -> Result { - let profile = active_profile(&state, &jar).await?; - let auth: &dyn AuthService = profile.auth.as_ref(); - let identity = auth.identity(&request.principal_subject_id).await?; - let grants = auth.grants_for(&identity.subject_id).await?; - let plan = harmony_auth::plan_grant(&identity, &grants, request)?; - auth.apply_grant(plan, "profile-operator").await?; - Ok(Redirect::to(&format!( - "/identities/{}", - identity.subject_id - ))) -} - -async fn role_review( - State(state): State, - jar: CookieJar, - Path(subject_id): Path, - Query(form): Query, -) -> Result { - let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&subject_id).await?; - let plan = requested_role_plan(&profile.auth, &identity, &form).await?; - Ok(views::role_review(&profile.view(), &identity, &plan)) -} - -async fn apply_role( - State(state): State, - jar: CookieJar, - Path(subject_id): Path, - Form(form): Form, -) -> Result { - let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&subject_id).await?; - let plan = requested_role_plan(&profile.auth, &identity, &form).await?; - profile.auth.apply_jwt_role(plan).await?; - Ok(Redirect::to(&format!( - "/identities/{}", - identity.subject_id - ))) -} - -async fn requested_role_plan( - auth: &backend::BackendAuth, - identity: &Identity, - form: &JwtRoleForm, -) -> Result { - let current = auth.jwt_role(&identity.subject_id).await?; - let plan = - harmony_auth::plan_jwt_role(identity, current.as_ref(), &form.audiences, &form.policies)?; - let available = auth.acl_policies().await?; - let missing = plan - .token_policies - .iter() - .filter(|policy| !available.contains(policy)) - .cloned() - .collect::>(); - if !missing.is_empty() { - return Err(AuthError::InvalidGrant(format!( - "OpenBao ACL policies do not exist: {}", - missing.join(", ") - ))); + let mut rows = Vec::new(); + let identities = profile.auth.identities(query.q.as_deref()).await?; + let access = profile.auth.access_for(&identities).await?; + for (identity, access) in identities.into_iter().zip(access) { + let kind_matches = query.kind.as_deref().is_none_or(|kind| { + kind.is_empty() + || matches!( + (&identity.kind, kind), + (IdentityKind::Human, "human") | (IdentityKind::Service, "service") + ) + }); + let tenant_matches = query.tenant.as_deref().is_none_or(|tenant| { + tenant.is_empty() + || access + .assignments + .iter() + .any(|assignment| assignment.scope.tenant == tenant) + || access.imported.iter().any(|imported| { + imported + .secret_paths + .iter() + .any(|path| path.starts_with(tenant)) + }) + }); + if kind_matches && tenant_matches { + rows.push((identity, access)); + } } - Ok(plan) -} - -async fn overview(State(state): State, jar: CookieJar) -> Result { - let profile = active_profile(&state, &jar).await?; - let identities = profile.auth.identities(None).await?; - let mut grants = 0; - for identity in &identities { - grants += profile.auth.grants_for(&identity.subject_id).await?.len(); - } - Ok(views::overview(&profile.view(), &identities, grants)) -} - -async fn identities( - State(state): State, - jar: CookieJar, - Query(search): Query, -) -> Result { - let profile = active_profile(&state, &jar).await?; - let identities = profile.auth.identities(search.q.as_deref()).await?; - Ok(views::identities( + Ok(views::dashboard( &profile.view(), - &identities, - search.q.as_deref(), + &rows, + query.q.as_deref(), + query.kind.as_deref(), + query.tenant.as_deref(), )) } @@ -480,44 +425,129 @@ async fn identity( ) -> Result { let profile = active_profile(&state, &jar).await?; let identity = profile.auth.identity(&subject_id).await?; - let grants = profile.auth.grants_for(&subject_id).await?; - let role = profile.auth.jwt_role(&subject_id).await?; - let policies = profile.auth.acl_policies().await?; - let policy = harmony_auth::render_policy(&grants); - Ok(views::identity( + let access = profile.auth.access(&subject_id).await?; + Ok(views::identity(&profile.view(), &identity, &access)) +} + +async fn assignment_review( + State(state): State, + jar: CookieJar, + Path(subject_id): Path, + Query(form): Query, +) -> Result { + let profile = active_profile(&state, &jar).await?; + let plan = profile + .auth + .plan_assignment(AssignmentRequest { + subject_id, + permission: form.permission, + tenant: form.tenant, + project: form.project, + }) + .await?; + Ok(views::assignment_review(&profile.view(), &plan)) +} + +async fn apply_assignment( + State(state): State, + jar: CookieJar, + Path(subject_id): Path, + Form(form): Form, +) -> Result { + let profile = active_profile(&state, &jar).await?; + let plan = profile + .auth + .plan_assignment(AssignmentRequest { + subject_id: subject_id.clone(), + permission: form.permission, + tenant: form.tenant, + project: form.project, + }) + .await?; + let assignment = profile.auth.apply_assignment(plan).await?; + tracing::info!( + %subject_id, + permission = assignment.permission.label(), + scope = %assignment.scope.path(), + "applied Harmony permission assignment" + ); + Ok(Redirect::to(&format!("/identities/{subject_id}"))) +} + +async fn remove_assignment( + State(state): State, + jar: CookieJar, + Path((subject_id, assignment_id)): Path<(String, Uuid)>, +) -> Result { + let profile = active_profile(&state, &jar).await?; + profile + .auth + .remove_assignment(&subject_id, assignment_id) + .await?; + tracing::info!(%subject_id, %assignment_id, "removed Harmony permission assignment"); + Ok(Redirect::to(&format!("/identities/{subject_id}"))) +} + +async fn remove_assignment_review( + State(state): State, + jar: CookieJar, + Path((subject_id, assignment_id)): Path<(String, Uuid)>, +) -> Result { + let profile = active_profile(&state, &jar).await?; + let identity = profile.auth.identity(&subject_id).await?; + let assignment = profile + .auth + .access(&subject_id) + .await? + .assignments + .into_iter() + .find(|assignment| assignment.id == assignment_id) + .ok_or_else(|| AuthError::Invalid("assignment does not exist".into()))?; + Ok(views::remove_assignment_review( &profile.view(), &identity, - &grants, - role.as_ref(), - &policies, - &policy, - true, + &assignment, )) } +async fn tenants(State(state): State, jar: CookieJar) -> Result { + let profile = active_profile(&state, &jar).await?; + Ok(views::tenants( + &profile.view(), + &profile.auth.tenants().await?, + )) +} + +async fn new_identity( + State(state): State, + jar: CookieJar, +) -> Result { + let profile = active_profile(&state, &jar).await?; + Ok(views::new_identity(&profile.view())) +} + +fn session_cookie(name: &'static str, value: String) -> Cookie<'static> { + Cookie::build((name, value)) + .http_only(true) + .same_site(SameSite::Lax) + .path("/") + .build() +} + +fn removal_cookie(name: &'static str) -> Cookie<'static> { + Cookie::build(name).path("/").build() +} + async fn css() -> impl IntoResponse { ( - [(axum::http::header::CONTENT_TYPE, "text/css; charset=utf-8")], - format!("{}{}", include_str!("app.css"), include_str!("a11y.css")), - ) -} - -async fn htmx() -> impl IntoResponse { - ( - [( - axum::http::header::CONTENT_TYPE, - "text/javascript; charset=utf-8", - )], - include_bytes!("../../fleet/harmony-fleet-operator/vendor/htmx.min.js").as_slice(), + [(header::CONTENT_TYPE, "text/css; charset=utf-8")], + include_str!("app.css"), ) } async fn profiles_js() -> impl IntoResponse { ( - [( - axum::http::header::CONTENT_TYPE, - "text/javascript; charset=utf-8", - )], + [(header::CONTENT_TYPE, "text/javascript; charset=utf-8")], include_str!("profiles.js"), ) } @@ -535,27 +565,33 @@ impl From for AppError { impl IntoResponse for AppError { fn into_response(self) -> Response { - if matches!(self, Self::Disconnected) { - return Redirect::to("/").into_response(); + match self { + Self::Disconnected => Redirect::to("/").into_response(), + Self::Auth(error) => { + let status = match &error { + AuthError::IdentityNotFound => StatusCode::NOT_FOUND, + AuthError::Invalid(_) => StatusCode::UNPROCESSABLE_ENTITY, + AuthError::Backend(_) => StatusCode::BAD_GATEWAY, + }; + match &error { + AuthError::Backend(_) => tracing::error!(%error, "backend operation failed"), + _ => tracing::warn!(%error, "request rejected"), + } + let detail = match &error { + AuthError::Invalid(message) => Some(message.as_str()), + _ => None, + }; + (status, views::error(status, detail)).into_response() + } } - let Self::Auth(error) = self else { - unreachable!() - }; - let status = match &error { - AuthError::IdentityNotFound => axum::http::StatusCode::NOT_FOUND, - AuthError::InvalidGrant(_) => axum::http::StatusCode::UNPROCESSABLE_ENTITY, - AuthError::Backend(_) => axum::http::StatusCode::BAD_GATEWAY, - }; - let detail = match &error { - AuthError::InvalidGrant(message) => Some(message.as_str()), - _ => None, - }; - (status, views::error(status, detail)).into_response() } } impl ConnectedProfile { fn view(&self) -> views::Profile<'_> { - views::Profile { name: &self.name } + views::Profile { + name: &self.name, + zitadel_url: self.auth.zitadel_url(), + } } } diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index ccea4f9f..c23f1a98 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -1,71 +1,64 @@ use axum::http::StatusCode; use harmony_auth::{ - AccessLevel, Grant, GrantPlan, Identity, IdentityKind, JwtRole, JwtRolePlan, Selector, + AssignmentPlan, Identity, IdentityAccess, IdentityKind, Permission, TenantSummary, }; use maud::{DOCTYPE, Markup, html}; pub struct Profile<'a> { pub name: &'a str, + pub zitadel_url: &'a str, } pub fn profiles(connection_error: Option<&str>) -> Markup { - layout( + shell( "Profiles", - "", None, + "", html! { - section class="hero profile-hero" { - div { - p class="eyebrow" { "INFRASTRUCTURE PROFILES" } - h1 { "Choose where you want to work." } - p class="lede" { "Profiles remember backend locations in this browser. Credentials stay in server memory only while connected." } - } + div class="page-head" { + div { h1 { "Profiles" } p { "Choose a Zitadel and OpenBao environment." } } button class="button primary" id="show-profile-form" type="button" { "Add profile" } } @if let Some(error) = connection_error { - div class="connection-error" role="alert" { strong { "Connection failed" } p { (error) } } + div class="alert error" role="alert" { strong { "Connection failed" } span { (error) } } } - section class="profile-workspace" { + div class="profile-layout" { div id="profile-list" class="profile-list" aria-live="polite" {} - form id="profile-editor" class="panel profile-editor" hidden { - div class="section-title" { div { h2 { "New profile" } p { "Only this non-secret metadata is saved in your browser." } } } - label { "Profile name" input name="name" required placeholder="NationTech production"; } + form id="profile-editor" class="card form" hidden { + h2 { "Profile details" } + label { "Name" input name="name" required placeholder="NationTech production"; } label { "Zitadel URL" input name="zitadel_url" type="url" required placeholder="https://sso.example.com"; } label { "OpenBao URL" input name="openbao_url" type="url" required placeholder="https://secrets.example.com"; } - div class="form-actions" { + div class="actions" { button class="button primary" type="submit" { "Save and connect" } button class="button" id="cancel-profile" type="button" { "Cancel" } } } } - dialog id="connect-dialog" class="connect-dialog" { + dialog id="connect-dialog" class="dialog" { form action="/profiles/connect" method="post" autocomplete="off" { input type="hidden" name="profile_id"; input type="hidden" name="name"; input type="hidden" name="zitadel_url"; input type="hidden" name="openbao_url"; - p class="eyebrow" { "CONNECT BACKENDS" } h2 id="connect-title" { "Connect profile" } - p { "Credentials are held in this server process and disappear when it restarts." } - p id="credential-instruction" {} + p id="credential-instruction" class="muted" {} label { "Zitadel service-account PAT" input name="zitadel_pat" type="password" required autocomplete="off"; } - small { "PATs belong to Zitadel service accounts. The account needs the administrator permissions you intend to use." } label { "OpenBao token" input name="openbao_token" type="password" required autocomplete="off"; } - small class="root-warning" { "The initial root token works for bootstrap. Prefer a temporary, non-renewable root-policy token for this session." } - details class="token-help" { - summary { "Create a temporary OpenBao administrator token" } - p { "Run this while authenticated as root, then give the resulting one-hour token to Harmony:" } + details { + summary { "Temporary OpenBao administrator token" } + p class="muted" { "Run while authenticated as root:" } pre tabindex="0" { code { "bao token create -policy=root -ttl=1h -explicit-max-ttl=1h -renewable=false -display-name=temporary-admin" } } } details { - summary { "How do I create a Zitadel PAT?" } + summary { "Create a Zitadel PAT" } ol { - li { "Sign in to the Zitadel Console with your administrator account." } - li { "Create a service account and grant its required administrator role." } - li { "Create an expiring Personal Access Token on that service account." } + li { "Create a Zitadel service account." } + li { "Grant its required administrator role." } + li { "Create an expiring Personal Access Token." } } } - div class="form-actions" { + div class="actions" { button class="button primary" type="submit" { "Validate and connect" } button class="button" id="cancel-connect" type="button" { "Cancel" } } @@ -76,92 +69,130 @@ pub fn profiles(connection_error: Option<&str>) -> Markup { ) } -pub fn overview(profile: &Profile<'_>, identities: &[Identity], grants: usize) -> Markup { - let active = identities.iter().filter(|identity| identity.active).count(); - layout( - "Overview", - "/dashboard", - Some(profile), - html! { - section class="hero" { - div { - p class="eyebrow" { "AUTHORIZATION CONTROL PLANE" } - h1 { "Understand direct secret access, identity by identity." } - p class="lede" { "Harmony makes direct access intent inspectable without copying identities or secret values." } - } - a class="button primary" href="/identities" { "Review identities" } - } - section class="metrics" aria-label="System summary" { - (metric("Identities", identities.len(), "Discovered from Zitadel")) - (metric("Active", active, "Eligible for new grants")) - (metric("Direct grants", grants, "Canonical access intent")) - article class="metric healthy" { - span { "Current mode" } - strong { "Inspect" } - small { "Mutations await JWT role configuration" } - } - } - section class="panel split" { - div { - p class="eyebrow" { "QUICK ANSWERS" } - h2 { "Start with an identity" } - p { "Find a person or machine, then inspect every secret path they can access and why." } - a class="text-link" href="/identities" { "Browse all identities" } - } - div class="rule-card" { - span class="rule-number" { "01" } - p { "Zitadel subject IDs remain the stable identity key." } - span class="rule-number" { "02" } - p { "Secret values stay hidden. This console manages access, not data." } - } - } - }, - ) -} - -pub fn identities(profile: &Profile<'_>, identities: &[Identity], search: Option<&str>) -> Markup { - layout( +pub fn dashboard( + profile: &Profile<'_>, + rows: &[(Identity, IdentityAccess)], + search: Option<&str>, + kind: Option<&str>, + tenant: Option<&str>, +) -> Markup { + shell( "Identities", - "/identities", Some(profile), + "/dashboard", html! { - (page_heading("Identities", "People and machines discovered from Zitadel. Email and names are display data; subject ID is authoritative.")) - section class="panel" { - form class="search" action="/identities" method="get" { - label for="identity-search" { "Search identities" } - div class="search-row" { - input id="identity-search" name="q" type="search" value=[search] placeholder="Name, login, or email"; - button class="button" type="submit" { "Search" } - } - } - div class="list-summary" { (identities.len()) " matching identities" } - div class="identity-list" { - @for identity in identities { - a class="identity-row" href=(format!("/identities/{}", identity.subject_id)) { - (avatar(identity)) - div class="identity-main" { - strong { (&identity.display_name) } - span { (&identity.login_name) } - @if let Some(email) = identity.email.as_deref() { - span class="identity-email" { (email) } - } - span class="mobile-meta" { - (if identity.kind == IdentityKind::Human { "Human" } else { "Machine" }) - " · " - (if identity.active { "Active" } else { "Suspended" }) + div class="page-head" { + div { h1 { "Identities" } p { "Humans and service accounts in this Zitadel realm." } } + a class="button primary" href="/identities/new" { "Create identity" } + } + form class="filters card" action="/dashboard" method="get" { + label class="search-field" { span class="sr-only" { "Search identities" } input name="q" type="search" value=[search] placeholder="Search name, login, or email"; } + label { span class="sr-only" { "Identity type" } select name="kind" { + option value="" selected[kind.unwrap_or_default().is_empty()] { "All types" } + option value="human" selected[kind == Some("human")] { "Humans" } + option value="service" selected[kind == Some("service")] { "Service accounts" } + } } + label { span class="sr-only" { "Tenant" } input name="tenant" value=[tenant] placeholder="Tenant"; } + button class="button" type="submit" { "Filter" } + @if search.is_some() || kind.is_some() || tenant.is_some() { a class="text-link" href="/dashboard" { "Clear" } } + } + section class="card table-card" { + div class="table-summary" { (rows.len()) " identities" } + table { + thead { tr { th { "Identity" } th { "Type" } th { "Status" } th { "Harmony permissions" } th { "Actions" } } } + tbody { + @for (identity, access) in rows { + tr { + td data-label="Identity" { strong { (&identity.display_name) } small { (&identity.login_name) } } + td data-label="Type" { (kind_badge(&identity.kind)) } + td data-label="Status" { span class=(if identity.active { "status active" } else { "status inactive" }) { (if identity.active { "Active" } else { "Suspended" }) } } + td data-label="Permissions" class="permission-cell" { + @for assignment in &access.assignments { span class="badge permission" { (assignment.permission.label()) " · " (assignment.scope.label()) } } + @for imported in &access.imported { span class="badge imported" { "Imported · " (imported.policy_name) } } + @if access.assignments.is_empty() && access.imported.is_empty() { span class="muted" { "None" } } } + td data-label="Actions" { a class="row-link" href=(format!("/identities/{}", identity.subject_id)) { "View " (&identity.display_name) } } } - span class=(if identity.kind == IdentityKind::Human { "kind human" } else { "kind machine" }) { - (if identity.kind == IdentityKind::Human { "Human" } else { "Machine" }) - } - span class=(if identity.active { "status active" } else { "status suspended" }) { - (if identity.active { "Active" } else { "Suspended" }) - } - span class="arrow" aria-hidden="true" { "→" } } } - @if identities.is_empty() { - div class="empty" { strong { "No identities found" } p { "Try a name, login, or email fragment." } } + } + @if rows.is_empty() { div class="empty" { strong { "No identities found" } p { "Change or clear the filters." } } } + } + }, + ) +} + +pub fn identity(profile: &Profile<'_>, identity: &Identity, access: &IdentityAccess) -> Markup { + let tenants = access + .assignments + .iter() + .map(|assignment| assignment.scope.tenant.as_str()) + .collect::>(); + shell( + &identity.display_name, + Some(profile), + "/dashboard", + html! { + a class="back" href="/dashboard" { "← Identities" } + div class="identity-head" { + div class="avatar" { (initials(&identity.display_name)) } + div { h1 { (&identity.display_name) } p { (&identity.login_name) " · " (if identity.kind == IdentityKind::Human { "Human" } else { "Service account" }) } } + span class=(if identity.active { "status active" } else { "status inactive" }) { (if identity.active { "Active" } else { "Suspended" }) } + } + div class="two-column" { + section class="card" { + div class="section-head" { h2 { "Permissions" } span class="count" { (access.assignments.len()) } } + @for assignment in &access.assignments { + article class="assignment" { + div { strong { (assignment.permission.label()) } p { (assignment.scope.label()) } small { (assignment.permission.description()) } } + a class="button danger" href=(format!("/identities/{}/assignments/{}/remove/review", identity.subject_id, assignment.id)) { "Remove" } + } + } + @if access.assignments.is_empty() { div class="empty compact" { "No Harmony permissions assigned." } } + } + section class="card" { + h2 { "Assign permission" } + @if identity.active { + form class="form assignment-form" action=(format!("/identities/{}/assignments/review", identity.subject_id)) method="get" autocomplete="off" { + label { "Permission" select name="permission" required { + @if identity.kind == IdentityKind::Human { + option value="tenant_admin" { "Tenant Admin" } + option value="read_only" { "Read-only" } + } @else { + option value="cd_deployer" { "CD Deployer" } + option value="read_only" { "Read-only" } + } + } } + label { "Tenant" input name="tenant" list="tenant-options" value="" required placeholder="devsights"; } + datalist id="tenant-options" { @for tenant in tenants { option value=(tenant) {} } } + label { "Project (optional)" input name="project" value="" placeholder="folk-timesheet"; } + button class="button primary" type="submit" { "Review assignment" } + } + } @else { p class="muted" { "Suspended identities cannot receive new permissions." } } + } + } + @if !access.imported.is_empty() { + section class="card imported-section" { + div class="section-head" { h2 { "Imported access" } span class="badge imported" { "Existing OpenBao" } } + p class="muted" { "These policies already issue access but are not yet managed as Harmony permissions." } + @for imported in &access.imported { + article class="imported-row" { + div { small { "Source policy" } strong { (&imported.policy_name) } } + div { small { "Scope" } span { @if imported.secret_paths.is_empty() { "Not recognized" } @else { (imported.secret_paths.join(", ")) } } } + div { small { "Effect" } span { (&imported.effect) } } + } + } + } + } + details class="card advanced" { + summary { "Advanced implementation details" } + @if access.roles.is_empty() { p { "No matching JWT role." } } + @for role in &access.roles { + dl { + (fact("JWT role", &role.name)) + (fact("Matched subject", &role.subject_id)) + (fact("Audiences", &values(&role.bound_audiences))) + (fact("Token policies", &values(&role.token_policies))) } } } @@ -169,287 +200,175 @@ pub fn identities(profile: &Profile<'_>, identities: &[Identity], search: Option ) } -pub fn identity( +pub fn assignment_review(profile: &Profile<'_>, plan: &AssignmentPlan) -> Markup { + shell( + "Review assignment", + Some(profile), + "/dashboard", + html! { + a class="back" href=(format!("/identities/{}", plan.assignment.subject_id)) { "← Cancel" } + div class="page-head" { div { h1 { "Review assignment" } p { (&plan.summary) } } } + section class="card review-card" { + dl { + (fact("Permission", plan.assignment.permission.label())) + (fact("Scope", &plan.assignment.scope.label())) + (fact("Effect", plan.assignment.permission.description())) + } + @if plan.assignment.permission == Permission::TenantAdmin { + div class="alert warning" { strong { "Destructive permission" } span { "This user can change and delete every secret in this scope." } } + } + form action=(format!("/identities/{}/assignments", plan.assignment.subject_id)) method="post" { + input type="hidden" name="permission" value=(permission_value(plan.assignment.permission)); + input type="hidden" name="tenant" value=(&plan.assignment.scope.tenant); + input type="hidden" name="project" value=(plan.assignment.scope.project.as_deref().unwrap_or("")); + button class="button primary" type="submit" { "Apply assignment" } + } + details class="advanced-inline" { + summary { "Implementation preview" } + code { (&plan.assignment.policy_name) } + pre tabindex="0" { code { (&plan.policy) } } + } + } + }, + ) +} + +pub fn remove_assignment_review( profile: &Profile<'_>, identity: &Identity, - grants: &[Grant], - role: Option<&JwtRole>, - available_policies: &[String], - policy: &str, - can_manage: bool, + assignment: &harmony_auth::Assignment, ) -> Markup { - layout( - &identity.display_name, - "/identities", + shell( + "Remove assignment", Some(profile), + "/dashboard", html! { - a class="back" href="/identities" { "← All identities" } - section class="identity-header" { - (avatar(identity)) - div { - div class="title-line" { - h1 { (&identity.display_name) } - span class=(if identity.active { "status active" } else { "status suspended" }) { - (if identity.active { "Active" } else { "Suspended" }) - } - } - p { (&identity.login_name) } + a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } + div class="page-head" { div { h1 { "Remove assignment" } p { "Confirm before revoking this permission." } } } + section class="card review-card" { + dl { + (fact("Identity", &identity.display_name)) + (fact("Permission", assignment.permission.label())) + (fact("Scope", &assignment.scope.label())) } - } - section class="detail-grid" { - article class="panel facts" { - h2 { "Identity" } - dl { - (fact("Type", if identity.kind == IdentityKind::Human { "Human" } else { "Machine" })) - (fact("Email", identity.email.as_deref().unwrap_or("Not applicable"))) - (fact("Zitadel subject ID", &identity.subject_id)) - } - } - article class="panel access" { - div class="section-title" { - div { h2 { "Harmony direct grants" } p { "Canonical authorization intent" } } - span class="count" { (grants.len()) } - } - @for grant in grants { - div class="grant" { - div class="path" { code { (&grant.mount) "/" (&grant.path) } } - div class="grant-meta" { - span class="access-level" { (if grant.access == AccessLevel::ReadOnly { "Read only" } else { "Read, write + delete" }) } - span { (if grant.selector == Selector::Exact { "Exact secret" } else { "Entire subtree" }) } - } - } - } - @if grants.is_empty() { - div class="empty compact" { strong { "No Harmony direct grants" } p { "This does not mean the identity has no access. Inspect its live JWT role below." } } - } - @if !policy.is_empty() { - details class="policy" { - summary { "Generated OpenBao policy" } - div class="policy-heading" { - span { "Policy name" } - code { (harmony_auth::policy_name(&identity.subject_id)) } - } - pre tabindex="0" { code { (policy) } } - } - } - @if identity.active && can_manage { - details class="grant-form" { - summary { "Grant direct access" } - form action="/grants/plan" method="post" { - input type="hidden" name="principal_subject_id" value=(&identity.subject_id); - label { "Secret mount" input name="mount" value="secret" required; } - label { "Logical path" input name="path" placeholder="tenants/acme/database" required; } - label { "Scope" select name="selector" { option value="exact" { "Exact secret" } option value="subtree" { "Entire subtree" } } } - label { "Access" select name="access" { option value="read_only" { "Read only" } option value="read_write" { "Read, write + delete" } } } - button class="button primary" type="submit" { "Review grant" } - } - } - } - } - } - section class="panel role-panel" { - div class="section-title" { - div { h2 { "OpenBao JWT role" } p { "JWT login configuration at auth/jwt/role/" (identity.subject_id) } } - @if role.is_some() { span class="status active" { "Configured" } } - } - @if let Some(role) = role { - dl class="role-facts" { - (fact("Role name", &role.name)) - (fact("Bound subject", &role.bound_subject)) - (fact("Role type", &role.role_type)) - (fact("User claim", &role.user_claim)) - (fact("Audiences", &display_values(&role.bound_audiences))) - (fact("Token policies", &display_values(&role.token_policies))) - } - p class="field-help" { "Token policies are attached on the next JWT login. Existing OpenBao tokens are unchanged." } - } @else { - div class="empty compact" { strong { "No subject JWT role" } p { "Create one to bind this Zitadel subject to OpenBao ACL policies." } } - } - @if can_manage { - details class="grant-form role-form" { - summary { (if role.is_some() { "Edit JWT role" } else { "Create JWT role" }) } - form action=(format!("/identities/{}/roles/review", identity.subject_id)) method="get" { - label { "Bound audiences" input name="audiences" value=(role.map(|role| role.bound_audiences.join(", ")).unwrap_or_default()) placeholder="Zitadel project resource ID"; } - label { "ACL policies" input name="policies" value=(role.map(|role| role.token_policies.join(", ")).unwrap_or_default()) required placeholder="policy-one, policy-two"; } - @if !available_policies.is_empty() { - p class="available-policies" { "Available: " (available_policies.join(", ")) } - } - p class="field-help" { "Harmony fixes role type to jwt, user claim to sub, and bound subject to this identity. Supported existing TTL, claim, CIDR, and token settings are preserved." } - button class="button primary" type="submit" { "Review role" } - } - } + div class="alert warning" { strong { "Access will be revoked" } span { "New JWT logins will no longer receive this permission. Existing tokens are unchanged until they expire or are revoked." } } + form action=(format!("/identities/{}/assignments/{}/remove", identity.subject_id, assignment.id)) method="post" { + button class="button danger" type="submit" { "Remove assignment" } } } }, ) } -pub fn role_review(profile: &Profile<'_>, identity: &Identity, plan: &JwtRolePlan) -> Markup { - layout( - "Review JWT role", - "/identities", +pub fn tenants(profile: &Profile<'_>, tenants: &[TenantSummary]) -> Markup { + shell( + "Tenants", Some(profile), + "/tenants", html! { - a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } - (page_heading("Review JWT role", "Confirm the live OpenBao login binding before applying it.")) - section class="detail-grid" { - article class="panel facts" { - h2 { (if plan.creates_role { "Create role" } else { "Update role" }) } - dl { - (fact("Identity", &identity.display_name)) - (fact("Role name", &plan.subject_id)) - (fact("Bound subject", &plan.subject_id)) - (fact("Role type", "jwt")) - (fact("User claim", "sub")) - } - } - article class="panel access" { - h2 { "Token authorization" } - dl { - (fact("Bound audiences", &display_values(&plan.bound_audiences))) - (fact("ACL policies", &display_values(&plan.token_policies))) - } - @if plan.bound_audiences.is_empty() { - p class="danger-note" { strong { "No audience constraint." } " JWTs carrying an aud claim may be rejected by OpenBao. Add the Zitadel project resource ID unless these tokens have no audience." } - } - @if plan.token_policies.iter().any(|policy| policy == "root") { - p class="danger-note" { strong { "Unrestricted OpenBao access." } " The root ACL policy allows this identity to perform any operation." } - } - form action=(format!("/identities/{}/roles", plan.subject_id)) method="post" { - input type="hidden" name="audiences" value=(plan.bound_audiences.join(",")); - input type="hidden" name="policies" value=(plan.token_policies.join(",")); - button class="button primary" type="submit" { (if plan.creates_role { "Create JWT role" } else { "Apply role changes" }) } - } + div class="page-head" { div { h1 { "Tenants" } p { "Business and project scopes discovered from assignments and existing policies." } } } + section class="card table-card" { + table { + thead { tr { th { "Tenant" } th { "Project" } th { "Humans" } th { "Services" } th { "Actions" } } } + tbody { @for tenant in tenants { tr { + td data-label="Tenant" { strong { (&tenant.scope.tenant) } } + td data-label="Project" { (tenant.scope.project.as_deref().unwrap_or("All projects")) } + td data-label="Humans" { (tenant.humans) } + td data-label="Services" { (tenant.services) } + td data-label="Actions" { a class="row-link" href=(format!("/dashboard?tenant={}", tenant.scope.tenant)) { "View " (&tenant.scope.tenant) " identities" } } + } } } } + @if tenants.is_empty() { div class="empty" { strong { "No tenant scopes discovered" } p { "Assign a permission or inspect imported access from an identity." } } } } }, ) } -pub fn grant_review(profile: &Profile<'_>, identity: &Identity, plan: &GrantPlan) -> Markup { - layout( - "Review grant", - "/identities", +pub fn new_identity(profile: &Profile<'_>) -> Markup { + shell( + "Create identity", Some(profile), + "/dashboard", html! { - a class="back" href=(format!("/identities/{}", identity.subject_id)) { "← Cancel" } - (page_heading("Review direct grant", "Confirm the authorization intent and generated enforcement change before applying it.")) - section class="detail-grid" { - article class="panel facts" { - h2 { "Authorization intent" } - dl { - (fact("Identity", &identity.display_name)) - (fact("Secret", &format!("{}/{}", plan.request.mount, plan.request.path))) - (fact("Scope", if plan.request.selector == Selector::Exact { "Exact secret" } else { "Entire subtree" })) - (fact("Access", if plan.request.access == AccessLevel::ReadOnly { "Read only" } else { "Read, write + delete" })) - } + a class="back" href="/dashboard" { "← Identities" } + div class="page-head" { div { h1 { "Create identity" } p { "Create it in Zitadel, then return here to assign Harmony permissions." } } } + div class="two-column creation" { + section class="card" { + span class="badge" { "Human" } + h2 { "Human user" } + ol { li { "Open the Zitadel Console." } li { "Create the human user and complete its login setup." } li { "Return and refresh identities." } } + a class="button primary" href=(profile.zitadel_url) target="_blank" rel="noopener" { "Open Zitadel" } } - article class="panel access" { - h2 { "Generated policy after apply" } - pre class="review-policy" tabindex="0" { code { (&plan.resulting_policy) } } - @if plan.request.access == AccessLevel::ReadWrite { - p class="danger-note" { - strong { "Destructive access." } - @if plan.request.selector == Selector::Subtree { - " This identity can create, change, and delete this secret and every secret below it." - } @else { - " This identity can create, change, and delete this secret." - } - } - } - form action="/grants" method="post" { - input type="hidden" name="principal_subject_id" value=(&plan.request.principal_subject_id); - input type="hidden" name="mount" value=(&plan.request.mount); - input type="hidden" name="path" value=(&plan.request.path); - input type="hidden" name="selector" value=(if plan.request.selector == Selector::Exact { "exact" } else { "subtree" }); - input type="hidden" name="access" value=(if plan.request.access == AccessLevel::ReadOnly { "read_only" } else { "read_write" }); - button class="button primary" type="submit" { "Apply direct grant" } - } + section class="card" { + span class="badge" { "Service" } + h2 { "CD service account" } + ol { li { "Open the Zitadel Console." } li { "Create a service account; keep its key outside Harmony." } li { "Return and refresh identities." } } + a class="button primary" href=(profile.zitadel_url) target="_blank" rel="noopener" { "Open Zitadel" } } } + a class="button" href="/dashboard" { "Refresh identities" } }, ) } pub fn error(status: StatusCode, detail: Option<&str>) -> Markup { - layout( + shell( "Request failed", - "", None, - html! { section class="panel empty" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { "Change could not be reviewed: " (detail) "." } } @else { p { "The requested authorization data could not be loaded." } } a class="button" href="/" { "Return to profiles" } } }, + "", + html! { section class="card empty error-page" { h1 { (status.as_u16()) } @if let Some(detail) = detail { p { (detail) } } @else { p { "The operation failed. Check the server log for details." } } a class="button" href="/" { "Return to profiles" } } }, ) } -fn layout(title: &str, current: &str, profile: Option<&Profile<'_>>, content: Markup) -> Markup { +fn shell(title: &str, profile: Option<&Profile<'_>>, current: &str, content: Markup) -> Markup { html! { (DOCTYPE) html lang="en" { - head { - meta charset="utf-8"; - meta name="viewport" content="width=device-width, initial-scale=1"; - title { (title) " · Harmony Auth" } - link rel="stylesheet" href="/static/app.css"; - script src="/static/htmx.min.js" defer {} - } + head { meta charset="utf-8"; meta name="viewport" content="width=device-width, initial-scale=1"; title { (title) " · Harmony Auth" } link rel="stylesheet" href="/static/app.css"; } body { header class="topbar" { - a class="brand" href="/" { span class="brand-mark" { "H" } span { "Harmony" } em { "AUTH" } } - nav aria-label="Primary" { - @if profile.is_some() { - (nav_link("/dashboard", "Overview", current)) - (nav_link("/identities", "Identities", current)) - } - } - @if let Some(profile) = profile { - a class="profile-switch" href="/" title="Switch or manage profiles" { - span class="signal" {} - span { (profile.name) } - small { "Switch" } - } - } + a class="brand" href=(if profile.is_some() { "/dashboard" } else { "/" }) { span class="brand-mark" { "H" } "Harmony Auth" } + @if profile.is_some() { nav aria-label="Primary" { (nav("/dashboard", "Identities", current)) (nav("/tenants", "Tenants", current)) } } + @if let Some(profile) = profile { a class="profile-switch" href="/" { span class="signal" {} (profile.name) small { "Switch" } } } } main { (content) } - footer { "Harmony Auth" span { "Authorization intent, made inspectable." } } } } } } -fn nav_link(href: &str, label: &str, current: &str) -> Markup { - let active = if href == "/" { - current == href - } else { - current.starts_with(href) - }; - html! { a href=(href) aria-current=[active.then_some("page")] { (label) } } +fn nav(href: &str, label: &str, current: &str) -> Markup { + html! { a href=(href) aria-current=[(current == href).then_some("page")] { (label) } } } -fn metric(label: &str, value: usize, note: &str) -> Markup { - html! { article class="metric" { span { (label) } strong { (value) } small { (note) } } } -} - -fn page_heading(title: &str, description: &str) -> Markup { - html! { section class="page-heading" { p class="eyebrow" { "ZITADEL DIRECTORY" } h1 { (title) } p { (description) } } } -} - -fn avatar(identity: &Identity) -> Markup { - let initials: String = identity - .display_name - .split_whitespace() - .filter_map(|part| part.chars().next()) - .take(2) - .collect(); - html! { span class=(if identity.kind == IdentityKind::Human { "avatar" } else { "avatar machine-avatar" }) { (initials.to_uppercase()) } } +fn kind_badge(kind: &IdentityKind) -> Markup { + html! { span class="badge" { (if *kind == IdentityKind::Human { "Human" } else { "Service" }) } } } fn fact(label: &str, value: &str) -> Markup { html! { div class="fact" { dt { (label) } dd { (value) } } } } -fn display_values(values: &[String]) -> String { +fn initials(name: &str) -> String { + name.split_whitespace() + .filter_map(|part| part.chars().next()) + .take(2) + .collect::() + .to_uppercase() +} + +fn values(values: &[String]) -> String { if values.is_empty() { "None".into() } else { values.join(", ") } } + +fn permission_value(permission: Permission) -> &'static str { + match permission { + Permission::TenantAdmin => "tenant_admin", + Permission::CdDeployer => "cd_deployer", + Permission::ReadOnly => "read_only", + } +} -- 2.39.5 From bd1554033d8aae52ef9a706022e64f2313f0ce78 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 20 Jul 2026 08:22:26 -0400 Subject: [PATCH 08/47] feat: improve auth identity discovery ux --- harmony_auth/src/lib.rs | 14 +++++++- harmony_auth_ui/src/app.css | 11 ++++++- harmony_auth_ui/src/backend.rs | 51 ++++++++++++++++++----------- harmony_auth_ui/src/dashboard.js | 55 ++++++++++++++++++++++++++++++++ harmony_auth_ui/src/main.rs | 8 +++++ harmony_auth_ui/src/views.rs | 48 ++++++++++++++++++++-------- 6 files changed, 153 insertions(+), 34 deletions(-) create mode 100644 harmony_auth_ui/src/dashboard.js diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index a0454e62..b1e6b9af 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -107,12 +107,24 @@ pub struct ImportedAccess { pub effect: String, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpenBaoPolicy { + pub name: String, + pub body: Option, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct JwtRole { pub name: String, pub subject_id: String, pub bound_audiences: Vec, - pub token_policies: Vec, + pub policies: Vec, +} + +impl JwtRole { + pub fn policy_names(&self) -> impl Iterator { + self.policies.iter().map(|policy| policy.name.as_str()) + } } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/harmony_auth_ui/src/app.css b/harmony_auth_ui/src/app.css index 14a9a59f..81b2a906 100644 --- a/harmony_auth_ui/src/app.css +++ b/harmony_auth_ui/src/app.css @@ -35,22 +35,28 @@ h2 { margin-bottom: 14px; font: 600 18px/1.25 Georgia, serif; } .button.danger, .quiet-danger { color: var(--red); } .text-link, .row-link, .back { font-weight: 700; text-decoration: none; font-size: 12px; } .back { display: inline-block; margin-bottom: 20px; } -.filters { display: flex; gap: 10px; margin-bottom: 14px; padding: 13px; } +.filters { display: flex; align-items: center; gap: 10px; margin-bottom: 14px; padding: 13px; } .filters input, .filters select, .form input, .form select, .dialog input { height: 38px; min-width: 0; border: 1px solid #bbb8ae; border-radius: 3px; background: white; padding: 0 10px; color: var(--ink); } .filters .search-field { flex: 1; } .filters .search-field input { width: 100%; } .filters label:not(.search-field) input { width: 150px; } +.clear-filter { height: 38px; } .table-card { padding: 0; overflow-x: auto; } .table-summary { padding: 13px 17px; border-bottom: 1px solid var(--line); color: var(--muted); font-size: 11px; } table { width: 100%; border-collapse: collapse; } th { color: var(--muted); font-size: 10px; text-align: left; text-transform: uppercase; letter-spacing: .06em; } th, td { padding: 13px 17px; border-bottom: 1px solid var(--line); vertical-align: middle; } tbody tr:last-child td { border-bottom: 0; } +.clickable-row { cursor: pointer; } +.clickable-row:hover, .clickable-row:focus { background: #f2f5ec; outline: none; } +.clickable-row:focus { box-shadow: inset 3px 0 var(--green); } +.identity-link { color: inherit; text-decoration: none; } td strong, td small { display: block; } td small { margin-top: 2px; color: var(--muted); } .permission-cell { min-width: 260px; } .badge { display: inline-flex; margin: 2px 5px 2px 0; padding: 3px 7px; border: 1px solid var(--line); border-radius: 99px; font-size: 10px; } .badge.permission { border-color: #bdd1a0; background: #edf3e5; color: #33542d; } +.badge.scope { border-color: #b9c8d8; background: #edf2f6; color: #294660; } .badge.imported { border-color: #d7c49e; background: #f5eedf; color: #6c5022; } .status { font-size: 11px; } .status::before { content: ""; display: inline-block; width: 6px; height: 6px; margin-right: 6px; border-radius: 50%; } @@ -77,6 +83,9 @@ td small { margin-top: 2px; color: var(--muted); } .imported-row small { display: block; margin-bottom: 3px; color: var(--muted); } details summary { cursor: pointer; color: var(--green); font-weight: 700; font-size: 12px; } .advanced dl, .review-card dl { margin: 15px 0; } +.role-details { margin-top: 14px; padding-top: 14px; border-top: 1px solid var(--line); } +.role-details h3 { margin: 16px 0 8px; font-size: 12px; } +.policy-details { margin: 7px 0; } .fact { display: grid; grid-template-columns: 150px minmax(0, 1fr); padding: 9px 0; border-bottom: 1px solid var(--line); } .fact dt { color: var(--muted); } .fact dd { margin: 0; overflow-wrap: anywhere; } diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth_ui/src/backend.rs index d79d57a6..e652bd46 100644 --- a/harmony_auth_ui/src/backend.rs +++ b/harmony_auth_ui/src/backend.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use async_trait::async_trait; use harmony_auth::{ Assignment, AssignmentPlan, AssignmentRequest, AuthError, AuthService, Identity, - IdentityAccess, IdentityKind, ImportedAccess, JwtRole, Scope, TenantSummary, + IdentityAccess, IdentityKind, ImportedAccess, JwtRole, OpenBaoPolicy, Scope, TenantSummary, }; use reqwest::{Client, Method, StatusCode}; use serde_json::{Map, Value, json}; @@ -151,7 +151,10 @@ impl BackendAuth { name, subject_id, bound_audiences: strings(&raw["bound_audiences"]), - token_policies: role_policies(&raw), + policies: role_policies(&raw) + .into_iter() + .map(|name| OpenBaoPolicy { name, body: None }) + .collect(), }, raw, }); @@ -254,7 +257,7 @@ impl BackendAuth { name: format!("harmony-{subject_id}"), subject_id: subject_id.into(), bound_audiences: strings(&raw["bound_audiences"]), - token_policies: vec![], + policies: vec![], }, raw, }) @@ -282,14 +285,23 @@ impl BackendAuth { let roles = self.roles().await?; let policy_names = roles .iter() - .flat_map(|role| role.role.token_policies.iter()) - .filter(|policy| policy.as_str() != "default") - .cloned() + .flat_map(|role| role.role.policy_names()) + .filter(|policy| *policy != "default") + .map(str::to_string) .collect::>(); let mut policy_details = BTreeMap::new(); for policy in policy_names { - let body = self.policy(&policy).await?.unwrap_or_default(); - policy_details.insert(policy, (secret_paths(&body), policy_effect(&body))); + if let Some(body) = self.policy(&policy).await? { + policy_details.insert(policy, (secret_paths(&body), policy_effect(&body), body)); + } + } + let mut roles = roles; + for role in &mut roles { + for policy in &mut role.role.policies { + policy.body = policy_details + .get(&policy.name) + .map(|details| details.2.clone()); + } } let mut access = Vec::with_capacity(identities.len()); for identity in identities { @@ -306,20 +318,20 @@ impl BackendAuth { .iter() .flat_map(|role| { role.role - .token_policies + .policies .iter() .filter(|policy| { - policy.as_str() != "default" && !managed.contains(policy.as_str()) + policy.name != "default" && !managed.contains(policy.name.as_str()) }) .map(|policy| ImportedAccess { role_name: role.role.name.clone(), - policy_name: policy.clone(), + policy_name: policy.name.clone(), secret_paths: policy_details - .get(policy) + .get(&policy.name) .map(|details| details.0.clone()) .unwrap_or_default(), effect: policy_details - .get(policy) + .get(&policy.name) .map(|details| details.1.clone()) .unwrap_or_else(|| "Custom OpenBao access".into()), }) @@ -447,7 +459,11 @@ impl AuthService for BackendAuth { .await? .error_for_status() .map_err(backend)?; - let mut policies = role.role.token_policies.clone(); + let mut policies = role + .role + .policy_names() + .map(str::to_string) + .collect::>(); if !policies.contains(&plan.assignment.policy_name) { policies.push(plan.assignment.policy_name.clone()); policies.sort_unstable(); @@ -487,10 +503,9 @@ impl AuthService for BackendAuth { let role = roles.pop().unwrap(); let policies = role .role - .token_policies - .iter() - .filter(|policy| *policy != &assignment.policy_name) - .cloned() + .policy_names() + .filter(|policy| *policy != assignment.policy_name.as_str()) + .map(str::to_string) .collect::>(); self.write_role(&role, &policies).await?; self.openbao( diff --git a/harmony_auth_ui/src/dashboard.js b/harmony_auth_ui/src/dashboard.js new file mode 100644 index 00000000..0aaa174e --- /dev/null +++ b/harmony_auth_ui/src/dashboard.js @@ -0,0 +1,55 @@ +const filter = document.querySelector("[data-live-filter]"); + +if (filter) { + let timeout; + let request; + + filter.addEventListener("submit", async (event) => { + event.preventDefault(); + request?.abort(); + request = new AbortController(); + const url = `${filter.action}?${new URLSearchParams(new FormData(filter))}`; + + try { + const response = await fetch(url, { signal: request.signal }); + if (!response.ok) { + window.location.assign(url); + return; + } + const page = new DOMParser().parseFromString(await response.text(), "text/html"); + document.querySelector(".table-card").replaceWith(page.querySelector(".table-card")); + filter.querySelector(".clear-filter").hidden = !Array.from(new FormData(filter).values()).some((value) => value); + window.history.replaceState(null, "", url); + } catch (error) { + if (error.name !== "AbortError") window.location.assign(url); + } + }); + filter.addEventListener("input", () => { + clearTimeout(timeout); + timeout = setTimeout(() => filter.requestSubmit(), 350); + }); + filter.addEventListener("change", () => { + clearTimeout(timeout); + filter.requestSubmit(); + }); + filter.querySelector(".clear-filter").addEventListener("click", (event) => { + event.preventDefault(); + filter.reset(); + filter.requestSubmit(); + }); +} + +document.addEventListener("click", (event) => { + const row = event.target.closest("[data-href]"); + if (row && !event.target.closest("a, button, input, select")) { + window.location.assign(row.dataset.href); + } +}); + +document.addEventListener("keydown", (event) => { + const row = event.target.closest("[data-href]"); + if (row && event.target === row && (event.key === "Enter" || event.key === " ")) { + event.preventDefault(); + window.location.assign(row.dataset.href); + } +}); diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index 4285ac65..7ab0ff64 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -127,6 +127,7 @@ fn router(state: AppState) -> Router { get(remove_assignment_review), ) .route("/static/app.css", get(css)) + .route("/static/dashboard.js", get(dashboard_js)) .route("/static/profiles.js", get(profiles_js)) .route("/favicon.ico", get(|| async { StatusCode::NO_CONTENT })) .layer(SetResponseHeaderLayer::overriding( @@ -552,6 +553,13 @@ async fn profiles_js() -> impl IntoResponse { ) } +async fn dashboard_js() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/javascript; charset=utf-8")], + include_str!("dashboard.js"), + ) +} + enum AppError { Auth(AuthError), Disconnected, diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index c23f1a98..ec2d745e 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -76,6 +76,9 @@ pub fn dashboard( kind: Option<&str>, tenant: Option<&str>, ) -> Markup { + let has_filters = search.is_some_and(|value| !value.is_empty()) + || kind.is_some_and(|value| !value.is_empty()) + || tenant.is_some_and(|value| !value.is_empty()); shell( "Identities", Some(profile), @@ -85,7 +88,7 @@ pub fn dashboard( div { h1 { "Identities" } p { "Humans and service accounts in this Zitadel realm." } } a class="button primary" href="/identities/new" { "Create identity" } } - form class="filters card" action="/dashboard" method="get" { + form class="filters card" action="/dashboard" method="get" data-live-filter { label class="search-field" { span class="sr-only" { "Search identities" } input name="q" type="search" value=[search] placeholder="Search name, login, or email"; } label { span class="sr-only" { "Identity type" } select name="kind" { option value="" selected[kind.unwrap_or_default().is_empty()] { "All types" } @@ -93,31 +96,35 @@ pub fn dashboard( option value="service" selected[kind == Some("service")] { "Service accounts" } } } label { span class="sr-only" { "Tenant" } input name="tenant" value=[tenant] placeholder="Tenant"; } - button class="button" type="submit" { "Filter" } - @if search.is_some() || kind.is_some() || tenant.is_some() { a class="text-link" href="/dashboard" { "Clear" } } + a class="button clear-filter" href="/dashboard" hidden[!has_filters] { "Clear" } } section class="card table-card" { div class="table-summary" { (rows.len()) " identities" } table { - thead { tr { th { "Identity" } th { "Type" } th { "Status" } th { "Harmony permissions" } th { "Actions" } } } + thead { tr { th { "Identity" } th { "Type" } th { "Status" } th { "Tenant / project" } th { "Access" } } } tbody { @for (identity, access) in rows { - tr { - td data-label="Identity" { strong { (&identity.display_name) } small { (&identity.login_name) } } + tr class="clickable-row" tabindex="0" data-href=(format!("/identities/{}", identity.subject_id)) { + td data-label="Identity" { strong { a class="identity-link" href=(format!("/identities/{}", identity.subject_id)) { (&identity.display_name) } } small { (&identity.login_name) } } td data-label="Type" { (kind_badge(&identity.kind)) } td data-label="Status" { span class=(if identity.active { "status active" } else { "status inactive" }) { (if identity.active { "Active" } else { "Suspended" }) } } - td data-label="Permissions" class="permission-cell" { - @for assignment in &access.assignments { span class="badge permission" { (assignment.permission.label()) " · " (assignment.scope.label()) } } + td data-label="Tenant / project" class="scope-cell" { + @for assignment in &access.assignments { span class="badge scope" { (assignment.scope.label()) } } + @for imported in &access.imported { @for path in &imported.secret_paths { span class="badge scope imported" { (path) } } } + @if access.assignments.is_empty() && access.imported.iter().all(|item| item.secret_paths.is_empty()) { span class="muted" { "None" } } + } + td data-label="Access" class="permission-cell" { + @for assignment in &access.assignments { span class="badge permission" { (assignment.permission.label()) } } @for imported in &access.imported { span class="badge imported" { "Imported · " (imported.policy_name) } } @if access.assignments.is_empty() && access.imported.is_empty() { span class="muted" { "None" } } } - td data-label="Actions" { a class="row-link" href=(format!("/identities/{}", identity.subject_id)) { "View " (&identity.display_name) } } } } } } @if rows.is_empty() { div class="empty" { strong { "No identities found" } p { "Change or clear the filters." } } } } + script src="/static/dashboard.js" defer {} }, ) } @@ -188,11 +195,24 @@ pub fn identity(profile: &Profile<'_>, identity: &Identity, access: &IdentityAcc summary { "Advanced implementation details" } @if access.roles.is_empty() { p { "No matching JWT role." } } @for role in &access.roles { - dl { - (fact("JWT role", &role.name)) - (fact("Matched subject", &role.subject_id)) - (fact("Audiences", &values(&role.bound_audiences))) - (fact("Token policies", &values(&role.token_policies))) + details class="role-details" { + summary { (&role.name) } + dl { + (fact("Matched subject", &role.subject_id)) + (fact("Audiences", &values(&role.bound_audiences))) + } + h3 { "OpenBao policies" } + @if role.policies.is_empty() { p class="muted" { "No policies attached." } } + @for policy in &role.policies { + details class="policy-details" { + summary { (&policy.name) } + @if let Some(body) = &policy.body { + pre tabindex="0" { code { (body) } } + } @else { + p class="muted" { "Policy body is not available." } + } + } + } } } } -- 2.39.5 From 3b7ebd0a9a1737c8c53183156b147047f01d8b3d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 20 Jul 2026 14:31:15 -0400 Subject: [PATCH 09/47] feat: Created a cli for harmony auth to manage teanants, accounts and permissions --- Cargo.lock | 16 +- Cargo.toml | 1 + docs/SUMMARY.md | 1 + docs/guides/harmony-auth-cli.md | 376 ++++++++++++++ harmony_auth/Cargo.toml | 2 + .../src/backend.rs | 311 ++++++++--- harmony_auth/src/lib.rs | 99 +++- harmony_auth_cli/Cargo.toml | 17 + harmony_auth_cli/src/main.rs | 481 ++++++++++++++++++ harmony_auth_ui/Cargo.toml | 3 - harmony_auth_ui/src/main.rs | 106 ++-- harmony_auth_ui/src/views.rs | 8 +- 12 files changed, 1264 insertions(+), 157 deletions(-) create mode 100644 docs/guides/harmony-auth-cli.md rename {harmony_auth_ui => harmony_auth}/src/backend.rs (68%) create mode 100644 harmony_auth_cli/Cargo.toml create mode 100644 harmony_auth_cli/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index f10435a2..45b1838b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4323,27 +4323,37 @@ version = "0.1.0" dependencies = [ "async-trait", "chrono", + "reqwest 0.12.28", "serde", + "serde_json", "sha2 0.10.9", "thiserror 2.0.18", "uuid", ] +[[package]] +name = "harmony_auth_cli" +version = "0.1.0" +dependencies = [ + "clap", + "harmony_auth", + "serde_json", + "tokio", + "tracing-subscriber", +] + [[package]] name = "harmony_auth_ui" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", "axum", "axum-extra", - "chrono", "clap", "harmony_auth", "maud", "reqwest 0.12.28", "serde", - "serde_json", "tokio", "tower-http", "tracing", diff --git a/Cargo.toml b/Cargo.toml index f4021894..5254ec6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "fleet/harmony-fleet-e2e", "harmony-reconciler-contracts", "harmony_auth", + "harmony_auth_cli", "harmony_auth_ui", "examples/fleet_server_install", "nats/jwt", diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 7706f74e..208aa2d0 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -22,6 +22,7 @@ - [Developer Guide](./guides/developer-guide.md) - [Application CLI — Use Cases & Commands](./guides/application-cli.md) +- [Harmony Auth CLI](./guides/harmony-auth-cli.md) - [Application Capabilities — .with(...)](./guides/application-capabilities.md) - [Writing a Score](./guides/writing-a-score.md) - [Writing a Topology](./guides/writing-a-topology.md) diff --git a/docs/guides/harmony-auth-cli.md b/docs/guides/harmony-auth-cli.md new file mode 100644 index 00000000..cf68b690 --- /dev/null +++ b/docs/guides/harmony-auth-cli.md @@ -0,0 +1,376 @@ +# Harmony Auth CLI + +> **Status: read-only commands implemented.** Mutation commands follow the +> ADR-027 group migration. + +`harmony-auth` inspects and manages the relationship between Zitadel +identities and OpenBao access. It presents tenants, identities, and Harmony +permissions first. JWT roles, policy names, subject claims, and HCL remain +available through advanced output. + +The CLI is the preferred interface while the web UI matures. Both interfaces +use the same `harmony_auth` operations and return the same effective access. + +## Mental model + +The CLI has two main views: + +- `identity`: who an identity is and what it can access +- `tenant`: who can access a tenant or project + +Harmony permissions are the primary authorization vocabulary: + +| Permission | Intended identity | Effect | +|---|---|---| +| `tenant-admin` | Human | Read, create, change, and delete secrets in a tenant or project | +| `cd-deployer` | Service account | Read deployment secrets in one project | +| `read-only` | Human or service account | Read secrets in a tenant or project | + +Existing OpenBao policies that do not correspond to a Harmony assignment are +shown as imported access. The CLI does not rename, rewrite, or hide them. + +## Command tree + +```text +harmony-auth +├── connection check +├── identity list +├── identity show +├── tenant list +└── tenant show +``` + +There are no flat aliases. `harmony-auth list` is not valid. + +## Connection and credentials + +Every command except `--help` and `--version` requires: + +| Flag | Environment | Meaning | +|---|---|---| +| `--zitadel-url` | `ZITADEL_URL` | Zitadel base URL | +| `--openbao-url` | `OPENBAO_URL` | OpenBao base URL | +| none | `ZITADEL_PAT` | Zitadel service-account PAT | +| none | `OPENBAO_TOKEN` | Temporary OpenBao administrator token | + +Secrets are environment-only because command-line arguments remain in shell +history and may be visible in the process list. Secret values never appear in +help output, normal output, JSON, or logs. + +Example: + +```sh +export ZITADEL_URL=https://sso.example.com +export ZITADEL_PAT=... +export OPENBAO_URL=https://secrets.example.com +export OPENBAO_TOKEN=... + +harmony-auth connection check +``` + +The CLI does not persist profiles or credentials. Browser profile storage and +session credential refresh remain web UI concerns. + +`connection check` attempts both backends even when one fails. It reports each +status without printing provider response bodies: + +```text +Zitadel connected +OpenBao connected +``` + +## Identity commands + +### List identities + +```sh +harmony-auth identity list +harmony-auth identity list --search folk +harmony-auth identity list --kind human +harmony-auth identity list --tenant devsights +harmony-auth identity list --tenant devsights --kind service +``` + +Filters combine with AND semantics. `--search` matches display name, login, or +email. `--kind` accepts `human` or `service`. Tenant matching uses parsed +assignment and imported-policy scopes, not string-prefix matching. + +Imported scopes are recognized only from wildcard roots: + +- `/data//*` is tenant-wide. +- `/data///*` is project-specific. +- Exact secret paths and wildcard paths below a project are custom access. They + remain visible but do not affect tenant filters or summaries. + +Tenant and project components must pass the same slug validation as managed +Harmony scopes. `harmony_auth` returns both the raw paths and parsed scopes; +frontends never infer scopes themselves. + +Human output keeps the tenant visible: + +```text +ACTIVE HUMAN Alice Example alice@example.com + subject 241696899342475267 + devsights Tenant Admin + +ACTIVE SERVICE Folk CD folk-cd + subject 241697058442100739 + devsights/folk-timesheet CD Deployer +``` + +An identity without recognized access is still listed with `No access`. + +### Show an identity + +```sh +harmony-auth identity show 241696899342475267 +``` + +Default output includes identity metadata, Harmony assignments, and imported +access: + +```text +Alice Example + Subject: 241696899342475267 + Login: alice@example.com + Kind: Human + Status: Active + +Harmony permissions + 019b... Tenant Admin devsights + +Imported OpenBao access + legacy-folk-reader devsights/folk-timesheet Read secrets +``` + +Use `--advanced` to inspect implementation details: + +```sh +harmony-auth identity show 241696899342475267 --advanced +``` + +Advanced output adds matching JWT roles, bound subject, audiences, attached +policy names, and the exact HCL returned by OpenBao. A built-in or inaccessible +policy remains listed with `Policy body unavailable`. + +## Planned group-based mutations + +The first release does not grant or revoke access. Existing per-subject JWT +roles are discovery input, not a writable authorization model. + +ADR-027 makes Zitadel roles named `:owner`, `:deployer`, and +`:viewer` authoritative. One shared OpenBao JWT role reads the `groups` +claim, and OpenBao external groups attach policies. Mutation commands ship only +after `harmony_auth` implements that model end to end. + +### Grant a permission (planned) + +```sh +harmony-auth identity grant 241696899342475267 \ + --permission tenant-admin \ + --tenant devsights +``` + +Grant is review-only by default: + +```text +Plan + Identity: Alice Example (241696899342475267) + Permission: Tenant Admin + Scope: devsights + Effect: Read, create, change, and delete secrets + +No changes applied. Re-run with --apply to continue. +``` + +Apply the reviewed request explicitly: + +```sh +harmony-auth identity grant 241696899342475267 \ + --permission tenant-admin \ + --tenant devsights \ + --apply +``` + +`cd-deployer` requires `--project`. Other permissions accept an optional +project. Permission applicability and scope validation come from +`harmony_auth`; the CLI does not duplicate those rules. + +Applying a grant changes the Zitadel role assignment. It does not create or +edit a per-subject OpenBao JWT role. Applying the same grant twice must report +`changed: false`. + +### Revoke a permission (planned) + +The assignment ID comes from `identity show`: + +```sh +harmony-auth identity revoke 241696899342475267 019b... +``` + +Revoke is also review-only by default. It shows identity, permission, scope, +and the warning that already-issued OpenBao tokens remain valid until expiry or +revocation. `--apply` performs the removal: + +```sh +harmony-auth identity revoke 241696899342475267 019b... --apply +``` + +Imported per-subject access cannot be revoked through this command because it +is outside the ADR-027 group model. Its OpenBao policy name remains visible for +manual migration. + +## Tenant commands + +### List tenants + +```sh +harmony-auth tenant list +``` + +Tenants and projects are discovered from managed assignments and recognized +OpenBao policy paths: + +```text +TENANT PROJECT HUMANS SERVICES +devsights All projects 2 0 +devsights folk-timesheet 1 1 +detexion harmony-fleet 0 2 +``` + +### Show a tenant + +```sh +harmony-auth tenant show devsights +harmony-auth tenant show devsights --project folk-timesheet +``` + +Output lists matching scopes and identities with their access. With +`--project`, tenant-wide access and access to that exact project are included; +other projects are excluded: + +```text +devsights/folk-timesheet + Alice Example Human Read-only + Folk CD Service CD Deployer +``` + +This is an authorization view, not a secret-value browser. Secret listing, +creation, update, and reveal are outside the first release. + +## JSON output + +Every implemented command accepts `--json`. JSON is written to stdout; +diagnostics and logs are written to stderr. The envelope is versioned: + +```json +{ + "schema_version": 1, + "command": "identity.list", + "result": {} +} +``` + +The first release uses these result shapes: + +| Command | `result` fields | +|---|---| +| `connection check` | `zitadel: { connected }`, `openbao: { connected }` | +| `identity list` | `identities: [{ identity, access }]` | +| `identity show` | `identity`, `access` | +| `tenant list` | `tenants: [{ scope, humans, services }]` | +| `tenant show` | `tenant`, `project`, `identities: [{ identity, access }]` | + +`identity` contains `subject_id`, `kind`, `display_name`, `login_name`, +`email`, and `active`. Identity kinds are `human` and `service`; `email` is a +string or `null`. + +`scope` contains `tenant` and `project`, where `project` is a string or `null`. +An assignment contains `id`, `subject_id`, `permission`, `scope`, +`policy_name`, and `created_at`. IDs are UUID strings, timestamps are RFC 3339, +and permissions are `tenant_admin`, `cd_deployer`, or `read_only`. + +`access` contains: + +- `assignments`: assignment objects as defined above +- `imported`: objects with `role_name`, `policy_name`, `secret_paths`, + `scopes`, and `effect` +- `roles`: objects with `name`, `subject_id`, `bound_audiences`, and `policies` + +Each role policy contains `name` and `body`. Policy bodies are strings only for +`identity show --advanced`; otherwise `body` is `null`. Imported +`secret_paths`, audiences, and scopes are arrays. `effect` is the +plain-language interpretation returned by `harmony_auth`. + +`connection check` always returns both status objects and exits `1` when either +`connected` value is false. A top-level error is used only when the command +cannot produce those statuses. + +Domain and backend errors use stdout when `--json` is active: + +```json +{ + "schema_version": 1, + "command": "identity.show", + "error": { + "kind": "not_found", + "message": "identity not found" + } +} +``` + +Error kinds are `not_found`, `invalid`, and `backend`. Clap usage errors remain +on stderr because command parsing fails before JSON dispatch. + +Planned grant and revoke results will add `applied` and `changed` when those +commands are implemented. Their JSON schema is not frozen by this release. + +## Exit and error behavior + +| Exit | Meaning | +|---|---| +| `0` | Query completed, plan produced, or mutation applied/no-op | +| `2` | Invalid command, missing connection value, not found, or invalid input | +| `1` | Zitadel or OpenBao request failed | + +Errors name the failed operation and backend but do not print credentials, +provider response bodies, or stack traces. `RUST_LOG=info` enables operation +logs on stderr. + +## Architecture boundary + +```text +harmony_auth_ui ─┐ + ├──> harmony_auth ──> Zitadel + OpenBao +harmony_auth_cli ─┘ +``` + +`harmony_auth` owns: + +- identities, scopes, permissions, assignments, roles, policies, and plans +- Zitadel identity discovery +- OpenBao role and policy discovery +- tenant and identity access queries +- permission applicability and scope validation +- assignment discovery and current legacy assignment operations used by the UI +- provider request construction and response interpretation + +`harmony_auth_cli` owns: + +- Clap arguments and environment mapping +- terminal and JSON rendering +- binary exit codes and logging setup + +`harmony_auth_ui` owns HTTP routes, browser sessions, cookies, forms, HTML, CSS, +and browser JavaScript. Neither frontend may parse OpenBao policies, infer +tenants, reconcile JWT roles, or implement permission rules. + +## First-release limits + +- Authorization discovers existing per-subject JWT roles as imported access. +- Grant and revoke wait for Zitadel role and OpenBao external-group operations. +- The CLI does not create Zitadel identities. +- The CLI does not provide a generic OpenBao policy editor. +- Tenant administrators are not yet authenticated as constrained actors; the + supplied OpenBao token determines backend authority. +- The CLI does not store profiles or credentials. diff --git a/harmony_auth/Cargo.toml b/harmony_auth/Cargo.toml index 70e7a46e..6cde4cde 100644 --- a/harmony_auth/Cargo.toml +++ b/harmony_auth/Cargo.toml @@ -8,7 +8,9 @@ license.workspace = true [dependencies] async-trait.workspace = true chrono = { workspace = true, features = ["serde"] } +reqwest.workspace = true serde.workspace = true +serde_json.workspace = true sha2.workspace = true thiserror.workspace = true uuid = { workspace = true, features = ["serde"] } diff --git a/harmony_auth_ui/src/backend.rs b/harmony_auth/src/backend.rs similarity index 68% rename from harmony_auth_ui/src/backend.rs rename to harmony_auth/src/backend.rs index e652bd46..831a0cb0 100644 --- a/harmony_auth_ui/src/backend.rs +++ b/harmony_auth/src/backend.rs @@ -1,10 +1,14 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use async_trait::async_trait; -use harmony_auth::{ - Assignment, AssignmentPlan, AssignmentRequest, AuthError, AuthService, Identity, - IdentityAccess, IdentityKind, ImportedAccess, JwtRole, OpenBaoPolicy, Scope, TenantSummary, +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, }; + +use crate::{ + Assignment, AssignmentPlan, AssignmentRequest, AuthError, AuthService, BackendConnection, + ConnectionStatus, Identity, IdentityAccess, IdentityFilter, IdentityKind, IdentityWithAccess, + ImportedAccess, JwtRole, OpenBaoPolicy, RemovalPlan, Scope, TenantSummary, +}; +use async_trait::async_trait; use reqwest::{Client, Method, StatusCode}; use serde_json::{Map, Value, json}; use uuid::Uuid; @@ -22,21 +26,31 @@ struct RoleRecord { raw: Value, } +struct PolicyDetails { + secret_paths: Vec, + scopes: Vec, + effect: String, + body: String, +} + impl BackendAuth { pub fn new( - client: Client, zitadel_url: String, zitadel_pat: String, openbao_url: String, openbao_token: String, - ) -> Self { - Self { - client, + ) -> Result { + Ok(Self { + client: Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(10)) + .build() + .map_err(|error| format!("could not configure HTTP client: {error}"))?, zitadel_url: zitadel_url.trim_end_matches('/').into(), zitadel_pat, openbao_url: openbao_url.trim_end_matches('/').into(), openbao_token, - } + }) } pub fn zitadel_url(&self) -> &str { @@ -44,41 +58,45 @@ impl BackendAuth { } pub async fn validate(&self) -> Result<(), String> { + let status = self.connection_status().await; + match (status.zitadel.connected, status.openbao.connected) { + (true, true) => Ok(()), + (false, true) => Err("Zitadel connection failed".into()), + (true, false) => Err("OpenBao connection failed".into()), + (false, false) => Err("Zitadel and OpenBao connections failed".into()), + } + } + + pub async fn connection_status(&self) -> ConnectionStatus { let zitadel = self .client .get(format!("{}/management/v1/orgs/me", self.zitadel_url)) .bearer_auth(&self.zitadel_pat) .send() .await - .map_err(|error| format!("Zitadel could not be reached: {error}"))?; - if !zitadel.status().is_success() { - return Err(format!( - "Zitadel rejected the service-account PAT ({})", - zitadel.status() - )); - } + .is_ok_and(|response| response.status().is_success()); let openbao = self .openbao(Method::GET, "auth/token/lookup-self", None) .await - .map_err(|error| error.to_string())?; - if !openbao.status().is_success() { - return Err(format!("OpenBao rejected the token ({})", openbao.status())); + .is_ok_and(|response| response.status().is_success()); + ConnectionStatus { + zitadel: BackendConnection { connected: zitadel }, + openbao: BackendConnection { connected: openbao }, } - Ok(()) } - pub fn with_credentials(&self, zitadel_pat: String, openbao_token: String) -> Self { - Self::new( - self.client.clone(), - self.zitadel_url.clone(), - zitadel_pat, - self.openbao_url.clone(), - openbao_token, - ) - } - - pub fn credentials(&self) -> (&str, &str) { - (&self.zitadel_pat, &self.openbao_token) + pub fn with_credentials( + &self, + zitadel_pat: Option, + openbao_token: Option, + ) -> Self { + Self { + client: self.client.clone(), + zitadel_url: self.zitadel_url.clone(), + zitadel_pat: zitadel_pat.unwrap_or_else(|| self.zitadel_pat.clone()), + openbao_url: self.openbao_url.clone(), + openbao_token: openbao_token.unwrap_or_else(|| self.openbao_token.clone()), + } } async fn all_identities(&self) -> Result, AuthError> { @@ -174,10 +192,22 @@ impl BackendAuth { } async fn policy(&self, name: &str) -> Result, AuthError> { - Ok(self - .json(&format!("sys/policies/acl/{name}")) - .await? - .and_then(|data| data["policy"].as_str().map(str::to_string))) + let response = self + .openbao(Method::GET, &format!("sys/policies/acl/{name}"), None) + .await?; + if matches!( + response.status(), + StatusCode::FORBIDDEN | StatusCode::NOT_FOUND + ) { + return Ok(None); + } + let body: Value = response + .error_for_status() + .map_err(backend)? + .json() + .await + .map_err(backend)?; + Ok(body["data"]["policy"].as_str().map(str::to_string)) } async fn assignments(&self, subject_id: &str) -> Result, AuthError> { @@ -281,6 +311,7 @@ impl BackendAuth { async fn access_for_identities( &self, identities: &[Identity], + include_policy_bodies: bool, ) -> Result, AuthError> { let roles = self.roles().await?; let policy_names = roles @@ -292,15 +323,25 @@ impl BackendAuth { let mut policy_details = BTreeMap::new(); for policy in policy_names { if let Some(body) = self.policy(&policy).await? { - policy_details.insert(policy, (secret_paths(&body), policy_effect(&body), body)); + let (secret_paths, scopes) = policy_access(&body); + policy_details.insert( + policy, + PolicyDetails { + secret_paths, + scopes, + effect: policy_effect(&body), + body, + }, + ); } } let mut roles = roles; for role in &mut roles { for policy in &mut role.role.policies { - policy.body = policy_details - .get(&policy.name) - .map(|details| details.2.clone()); + policy.body = include_policy_bodies + .then(|| policy_details.get(&policy.name)) + .flatten() + .map(|details| details.body.clone()); } } let mut access = Vec::with_capacity(identities.len()); @@ -328,11 +369,15 @@ impl BackendAuth { policy_name: policy.name.clone(), secret_paths: policy_details .get(&policy.name) - .map(|details| details.0.clone()) + .map(|details| details.secret_paths.clone()) + .unwrap_or_default(), + scopes: policy_details + .get(&policy.name) + .map(|details| details.scopes.clone()) .unwrap_or_default(), effect: policy_details .get(&policy.name) - .map(|details| details.1.clone()) + .map(|details| details.effect.clone()) .unwrap_or_else(|| "Custom OpenBao access".into()), }) }) @@ -367,6 +412,33 @@ impl AuthService for BackendAuth { .collect()) } + async fn identities_with_access( + &self, + filter: &IdentityFilter, + ) -> Result, AuthError> { + let identities = self.identities(filter.search.as_deref()).await?; + let access = self.access_for_identities(&identities, false).await?; + let tenant = filter + .tenant + .as_deref() + .map(|tenant| Scope::new(tenant, None)) + .transpose()?; + Ok(identities + .into_iter() + .zip(access) + .filter(|(identity, access)| { + filter + .kind + .as_ref() + .is_none_or(|kind| kind == &identity.kind) + && tenant + .as_ref() + .is_none_or(|scope| access_matches_scope(access, scope)) + }) + .map(|(identity, access)| IdentityWithAccess { identity, access }) + .collect()) + } + async fn identity(&self, subject_id: &str) -> Result { self.all_identities() .await? @@ -375,34 +447,36 @@ impl AuthService for BackendAuth { .ok_or(AuthError::IdentityNotFound) } - async fn access(&self, subject_id: &str) -> Result { + async fn identity_with_access( + &self, + subject_id: &str, + include_policy_bodies: bool, + ) -> Result { let identity = self.identity(subject_id).await?; - self.access_for_identities(&[identity]) + let access = self + .access_for_identities(std::slice::from_ref(&identity), include_policy_bodies) .await? .pop() - .ok_or(AuthError::IdentityNotFound) - } - - async fn access_for(&self, identities: &[Identity]) -> Result, AuthError> { - self.access_for_identities(identities).await + .ok_or(AuthError::IdentityNotFound)?; + Ok(IdentityWithAccess { identity, access }) } async fn tenants(&self) -> Result, AuthError> { let identities = self.all_identities().await?; let mut tenants: BTreeMap, BTreeSet)> = BTreeMap::new(); - let access = self.access_for_identities(&identities).await?; + let access = self.access_for_identities(&identities, false).await?; for (identity, access) in identities.into_iter().zip(access) { for scope in access .assignments .iter() .map(|assignment| assignment.scope.clone()) - .chain(access.imported.iter().flat_map(|access| { - access.secret_paths.iter().filter_map(|path| { - let mut parts = path.split('/'); - Scope::new(parts.next()?, parts.next()).ok() - }) - })) + .chain( + access + .imported + .iter() + .flat_map(|access| access.scopes.iter().cloned()), + ) { let entry = tenants .entry(scope.path()) @@ -424,12 +498,37 @@ impl AuthService for BackendAuth { .collect()) } + async fn tenant_access(&self, scope: &Scope) -> Result, AuthError> { + let mut rows = self + .identities_with_access(&IdentityFilter { + tenant: Some(scope.tenant.clone()), + ..IdentityFilter::default() + }) + .await? + .into_iter() + .filter(|row| access_matches_scope(&row.access, scope)) + .collect::>(); + for row in &mut rows { + row.access + .assignments + .retain(|assignment| scope_matches_target(&assignment.scope, scope)); + row.access.imported.retain(|access| { + access + .scopes + .iter() + .any(|candidate| scope_matches_target(candidate, scope)) + }); + row.access.roles.clear(); + } + Ok(rows) + } + async fn plan_assignment( &self, request: AssignmentRequest, ) -> Result { let identity = self.identity(&request.subject_id).await?; - let mut plan = harmony_auth::plan_assignment(&identity, request, "secret")?; + let mut plan = crate::plan_assignment(&identity, request, "secret")?; if let Some(existing) = self .assignments(&identity.subject_id) .await? @@ -446,7 +545,8 @@ impl AuthService for BackendAuth { Ok(plan) } - async fn apply_assignment(&self, plan: AssignmentPlan) -> Result { + async fn apply_assignment(&self, request: AssignmentRequest) -> Result { + let plan = self.plan_assignment(request).await?; let role = self .role_for_assignment(&plan.assignment.subject_id) .await?; @@ -483,6 +583,24 @@ impl AuthService for BackendAuth { Ok(plan.assignment) } + async fn plan_removal( + &self, + subject_id: &str, + assignment_id: Uuid, + ) -> Result { + let identity = self.identity(subject_id).await?; + let assignment = self + .assignments(subject_id) + .await? + .into_iter() + .find(|assignment| assignment.id == assignment_id) + .ok_or_else(|| AuthError::Invalid("assignment does not exist".into()))?; + Ok(RemovalPlan { + identity, + assignment, + }) + } + async fn remove_assignment( &self, subject_id: &str, @@ -580,16 +698,49 @@ fn strings(value: &Value) -> Vec { .collect() } -fn secret_paths(policy: &str) -> Vec { - policy +fn policy_access(policy: &str) -> (Vec, Vec) { + let paths = policy .lines() .filter_map(|line| line.split_once("path \"").map(|(_, rest)| rest)) .filter_map(|rest| rest.split_once('"').map(|(path, _)| path)) .filter_map(|path| path.split_once("/data/").map(|(_, logical)| logical)) + .collect::>(); + let secret_paths = paths + .iter() .map(|path| path.trim_end_matches("/*").to_string()) + .collect(); + let scopes = paths + .into_iter() + .filter_map(|path| path.strip_suffix("/*")) + .filter_map(|path| { + let parts = path.split('/').collect::>(); + match parts.as_slice() { + [tenant] => Scope::new(tenant, None).ok(), + [tenant, project] => Scope::new(tenant, Some(project)).ok(), + _ => None, + } + }) .collect::>() .into_iter() - .collect() + .collect(); + (secret_paths, scopes) +} + +fn access_matches_scope(access: &IdentityAccess, target: &Scope) -> bool { + access + .assignments + .iter() + .map(|assignment| &assignment.scope) + .chain(access.imported.iter().flat_map(|access| &access.scopes)) + .any(|scope| scope_matches_target(scope, target)) +} + +fn scope_matches_target(scope: &Scope, target: &Scope) -> bool { + scope.tenant == target.tenant + && target + .project + .as_ref() + .is_none_or(|project| scope.project.as_ref().is_none_or(|value| value == project)) } fn policy_effect(policy: &str) -> String { @@ -665,11 +816,41 @@ mod tests { let role = json!({ "bound_subject": "n/a", "bound_claims": { "sub": "123" } }); assert_eq!(role_subject(&role), "123"); assert_eq!( - secret_paths( + policy_access( "path \"secret/data/devsights/folk/*\" { capabilities = [\"read\"] }\n\ path \"secret/metadata/devsights/folk/*\" { capabilities = [\"list\"] }" - ), + ) + .0, ["devsights/folk"] ); } + + #[test] + fn recognizes_only_tenant_and_project_wildcard_roots() { + let (_, scopes) = policy_access( + "path \"secret/data/devsights/*\" {}\n\ + path \"secret/data/devsights/folk/*\" {}\n\ + path \"secret/data/devsights/folk/database/*\" {}\n\ + path \"secret/data/devsights/api-key\" {}", + ); + + assert_eq!( + scopes, + [ + Scope::new("devsights", None).unwrap(), + Scope::new("devsights", Some("folk")).unwrap(), + ] + ); + } + + #[test] + fn tenant_wide_scope_covers_projects_but_other_projects_do_not() { + let tenant = Scope::new("devsights", None).unwrap(); + let folk = Scope::new("devsights", Some("folk")).unwrap(); + let fleet = Scope::new("devsights", Some("fleet")).unwrap(); + + assert!(scope_matches_target(&tenant, &folk)); + assert!(scope_matches_target(&folk, &folk)); + assert!(!scope_matches_target(&fleet, &folk)); + } } diff --git a/harmony_auth/src/lib.rs b/harmony_auth/src/lib.rs index b1e6b9af..48d04f62 100644 --- a/harmony_auth/src/lib.rs +++ b/harmony_auth/src/lib.rs @@ -5,6 +5,10 @@ use sha2::{Digest, Sha256}; use thiserror::Error; use uuid::Uuid; +mod backend; + +pub use backend::BackendAuth; + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum IdentityKind { @@ -54,9 +58,16 @@ impl Permission { Self::ReadOnly => "Read secrets", } } + + pub fn available_for(kind: &IdentityKind) -> &'static [Self] { + match kind { + IdentityKind::Human => &[Self::TenantAdmin, Self::ReadOnly], + IdentityKind::Service => &[Self::CdDeployer, Self::ReadOnly], + } + } } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct Scope { pub tenant: String, pub project: Option, @@ -99,21 +110,22 @@ pub struct Assignment { pub created_at: DateTime, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct ImportedAccess { pub role_name: String, pub policy_name: String, pub secret_paths: Vec, + pub scopes: Vec, pub effect: String, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct OpenBaoPolicy { pub name: String, pub body: Option, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct JwtRole { pub name: String, pub subject_id: String, @@ -127,20 +139,44 @@ impl JwtRole { } } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct IdentityAccess { pub assignments: Vec, pub imported: Vec, pub roles: Vec, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub struct TenantSummary { pub scope: Scope, pub humans: usize, pub services: usize, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct IdentityWithAccess { + pub identity: Identity, + pub access: IdentityAccess, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct IdentityFilter { + pub search: Option, + pub kind: Option, + pub tenant: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct BackendConnection { + pub connected: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ConnectionStatus { + pub zitadel: BackendConnection, + pub openbao: BackendConnection, +} + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct AssignmentRequest { pub subject_id: String, @@ -156,6 +192,12 @@ pub struct AssignmentPlan { pub policy: String, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RemovalPlan { + pub identity: Identity, + pub assignment: Assignment, +} + #[derive(Debug, Error)] pub enum AuthError { #[error("identity not found")] @@ -169,15 +211,28 @@ pub enum AuthError { #[async_trait] pub trait AuthService: Send + Sync { async fn identities(&self, search: Option<&str>) -> Result, AuthError>; + async fn identities_with_access( + &self, + filter: &IdentityFilter, + ) -> Result, AuthError>; async fn identity(&self, subject_id: &str) -> Result; - async fn access(&self, subject_id: &str) -> Result; - async fn access_for(&self, identities: &[Identity]) -> Result, AuthError>; + async fn identity_with_access( + &self, + subject_id: &str, + include_policy_bodies: bool, + ) -> Result; async fn tenants(&self) -> Result, AuthError>; + async fn tenant_access(&self, scope: &Scope) -> Result, AuthError>; async fn plan_assignment( &self, request: AssignmentRequest, ) -> Result; - async fn apply_assignment(&self, plan: AssignmentPlan) -> Result; + async fn apply_assignment(&self, request: AssignmentRequest) -> Result; + async fn plan_removal( + &self, + subject_id: &str, + assignment_id: Uuid, + ) -> Result; async fn remove_assignment( &self, subject_id: &str, @@ -196,10 +251,11 @@ pub fn plan_assignment( if identity.subject_id != request.subject_id { return Err(AuthError::Invalid("identity does not match request".into())); } - if request.permission == Permission::CdDeployer && identity.kind != IdentityKind::Service { - return Err(AuthError::Invalid( - "CD Deployer is intended for a service account".into(), - )); + if !Permission::available_for(&identity.kind).contains(&request.permission) { + return Err(AuthError::Invalid(format!( + "{} is not available for this identity type", + request.permission.label() + ))); } if request.permission == Permission::CdDeployer && request.project.trim().is_empty() { return Err(AuthError::Invalid("CD Deployer requires a project".into())); @@ -310,4 +366,21 @@ mod tests { Err(AuthError::Invalid(_)) )); } + + #[test] + fn rejects_tenant_admin_for_service_account() { + assert!(matches!( + plan_assignment( + &service(), + AssignmentRequest { + subject_id: "123".into(), + permission: Permission::TenantAdmin, + tenant: "devsights".into(), + project: String::new(), + }, + "secret" + ), + Err(AuthError::Invalid(_)) + )); + } } diff --git a/harmony_auth_cli/Cargo.toml b/harmony_auth_cli/Cargo.toml new file mode 100644 index 00000000..fad83d54 --- /dev/null +++ b/harmony_auth_cli/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "harmony_auth_cli" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[[bin]] +name = "harmony-auth" +path = "src/main.rs" + +[dependencies] +harmony_auth = { path = "../harmony_auth" } +clap.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/harmony_auth_cli/src/main.rs b/harmony_auth_cli/src/main.rs new file mode 100644 index 00000000..0f783e4c --- /dev/null +++ b/harmony_auth_cli/src/main.rs @@ -0,0 +1,481 @@ +use std::{env, process::ExitCode}; + +use clap::{Parser, Subcommand, ValueEnum}; +use harmony_auth::{ + AuthError, AuthService, BackendAuth, ConnectionStatus, IdentityAccess, IdentityFilter, + IdentityKind, IdentityWithAccess, Scope, TenantSummary, +}; +use serde_json::{Value, json}; + +#[derive(Parser)] +#[command( + name = "harmony-auth", + version, + about = "Inspect Harmony authorization" +)] +struct Cli { + #[arg(long, env = "ZITADEL_URL", global = true)] + zitadel_url: Option, + + #[arg(long, env = "OPENBAO_URL", global = true)] + openbao_url: Option, + + #[arg(long, global = true)] + json: bool, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + Connection { + #[command(subcommand)] + command: ConnectionCommand, + }, + Identity { + #[command(subcommand)] + command: IdentityCommand, + }, + Tenant { + #[command(subcommand)] + command: TenantCommand, + }, +} + +#[derive(Subcommand)] +enum ConnectionCommand { + Check, +} + +#[derive(Subcommand)] +enum IdentityCommand { + List { + #[arg(long)] + search: Option, + #[arg(long)] + kind: Option, + #[arg(long)] + tenant: Option, + }, + Show { + subject_id: String, + #[arg(long)] + advanced: bool, + }, +} + +#[derive(Subcommand)] +enum TenantCommand { + List, + Show { + tenant: String, + #[arg(long)] + project: Option, + }, +} + +#[derive(Clone, ValueEnum)] +enum Kind { + Human, + Service, +} + +enum Output { + Connection(ConnectionStatus), + IdentityList(Vec), + IdentityShow(IdentityWithAccess, bool), + TenantList(Vec), + TenantShow(Scope, Vec), +} + +#[tokio::main] +async fn main() -> ExitCode { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .init(); + let cli = Cli::parse(); + let command = cli.command.name(); + match run(&cli).await { + Ok(output) => { + output.print(cli.json); + if output.succeeded() { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } + } + Err(error) => { + if cli.json { + println!( + "{}", + json!({ + "schema_version": 1, + "command": command, + "error": { + "kind": error_kind(&error), + "message": error.to_string(), + } + }) + ); + } else { + eprintln!("{error}"); + } + ExitCode::from(if matches!(error, AuthError::Backend(_)) { + 1 + } else { + 2 + }) + } + } +} + +async fn run(cli: &Cli) -> Result { + let auth = BackendAuth::new( + connection_value(cli.zitadel_url.as_ref(), "ZITADEL_URL")?, + credential("ZITADEL_PAT")?, + connection_value(cli.openbao_url.as_ref(), "OPENBAO_URL")?, + credential("OPENBAO_TOKEN")?, + ) + .map_err(AuthError::Backend)?; + + match &cli.command { + Command::Connection { + command: ConnectionCommand::Check, + } => Ok(Output::Connection(auth.connection_status().await)), + Command::Identity { + command: + IdentityCommand::List { + search, + kind, + tenant, + }, + } => { + let rows = auth + .identities_with_access(&IdentityFilter { + search: search.clone(), + kind: kind.as_ref().map(|kind| match kind { + Kind::Human => IdentityKind::Human, + Kind::Service => IdentityKind::Service, + }), + tenant: tenant.clone(), + }) + .await?; + Ok(Output::IdentityList(rows)) + } + Command::Identity { + command: + IdentityCommand::Show { + subject_id, + advanced, + }, + } => { + let row = auth.identity_with_access(subject_id, *advanced).await?; + Ok(Output::IdentityShow(row, *advanced)) + } + Command::Tenant { + command: TenantCommand::List, + } => Ok(Output::TenantList(auth.tenants().await?)), + Command::Tenant { + command: TenantCommand::Show { tenant, project }, + } => { + let scope = Scope::new(tenant, project.as_deref())?; + let rows = auth.tenant_access(&scope).await?; + Ok(Output::TenantShow(scope, rows)) + } + } +} + +fn credential(name: &str) -> Result { + env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .ok_or_else(|| AuthError::Invalid(format!("{name} is required"))) +} + +fn connection_value(value: Option<&String>, name: &str) -> Result { + value + .filter(|value| !value.is_empty()) + .cloned() + .ok_or_else(|| AuthError::Invalid(format!("{name} is required"))) +} + +impl Command { + fn name(&self) -> &'static str { + match self { + Self::Connection { .. } => "connection.check", + Self::Identity { + command: IdentityCommand::List { .. }, + } => "identity.list", + Self::Identity { + command: IdentityCommand::Show { .. }, + } => "identity.show", + Self::Tenant { + command: TenantCommand::List, + } => "tenant.list", + Self::Tenant { + command: TenantCommand::Show { .. }, + } => "tenant.show", + } + } +} + +impl Output { + fn succeeded(&self) -> bool { + match self { + Self::Connection(status) => status.zitadel.connected && status.openbao.connected, + _ => true, + } + } + + fn print(&self, json_output: bool) { + if json_output { + println!( + "{}", + json!({ + "schema_version": 1, + "command": self.command(), + "result": self.value(), + }) + ); + } else { + self.print_human(); + } + } + + fn command(&self) -> &'static str { + match self { + Self::Connection(_) => "connection.check", + Self::IdentityList(_) => "identity.list", + Self::IdentityShow(_, _) => "identity.show", + Self::TenantList(_) => "tenant.list", + Self::TenantShow(_, _) => "tenant.show", + } + } + + fn value(&self) -> Value { + match self { + Self::Connection(status) => json!(status), + Self::IdentityList(identities) => json!({ "identities": identities }), + Self::IdentityShow(row, _) => { + json!({ "identity": row.identity, "access": row.access }) + } + Self::TenantList(tenants) => json!({ "tenants": tenants }), + Self::TenantShow(scope, identities) => json!({ + "tenant": scope.tenant, + "project": scope.project, + "identities": identities, + }), + } + } + + fn print_human(&self) { + match self { + Self::Connection(status) => { + println!("Zitadel {}", connection_label(status.zitadel.connected)); + println!("OpenBao {}", connection_label(status.openbao.connected)); + } + Self::IdentityList(rows) => { + if rows.is_empty() { + println!("No identities found."); + } + for row in rows { + print_identity_list_item(row); + } + } + Self::IdentityShow(row, advanced) => print_identity_show(row, *advanced), + Self::TenantList(tenants) => { + println!("TENANT\tPROJECT\tHUMANS\tSERVICES"); + for tenant in tenants { + println!( + "{}\t{}\t{}\t{}", + tenant.scope.tenant, + tenant.scope.project.as_deref().unwrap_or("All projects"), + tenant.humans, + tenant.services + ); + } + } + Self::TenantShow(scope, rows) => { + println!("{}", scope.label()); + if rows.is_empty() { + println!(" No identities found."); + } + for row in rows { + println!( + " {} {}", + row.identity.display_name, + kind_label(&row.identity.kind) + ); + print_access(&row.access, " "); + } + } + } + } +} + +fn print_identity_list_item(row: &IdentityWithAccess) { + println!( + "{} {} {} {}", + if row.identity.active { + "ACTIVE" + } else { + "SUSPENDED" + }, + kind_label(&row.identity.kind).to_uppercase(), + row.identity.display_name, + row.identity.login_name + ); + println!(" subject {}", row.identity.subject_id); + print_access(&row.access, " "); + println!(); +} + +fn print_identity_show(row: &IdentityWithAccess, advanced: bool) { + println!("{}", row.identity.display_name); + println!(" Subject: {}", row.identity.subject_id); + println!(" Login: {}", row.identity.login_name); + println!( + " Email: {}", + row.identity.email.as_deref().unwrap_or("None") + ); + println!(" Kind: {}", kind_label(&row.identity.kind)); + println!( + " Status: {}", + if row.identity.active { + "Active" + } else { + "Suspended" + } + ); + + println!("\nHarmony permissions"); + if row.access.assignments.is_empty() { + println!(" None"); + } + for assignment in &row.access.assignments { + println!( + " {} {} {}", + assignment.id, + assignment.permission.label(), + assignment.scope.label() + ); + } + + println!("\nImported OpenBao access"); + if row.access.imported.is_empty() { + println!(" None"); + } + for imported in &row.access.imported { + let scopes = if imported.scopes.is_empty() { + imported.secret_paths.join(", ") + } else { + imported + .scopes + .iter() + .map(Scope::label) + .collect::>() + .join(", ") + }; + println!( + " {} {} {}", + imported.policy_name, scopes, imported.effect + ); + } + + if advanced { + println!("\nAdvanced OpenBao details"); + print_roles(&row.access, " "); + } +} + +fn print_access(access: &IdentityAccess, indent: &str) { + if access.assignments.is_empty() && access.imported.is_empty() { + println!("{indent}No access"); + } + for assignment in &access.assignments { + println!( + "{indent}{} {} {}", + assignment.id, + assignment.scope.label(), + assignment.permission.label() + ); + } + for imported in &access.imported { + let scopes = if imported.scopes.is_empty() { + imported.secret_paths.join(", ") + } else { + imported + .scopes + .iter() + .map(Scope::label) + .collect::>() + .join(", ") + }; + println!( + "{indent}Imported {} {} {}", + imported.policy_name, scopes, imported.effect + ); + } +} + +fn print_roles(access: &IdentityAccess, indent: &str) { + if access.roles.is_empty() { + println!("{indent}No matching JWT role"); + } + for role in &access.roles { + println!("{indent}JWT role {}", role.name); + println!("{indent} Subject: {}", role.subject_id); + println!( + "{indent} Audiences: {}", + if role.bound_audiences.is_empty() { + "None".into() + } else { + role.bound_audiences.join(", ") + } + ); + for policy in &role.policies { + println!("{indent} Policy: {}", policy.name); + if let Some(body) = &policy.body { + for line in body.lines() { + println!("{indent} {line}"); + } + } else { + println!("{indent} Policy body unavailable"); + } + } + } +} + +fn kind_label(kind: &IdentityKind) -> &'static str { + match kind { + IdentityKind::Human => "Human", + IdentityKind::Service => "Service", + } +} + +fn connection_label(connected: bool) -> &'static str { + if connected { "connected" } else { "failed" } +} + +fn error_kind(error: &AuthError) -> &'static str { + match error { + AuthError::IdentityNotFound => "not_found", + AuthError::Invalid(_) => "invalid", + AuthError::Backend(_) => "backend", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_identity_first_command_tree_without_connection_values() { + let cli = + Cli::try_parse_from(["harmony-auth", "identity", "list", "--kind", "human"]).unwrap(); + + assert_eq!(cli.command.name(), "identity.list"); + } +} diff --git a/harmony_auth_ui/Cargo.toml b/harmony_auth_ui/Cargo.toml index d9da85ca..2b79332b 100644 --- a/harmony_auth_ui/Cargo.toml +++ b/harmony_auth_ui/Cargo.toml @@ -8,15 +8,12 @@ license.workspace = true [dependencies] harmony_auth = { path = "../harmony_auth" } anyhow.workspace = true -async-trait.workspace = true axum = "0.8" axum-extra = { version = "0.10", features = ["cookie"] } -chrono = { workspace = true, features = ["serde"] } clap.workspace = true maud = { version = "0.27", features = ["axum"] } reqwest.workspace = true serde.workspace = true -serde_json.workspace = true tokio.workspace = true tower-http = { version = "0.6", features = ["set-header"] } tracing.workspace = true diff --git a/harmony_auth_ui/src/main.rs b/harmony_auth_ui/src/main.rs index 7ab0ff64..7ef77437 100644 --- a/harmony_auth_ui/src/main.rs +++ b/harmony_auth_ui/src/main.rs @@ -1,7 +1,6 @@ -mod backend; mod views; -use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; use anyhow::Result; use axum::{ @@ -15,7 +14,10 @@ use axum::{ }; use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite}; use clap::Parser; -use harmony_auth::{AssignmentRequest, AuthError, AuthService, IdentityKind, Permission}; +use harmony_auth::{ + AssignmentRequest, AuthError, AuthService, BackendAuth, IdentityFilter, IdentityKind, + Permission, +}; use serde::Deserialize; use tokio::sync::RwLock; use tower_http::set_header::SetResponseHeaderLayer; @@ -29,7 +31,6 @@ struct Args { #[derive(Clone)] struct AppState { - client: reqwest::Client, sessions: Arc>>>, connection_errors: Arc>>, } @@ -37,7 +38,7 @@ struct AppState { #[derive(Clone)] struct ConnectedProfile { name: String, - auth: Arc, + auth: Arc, } #[derive(Default, Deserialize)] @@ -85,10 +86,6 @@ async fn main() -> Result<()> { .init(); let args = Args::parse(); let state = AppState { - client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(10)) - .build()?, sessions: Arc::default(), connection_errors: Arc::default(), }; @@ -237,18 +234,9 @@ async fn connect_profile( .and_then(|profiles| profiles.get(&form.profile_id)) .cloned(); let backend = if let Some(existing) = &existing { - let (old_zitadel, old_openbao) = existing.auth.credentials(); Arc::new(existing.auth.with_credentials( - if form.zitadel_pat.is_empty() { - old_zitadel.into() - } else { - form.zitadel_pat - }, - if form.openbao_token.is_empty() { - old_openbao.into() - } else { - form.openbao_token - }, + (!form.zitadel_pat.is_empty()).then_some(form.zitadel_pat), + (!form.openbao_token.is_empty()).then_some(form.openbao_token), )) } else if form.zitadel_pat.is_empty() || form.openbao_token.is_empty() { return connection_error( @@ -259,13 +247,16 @@ async fn connect_profile( ) .await; } else { - Arc::new(backend::BackendAuth::new( - state.client.clone(), + let backend = match BackendAuth::new( form.zitadel_url, form.zitadel_pat, form.openbao_url, form.openbao_token, - )) + ) { + Ok(backend) => backend, + Err(error) => return connection_error(&state, jar, session_id, error).await, + }; + Arc::new(backend) }; if let Err(error) = backend.validate().await { return connection_error(&state, jar, session_id, error).await; @@ -382,34 +373,22 @@ async fn dashboard( Query(query): Query, ) -> Result { let profile = active_profile(&state, &jar).await?; - let mut rows = Vec::new(); - let identities = profile.auth.identities(query.q.as_deref()).await?; - let access = profile.auth.access_for(&identities).await?; - for (identity, access) in identities.into_iter().zip(access) { - let kind_matches = query.kind.as_deref().is_none_or(|kind| { - kind.is_empty() - || matches!( - (&identity.kind, kind), - (IdentityKind::Human, "human") | (IdentityKind::Service, "service") - ) - }); - let tenant_matches = query.tenant.as_deref().is_none_or(|tenant| { - tenant.is_empty() - || access - .assignments - .iter() - .any(|assignment| assignment.scope.tenant == tenant) - || access.imported.iter().any(|imported| { - imported - .secret_paths - .iter() - .any(|path| path.starts_with(tenant)) - }) - }); - if kind_matches && tenant_matches { - rows.push((identity, access)); - } - } + let kind = match query.kind.as_deref() { + Some("human") => Some(IdentityKind::Human), + Some("service") => Some(IdentityKind::Service), + _ => None, + }; + let rows = profile + .auth + .identities_with_access(&IdentityFilter { + search: query.q.clone(), + kind, + tenant: query.tenant.clone(), + }) + .await? + .into_iter() + .map(|row| (row.identity, row.access)) + .collect::>(); Ok(views::dashboard( &profile.view(), &rows, @@ -425,9 +404,8 @@ async fn identity( Path(subject_id): Path, ) -> Result { let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&subject_id).await?; - let access = profile.auth.access(&subject_id).await?; - Ok(views::identity(&profile.view(), &identity, &access)) + let row = profile.auth.identity_with_access(&subject_id, true).await?; + Ok(views::identity(&profile.view(), &row.identity, &row.access)) } async fn assignment_review( @@ -456,16 +434,15 @@ async fn apply_assignment( Form(form): Form, ) -> Result { let profile = active_profile(&state, &jar).await?; - let plan = profile + let assignment = profile .auth - .plan_assignment(AssignmentRequest { + .apply_assignment(AssignmentRequest { subject_id: subject_id.clone(), permission: form.permission, tenant: form.tenant, project: form.project, }) .await?; - let assignment = profile.auth.apply_assignment(plan).await?; tracing::info!( %subject_id, permission = assignment.permission.label(), @@ -495,19 +472,14 @@ async fn remove_assignment_review( Path((subject_id, assignment_id)): Path<(String, Uuid)>, ) -> Result { let profile = active_profile(&state, &jar).await?; - let identity = profile.auth.identity(&subject_id).await?; - let assignment = profile + let plan = profile .auth - .access(&subject_id) - .await? - .assignments - .into_iter() - .find(|assignment| assignment.id == assignment_id) - .ok_or_else(|| AuthError::Invalid("assignment does not exist".into()))?; + .plan_removal(&subject_id, assignment_id) + .await?; Ok(views::remove_assignment_review( &profile.view(), - &identity, - &assignment, + &plan.identity, + &plan.assignment, )) } diff --git a/harmony_auth_ui/src/views.rs b/harmony_auth_ui/src/views.rs index ec2d745e..97556c9d 100644 --- a/harmony_auth_ui/src/views.rs +++ b/harmony_auth_ui/src/views.rs @@ -162,12 +162,8 @@ pub fn identity(profile: &Profile<'_>, identity: &Identity, access: &IdentityAcc @if identity.active { form class="form assignment-form" action=(format!("/identities/{}/assignments/review", identity.subject_id)) method="get" autocomplete="off" { label { "Permission" select name="permission" required { - @if identity.kind == IdentityKind::Human { - option value="tenant_admin" { "Tenant Admin" } - option value="read_only" { "Read-only" } - } @else { - option value="cd_deployer" { "CD Deployer" } - option value="read_only" { "Read-only" } + @for permission in Permission::available_for(&identity.kind) { + option value=(permission_value(*permission)) { (permission.label()) } } } } label { "Tenant" input name="tenant" list="tenant-options" value="" required placeholder="devsights"; } -- 2.39.5 From f7feba36e6bf277dcc611073c828669456addceb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 20 Jul 2026 22:53:36 -0400 Subject: [PATCH 10/47] feat: fleet deploy supports alternative credential store --- fleet/harmony-fleet-deploy/src/app.rs | 45 +++++++++++++++++---------- fleet/harmony-fleet-deploy/src/lib.rs | 2 +- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/fleet/harmony-fleet-deploy/src/app.rs b/fleet/harmony-fleet-deploy/src/app.rs index d8cc8293..b20e4306 100644 --- a/fleet/harmony-fleet-deploy/src/app.rs +++ b/fleet/harmony-fleet-deploy/src/app.rs @@ -167,31 +167,27 @@ impl HarmonyApp for FleetCrdsApp { pub struct FleetTenantProvisionApp { tenant: TenantConfig, - credential_store: Arc, + credential_store: TenantCredentialStore, +} + +enum TenantCredentialStore { + Client(Arc), + OpenBao(OpenBaoClusterAccess), } impl FleetTenantProvisionApp { pub fn new(tenant: TenantConfig, credential_store: Arc) -> Self { Self { tenant, - credential_store, + credential_store: TenantCredentialStore::Client(credential_store), } } - pub async fn from_openbao( - tenant: TenantConfig, - credential_store: &OpenBaoClusterAccess, - ) -> Result { - let source = harmony_config::openbao_source( - credential_store.namespace.as_ref(), - Some(credential_store.url.to_string()), - Some(credential_store.zitadel_url.to_string()), - Some(credential_store.zitadel_audience.to_string()), - Some(credential_store.role.to_string()), - ) - .await - .ok_or_else(|| anyhow::anyhow!("tenant credential store is unavailable"))?; - Ok(Self::new(tenant, Arc::new(ConfigClient::new(vec![source])))) + pub fn from_openbao(tenant: TenantConfig, credential_store: OpenBaoClusterAccess) -> Self { + Self { + tenant, + credential_store: TenantCredentialStore::OpenBao(credential_store), + } } } @@ -209,6 +205,21 @@ impl HarmonyApp for FleetTenantProvisionApp { _ctx: &AppContext, _images: &ImageRefs, ) -> Result>>, AppError> { + let credential_store = match &self.credential_store { + TenantCredentialStore::Client(client) => client.clone(), + TenantCredentialStore::OpenBao(store) => { + let source = harmony_config::openbao_source( + store.namespace.as_ref(), + Some(store.url.to_string()), + Some(store.zitadel_url.to_string()), + Some(store.zitadel_audience.to_string()), + Some(store.role.to_string()), + ) + .await + .ok_or_else(|| AppError::Deploy("tenant credential store is unavailable".into()))?; + Arc::new(ConfigClient::new(vec![source])) + } + }; let namespace = self .tenant .name @@ -224,7 +235,7 @@ impl HarmonyApp for FleetTenantProvisionApp { .parse() .expect("static Kubernetes name is valid"), fleet_deployer_rules(), - self.credential_store.clone(), + credential_store, )), ]) } diff --git a/fleet/harmony-fleet-deploy/src/lib.rs b/fleet/harmony-fleet-deploy/src/lib.rs index 85de10df..790642a5 100644 --- a/fleet/harmony-fleet-deploy/src/lib.rs +++ b/fleet/harmony-fleet-deploy/src/lib.rs @@ -38,7 +38,7 @@ pub async fn provision_fleet_tenant_with_context( tenant: harmony::topology::tenant::TenantConfig, credential_store: harmony_app::OpenBaoClusterAccess, ) -> anyhow::Result<()> { - let app = FleetTenantProvisionApp::from_openbao(tenant, &credential_store).await?; + let app = FleetTenantProvisionApp::from_openbao(tenant, credential_store); let contexts = harmony_app::ContextCatalog::new([context])?; harmony_cli::app::app_main(app, contexts).await } -- 2.39.5 From 33900faa1a4f9c512c0d563d1b2673ce07e7bb23 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 21 Jul 2026 14:26:13 -0400 Subject: [PATCH 11/47] feat: complete fleet deployment flow --- Cargo.lock | 2 + docs/guides/fleet-staging-install.md | 14 +- docs/guides/operator-dashboard-sso.md | 30 +- .../src/bin/tenant-provision.rs | 20 +- examples/fleet_typed_deploy/src/lib.rs | 20 - examples/harmony_sso/src/main.rs | 1 + examples/monitoring_with_tenant/src/main.rs | 1 + examples/openbao/src/main.rs | 2 + fleet/harmony-fleet-deploy/Cargo.toml | 4 + fleet/harmony-fleet-deploy/src/app.rs | 224 +++++++- .../src/bin/harmony-fleet-release.rs | 30 + fleet/harmony-fleet-deploy/src/lib.rs | 32 ++ .../src/operator/chart.rs | 35 +- .../src/operator/score.rs | 326 ++++++----- .../harmony-fleet-e2e/tests/openbao_groups.rs | 2 + .../harmony-fleet-e2e/tests/openbao_policy.rs | 2 + harmony-k8s/src/client.rs | 30 +- harmony/Cargo.toml | 1 + harmony/src/domain/topology/tenant/k8s.rs | 29 +- harmony/src/domain/topology/tenant/mod.rs | 3 + harmony/src/modules/nats/score_nats.rs | 30 +- harmony/src/modules/openbao/mod.rs | 29 +- harmony/src/modules/openbao/setup.rs | 517 ++++++++++-------- harmony/src/modules/tenant/credentials.rs | 10 + harmony/src/modules/zitadel/setup.rs | 45 +- harmony_app/src/context.rs | 22 + harmony_app/src/publish.rs | 32 +- harmony_cli/src/cli_logger.rs | 6 +- harmony_zitadel_auth/src/config.rs | 26 + harmony_zitadel_jwt/Cargo.toml | 1 + harmony_zitadel_jwt/src/lib.rs | 35 +- 31 files changed, 1085 insertions(+), 476 deletions(-) create mode 100644 fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs diff --git a/Cargo.lock b/Cargo.lock index 45b1838b..7c16a1e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4012,6 +4012,7 @@ dependencies = [ "tokio-util", "url", "uuid", + "vaultrs", "virt", "walkdir", "webbrowser", @@ -4613,6 +4614,7 @@ name = "harmony_zitadel_jwt" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "chrono", "jsonwebtoken", "reqwest 0.12.28", diff --git a/docs/guides/fleet-staging-install.md b/docs/guides/fleet-staging-install.md index f530b4ad..f2a3ea42 100644 --- a/docs/guides/fleet-staging-install.md +++ b/docs/guides/fleet-staging-install.md @@ -72,10 +72,12 @@ old releases. Disposable local and E2E clusters should be recreated instead. ## Provision the tenant -Run the tenant provisioning binary with a platform-admin context. It applies -`TenantScore`, creates the Fleet deployer ServiceAccount and RBAC, issues its -kubeconfig, and stores `ClusterAccess` through Harmony Config. The credential -store role must be able to write the tenant's OpenBao namespace. +Run the tenant provisioning binary with an explicit administrator kubeconfig. +It applies `TenantScore`, creates the Fleet deployer ServiceAccount and RBAC, +issues its namespace-scoped kubeconfig, and stores `ClusterAccess` through +Harmony Config. Do not store the administrator kubeconfig in OpenBao or expose +it to the tenant CI/CD identity. The provisioning token only needs write access +to the tenant's OpenBao namespace. The deployer can create namespaced Roles and RoleBindings because Helm installs the Fleet operator's runtime RBAC. Kubernetes prevents it from binding rights it @@ -88,8 +90,8 @@ provisioning to create and store a replacement. This remains the rotation path until short-lived TokenRequest brokerage is implemented. ```bash -cargo run --release --bin tenant-provision -- \ - deploy --context platform-admin +export KUBECONFIG=/secure/path/platform-admin.kubeconfig +cargo run --release --bin tenant-provision ``` Fleet creates the immutable NATS callout credential Secret on first deploy and diff --git a/docs/guides/operator-dashboard-sso.md b/docs/guides/operator-dashboard-sso.md index c22eafb2..b3c6b6ee 100644 --- a/docs/guides/operator-dashboard-sso.md +++ b/docs/guides/operator-dashboard-sso.md @@ -5,21 +5,14 @@ public client). Distinct from the agent/callout machine auth ([fleet-zitadel-faq](./fleet-zitadel-faq.md)); the security rationale is in [web-auth-security](./web-auth-security.md). Code: `harmony_zitadel_auth/`. -## Quickstart (staging) +## Deployment -1. **Zitadel app** — create a **Web** application, auth method **PKCE** (no client - secret), redirect URI `https://fleet-stg./auth/callback`, post-logout URI - `https://fleet-stg./`. Copy its **Client ID**. -2. **Seed config** in OpenBao (namespace `fleet-staging`) — the deploy derives every - host from `base_domain`, so you set only: - - `FleetDeployConfig.operator_oidc_client_id` = the Client ID - - `FleetDeployConfig.operator_trusted_audiences` = `[""]` - - `FleetDeploySecrets.operator_cookie_key_b64` = `openssl rand -base64 64` -3. **Deploy**: `./fleet/scripts/dev-deploy-operator.sh` -4. Open `https://fleet-stg./` → Zitadel login → back to the dashboard. - -`fleet_staging_install` already generates the cookie key, so a fresh install needs -only the Client ID + audiences. +`FleetApp` declares a dedicated dashboard Web PKCE application with the +dashboard callback and logout URLs. The operator's Device Code application is +separate. `ZitadelSetupScore` reconciles both and publishes the dashboard client +ID. `FleetOperatorScore` builds `ZitadelAuthConfig` from that output and the +dashboard Ingress, then generates and retains the session cookie key in the +operator Secret. No client ID or cookie key is entered by hand. ## Local dev (`serve-web`) @@ -42,11 +35,10 @@ on the app's **Development Mode** (Zitadel rejects non-HTTPS redirects otherwise ## Config reference -The operator reads `ZitadelAuthConfig` + `OperatorCookieKey` via ConfigClient. The -deploy derives `zitadel_base` / `base_url` / `logout_redirect_uri` from `base_domain` -(`https://sso-stg.`, `https://fleet-stg.`, `…/`) and fixes -`scope = openid profile email`; you supply `client_id`, `trusted_audiences`, -`cookie_key_b64`. All endpoints derive from `zitadel_base`: +The operator reads `ZitadelAuthConfig` and `OperatorCookieKey` through +ConfigClient. The deploy derives `zitadel_base`, `base_url`, client ID, trusted +audience, logout URI, and `scope = openid profile email`. All endpoints derive +from `zitadel_base`: `/.well-known/openid-configuration`, `/oauth/v2/authorize`, `/oauth/v2/token`, `/oidc/v1/end_session`. diff --git a/examples/fleet_typed_deploy/src/bin/tenant-provision.rs b/examples/fleet_typed_deploy/src/bin/tenant-provision.rs index 8c0c63cb..c0236c38 100644 --- a/examples/fleet_typed_deploy/src/bin/tenant-provision.rs +++ b/examples/fleet_typed_deploy/src/bin/tenant-provision.rs @@ -1,8 +1,20 @@ -use example_fleet_typed_deploy::{credential_store, platform_context, tenant_config}; -use harmony_fleet_deploy::provision_fleet_tenant_with_context; +use std::{env, path::PathBuf}; + +use anyhow::Context; +use example_fleet_typed_deploy::{credential_store, fleet_context, tenant_config}; +use harmony_fleet_deploy::provision_fleet_tenant_with_kubeconfig; #[tokio::main] async fn main() -> anyhow::Result<()> { - provision_fleet_tenant_with_context(platform_context()?, tenant_config(), credential_store()?) - .await + let kubeconfig = env::var_os("KUBECONFIG") + .map(PathBuf::from) + .context("set KUBECONFIG to your cluster-admin kubeconfig")?; + provision_fleet_tenant_with_kubeconfig( + fleet_context()?, + kubeconfig, + tenant_config(), + credential_store()?, + false, + ) + .await } diff --git a/examples/fleet_typed_deploy/src/lib.rs b/examples/fleet_typed_deploy/src/lib.rs index d826f5d9..f9b45940 100644 --- a/examples/fleet_typed_deploy/src/lib.rs +++ b/examples/fleet_typed_deploy/src/lib.rs @@ -18,26 +18,6 @@ pub fn fleet_context() -> anyhow::Result { }) } -pub fn platform_context() -> anyhow::Result { - Ok(Context { - name: context_name!("platform-admin"), - namespace: "platform-system".parse()?, - spec: ContextSpec::Remote(RemoteContext { - registry: oci_registry!("registry.example.com"), - repository: oci_repository!("customer/fleet"), - domain: domain!("fleet.example.com"), - image_pull_secret: None, - access: OpenBaoClusterAccess { - namespace: openbao_namespace!("platform/admin"), - url: http_url!("https://secrets.example.com"), - role: "platform-admin".parse()?, - zitadel_url: http_url!("https://identity.example.com"), - zitadel_audience: "openbao".parse()?, - }, - }), - }) -} - pub fn tenant_config() -> TenantConfig { TenantConfig { id: "customer-fleet".into(), diff --git a/examples/harmony_sso/src/main.rs b/examples/harmony_sso/src/main.rs index dc81b795..917c4bef 100644 --- a/examples/harmony_sso/src/main.rs +++ b/examples/harmony_sso/src/main.rs @@ -250,6 +250,7 @@ async fn main() -> anyhow::Result<()> { openshift: false, tls_issuer: None, node_port: None, + create_namespace: true, } .interpret(&Inventory::autoload(), &topology) .await diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index f67f9d8a..88996df4 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -37,6 +37,7 @@ async fn main() { memory_request_gb: 4.0, memory_limit_gb: 4.0, storage_total_gb: 10.0, + service_limit: 10, }, network_policy: TenantNetworkPolicy::default(), }, diff --git a/examples/openbao/src/main.rs b/examples/openbao/src/main.rs index 28f51fa6..e8af8f9e 100644 --- a/examples/openbao/src/main.rs +++ b/examples/openbao/src/main.rs @@ -95,6 +95,7 @@ async fn main() -> Result<()> { openshift: cfg.openshift, tls_issuer: (!cfg.tls_issuer.is_empty()).then(|| cfg.tls_issuer.clone()), node_port: None, + create_namespace: true, }; // JWT auth composes in only when both issuer and audience are set; it @@ -130,6 +131,7 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"# users: vec![], jwt_auth, oidc_application: None, + endpoint: None, }; let scores: Vec>> = vec![Box::new(deploy), Box::new(setup)]; diff --git a/fleet/harmony-fleet-deploy/Cargo.toml b/fleet/harmony-fleet-deploy/Cargo.toml index 0162e64c..7833f995 100644 --- a/fleet/harmony-fleet-deploy/Cargo.toml +++ b/fleet/harmony-fleet-deploy/Cargo.toml @@ -20,6 +20,10 @@ path = "src/main.rs" name = "harmony-fleet-crds-deploy" path = "src/bin/harmony-fleet-crds-deploy.rs" +[[bin]] +name = "harmony-fleet-release" +path = "src/bin/harmony-fleet-release.rs" + [dependencies] harmony = { path = "../../harmony", features = ["podman"] } harmony_cli = { path = "../../harmony_cli" } diff --git a/fleet/harmony-fleet-deploy/src/app.rs b/fleet/harmony-fleet-deploy/src/app.rs index b20e4306..c0d5fc35 100644 --- a/fleet/harmony-fleet-deploy/src/app.rs +++ b/fleet/harmony-fleet-deploy/src/app.rs @@ -22,11 +22,18 @@ const PROJECT: &str = "fleet"; const ADMIN_ROLE: &str = "fleet-admin"; const DEVICE_ROLE: &str = "device"; const OPERATOR_APP: &str = "harmony-fleet-operator"; +const DASHBOARD_APP: &str = "harmony-fleet-dashboard"; const OPERATOR_USER: &str = "fleet-operator"; const NATS_ACCOUNT: &str = "FLEET"; pub struct FleetApp; +impl FleetApp { + pub fn official_images(tag: &str) -> Vec { + fleet_images(|name| format!("hub.nationtech.io/harmony/{name}:{tag}")) + } +} + #[async_trait] impl HarmonyApp for FleetApp { fn identity(&self, ctx: &AppContext) -> AppIdentity { @@ -37,22 +44,7 @@ impl HarmonyApp for FleetApp { } fn images(&self, ctx: &AppContext) -> Result, AppError> { - Ok(vec![ - ImageSpec { - name: "operator".to_string(), - image: ctx.image("harmony-fleet-operator"), - context: ".".into(), - dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(), - build_args: Vec::new(), - }, - ImageSpec { - name: "callout".to_string(), - image: ctx.image("harmony-nats-callout"), - context: ".".into(), - dockerfile: "nats/callout/Dockerfile".into(), - build_args: Vec::new(), - }, - ]) + Ok(fleet_images(|name| ctx.image(name))) } async fn scores( @@ -60,6 +52,12 @@ impl HarmonyApp for FleetApp { ctx: &AppContext, images: &ImageRefs, ) -> Result>>, AppError> { + let operator_image = images + .get("operator") + .map_or_else(|| ctx.image("harmony-fleet-operator"), str::to_owned); + let callout_image = images + .get("callout") + .map_or_else(|| ctx.image("harmony-nats-callout"), str::to_owned); let namespace = ctx.namespace(); let image_pull_secret = ctx.image_pull_secret(); @@ -73,16 +71,37 @@ impl HarmonyApp for FleetApp { zitadel = zitadel.http(Some(8080)); } let provider = zitadel.provider_ref(); + let dashboard_host = + (ctx.profile() == Profile::Prod).then(|| ctx.service_host("dashboard")); let identity = ZitadelSetupScore::for_provider(&provider, namespace, namespace) .application(PROJECT, OPERATOR_APP, ZitadelAppType::DeviceCode) .api_application(PROJECT, "nats") .role(PROJECT, ADMIN_ROLE, "Fleet Admin") .role(PROJECT, DEVICE_ROLE, "Device") - .machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE]) - .port_forward("zitadel") - .groups_claim(); + .machine_identity(PROJECT, OPERATOR_USER, "Fleet Operator", [ADMIN_ROLE]); + let identity = if let Some(host) = &dashboard_host { + identity.application( + PROJECT, + DASHBOARD_APP, + ZitadelAppType::WebPkce { + redirect_uris: vec![format!("https://{host}/auth/callback")], + post_logout_redirect_uris: vec![format!("https://{host}/")], + }, + ) + } else { + identity + }; + let identity = if ctx.profile() == Profile::Local { + identity.port_forward("zitadel") + } else { + identity + } + .groups_claim(); let application = identity.application_ref(OPERATOR_APP); + let dashboard_application = dashboard_host + .as_ref() + .map(|_| identity.application_ref(DASHBOARD_APP)); let operator_identity = identity.machine_identity_ref(OPERATOR_USER); let credentials = @@ -97,7 +116,8 @@ impl HarmonyApp for FleetApp { // I feel like this should not be a standalone nats score but rather be configuration passed // to the main nats score that is installing the nats cluster let nats = NatsScore::callout_account("fleet-nats", namespace, service, NATS_ACCOUNT) - .with_jetstream_size("2Gi"); + .with_jetstream_size("2Gi") + .create_namespace(false); let nats = if ctx.profile() == Profile::Prod { nats.websocket(ctx.service_host("nats"), "letsencrypt-prod") } else { @@ -108,15 +128,17 @@ impl HarmonyApp for FleetApp { NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth") .credentials(&credentials.credentials_ref()) .with_oidc(&provider, &application) - .image(images.require("callout")?) + .image(callout_image) .image_pull_secret(image_pull_secret.clone()) .admin_role(ADMIN_ROLE) .device_role(DEVICE_ROLE) .device_id_claim("client_id"); let nats = nats.with_auth_callout(&callout.auth_callout_ref()); - let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao")); + let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao")) + .create_namespace(false); if ctx.profile() == Profile::Prod { + openbao.openshift = true; openbao = openbao.tls("letsencrypt-prod"); } let openbao_setup = OpenbaoSetupScore::new(openbao.instance.clone()).with_oidc_application( @@ -124,12 +146,26 @@ impl HarmonyApp for FleetApp { &application, OpenbaoJwtAuth::oidc("fleet-device"), ); + let openbao_setup = if ctx.profile() == Profile::Prod { + openbao_setup.endpoint(format!("https://{}", ctx.service_host("openbao"))) + } else { + openbao_setup + }; - let operator = FleetOperatorScore::new(images.require("operator")?) + let operator = FleetOperatorScore::new(operator_image) .namespace(namespace) .image_pull_secret(image_pull_secret) .messaging(&nats.client_ref()) .identity(&provider, &application, &operator_identity); + let operator = if let Some((host, dashboard_application)) = + dashboard_host.zip(dashboard_application) + { + operator + .ingress(host, Some("letsencrypt-prod".to_string())) + .web_auth(&dashboard_application) + } else { + operator + }; Ok(vec![ Box::new(postgres), @@ -145,6 +181,25 @@ impl HarmonyApp for FleetApp { } } +fn fleet_images(image: impl Fn(&str) -> String) -> Vec { + vec![ + ImageSpec { + name: "operator".to_string(), + image: image("harmony-fleet-operator"), + context: ".".into(), + dockerfile: "fleet/harmony-fleet-operator/Dockerfile".into(), + build_args: Vec::new(), + }, + ImageSpec { + name: "callout".to_string(), + image: image("harmony-nats-callout"), + context: ".".into(), + dockerfile: "nats/callout/Dockerfile".into(), + build_args: Vec::new(), + }, + ] +} + pub struct FleetCrdsApp; #[async_trait] @@ -168,6 +223,7 @@ impl HarmonyApp for FleetCrdsApp { pub struct FleetTenantProvisionApp { tenant: TenantConfig, credential_store: TenantCredentialStore, + allow_insecure_source: bool, } enum TenantCredentialStore { @@ -180,6 +236,7 @@ impl FleetTenantProvisionApp { Self { tenant, credential_store: TenantCredentialStore::Client(credential_store), + allow_insecure_source: false, } } @@ -187,8 +244,14 @@ impl FleetTenantProvisionApp { Self { tenant, credential_store: TenantCredentialStore::OpenBao(credential_store), + allow_insecure_source: false, } } + + pub fn allow_insecure_source(mut self) -> Self { + self.allow_insecure_source = true; + self + } } #[async_trait] @@ -236,6 +299,7 @@ impl HarmonyApp for FleetTenantProvisionApp { .expect("static Kubernetes name is valid"), fleet_deployer_rules(), credential_store, + self.allow_insecure_source, )), ]) } @@ -312,6 +376,12 @@ fn fleet_deployer_rules() -> Vec { verbs: verbs(), ..Default::default() }, + PolicyRule { + api_groups: Some(vec!["route.openshift.io".to_string()]), + resources: Some(vec!["routes/custom-host".to_string()]), + verbs: vec!["create".to_string()], + ..Default::default() + }, PolicyRule { api_groups: Some(vec!["policy".to_string()]), resources: Some(vec!["poddisruptionbudgets".to_string()]), @@ -350,11 +420,106 @@ fn fleet_deployer_rules() -> Vec { } #[cfg(test)] -mod tenant_tests { +mod tests { use super::*; use harmony::topology::tenant::TenantNetworkPolicy; use harmony_types::id::Id; + fn prod_context() -> harmony_app::Context { + harmony_app::Context { + name: "prod".parse().unwrap(), + namespace: "fleet".parse().unwrap(), + spec: harmony_app::ContextSpec::Remote(harmony_app::RemoteContext { + registry: "hub.nationtech.io".parse().unwrap(), + repository: "harmony".parse().unwrap(), + domain: "fleet.example.com".parse().unwrap(), + image_pull_secret: None, + access: OpenBaoClusterAccess { + namespace: "customer/fleet".parse().unwrap(), + url: "https://secrets.example.com".parse().unwrap(), + role: "fleet-deployer".parse().unwrap(), + zitadel_url: "https://identity.example.com".parse().unwrap(), + zitadel_audience: "openbao".parse().unwrap(), + }, + }), + } + } + + async fn serialized_fleet(context: harmony_app::Context, images: ImageRefs) -> String { + let context = AppContext::load_metadata(&context, "test", None); + FleetApp + .scores(&context, &images) + .await + .unwrap() + .iter() + .map(|score| serde_json::to_string(&score.serialize()).unwrap()) + .collect() + } + + #[test] + fn official_images_share_the_release_tag() { + let images = FleetApp::official_images("0.0.6"); + assert_eq!(images.len(), 2); + assert!(images.iter().all( + |image| image.image.starts_with("hub.nationtech.io/harmony/") + && image.image.ends_with(":0.0.6") + )); + } + + #[tokio::test] + async fn fleet_uses_supplied_images() { + let serialized = serialized_fleet( + prod_context(), + ImageRefs::new([ + ( + "operator".to_string(), + "registry.example/operator@sha256:operator".to_string(), + ), + ( + "callout".to_string(), + "registry.example/callout@sha256:callout".to_string(), + ), + ]), + ) + .await; + + assert!(serialized.contains("registry.example/operator@sha256:operator")); + assert!(serialized.contains("registry.example/callout@sha256:callout")); + assert!(!serialized.contains("\"port_forward_service\":\"zitadel\"")); + assert!(serialized.contains("https://openbao.fleet.example.com")); + assert!(serialized.contains("dashboard.fleet.example.com")); + assert!(serialized.contains("https://dashboard.fleet.example.com/auth/callback")); + assert!( + serialized + .contains("\"app_name\":\"harmony-fleet-operator\",\"app_type\":\"DeviceCode\"") + ); + assert!(serialized.contains("\"app_name\":\"harmony-fleet-dashboard\"")); + assert!(serialized.contains("zitadel-harmony-fleet-dashboard-oidc")); + } + + #[tokio::test] + async fn fleet_deploy_uses_the_context_release_tag() { + let serialized = serialized_fleet(prod_context(), ImageRefs::default()).await; + + assert!(serialized.contains("hub.nationtech.io/harmony/harmony-fleet-operator:test")); + assert!(serialized.contains("hub.nationtech.io/harmony/harmony-nats-callout:test")); + } + + #[tokio::test] + async fn local_fleet_uses_zitadel_port_forward() { + let serialized = serialized_fleet( + harmony_app::Context { + name: "local".parse().unwrap(), + namespace: "fleet".parse().unwrap(), + spec: harmony_app::ContextSpec::Local(harmony_app::LocalContext::ManagedK3d), + }, + ImageRefs::default(), + ) + .await; + + assert!(serialized.contains("\"port_forward_service\":\"zitadel\"")); + } + #[test] fn deployer_permissions_exclude_cluster_resources() { let rules = fleet_deployer_rules(); @@ -366,6 +531,17 @@ mod tenant_tests { rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()]) && rule.resources.as_deref() == Some(&["routes".to_string()]) })); + assert!(rules.iter().any(|rule| { + rule.api_groups.as_deref() == Some(&[String::new()]) + && rule.resources.as_deref() + == Some(&["pods/exec".to_string(), "pods/portforward".to_string()]) + && rule.verbs == ["create".to_string()] + })); + assert!(rules.iter().any(|rule| { + rule.api_groups.as_deref() == Some(&["route.openshift.io".to_string()]) + && rule.resources.as_deref() == Some(&["routes/custom-host".to_string()]) + && rule.verbs == ["create".to_string()] + })); assert!(rules.iter().all(|rule| { !rule.resources.as_ref().is_some_and(|resources| { resources diff --git a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs new file mode 100644 index 00000000..0f9c421d --- /dev/null +++ b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs @@ -0,0 +1,30 @@ +use clap::Parser; +use harmony_app::{PublicationTopology, publish::build_images}; +use harmony_fleet_deploy::FleetApp; + +#[derive(Parser)] +struct Args { + #[arg(long)] + tag: String, + #[arg(long)] + push: bool, +} + +fn main() -> anyhow::Result<()> { + harmony_cli::cli_logger::init(); + let args = Args::parse(); + let images = FleetApp::official_images(&args.tag); + let registry = PublicationTopology::Registry { + registry: "hub.nationtech.io".to_string(), + }; + let refs = build_images(&images, ®istry)?; + let refs = if args.push { + harmony_app::publish::publish_images(&images, &refs, ®istry)? + } else { + refs + }; + for (name, image) in refs.iter() { + println!("{name}={image}"); + } + Ok(()) +} diff --git a/fleet/harmony-fleet-deploy/src/lib.rs b/fleet/harmony-fleet-deploy/src/lib.rs index 790642a5..4dfdfd7a 100644 --- a/fleet/harmony-fleet-deploy/src/lib.rs +++ b/fleet/harmony-fleet-deploy/src/lib.rs @@ -33,6 +33,13 @@ pub async fn deploy_fleet_crds_with_context(context: harmony_app::Context) -> an harmony_cli::app::app_main(FleetCrdsApp, contexts).await } +pub async fn deploy_fleet_crds_with_kubeconfig( + context: harmony_app::Context, + kubeconfig: std::path::PathBuf, +) -> anyhow::Result<()> { + deploy_with_kubeconfig(&FleetCrdsApp, context, kubeconfig).await +} + pub async fn provision_fleet_tenant_with_context( context: harmony_app::Context, tenant: harmony::topology::tenant::TenantConfig, @@ -42,3 +49,28 @@ pub async fn provision_fleet_tenant_with_context( let contexts = harmony_app::ContextCatalog::new([context])?; harmony_cli::app::app_main(app, contexts).await } + +pub async fn provision_fleet_tenant_with_kubeconfig( + context: harmony_app::Context, + kubeconfig: std::path::PathBuf, + tenant: harmony::topology::tenant::TenantConfig, + credential_store: harmony_app::OpenBaoClusterAccess, + allow_insecure_source: bool, +) -> anyhow::Result<()> { + let mut app = FleetTenantProvisionApp::from_openbao(tenant, credential_store); + if allow_insecure_source { + app = app.allow_insecure_source(); + } + deploy_with_kubeconfig(&app, context, kubeconfig).await +} + +async fn deploy_with_kubeconfig( + app: &dyn harmony_app::HarmonyApp, + context: harmony_app::Context, + kubeconfig: std::path::PathBuf, +) -> anyhow::Result<()> { + harmony_cli::cli_logger::init(); + let ctx = harmony_app::AppContext::from_kubeconfig(&context, "bootstrap", kubeconfig)?; + harmony_app::deploy(app, ctx.topology(), &ctx).await?; + Ok(()) +} diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 9f9d3d7b..272a9686 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -67,8 +67,7 @@ pub struct ChartOptions { /// at `…/harmony-fleet-operator-chart:` matching the image tag. pub chart_version: Option, /// JSON of the dashboard's `ZitadelAuthConfig`, stored in the - /// operator Secret under [`ENV_WEB_AUTH_CONFIG`]. `None` leaves the - /// dashboard unauthenticated (dev/e2e). + /// operator Secret under [`ENV_WEB_AUTH_CONFIG`]. pub web_auth_config_json: Option, /// JSON of the dashboard's `OperatorCookieKey`, stored under /// [`ENV_WEB_COOKIE_KEY`]. @@ -183,12 +182,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result { /// (with the JSON keyfile inlined under `key_json`). Returns `None` /// when no credentials are configured (no-auth dev mode). pub fn operator_secret(opts: &ChartOptions) -> Option { - let creds = opts.credentials.as_ref()?; + if opts.credentials.is_none() + && opts.web_auth_config_json.is_none() + && opts.web_cookie_key_json.is_none() + { + return None; + } let mut data: BTreeMap = BTreeMap::new(); - data.insert( - SECRET_KEY_CREDENTIALS_TOML.to_string(), - ByteString(creds.credentials_toml.as_bytes().to_vec()), - ); + if let Some(creds) = &opts.credentials { + data.insert( + SECRET_KEY_CREDENTIALS_TOML.to_string(), + ByteString(creds.credentials_toml.as_bytes().to_vec()), + ); + } // Dashboard auth config + cookie key (when configured) ride in the // same Secret; the Deployment sources them as HARMONY_CONFIG_* env // for the operator's ConfigClient. @@ -715,4 +721,19 @@ mod tests { format!("HARMONY_CONFIG_{}", OperatorCookieKey::KEY) ); } + + #[test] + fn web_auth_does_not_require_static_nats_credentials() { + let secret = operator_secret(&ChartOptions { + web_auth_config_json: Some("auth".to_string()), + web_cookie_key_json: Some("cookie".to_string()), + ..Default::default() + }) + .expect("web auth requires an operator Secret"); + let data = secret.data.unwrap(); + + assert_eq!(data[ENV_WEB_AUTH_CONFIG].0, b"auth"); + assert_eq!(data[ENV_WEB_COOKIE_KEY].0, b"cookie"); + assert!(!data.contains_key(SECRET_KEY_CREDENTIALS_TOML)); + } } diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index ee9a4242..57833e2e 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -163,21 +163,12 @@ pub struct FleetOperatorScore { /// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no /// host) serves plain HTTP — the right default on issuer-less k3d. pub cluster_issuer: Option, - /// Dashboard SSO config + cookie key, baked into the operator Secret - /// for the pod's `ConfigClient` to read. `None` leaves the dashboard - /// unauthenticated (dev/e2e). - pub web_auth: Option, + /// Dashboard Web PKCE application used to derive SSO configuration. + pub web_auth: Option, pub identity: Option, pub image_pull_secret: Option, } -/// The dashboard's auth inputs the operator reads via `ConfigClient`. -#[derive(Debug, Clone, Serialize)] -pub struct WebAuth { - pub config: harmony_zitadel_auth::ZitadelAuthConfig, - pub cookie: harmony_zitadel_auth::OperatorCookieKey, -} - impl FleetOperatorScore { /// Build a score targeting the `fleet-system` namespace with the /// dev-default image and NATS URL. Use the builders to override. @@ -210,14 +201,9 @@ impl FleetOperatorScore { self } - /// Configure dashboard SSO: the `ZitadelAuthConfig` + cookie key are - /// baked into the operator Secret for the pod's `ConfigClient`. - pub fn web_auth( - mut self, - config: harmony_zitadel_auth::ZitadelAuthConfig, - cookie: harmony_zitadel_auth::OperatorCookieKey, - ) -> Self { - self.web_auth = Some(WebAuth { config, cookie }); + /// Configure dashboard SSO from the declared identity and Ingress. + pub fn web_auth(mut self, application: &OidcApplicationRef) -> Self { + self.web_auth = Some(application.clone()); self } @@ -296,9 +282,11 @@ pub struct FleetOperatorInterpret { async fn smoke_test_operator( namespace: &str, expected_config_hash: &str, + require_dashboard: bool, topology: &T, ) -> Result<(), InterpretError> { let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; + let mut last_observation = "no operator pod observed".to_string(); tokio::time::timeout(Duration::from_secs(180), async { loop { let pods = k8s @@ -310,32 +298,62 @@ async fn smoke_test_operator( ), ) .await; - if let Ok(pods) = pods { - for pod in pods.items { - let has_expected_config = pod - .metadata - .annotations - .as_ref() - .and_then(|annotations| { - annotations.get("harmony.nationtech.io/config-hash") - }) - .is_some_and(|hash| hash == expected_config_hash); - if let Some(name) = pod.metadata.name - && has_expected_config - && k8s - .pod_logs(namespace, &name, Some(100)) - .await - .is_ok_and(|logs| logs.contains("KV bucket ready")) - { - return; + match pods { + Ok(pods) if pods.items.is_empty() => { + last_observation = "no operator pod matched the release label".to_string(); + } + Ok(pods) => { + for pod in pods.items { + let name = pod.metadata.name.unwrap_or_default(); + let has_expected_config = pod + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get("harmony.nationtech.io/config-hash") + }) + .is_some_and(|hash| hash == expected_config_hash); + if !has_expected_config { + last_observation = format!("pod {name} has stale configuration"); + continue; + } + match k8s.pod_logs(namespace, &name, Some(100)).await { + Ok(logs) + if logs.contains("KV bucket ready") + && (!require_dashboard + || logs.contains( + "fleet operator web frontend listening", + )) => + { + return; + } + Ok(logs) => { + last_observation = if !logs.contains("KV bucket ready") { + format!("pod {name} logs lack `KV bucket ready`") + } else { + format!( + "pod {name} logs lack `fleet operator web frontend listening`" + ) + }; + } + Err(error) => { + last_observation = + format!("reading pod {name} logs failed: {error}"); + } + } } } + Err(error) => last_observation = format!("listing operator pods failed: {error}"), } tokio::time::sleep(Duration::from_secs(2)).await; } }) .await - .map_err(|_| InterpretError::new("operator did not initialize authenticated NATS".to_string())) + .map_err(|_| { + InterpretError::new(format!( + "operator did not become ready: {last_observation}" + )) + }) } #[async_trait] @@ -360,6 +378,14 @@ impl Interpret for FleetOperatorInterp self.score.namespace ))); } + if let Some(web_auth) = &self.score.web_auth + && web_auth.namespace() != self.score.namespace + { + return Err(InterpretError::new(format!( + "dashboard application output must be in namespace '{}'", + self.score.namespace + ))); + } let k8s = topology.k8s_client().await.map_err(InterpretError::new)?; k8s.ensure_namespace(&self.score.namespace) .await @@ -369,99 +395,119 @@ impl Interpret for FleetOperatorInterp self.score.namespace )) })?; - let identity_version = if let Some(identity) = &self.score.identity { - Some( - tokio::time::timeout(Duration::from_secs(180), async { - loop { - let application = k8s - .get_resource::( - identity.application.config_map_name(), - Some(identity.application.namespace()), - ) - .await - .ok() - .flatten(); - let machine = k8s - .get_resource::( - identity.machine.secret_name(), - Some(identity.machine.namespace()), - ) - .await - .ok() - .flatten(); - if let (Some(application), Some(machine)) = (application, machine) - && let Some(project_id) = application - .data - .as_ref() - .and_then(|data| data.get(identity.application.project_id_key())) - { - return format!( + let (identity_version, web_client_id) = if let Some(identity) = &self.score.identity { + let (version, client_id) = tokio::time::timeout(Duration::from_secs(180), async { + loop { + let application = k8s + .get_resource::( + identity.application.config_map_name(), + Some(identity.application.namespace()), + ) + .await + .ok() + .flatten(); + let machine = k8s + .get_resource::( + identity.machine.secret_name(), + Some(identity.machine.namespace()), + ) + .await + .ok() + .flatten(); + let web_client_id = if let Some(web_auth) = &self.score.web_auth { + k8s.get_resource::( + web_auth.config_map_name(), + Some(web_auth.namespace()), + ) + .await + .ok() + .flatten() + .and_then(|config_map| config_map.data) + .and_then(|data| data.get(web_auth.client_id_key()).cloned()) + .map(Some) + } else { + Some(None) + }; + if let (Some(application), Some(machine)) = (application, machine) + && let Some(data) = application.data.as_ref() + && let Some(project_id) = data.get(identity.application.project_id_key()) + && let Some(web_client_id) = web_client_id + { + return ( + format!( "{}:{}", project_id, machine.metadata.resource_version.unwrap_or_default() - ); - } - tokio::time::sleep(Duration::from_secs(2)).await; + ), + web_client_id, + ); } - }) - .await - .map_err(|_| { - InterpretError::new("timed out waiting for operator identity refs".to_string()) - })?, - ) + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await + .map_err(|_| { + InterpretError::new("timed out waiting for operator identity refs".to_string()) + })?; + (Some(version), client_id) } else { - None + (None, None) }; let credentials = self.score.credentials.clone(); - // Apply the credentials Secret BEFORE the helm install (the - // chart's Deployment references it via secretKeyRef). Applied - // directly, not via the chart — it's environment-specific. The - // published-chart CD path runs without credentials today, so - // this is a no-op there. - let (web_auth_config_json, web_cookie_key_json) = match &self.score.web_auth { - Some(w) => ( - Some(serde_json::to_string(&w.config).map_err(|e| { + // Apply environment-specific credentials before Helm creates the pod. + // Keeping the Secret outside the chart avoids competing field owners. + let (web_auth_config_json, web_cookie_key_json) = if self.score.web_auth.is_some() { + let identity = self.score.identity.as_ref().ok_or_else(|| { + InterpretError::new("dashboard web auth requires an operator identity".to_string()) + })?; + let host = self.score.operator_ui_host.as_ref().ok_or_else(|| { + InterpretError::new("dashboard web auth requires an Ingress".to_string()) + })?; + let client_id = web_client_id.as_ref().expect("identity resolved above"); + let scheme = if self.score.cluster_issuer.is_some() { + "https" + } else { + "http" + }; + let base_url = format!("{scheme}://{host}"); + let config = harmony_zitadel_auth::ZitadelAuthConfig { + zitadel_base: identity.provider.issuer(), + base_url: base_url.clone(), + client_id: client_id.clone(), + scope: "openid profile email".to_string(), + trusted_audiences: vec![client_id.clone()], + logout_redirect_uri: format!("{base_url}/"), + }; + let existing_secret = k8s + .get_resource::(chart::SECRET_NAME, Some(&self.score.namespace)) + .await + .map_err(|e| InterpretError::new(format!("read operator Secret: {e}")))?; + let cookie = existing_secret + .as_ref() + .and_then(|secret| secret.data.as_ref()) + .and_then(|data| data.get(chart::ENV_WEB_COOKIE_KEY)) + .map(|value| { + serde_json::from_slice::(&value.0) + .map_err(|e| { + InterpretError::new(format!("parse existing OperatorCookieKey: {e}")) + }) + }) + .transpose()? + .unwrap_or_else(harmony_zitadel_auth::OperatorCookieKey::generate); + ( + Some(serde_json::to_string(&config).map_err(|e| { InterpretError::new(format!("serialize ZitadelAuthConfig: {e}")) })?), - Some(serde_json::to_string(&w.cookie).map_err(|e| { + Some(serde_json::to_string(&cookie).map_err(|e| { InterpretError::new(format!("serialize OperatorCookieKey: {e}")) })?), - ), - None => (None, None), + ) + } else { + (None, None) }; - let expected_config_hash = chart::config_hash(&ChartOptions { - credentials: credentials.clone(), - web_auth_config_json: web_auth_config_json.clone(), - web_cookie_key_json: web_cookie_key_json.clone(), - identity: self.score.identity.clone(), - identity_version: identity_version.clone(), - image_pull_secret: self.score.image_pull_secret.clone(), - ..ChartOptions::default() - }); - if let Some(creds) = &credentials - && let Some(secret) = operator_secret(&ChartOptions { - credentials: Some(creds.clone()), - web_auth_config_json: web_auth_config_json.clone(), - web_cookie_key_json: web_cookie_key_json.clone(), - identity: self.score.identity.clone(), - identity_version: identity_version.clone(), - image_pull_secret: self.score.image_pull_secret.clone(), - ..ChartOptions::default() - }) - { - info!( - "Applying operator credentials Secret '{}' in {}", - chart::SECRET_NAME, - self.score.namespace - ); - K8sResourceScore::single(secret, Some(self.score.namespace.clone())) - .interpret(inventory, topology) - .await?; - } - let tmp = tempfile::tempdir() .map_err(|e| InterpretError::new(format!("operator chart tempdir: {e}")))?; - let chart_path = build_chart(&ChartOptions { + let chart_options = ChartOptions { output_dir: tmp.path().to_path_buf(), image: self.score.image.clone(), image_pull_policy: self.score.image_pull_policy.clone(), @@ -474,8 +520,21 @@ impl Interpret for FleetOperatorInterp identity: self.score.identity.clone(), identity_version, image_pull_secret: self.score.image_pull_secret.clone(), - }) - .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?; + }; + let expected_config_hash = chart::config_hash(&chart_options); + if let Some(secret) = operator_secret(&chart_options) { + info!( + "Applying operator credentials Secret '{}' in {}", + chart::SECRET_NAME, + self.score.namespace + ); + K8sResourceScore::single(secret, Some(self.score.namespace.clone())) + .interpret(inventory, topology) + .await?; + } + + let chart_path = build_chart(&chart_options) + .map_err(|e| InterpretError::new(format!("build operator chart: {e}")))?; let chart_path_str = chart_path .to_str() .ok_or_else(|| InterpretError::new("operator chart path is not utf-8".to_string()))?; @@ -507,10 +566,6 @@ impl Interpret for FleetOperatorInterp )) })?; - if credentials.is_some() || self.score.identity.is_some() { - smoke_test_operator(&self.score.namespace, &expected_config_hash, topology).await?; - } - // Expose the UI. Applied after the chart so the backing Service // (shipped in the chart) exists. Skipped when no host is set — // dev/e2e harnesses keep the operator cluster-internal. @@ -545,6 +600,31 @@ impl Interpret for FleetOperatorInterp details.push(format!("operator UI: {scheme}://{host}")); } + if credentials.is_some() || self.score.identity.is_some() { + smoke_test_operator( + &self.score.namespace, + &expected_config_hash, + self.score.web_auth.is_some(), + topology, + ) + .await?; + } + + info!( + r#" +===== FLEET VERIFICATION ===== +kubectl -n {namespace} get \ + pods,statefulsets,deployments,pvc,services,routes,certificates + +kubectl -n {namespace} logs \ + deploy/{release} --tail=200 | grep 'KV bucket ready' + +kubectl -n {namespace} get resourcequota +=============================="#, + namespace = self.score.namespace, + release = self.score.release_name, + ); + Ok(Outcome::success_with_details(helm_outcome.message, details)) } diff --git a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs index c5058878..ea7119db 100644 --- a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs +++ b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs @@ -169,6 +169,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result { openshift: false, tls_issuer: None, node_port: None, + create_namespace: true, }), Box::new(OpenbaoSetupScore { instance: instance.clone(), @@ -188,6 +189,7 @@ async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result { max_ttl: "1h".to_string(), }), oidc_application: None, + endpoint: None, }), ]; for score in scores { diff --git a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs index 4a376326..f3da0d54 100644 --- a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs +++ b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs @@ -90,6 +90,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"# openshift: false, tls_issuer: None, node_port: None, + create_namespace: true, }), Box::new(OpenbaoSetupScore { instance: instance.clone(), @@ -105,6 +106,7 @@ path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"# }], jwt_auth: None, oidc_application: None, + endpoint: None, }), ]; diff --git a/harmony-k8s/src/client.rs b/harmony-k8s/src/client.rs index 0da8c81e..d6dc4b70 100644 --- a/harmony-k8s/src/client.rs +++ b/harmony-k8s/src/client.rs @@ -14,6 +14,7 @@ pub struct ClusterConnection { pub server: String, pub tls_server_name: Option, pub proxy_url: Option, + pub tls_verified: bool, } impl ClusterConnection { @@ -39,9 +40,6 @@ impl ClusterConnection { .find(|cluster| &cluster.name == cluster_name) .and_then(|cluster| cluster.cluster.as_ref()) .ok_or_else(|| format!("kubeconfig cluster '{cluster_name}' not found"))?; - if cluster.insecure_skip_tls_verify == Some(true) { - return Err("cannot issue tenant credentials for an insecure cluster".to_string()); - } let server = safe_endpoint( cluster .server @@ -60,6 +58,7 @@ impl ClusterConnection { server, tls_server_name: cluster.tls_server_name.clone(), proxy_url, + tls_verified: cluster.insecure_skip_tls_verify != Some(true), }) } } @@ -232,10 +231,35 @@ users: server: "https://api.example.com:6443".to_string(), tls_server_name: None, proxy_url: None, + tls_verified: true, } ); } + #[test] + fn connection_records_insecure_tls() { + let kubeconfig: Kubeconfig = serde_yaml::from_str( + r#" +current-context: admin +contexts: + - name: admin + context: { cluster: shared } +clusters: + - name: shared + cluster: + server: https://api.example.com + insecure-skip-tls-verify: true +"#, + ) + .unwrap(); + + assert!( + !ClusterConnection::from_kubeconfig(&kubeconfig, &KubeConfigOptions::default()) + .unwrap() + .tls_verified + ); + } + #[test] fn connection_rejects_credentials_in_server_url() { let kubeconfig: Kubeconfig = serde_yaml::from_str( diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 048ff3e4..9e36d25e 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -47,6 +47,7 @@ harmony-reconciler-contracts = { path = "../harmony-reconciler-contracts" } anyhow.workspace = true uuid.workspace = true url.workspace = true +vaultrs = "0.7.4" kube = { workspace = true, features = ["derive"] } k8s-openapi.workspace = true serde_yaml.workspace = true diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index b4c8b174..65c2cd8b 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -120,7 +120,7 @@ impl K8sTenantManager { "requests.memory": format!("{:.3}Gi", config.resource_limits.memory_request_gb), "requests.storage": format!("{:.3}Gi", config.resource_limits.storage_total_gb), "pods": "20", - "services": "10", + "services": config.resource_limits.service_limit.to_string(), "configmaps": "60", "secrets": "60", "persistentvolumeclaims": "15", @@ -178,13 +178,13 @@ impl K8sTenantManager { }) } - fn build_network_policy(&self, config: &TenantConfig) -> Result { + fn build_network_policy(config: &TenantConfig) -> Result { let network_policy = json!({ "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": { "name": format!("{}-network-policy", config.name), - "namespace": self.get_namespace_name(config), + "namespace": config.name, }, "spec": { "podSelector": {}, @@ -277,7 +277,7 @@ impl K8sTenantManager { .map(|ci| { json!({ "ipBlock": { - "cidr": ci.to_string(), + "cidr": network_policy_cidr(ci), } }) }) @@ -334,7 +334,7 @@ impl K8sTenantManager { .map(|ci| { json!({ "ipBlock": { - "cidr": ci.to_string(), + "cidr": network_policy_cidr(ci), } }) }) @@ -387,6 +387,23 @@ impl K8sTenantManager { } } +fn network_policy_cidr(cidr: &cidr::Ipv4Cidr) -> String { + format!("{cidr:#}") +} + +#[cfg(test)] +mod tests { + use super::network_policy_cidr; + + #[test] + fn network_policy_cidr_preserves_host_prefix() { + assert_eq!( + network_policy_cidr(&"192.168.3.1/32".parse().unwrap()), + "192.168.3.1/32" + ); + } +} + impl Clone for K8sTenantManager { fn clone(&self) -> Self { Self { @@ -403,7 +420,7 @@ impl TenantManager for K8sTenantManager { let namespace = self.build_namespace(config)?; let resource_quota = self.build_resource_quota(config)?; - let network_policy = self.build_network_policy(config)?; + let network_policy = Self::build_network_policy(config)?; let network_policy = self .network_policy_strategy .adjust_policy(network_policy, config); diff --git a/harmony/src/domain/topology/tenant/mod.rs b/harmony/src/domain/topology/tenant/mod.rs index 1ce1bcb1..98b6d5fd 100644 --- a/harmony/src/domain/topology/tenant/mod.rs +++ b/harmony/src/domain/topology/tenant/mod.rs @@ -54,6 +54,8 @@ pub struct ResourceLimits { /// Total persistent storage allocation in Gigabytes across all volumes. pub storage_total_gb: f32, + /// Maximum number of Services, including temporary ACME solver Services. + pub service_limit: u32, } impl Default for ResourceLimits { @@ -64,6 +66,7 @@ impl Default for ResourceLimits { memory_request_gb: 4.0, memory_limit_gb: 4.0, storage_total_gb: 20.0, + service_limit: 10, } } } diff --git a/harmony/src/modules/nats/score_nats.rs b/harmony/src/modules/nats/score_nats.rs index 611ea034..22715c7d 100644 --- a/harmony/src/modules/nats/score_nats.rs +++ b/harmony/src/modules/nats/score_nats.rs @@ -81,6 +81,7 @@ pub struct NatsScore { pub jetstream_size: String, pub image: Option, pub websocket: Option, + pub create_namespace: bool, } #[derive(Debug, Clone, Copy, Serialize)] @@ -100,6 +101,7 @@ impl NatsScore { jetstream_size: "10Gi".to_string(), image: None, websocket: None, + create_namespace: true, } } @@ -128,6 +130,7 @@ impl NatsScore { jetstream_size: "10Gi".to_string(), image: None, websocket: None, + create_namespace: true, } } @@ -148,6 +151,7 @@ impl NatsScore { jetstream_size: "10Gi".to_string(), image: None, websocket: None, + create_namespace: true, } } @@ -161,6 +165,11 @@ impl NatsScore { self } + pub fn create_namespace(mut self, create: bool) -> Self { + self.create_namespace = create; + self + } + pub fn image(mut self, image: impl Into) -> Self { self.image = Some(image.into()); self @@ -249,6 +258,13 @@ struct NatsValues { service: Option, #[serde(skip_serializing_if = "Option::is_none")] container: Option, + #[serde(rename = "natsBox")] + nats_box: NatsValuesNatsBox, +} + +#[derive(Debug, Serialize)] +struct NatsValuesNatsBox { + enabled: bool, } #[derive(Debug, Serialize)] @@ -556,6 +572,7 @@ fn build_values( }), (container, None) => container, }, + nats_box: NatsValuesNatsBox { enabled: false }, }) } @@ -600,11 +617,12 @@ impl Interpret for NatsInterpret { .score .values_yaml() .map_err(|e| InterpretError::new(format!("NATS values: {e}")))?; - let helm = NatsHelmChartScore::new( + let mut helm = NatsHelmChartScore::new( self.score.release_name.clone(), self.score.namespace.clone(), values_yaml, ); + helm.create_namespace = self.score.create_namespace; let outcome = helm.interpret(inventory, topology).await?; topology .k8s_client() @@ -717,6 +735,7 @@ mod tests { v["service"]["merge"]["spec"]["ports"][0]["name"], serde_yaml::Value::String("nats".into()) ); + assert_eq!(v["natsBox"]["enabled"], serde_yaml::Value::Bool(false)); } #[test] @@ -773,6 +792,15 @@ mod tests { ); } + #[test] + fn namespace_creation_can_be_disabled() { + assert!( + !NatsScore::new("nats", "tenant") + .create_namespace(false) + .create_namespace + ); + } + #[test] fn typed_callout_cycle_exposes_account_and_client_refs() { let nats = NatsScore::callout_account( diff --git a/harmony/src/modules/openbao/mod.rs b/harmony/src/modules/openbao/mod.rs index 001ad2fe..5155fb75 100644 --- a/harmony/src/modules/openbao/mod.rs +++ b/harmony/src/modules/openbao/mod.rs @@ -69,6 +69,7 @@ pub struct OpenbaoScore { pub tls_issuer: Option, #[serde(default)] pub node_port: Option, + pub create_namespace: bool, } impl OpenbaoScore { @@ -86,6 +87,7 @@ impl OpenbaoScore { openshift: false, tls_issuer: None, node_port: None, + create_namespace: true, } } @@ -99,12 +101,18 @@ impl OpenbaoScore { self } + pub fn create_namespace(mut self, create: bool) -> Self { + self.create_namespace = create; + self + } + fn values(&self) -> String { let Self { host, openshift, tls_issuer, node_port, + create_namespace: _, instance: _, } = self; // Edge TLS: the listener stays plain HTTP behind the ingress, which @@ -209,7 +217,7 @@ impl Interpret for OpenbaoInterpret { chart_version: None, values_overrides: None, values_yaml: Some(self.score.values()), - create_namespace: true, + create_namespace: self.score.create_namespace, install_only: false, force_conflicts: false, repository: Some(HelmRepository::new( @@ -255,6 +263,15 @@ impl Interpret for OpenbaoInterpret { mod tests { use super::*; + #[test] + fn namespace_creation_can_be_disabled() { + assert!( + !OpenbaoScore::new("tenant", "openbao", "bao.example") + .create_namespace(false) + .create_namespace + ); + } + #[test] fn no_issuer_renders_plain_ingress() { let v = OpenbaoScore { @@ -263,6 +280,7 @@ mod tests { openshift: false, tls_issuer: None, node_port: Some(30425), + create_namespace: true, } .values(); assert!(!v.contains("cert-manager.io/cluster-issuer")); @@ -278,10 +296,19 @@ mod tests { openshift: false, tls_issuer: Some("letsencrypt".into()), node_port: None, + create_namespace: true, } .values(); assert!(v.contains("cert-manager.io/cluster-issuer: letsencrypt")); assert!(v.contains("- hosts: [bao.example]")); assert!(v.contains("secretName: openbao-tls")); } + + #[test] + fn openshift_mode_is_rendered() { + let mut score = OpenbaoScore::new("tenant", "openbao", "bao.example"); + score.openshift = true; + + assert!(score.values().contains("openshift: true")); + } } diff --git a/harmony/src/modules/openbao/setup.rs b/harmony/src/modules/openbao/setup.rs index e9730654..dea95358 100644 --- a/harmony/src/modules/openbao/setup.rs +++ b/harmony/src/modules/openbao/setup.rs @@ -1,4 +1,8 @@ -use std::path::PathBuf; +use std::{ + collections::HashMap, + io::Write, + path::{Path, PathBuf}, +}; use async_trait::async_trait; use harmony_config::{Config, ConfigError}; @@ -6,6 +10,18 @@ use k8s_openapi::api::core::v1::ConfigMap; use log::{info, warn}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use vaultrs::{ + api::{ + auth::{ + oidc::requests::{SetConfigurationRequestBuilder, SetRoleRequestBuilder}, + userpass::requests::CreateUserRequestBuilder, + }, + sys::requests::EnableEngineRequestBuilder, + }, + auth::{oidc, userpass}, + client::{Client, VaultClient, VaultClientSettingsBuilder}, + sys, +}; use crate::{ data::Version, @@ -125,6 +141,11 @@ pub struct OpenbaoSetupScore { pub jwt_auth: Option, #[serde(default)] pub oidc_application: Option, + + /// Public API endpoint. When absent, setup uses a temporary pod + /// port-forward for local clusters. + #[serde(default)] + pub endpoint: Option, } fn default_kv_mount() -> String { @@ -140,6 +161,7 @@ impl Default for OpenbaoSetupScore { users: Vec::new(), jwt_auth: None, oidc_application: None, + endpoint: None, } } } @@ -175,6 +197,11 @@ impl OpenbaoSetupScore { score.oidc_application = Some(application.clone()); score } + + pub fn endpoint(mut self, endpoint: impl Into) -> Self { + self.endpoint = Some(endpoint.into()); + self + } } impl Score for OpenbaoSetupScore { @@ -205,6 +232,12 @@ struct InitOutput { root_token: String, } +#[derive(Debug, Deserialize)] +struct OpenbaoStatus { + initialized: bool, + sealed: bool, +} + #[derive(Debug, Serialize, Deserialize, JsonSchema, Config)] #[config(secret)] struct OpenbaoRecovery { @@ -230,6 +263,36 @@ fn keys_file(instance: &OpenbaoInstance) -> PathBuf { )) } +fn write_recovery_file(path: &Path, init: &InitOutput) -> Result<(), InterpretError> { + #[cfg(unix)] + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + std::fs::create_dir_all( + path.parent() + .ok_or_else(|| InterpretError::new(format!("invalid recovery path {path:?}")))?, + ) + .map_err(|e| InterpretError::new(format!("create recovery directory: {e}")))?; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options + .open(path) + .map_err(|e| InterpretError::new(format!("open {path:?}: {e}")))?; + file.write_all( + serde_json::to_string_pretty(init) + .map_err(|e| InterpretError::new(format!("serialize recovery: {e}")))? + .as_bytes(), + ) + .map_err(|e| InterpretError::new(format!("write {path:?}: {e}")))?; + file.sync_all() + .map_err(|e| InterpretError::new(format!("sync {path:?}: {e}")))?; + #[cfg(unix)] + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| InterpretError::new(format!("secure {path:?}: {e}")))?; + Ok(()) +} + /// The root token from the cached unseal-keys file written at init. /// Dev/staging convenience for callers that need to seed OpenBao right /// after [`OpenbaoSetupScore`] runs; production uses auto-unseal and @@ -244,6 +307,8 @@ pub fn cached_root_token(instance: &OpenbaoInstance) -> Result { impl OpenbaoSetupInterpret { async fn save_recovery(&self, init: &InitOutput) -> Result<(), InterpretError> { + let path = keys_file(&self.score.instance); + write_recovery_file(&path, init)?; let recovery = OpenbaoRecovery { namespace: self.score.instance.namespace.clone(), release: self.score.instance.release.clone(), @@ -251,27 +316,20 @@ impl OpenbaoSetupInterpret { root_token: init.root_token.clone(), }; match harmony_config::set(&recovery).await { - Ok(()) => Ok(()), - Err(ConfigError::NoSources) => { - let path = keys_file(&self.score.instance); - std::fs::create_dir_all(keys_dir()) - .map_err(|e| InterpretError::new(format!("create recovery directory: {e}")))?; - std::fs::write( - &path, - serde_json::to_string_pretty(init) - .map_err(|e| InterpretError::new(format!("serialize recovery: {e}")))?, - ) - .map_err(|e| InterpretError::new(format!("write {path:?}: {e}")))?; - #[cfg(unix)] + Ok(()) => { + if let Err(error) = std::fs::remove_file(&path) + && error.kind() != std::io::ErrorKind::NotFound { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) - .map_err(|e| InterpretError::new(format!("secure {path:?}: {e}")))?; + warn!( + "OpenBao recovery was persisted externally but local cleanup failed for \ + {path:?}: {error}" + ); } Ok(()) } + Err(ConfigError::NoSources) => Ok(()), Err(e) => Err(InterpretError::new(format!( - "persist OpenBao recovery material: {e}" + "persist OpenBao recovery material: {e}; recovery retained at {path:?}" ))), } } @@ -312,70 +370,63 @@ impl OpenbaoSetupInterpret { } } - async fn exec( - &self, - k8s: &harmony_k8s::K8sClient, - command: Vec<&str>, - ) -> Result { - k8s.exec_pod_capture_output( - &self.score.instance.pod(), - Some(&self.score.instance.namespace), - command, - ) + fn client(endpoint: &str) -> Result { + let settings = VaultClientSettingsBuilder::default() + .address(endpoint) + .build() + .map_err(|e| InterpretError::new(format!("OpenBao client settings: {e}")))?; + VaultClient::new(settings).map_err(|e| InterpretError::new(format!("OpenBao client: {e}"))) + } + + async fn status(endpoint: &str) -> Result { + let response = reqwest::get(format!("{}/v1/sys/health", endpoint.trim_end_matches('/'))) + .await + .map_err(|e| e.to_string())?; + response.json().await.map_err(|e| e.to_string()) + } + + async fn wait_for_api(endpoint: &str) -> Result<(), InterpretError> { + let mut last_error = String::new(); + tokio::time::timeout(std::time::Duration::from_secs(300), async { + loop { + match Self::status(endpoint).await { + Ok(_) => return, + Err(error) => last_error = error.to_string(), + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + }) .await - } - - async fn bao_command( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - shell_cmd: &str, - ) -> Result { - let full = format!("export VAULT_TOKEN={} && {}", root_token, shell_cmd); - self.exec(k8s, vec!["sh", "-c", &full]).await - } - - async fn bao( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - args: &[&str], - ) -> Result { - self.bao_command(k8s, root_token, &args.join(" ")).await + .map_err(|_| InterpretError::new(format!("OpenBao API not ready: {last_error}")))?; + Ok(()) } // -- Step 1: Init --------------------------------------------------------- - async fn init(&self, k8s: &harmony_k8s::K8sClient) -> Result { - // Source of truth for "is this vault initialized?" is OpenBao itself, - // not a `bao status` pre-check parsed from stderr — that probe is - // brittle because `exec_pod_capture_output` discards stdout on - // non-zero exit, and `bao status` exits 2 on a fresh-and-uninitialised - // vault. So we just attempt `operator init` and let OpenBao tell us - // authoritatively via its error message. - info!("[OpenbaoSetup] Probing init state via `bao operator init`..."); - let output = self - .exec(k8s, vec!["bao", "operator", "init", "-format=json"]) - .await; - - match output { - Ok(stdout) => { - // Fresh init — parse, persist, return root token. Overwrites - // any stale cached keys (warning already emitted above). - let init: InitOutput = serde_json::from_str(&stdout).map_err(|e| { - InterpretError::new(format!("Failed to parse init output: {e}")) - })?; - self.save_recovery(&init).await?; - info!("[OpenbaoSetup] Initialized"); - Ok(init) - } - Err(e) if e.contains("already initialized") => { - info!("[OpenbaoSetup] Vault is already initialized; loading recovery material"); - self.load_recovery().await - } - Err(e) => Err(InterpretError::new(format!( - "OpenBao operator init failed: {e}" - ))), + async fn init( + &self, + client: &VaultClient, + endpoint: &str, + ) -> Result { + info!("[OpenbaoSetup] Probing init state via OpenBao API..."); + if !Self::status(endpoint) + .await + .map_err(|e| InterpretError::new(format!("OpenBao status failed: {e}")))? + .initialized + { + let output = sys::start_initialization(client, 5, 3, None) + .await + .map_err(|e| InterpretError::new(format!("OpenBao initialization failed: {e}")))?; + let init = InitOutput { + keys: output.keys_base64, + root_token: output.root_token, + }; + self.save_recovery(&init).await?; + info!("[OpenbaoSetup] Initialized"); + Ok(init) + } else { + info!("[OpenbaoSetup] Vault is already initialized; loading recovery material"); + self.load_recovery().await } } @@ -383,47 +434,14 @@ impl OpenbaoSetupInterpret { async fn unseal( &self, - k8s: &harmony_k8s::K8sClient, + client: &VaultClient, + endpoint: &str, init: &InitOutput, ) -> Result<(), InterpretError> { - #[derive(Deserialize)] - struct Status { - sealed: bool, - } - - // `bao status -format=json` exits 2 on a sealed-but-initialised - // vault but still emits its JSON payload on stdout. Use - // exec_pod_capture so we read both streams regardless of exit - // status and parse the `sealed` field authoritatively. - let sealed = match k8s - .exec_pod_capture( - &self.score.instance.pod(), - Some(&self.score.instance.namespace), - vec!["bao", "status", "-format=json"], - ) + let sealed = Self::status(endpoint) .await - { - Ok(output) => serde_json::from_str::(&output.stdout) - .map(|s| s.sealed) - .unwrap_or_else(|_| { - // JSON missing or unparseable — fall back to the - // conservative default (treat as sealed) so we - // attempt unseal rather than silently skipping. - warn!( - "[OpenbaoSetup] Could not parse `bao status` JSON \ - (stderr: {}); assuming vault is sealed", - output.stderr.trim() - ); - true - }), - Err(e) => { - warn!( - "[OpenbaoSetup] `bao status` exec failed ({e}); \ - assuming vault is sealed" - ); - true - } - }; + .map_err(|e| InterpretError::new(format!("OpenBao status failed: {e}")))? + .sealed; if !sealed { info!("[OpenbaoSetup] Already unsealed"); @@ -432,7 +450,7 @@ impl OpenbaoSetupInterpret { info!("[OpenbaoSetup] Unsealing..."); for key in &init.keys[0..3] { - self.exec(k8s, vec!["bao", "operator", "unseal", key]) + sys::unseal(client, Some(key.clone()), None, None) .await .map_err(|e| InterpretError::new(format!("Unseal failed: {e}")))?; } @@ -443,57 +461,46 @@ impl OpenbaoSetupInterpret { // -- Step 3: Enable KV v2 ------------------------------------------------- - async fn enable_kv( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - ) -> Result<(), InterpretError> { + async fn enable_kv(&self, client: &VaultClient) -> Result<(), InterpretError> { let mount = &self.score.kv_mount; - let _ = self - .bao( - k8s, - root_token, - &[ - "bao", - "secrets", - "enable", - &format!("-path={mount}"), - "kv-v2", - ], - ) - .await; // ignore "already enabled" + if !sys::mount::list(client) + .await + .map_err(|e| InterpretError::new(format!("List OpenBao mounts failed: {e}")))? + .contains_key(&format!("{mount}/")) + { + let mut options = EnableEngineRequestBuilder::default(); + options.options(HashMap::from([("version".to_string(), "2".to_string())])); + sys::mount::enable(client, mount, "kv", Some(&mut options)) + .await + .map_err(|e| InterpretError::new(format!("Enable KV v2 failed: {e}")))?; + } Ok(()) } // -- Step 4: Enable userpass auth ----------------------------------------- - async fn enable_userpass( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - ) -> Result<(), InterpretError> { - let _ = self - .bao(k8s, root_token, &["bao", "auth", "enable", "userpass"]) - .await; + async fn enable_auth(&self, client: &VaultClient, mount: &str) -> Result<(), InterpretError> { + if !sys::auth::list(client) + .await + .map_err(|e| InterpretError::new(format!("List OpenBao auth methods failed: {e}")))? + .contains_key(&format!("{mount}/")) + { + sys::auth::enable(client, mount, mount, None) + .await + .map_err(|e| InterpretError::new(format!("Enable {mount} auth failed: {e}")))?; + } Ok(()) } // -- Step 5: Policies ----------------------------------------------------- - async fn apply_policies( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - ) -> Result<(), InterpretError> { + async fn apply_policies(&self, client: &VaultClient) -> Result<(), InterpretError> { for policy in &self.score.policies { - let escaped_hcl = policy.hcl.replace('\'', "'\\''"); - let cmd = format!( - "printf '{}' | bao policy write {} -", - escaped_hcl, policy.name - ); - self.bao_command(k8s, root_token, &cmd).await.map_err(|e| { - InterpretError::new(format!("Failed to create policy '{}': {e}", policy.name)) - })?; + sys::policy::set(client, &policy.name, &policy.hcl) + .await + .map_err(|e| { + InterpretError::new(format!("Failed to create policy '{}': {e}", policy.name)) + })?; info!("[OpenbaoSetup] Policy '{}' applied", policy.name); } Ok(()) @@ -501,23 +508,16 @@ impl OpenbaoSetupInterpret { // -- Step 6: Users -------------------------------------------------------- - async fn create_users( - &self, - k8s: &harmony_k8s::K8sClient, - root_token: &str, - ) -> Result<(), InterpretError> { + async fn create_users(&self, client: &VaultClient) -> Result<(), InterpretError> { for user in &self.score.users { - let policies = user.policies.join(","); - self.bao( - k8s, - root_token, - &[ - "bao", - "write", - &format!("auth/userpass/users/{}", user.username), - &format!("password={}", user.password), - &format!("policies={}", policies), - ], + let mut options = CreateUserRequestBuilder::default(); + options.token_policies(user.policies.clone()); + userpass::user::set( + client, + "userpass", + &user.username, + &user.password, + Some(&mut options), ) .await .map_err(|e| { @@ -525,7 +525,8 @@ impl OpenbaoSetupInterpret { })?; info!( "[OpenbaoSetup] User '{}' created (policies: {})", - user.username, policies + user.username, + user.policies.join(",") ); } Ok(()) @@ -536,7 +537,7 @@ impl OpenbaoSetupInterpret { async fn configure_jwt( &self, k8s: &harmony_k8s::K8sClient, - root_token: &str, + client: &VaultClient, ) -> Result<(), InterpretError> { let jwt = match &self.score.jwt_auth { Some(j) => j, @@ -569,69 +570,47 @@ impl OpenbaoSetupInterpret { jwt.bound_audiences.clone() }; - let _ = self - .bao(k8s, root_token, &["bao", "auth", "enable", JWT_AUTH_MOUNT]) - .await; + self.enable_auth(client, JWT_AUTH_MOUNT).await?; - // Configure JWT validation: static public keys when provided, - // otherwise OIDC discovery. Discovery may fail if the URL is not - // reachable from inside the cluster (e.g., Zitadel's ExternalDomain - // isn't resolvable). Non-fatal — warn and continue. - // Single-quote the PEM: it carries newlines the pod-exec shell - // would otherwise split into separate "commands". - let validation_arg = if jwt.jwt_validation_pubkeys.is_empty() { - format!("oidc_discovery_url={}", jwt.oidc_discovery_url) + let mut config = SetConfigurationRequestBuilder::default(); + config.bound_issuer(jwt.bound_issuer.clone()); + if jwt.jwt_validation_pubkeys.is_empty() { + config.oidc_discovery_url(jwt.oidc_discovery_url.clone()); } else { - format!( - "jwt_validation_pubkeys='{}'", - jwt.jwt_validation_pubkeys.replace('\'', "'\\''") - ) - }; - let config_result = self - .bao( - k8s, - root_token, - &[ - "bao", - "write", - &format!("auth/{JWT_AUTH_MOUNT}/config"), - &validation_arg, - &format!("bound_issuer={}", jwt.bound_issuer), - ], - ) - .await; - - match config_result { - Ok(_) => { - info!( - "[OpenbaoSetup] JWT auth configured (issuer: {})", - jwt.bound_issuer - ); - } - Err(e) => { - return Err(InterpretError::new(format!( + config.jwt_validation_pubkeys(vec![jwt.jwt_validation_pubkeys.clone()]); + } + oidc::config::set(client, JWT_AUTH_MOUNT, Some(&mut config)) + .await + .map_err(|e| { + InterpretError::new(format!( "Failed to configure JWT auth from '{}': {e}", jwt.oidc_discovery_url - ))); - } - } + )) + })?; + info!( + "[OpenbaoSetup] JWT auth configured (issuer: {})", + jwt.bound_issuer + ); - let mut role_args = vec![ - "bao".to_string(), - "write".to_string(), - format!("auth/{JWT_AUTH_MOUNT}/role/{}", jwt.role_name), - "role_type=jwt".to_string(), - format!("bound_audiences={bound_audiences}"), - format!("user_claim={}", jwt.user_claim), - format!("ttl={}", jwt.ttl), - format!("max_ttl={}", jwt.max_ttl), - format!("token_type={}", jwt.token_type), - ]; + let mut role = SetRoleRequestBuilder::default(); + role.role_type("jwt") + .bound_audiences(vec![bound_audiences]) + .token_ttl(jwt.ttl.clone()) + .token_max_ttl(jwt.max_ttl.clone()) + .token_type(jwt.token_type.clone()); if !jwt.groups_claim.is_empty() { - role_args.push(format!("groups_claim={}", jwt.groups_claim)); + role.groups_claim(jwt.groups_claim.clone()); } - let role_args: Vec<&str> = role_args.iter().map(String::as_str).collect(); - self.bao(k8s, root_token, &role_args).await.map_err(|e| { + oidc::role::set( + client, + JWT_AUTH_MOUNT, + &jwt.role_name, + &jwt.user_claim, + Vec::new(), + Some(&mut role), + ) + .await + .map_err(|e| { InterpretError::new(format!( "Failed to create JWT role '{}': {e}", jwt.role_name @@ -668,18 +647,41 @@ impl Interpret for OpenbaoSetupInterpret { )) })?; - let recovery = self.init(&k8s).await?; - self.unseal(&k8s, &recovery).await?; + let _port_forward; + let endpoint = if let Some(endpoint) = &self.score.endpoint { + _port_forward = None; + endpoint.clone() + } else { + let handle = k8s + .port_forward( + &self.score.instance.pod(), + &self.score.instance.namespace, + 0, + 8200, + ) + .await + .map_err(|e| InterpretError::new(format!("OpenBao port-forward failed: {e}")))?; + let endpoint = format!("http://127.0.0.1:{}", handle.port()); + info!("[OpenbaoSetup] Provisioning via port-forward {endpoint}"); + _port_forward = Some(handle); + endpoint + }; + let mut client = Self::client(&endpoint)?; + Self::wait_for_api(&endpoint).await?; + + let recovery = self.init(&client, &endpoint).await?; + self.unseal(&client, &endpoint, &recovery).await?; let root_token = recovery.root_token; - self.enable_kv(&k8s, &root_token).await?; + client.set_token(&root_token); + self.enable_kv(&client).await?; if !self.score.users.is_empty() { - self.enable_userpass(&k8s, &root_token).await?; + self.enable_auth(&client, "userpass").await?; } - self.apply_policies(&k8s, &root_token).await?; - self.create_users(&k8s, &root_token).await?; - self.configure_jwt(&k8s, &root_token).await?; + self.apply_policies(&client).await?; + self.create_users(&client).await?; + self.configure_jwt(&k8s, &client).await?; let mut details = vec![format!("kv_mount={}", self.score.kv_mount)]; for user in &self.score.users { @@ -736,6 +738,51 @@ mod tests { assert_eq!(s.instance.namespace, "openbao"); assert_eq!(s.instance.pod(), "openbao-0"); assert_eq!(s.kv_mount, "secret"); + assert_eq!(s.endpoint, None); + } + + #[test] + fn endpoint_selects_direct_api_access() { + let score = OpenbaoSetupScore::default().endpoint("https://openbao.example.com"); + assert_eq!( + score.endpoint.as_deref(), + Some("https://openbao.example.com") + ); + } + + #[test] + fn openbao_health_omits_vault_enterprise_fields() { + let status: OpenbaoStatus = serde_json::from_str( + r#"{"initialized":true,"sealed":false,"standby":false,"version":"2.6.0"}"#, + ) + .unwrap(); + assert!(status.initialized); + assert!(!status.sealed); + } + + #[test] + fn recovery_file_is_complete_and_private() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("recovery.json"); + let expected = InitOutput { + keys: vec!["key-1".to_string(), "key-2".to_string()], + root_token: "root-token".to_string(), + }; + + write_recovery_file(&path, &expected).unwrap(); + + let actual: InitOutput = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(actual.keys, expected.keys); + assert_eq!(actual.root_token, expected.root_token); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } } #[test] diff --git a/harmony/src/modules/tenant/credentials.rs b/harmony/src/modules/tenant/credentials.rs index 6d597378..2dd20948 100644 --- a/harmony/src/modules/tenant/credentials.rs +++ b/harmony/src/modules/tenant/credentials.rs @@ -34,6 +34,7 @@ pub struct TenantCredentialScore { rules: Vec, #[serde(skip)] store: Arc, + allow_insecure_source: bool, } impl std::fmt::Debug for TenantCredentialScore { @@ -52,12 +53,14 @@ impl TenantCredentialScore { name: K8sName, rules: Vec, store: Arc, + allow_insecure_source: bool, ) -> Self { Self { namespace, name, rules, store, + allow_insecure_source, } } @@ -200,6 +203,11 @@ impl Interpret for TenantCredentialInterpret { "tenant credentials require a secure topology loaded from a kubeconfig".to_string(), ) })?; + if !connection.tls_verified && !self.score.allow_insecure_source { + return Err(InterpretError::new( + "tenant credentials require a TLS-verified source kubeconfig".to_string(), + )); + } let namespace = self.score.namespace.as_ref(); client .apply(&self.score.service_account(), Some(namespace)) @@ -352,6 +360,7 @@ mod tests { server: "https://api.example.com:6443".to_string(), tls_server_name: None, proxy_url: None, + tls_verified: true, }, "customer-fleet", "fleet-deployer", @@ -372,6 +381,7 @@ mod tests { "fleet-deployer".parse().unwrap(), Vec::new(), Arc::new(ConfigClient::new(Vec::new())), + false, ); let serialized = serde_json::to_string(&score).unwrap(); diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index 4da19905..a458b0ba 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -450,14 +450,16 @@ impl ZitadelSetupScore { /// Function name doubles as the Action name — Zitadel requires the /// script's entry function to match. pub const GROUPS_CLAIM_ACTION_NAME: &str = "harmonyGroupsClaim"; +const GROUPS_CLAIM_ACTION_ALLOWED_TO_FAIL: bool = true; pub const GROUPS_CLAIM_ACTION_SCRIPT: &str = r#"function harmonyGroupsClaim(ctx, api) { - if (ctx.v1.user.grants === undefined || ctx.v1.user.grants.count == 0) { + var grants = ctx && ctx.v1 && ctx.v1.user && ctx.v1.user.grants; + if (!grants || !grants.grants || grants.grants.length === 0) { return; } let groups = []; - ctx.v1.user.grants.grants.forEach(grant => { - grant.roles.forEach(role => groups.push(role)); + grants.grants.forEach(grant => { + (grant.roles || []).forEach(role => groups.push(role)); }); api.v1.claims.setClaim('groups', groups); }"#; @@ -1743,6 +1745,13 @@ impl ZitadelSetupInterpret { // creation, 5 = Pre access token creation. const FLOW_COMPLEMENT_TOKEN: &str = "2"; const TRIGGERS: [&str; 2] = ["4", "5"]; + let action_body = serde_json::json!({ + "name": GROUPS_CLAIM_ACTION_NAME, + "script": GROUPS_CLAIM_ACTION_SCRIPT, + "timeout": "10s", + // Claim enrichment must never prevent Zitadel from issuing a token. + "allowedToFail": GROUPS_CLAIM_ACTION_ALLOWED_TO_FAIL, + }); let action_id = match self .find_action_id(client, pat, GROUPS_CLAIM_ACTION_NAME) @@ -1750,19 +1759,27 @@ impl ZitadelSetupInterpret { .map_err(InterpretError::new)? { Some(id) => { - debug!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' already exists"); + let resp = self + .put(client, &format!("/management/v1/actions/{id}")) + .bearer_auth(pat) + .json(&action_body) + .send() + .await + .map_err(|e| InterpretError::new(format!("Update action: {e}")))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + if !is_zitadel_no_changes(&body) { + return Err(InterpretError::new(format!("Update action failed: {body}"))); + } + } + info!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' reconciled"); id } None => { let resp = self .post(client, "/management/v1/actions") .bearer_auth(pat) - .json(&serde_json::json!({ - "name": GROUPS_CLAIM_ACTION_NAME, - "script": GROUPS_CLAIM_ACTION_SCRIPT, - "timeout": "10s", - "allowedToFail": false, - })) + .json(&action_body) .send() .await .map_err(|e| InterpretError::new(format!("Create action: {e}")))?; @@ -2928,6 +2945,14 @@ impl Interpret for ZitadelCredentialsExportInterpret mod tests { use super::*; + #[test] + fn groups_claim_action_guards_token_flows_without_user_grants() { + assert!(GROUPS_CLAIM_ACTION_ALLOWED_TO_FAIL); + assert!(GROUPS_CLAIM_ACTION_SCRIPT.contains("ctx && ctx.v1 && ctx.v1.user")); + assert!(GROUPS_CLAIM_ACTION_SCRIPT.contains("if (!grants || !grants.grants")); + assert!(!GROUPS_CLAIM_ACTION_SCRIPT.contains("ctx.v1.user.grants.count")); + } + /// Live validation against a local Zitadel (docker) + k3d. Provisions a /// PKCE app and exports its client_id to a ConfigMap, then the caller /// checks both via the Zitadel API / `kubectl`. Run: diff --git a/harmony_app/src/context.rs b/harmony_app/src/context.rs index f40e1976..6b1cc5b2 100644 --- a/harmony_app/src/context.rs +++ b/harmony_app/src/context.rs @@ -133,6 +133,28 @@ pub struct AppContext { } impl AppContext { + /// Use operator-owned cluster credentials without adding them to an app's + /// configured secret sources. + pub fn from_kubeconfig( + context: &Context, + version: impl Into, + kubeconfig: PathBuf, + ) -> Result { + let contents = std::fs::read_to_string(&kubeconfig) + .map_err(|e| io(format!("read kubeconfig '{}'", kubeconfig.display()), e))?; + let cluster_target = Some(kubeconfig_target(&contents)?); + let mut ctx = Self::new( + context, + version.into(), + None, + Arc::new(ConfigClient::new(Vec::new())), + None, + cluster_target, + ); + ctx.kubeconfig = Some(kubeconfig); + Ok(ctx) + } + /// Load context metadata without contacting the cluster credential source. pub fn load_metadata( context: &Context, diff --git a/harmony_app/src/publish.rs b/harmony_app/src/publish.rs index 803b9ebc..648793b3 100644 --- a/harmony_app/src/publish.rs +++ b/harmony_app/src/publish.rs @@ -33,6 +33,10 @@ impl ImageRefs { }) } + pub fn get(&self, name: &str) -> Option<&str> { + self.0.get(name).map(String::as_str) + } + pub fn iter(&self) -> impl Iterator { self.0 .iter() @@ -42,7 +46,7 @@ impl ImageRefs { pub trait ImagePublisher { fn archive_type(&self) -> &'static str; - fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result<(), ImageError>; + fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result; } pub enum PublicationTopology { @@ -58,9 +62,12 @@ impl ImagePublisher for PublicationTopology { } } - fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result<(), ImageError> { + fn publish(&self, specs: &[ImageSpec], images: &ImageRefs) -> Result { match self { - Self::K3d { cluster } => import_images_to_k3d(specs, images, cluster), + Self::K3d { cluster } => { + import_images_to_k3d(specs, images, cluster)?; + Ok(images.clone()) + } Self::Registry { registry } => push_images(specs, images, registry), } } @@ -143,16 +150,20 @@ pub fn publish_images( for spec in specs { images.require(&spec.name)?; } - publisher.publish(specs, images)?; - Ok(images.clone()) + publisher.publish(specs, images) } -fn push_images(specs: &[ImageSpec], images: &ImageRefs, registry: &str) -> Result<(), ImageError> { +fn push_images( + specs: &[ImageSpec], + images: &ImageRefs, + registry: &str, +) -> Result { let user = std::env::var("REGISTRY_USER") .map_err(|_| ImageError::Missing("REGISTRY_USER required to push".to_string()))?; let token = std::env::var("REGISTRY_TOKEN") .map_err(|_| ImageError::Missing("REGISTRY_TOKEN required to push".to_string()))?; registry_login(registry, &user, &token)?; + let mut published = BTreeMap::new(); for spec in specs { let reference = images.require(&spec.name)?; let (repository, expected_digest) = reference.split_once('@').ok_or_else(|| { @@ -195,13 +206,14 @@ fn push_images(specs: &[ImageSpec], images: &ImageRefs, registry: &str) -> Resul ImageError::Missing("docker push did not report a digest".to_string()) })?; if pushed_digest != expected_digest { - return Err(ImageError::Invalid(format!( - "docker pushed {pushed_digest} for '{}', expected {expected_digest}", + log::info!( + "registry digest for '{}' is {pushed_digest} (local archive {expected_digest})", spec.name - ))); + ); } + published.insert(spec.name.clone(), format!("{repository}@{pushed_digest}")); } - Ok(()) + Ok(ImageRefs::new(published)) } fn parse_pushed_digest(output: &str) -> Option<&str> { diff --git a/harmony_cli/src/cli_logger.rs b/harmony_cli/src/cli_logger.rs index db53008f..49b79f6e 100644 --- a/harmony_cli/src/cli_logger.rs +++ b/harmony_cli/src/cli_logger.rs @@ -17,8 +17,8 @@ pub fn init() { // The framework still emits via the `log` crate; tracing-subscriber's default // `tracing-log` bridge captures those records, so this subscriber covers both. -// Normal runs stay terse (level + message, ANSI-coloured); debug/trace adds the -// timestamp + target needed to actually debug — matching the old env_logger UX. +// Normal runs include the target so failures identify their owning module; +// debug/trace also adds timestamps. fn configure_logger() { let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); let verbose = std::env::var("RUST_LOG") @@ -28,7 +28,7 @@ fn configure_logger() { let _ = if verbose { builder.with_target(true).try_init() } else { - builder.without_time().with_target(false).try_init() + builder.without_time().with_target(true).try_init() }; } diff --git a/harmony_zitadel_auth/src/config.rs b/harmony_zitadel_auth/src/config.rs index c09ad04d..90d6d126 100644 --- a/harmony_zitadel_auth/src/config.rs +++ b/harmony_zitadel_auth/src/config.rs @@ -49,6 +49,19 @@ pub struct OperatorCookieKey { pub cookie_key_b64: String, } +impl OperatorCookieKey { + pub fn generate() -> Self { + use base64::{Engine, engine::general_purpose::STANDARD}; + use rand::RngCore; + + let mut bytes = [0; 64]; + rand::rng().fill_bytes(&mut bytes); + Self { + cookie_key_b64: STANDARD.encode(bytes), + } + } +} + #[cfg(feature = "axum")] impl OperatorCookieKey { pub fn key(&self) -> anyhow::Result { @@ -64,3 +77,16 @@ impl OperatorCookieKey { Ok(axum_extra::extract::cookie::Key::from(&bytes)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_cookie_key_has_64_bytes() { + use base64::{Engine, engine::general_purpose::STANDARD}; + + let key = OperatorCookieKey::generate(); + assert_eq!(STANDARD.decode(key.cookie_key_b64).unwrap().len(), 64); + } +} diff --git a/harmony_zitadel_jwt/Cargo.toml b/harmony_zitadel_jwt/Cargo.toml index 0ddbd2cc..cba34ab1 100644 --- a/harmony_zitadel_jwt/Cargo.toml +++ b/harmony_zitadel_jwt/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] anyhow.workspace = true +base64.workspace = true chrono = { workspace = true } jsonwebtoken = "9" reqwest = { workspace = true, features = ["json"] } diff --git a/harmony_zitadel_jwt/src/lib.rs b/harmony_zitadel_jwt/src/lib.rs index a07abbe1..bd288fe4 100644 --- a/harmony_zitadel_jwt/src/lib.rs +++ b/harmony_zitadel_jwt/src/lib.rs @@ -2,6 +2,7 @@ use serde::Deserialize; use std::sync::Mutex; use anyhow::{Context, Result}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader}; pub struct ZitadelJwtBearer { @@ -38,9 +39,7 @@ pub struct MachineKeyFile { #[derive(Debug, Clone)] pub struct CachedToken { pub(crate) access_token: String, - /// Unix seconds at which the token is no longer trusted by - /// `cached_if_fresh`. Computed from the OAuth response's `expires_in` - /// and the local clock at mint time. + /// Unix seconds at which the token is no longer trusted. pub(crate) expires_at_unix: i64, } @@ -129,7 +128,7 @@ impl ZitadelJwtBearer { *self.cache.lock().unwrap() = Some(CachedToken { access_token: token.clone(), - expires_at_unix: now + expires_in, + expires_at_unix: cache_expiry(&token, expires_in, now)?, }); Ok(token) @@ -149,6 +148,24 @@ impl ZitadelJwtBearer { } } +fn cache_expiry(token: &str, expires_in: i64, now: i64) -> Result { + let payload = token + .split('.') + .nth(1) + .context("Zitadel id_token is not a JWT")?; + let claims: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(payload) + .context("decoding Zitadel id_token claims")?, + ) + .context("parsing Zitadel id_token claims")?; + let jwt_exp = claims["exp"] + .as_i64() + .context("Zitadel id_token has no numeric exp claim")?; + + Ok(jwt_exp.min(now.saturating_add(expires_in))) +} + /// Build the JWT-bearer assertion. Split out from the network path so /// the claims + header shape can be unit-tested without an HTTP server, /// and split internally into the (pure) claim/header builders so they @@ -289,6 +306,16 @@ mod tests { assert_eq!(bearer().cached_if_fresh(), None); } + #[test] + fn cache_expiry_never_outlives_the_id_token() { + let now = 1_700_000_000; + let claims = URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{}}}"#, now + 600)); + let token = format!("e30.{claims}.signature"); + + assert_eq!(cache_expiry(&token, 3600, now).unwrap(), now + 600); + assert_eq!(cache_expiry(&token, 300, now).unwrap(), now + 300); + } + #[test] fn assertion_claims_carry_iss_sub_aud_exp_iat() { let now = 1_700_000_000; -- 2.39.5 From 3b57c658d0396780a17f22412a64335a63e38ce7 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 21 Jul 2026 15:07:02 -0400 Subject: [PATCH 12/47] fix: device name in nats callout --- Cargo.lock | 10 ++ fleet/harmony-fleet-agent/Cargo.toml | 1 + .../src/fleet_publisher.rs | 13 +-- fleet/harmony-fleet-agent/src/main.rs | 5 +- fleet/harmony-fleet-deploy/src/app.rs | 3 +- .../harmony-fleet-deploy/src/device_setup.rs | 10 +- harmony/src/modules/nats/score_nats.rs | 30 +++--- harmony/src/modules/nats/score_nats_k8s.rs | 101 +++++++----------- 8 files changed, 83 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c16a1e2..a1d9f778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4032,6 +4032,7 @@ dependencies = [ "harmony-reconciler-contracts", "harmony_secret", "podman-api", + "sd-notify", "serde", "serde_json", "sha2 0.10.9", @@ -7951,6 +7952,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sd-notify" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" +dependencies = [ + "libc", +] + [[package]] name = "sec1" version = "0.3.0" diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 6b73ad20..64c88ebe 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -23,3 +23,4 @@ clap = { workspace = true } toml = { workspace = true } thiserror = { workspace = true } podman-api = "0.9" +sd-notify = "0.4" diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index f0e82d81..eb5f10cf 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -74,7 +74,7 @@ impl FleetPublisher { &self, labels: BTreeMap, inventory: Option, - ) { + ) -> anyhow::Result<()> { let info = DeviceInfo { device_id: self.device_id.clone(), labels, @@ -82,14 +82,9 @@ impl FleetPublisher { updated_at: chrono::Utc::now(), }; let key = device_info_key(&self.device_id.to_string()); - match serde_json::to_vec(&info) { - Ok(payload) => { - if let Err(e) = self.info_bucket.put(&key, payload.into()).await { - tracing::warn!(%key, error = %e, "publish_device_info: kv put failed"); - } - } - Err(e) => tracing::warn!(error = %e, "publish_device_info: serialize failed"), - } + let payload = serde_json::to_vec(&info)?; + self.info_bucket.put(&key, payload.into()).await?; + Ok(()) } /// Tiny liveness ping. Called every 30s. diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 26fad5ff..62fdf5d1 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -280,7 +280,10 @@ async fn main() -> Result<()> { .or_insert_with(|| device_id.to_string()); fleet .publish_device_info(startup_labels, Some(inventory_snapshot.clone())) - .await; + .await + .context("publishing device registration")?; + sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) + .context("notifying systemd that registration completed")?; // Reconciler exists only when a podman topology is available. // Without it, the desired-state watch + periodic reconcile arms diff --git a/fleet/harmony-fleet-deploy/src/app.rs b/fleet/harmony-fleet-deploy/src/app.rs index c0d5fc35..e70eade2 100644 --- a/fleet/harmony-fleet-deploy/src/app.rs +++ b/fleet/harmony-fleet-deploy/src/app.rs @@ -124,7 +124,7 @@ impl HarmonyApp for FleetApp { nats }; let account = nats.account_ref(); - let callout = + let mut callout = NatsAuthCalloutScore::for_account("fleet-callout", namespace, &account, "auth") .credentials(&credentials.credentials_ref()) .with_oidc(&provider, &application) @@ -133,6 +133,7 @@ impl HarmonyApp for FleetApp { .admin_role(ADMIN_ROLE) .device_role(DEVICE_ROLE) .device_id_claim("client_id"); + callout.device_id_prefix_strip = "device-".to_string(); let nats = nats.with_auth_callout(&callout.auth_callout_ref()); let mut openbao = OpenbaoScore::new(namespace, "openbao", ctx.service_host("openbao")) diff --git a/fleet/harmony-fleet-deploy/src/device_setup.rs b/fleet/harmony-fleet-deploy/src/device_setup.rs index 57fdcfec..42504706 100644 --- a/fleet/harmony-fleet-deploy/src/device_setup.rs +++ b/fleet/harmony-fleet-deploy/src/device_setup.rs @@ -292,7 +292,8 @@ After=network-online.target Wants=network-online.target [Service] -Type=simple +Type=notify +NotifyAccess=main User=fleet-agent Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml Environment=RUST_LOG=info @@ -1109,4 +1110,11 @@ mod tests { let toml = cfg.render_toml(); assert!(toml.contains("danger_accept_invalid_certs = true")); } + + #[test] + fn systemd_service_is_ready_only_after_agent_notification() { + let unit = base_config(BTreeMap::new()).render_systemd_unit(); + assert!(unit.contains("Type=notify\n")); + assert!(unit.contains("NotifyAccess=main\n")); + } } diff --git a/harmony/src/modules/nats/score_nats.rs b/harmony/src/modules/nats/score_nats.rs index 22715c7d..92470cdc 100644 --- a/harmony/src/modules/nats/score_nats.rs +++ b/harmony/src/modules/nats/score_nats.rs @@ -9,14 +9,13 @@ use serde::Serialize; use crate::data::Version; use crate::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; use crate::inventory::Inventory; -use crate::modules::nats::score_nats_k8s::{WebSocketRouteCfg, websocket_route_score}; +use crate::modules::nats::score_nats_k8s::{WebSocketRouteCfg, websocket_ingress_score}; use crate::modules::nats::{ NatsAccountRef, NatsAuthCalloutCredentialsRef, NatsAuthCalloutRef, NatsClientRef, NatsHelmChartScore, }; use crate::score::Score; use crate::topology::{HelmCommand, K8sclient, Topology}; -use harmony_k8s::KubernetesDistribution; /// Authentication mode the deployed NATS server enforces. #[derive(Debug, Clone, Serialize)] @@ -636,23 +635,12 @@ impl Interpret for NatsInterpret { .await .map_err(InterpretError::new)?; if let Some(websocket) = &self.score.websocket { - let client = topology.k8s_client().await.map_err(InterpretError::new)?; - if client - .get_k8s_distribution() - .await - .map_err(|error| InterpretError::new(error.to_string()))? - != KubernetesDistribution::OpenshiftFamily - { - return Err(InterpretError::new( - "NATS WebSocket exposure currently requires OpenShift".to_string(), - )); - } - websocket_route_score( + websocket_ingress_score( &self.score.release_name, &self.score.namespace, &websocket.host, &websocket.cluster_issuer, - ) + )? .interpret(inventory, topology) .await?; } @@ -750,6 +738,18 @@ mod tests { values["config"]["websocket"]["no_tls"], serde_yaml::Value::Bool(true) ); + + let ingress = websocket_ingress_score( + &score.release_name, + &score.namespace, + "nats.example.com", + "letsencrypt-prod", + ) + .expect("valid ingress"); + assert_eq!(ingress.host.to_string(), "nats.example.com"); + assert_eq!(ingress.backend_service.to_string(), "fleet-nats"); + assert_eq!(ingress.port, 8080); + assert_eq!(ingress.cluster_issuer.as_deref(), Some("letsencrypt-prod")); } #[test] diff --git a/harmony/src/modules/nats/score_nats_k8s.rs b/harmony/src/modules/nats/score_nats_k8s.rs index bff563e9..272ffc50 100644 --- a/harmony/src/modules/nats/score_nats_k8s.rs +++ b/harmony/src/modules/nats/score_nats_k8s.rs @@ -29,8 +29,8 @@ use crate::{ topology::{HelmCommand, K8sclient, TlsRouter, Topology}, }; -/// Public WebSocket access for the NATS server, terminated as an -/// OpenShift Route with edge-TLS. The Route itself does not need TLS +/// Public WebSocket access for the NATS server, terminated by a +/// cert-manager-backed Ingress with edge-TLS. The backend does not need TLS /// for the upstream connection; the chart's `config.websocket.no_tls` /// is set so the WebSocket listener is plain HTTP inside the cluster /// and the Route owns the TLS handshake. @@ -39,38 +39,33 @@ pub struct WebSocketRouteCfg { /// Public hostname the Route answers on, e.g. /// `nats-fleet-staging.cb1.nationtech.io`. pub host: String, - /// cert-manager `ClusterIssuer` name. Annotation drives an - /// automatic Route certificate. Defaults to `letsencrypt-prod` — + /// cert-manager `ClusterIssuer` name. Defaults to `letsencrypt-prod` — /// override per cluster. pub cluster_issuer: String, } -pub(crate) fn websocket_route_score( +pub(crate) fn websocket_ingress_score( name: &str, namespace: &str, host: &str, cluster_issuer: &str, -) -> OKDRouteScore { - OKDRouteScore::new( - &format!("{name}-ws"), - namespace, - RouteSpec { - to: RouteTargetReference { - kind: "Service".to_string(), - name: name.to_string(), - weight: Some(100), - }, - host: Some(host.to_string()), - port: Some(RoutePort { target_port: 8080 }), - tls: Some(TLSConfig { - termination: "edge".to_string(), - insecure_edge_termination_policy: Some("Redirect".to_string()), - ..Default::default() - }), - ..Default::default() - }, - ) - .with_annotation("cert-manager.io/cluster-issuer", cluster_issuer) +) -> Result { + let fqdn = |value: &str| { + value.parse().map_err(|error| { + InterpretError::new(format!("invalid ingress name '{value}': {error}")) + }) + }; + Ok(K8sIngressScore { + name: fqdn(&format!("{name}-ws"))?, + host: fqdn(host)?, + backend_service: fqdn(name)?, + port: 8080, + path: None, + path_type: None, + namespace: Some(fqdn(namespace)?), + ingress_class_name: None, + cluster_issuer: Some(cluster_issuer.to_string()), + }) } /// Auth-callout configuration for the NATS server. When `Some`, the @@ -174,15 +169,9 @@ impl Interpr .await?; } if let Some(ws) = &self.score.websocket { - info!("creating websocket Route at host {}", ws.host); - self.create_websocket_route( - topology, - inventory, - self.score.distribution.clone(), - self.score.cluster.clone(), - ws, - ) - .await?; + info!("creating websocket Ingress at host {}", ws.host); + self.create_websocket_ingress(topology, inventory, self.score.cluster.clone(), ws) + .await?; } let domain = NatsEndpoint { host: domain }; @@ -283,40 +272,26 @@ impl NatsK8sInterpret { } } - /// WebSocket Route for the public single-instance shape. - /// Edge-TLS termination at the OKD router; the chart's WS + /// WebSocket Ingress for the public single-instance shape. + /// Edge-TLS termination at the ingress; the chart's WS /// listener runs plain HTTP inside the cluster - /// (`config.websocket.no_tls`). cert-manager picks up the Route - /// via the `cert-manager.io/cluster-issuer` annotation and - /// provisions the certificate. - async fn create_websocket_route( + /// (`config.websocket.no_tls`). cert-manager provisions the + /// certificate named by the Ingress TLS block. + async fn create_websocket_ingress( &self, topology: &T, inventory: &Inventory, - distribution: KubernetesDistribution, nats_cluster: NatsCluster, ws: &WebSocketRouteCfg, ) -> Result { - match distribution { - KubernetesDistribution::OpenshiftFamily => { - websocket_route_score( - &nats_cluster.name, - &nats_cluster.namespace, - &ws.host, - &ws.cluster_issuer, - ) - .interpret(inventory, topology) - .await - } - KubernetesDistribution::K3sFamily | KubernetesDistribution::Default => { - // Local k3d / non-OKD path is not needed for the - // staging install — the public-WS shape is OKD-only - // for now. Surface it as a hard error if anyone tries. - Err(InterpretError::new( - "WebSocketRouteCfg only implemented for OpenshiftFamily today".to_string(), - )) - } - } + websocket_ingress_score( + &nats_cluster.name, + &nats_cluster.namespace, + &ws.host, + &ws.cluster_issuer, + )? + .interpret(inventory, topology) + .await } async fn create_ca_bundle_secret( @@ -504,7 +479,7 @@ impl NatsK8sInterpret { // ---- websocket block ------------------------------------------------- // The chart's WS listener defaults to port 8080. With `no_tls` // the listener is plain HTTP inside the cluster; the OKD - // Route emitted by `create_websocket_route` terminates TLS. + // The WebSocket Ingress terminates TLS. let websocket_block = match self.score.websocket { Some(_) => String::from( " websocket: -- 2.39.5 From 3498588810f9ae891ff01931f3f549facf702577 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 21 Jul 2026 23:22:47 -0400 Subject: [PATCH 13/47] fix: harden fleet device status reporting --- examples/fleet_load_test/src/main.rs | 1 + .../src/fleet_publisher.rs | 1 + fleet/harmony-fleet-operator/src/crd.rs | 11 +- .../src/device_status.rs | 172 ++++++++++++++---- .../src/frontend/views/deployments.rs | 2 +- .../src/frontend/views/devices.rs | 12 +- fleet/harmony-fleet-operator/src/main.rs | 5 +- .../src/service/mock.rs | 1 + .../harmony-fleet-operator/src/service/mod.rs | 2 + .../src/service/real.rs | 3 + harmony-reconciler-contracts/src/fleet.rs | 12 +- 11 files changed, 167 insertions(+), 55 deletions(-) diff --git a/examples/fleet_load_test/src/main.rs b/examples/fleet_load_test/src/main.rs index fec77d19..cd2845ee 100644 --- a/examples/fleet_load_test/src/main.rs +++ b/examples/fleet_load_test/src/main.rs @@ -529,6 +529,7 @@ async fn simulate_heartbeat_loop( let hb = HeartbeatPayload { device_id: Id::from(device.device_id.clone()), at: Utc::now(), + agent_version: None, }; if let Ok(payload) = serde_json::to_vec(&hb) { if bucket.put(&hb_key, payload.into()).await.is_ok() { diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index eb5f10cf..20f0ff80 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -92,6 +92,7 @@ impl FleetPublisher { let hb = HeartbeatPayload { device_id: self.device_id.clone(), at: chrono::Utc::now(), + agent_version: Some(env!("CARGO_PKG_VERSION").to_string()), }; let key = device_heartbeat_key(&self.device_id.to_string()); match serde_json::to_vec(&hb) { diff --git a/fleet/harmony-fleet-operator/src/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs index 30e325b9..2672b1a6 100644 --- a/fleet/harmony-fleet-operator/src/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -90,11 +90,8 @@ pub struct AggregateLastError { /// reflects it here. /// /// `metadata.labels` carries the device's routing labels. `spec. -/// inventory` holds the hardware/OS snapshot. No status subresource -/// today — liveness is queried from the NATS `device-heartbeat` -/// bucket directly; when a CR-side reflection (Reachable / Stale -/// conditions) becomes useful, it'll land with its own reconciler -/// rather than sitting here as speculative surface. +/// inventory` holds the hardware/OS snapshot. The status subresource +/// reflects heartbeat liveness and the running agent version. #[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)] #[kube( group = "fleet.nationtech.io", @@ -117,14 +114,14 @@ pub struct DeviceSpec { /// `device-heartbeat` bucket onto the CR, so `kubectl get devices` and /// the dashboard see reachability without reading NATS. Written by the /// device-status reconciler. -#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)] +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeviceStatus { /// RFC 3339 timestamp of the last heartbeat seen. `None` until the /// device has pinged at least once. - #[serde(skip_serializing_if = "Option::is_none")] pub last_heartbeat: Option, pub reachability: Reachability, + pub current_version: Option, } /// Coarse liveness derived from heartbeat freshness. Failing/Pending diff --git a/fleet/harmony-fleet-operator/src/device_status.rs b/fleet/harmony-fleet-operator/src/device_status.rs index 8caebc0d..81482e5d 100644 --- a/fleet/harmony-fleet-operator/src/device_status.rs +++ b/fleet/harmony-fleet-operator/src/device_status.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use std::time::Duration; -use anyhow::Result; +use anyhow::{Context, Result, bail}; use async_nats::jetstream::kv::{Operation, Store}; use chrono::{DateTime, Utc}; use futures_util::StreamExt; @@ -24,7 +24,52 @@ use kube::{Client, ResourceExt}; use serde_json::json; use tokio::sync::Mutex; -use crate::crd::{Device, Reachability}; +use crate::crd::{Device, DeviceStatus, Reachability}; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ObservedHeartbeat { + received_at: DateTime, + agent_version: Option, +} + +fn heartbeat_status(heartbeat: Option, now: DateTime) -> DeviceStatus { + match heartbeat { + Some(heartbeat) => DeviceStatus { + last_heartbeat: Some(heartbeat.received_at.to_rfc3339()), + reachability: reachability(heartbeat.received_at, now), + current_version: heartbeat.agent_version, + }, + None => DeviceStatus { + last_heartbeat: None, + reachability: Reachability::Unknown, + current_version: None, + }, + } +} + +fn observe_heartbeat( + key: &str, + payload: &[u8], + received_at: DateTime, +) -> Result<(String, ObservedHeartbeat)> { + let device_id = key + .strip_prefix("heartbeat.") + .context("heartbeat key has no heartbeat. prefix")?; + let heartbeat: HeartbeatPayload = serde_json::from_slice(payload)?; + if heartbeat.device_id.to_string() != device_id { + bail!( + "heartbeat payload device {} does not match authorized key {device_id}", + heartbeat.device_id + ); + } + Ok(( + device_id.to_string(), + ObservedHeartbeat { + received_at, + agent_version: heartbeat.agent_version, + }, + )) +} /// A device with no heartbeat within this window is `Stale`. Agents /// ping every 30 s, so this tolerates ~2 missed pings. @@ -44,7 +89,7 @@ pub async fn run( }) .await?; - let heartbeats: Mutex>> = Mutex::new(HashMap::new()); + let heartbeats: Mutex>> = Mutex::new(HashMap::new()); let devices: Api = Api::namespaced(client, namespace); tokio::try_join!( @@ -56,7 +101,7 @@ pub async fn run( async fn watch_heartbeats( bucket: &Store, - heartbeats: &Mutex>>, + heartbeats: &Mutex>>, ) -> Result<()> { let mut watch = bucket.watch_with_history(">").await?; tracing::info!("device-status: watching device-heartbeat KV"); @@ -70,46 +115,63 @@ async fn watch_heartbeats( }; match entry.operation { Operation::Put => { - if let Ok(hb) = serde_json::from_slice::(&entry.value) { - heartbeats - .lock() - .await - .insert(hb.device_id.to_string(), hb.at); + let Some(received_at) = DateTime::from_timestamp( + entry.created.unix_timestamp(), + entry.created.nanosecond(), + ) else { + tracing::warn!(key = %entry.key, "device-status: invalid NATS timestamp"); + continue; + }; + match observe_heartbeat(&entry.key, &entry.value, received_at) { + Ok((device_id, heartbeat)) => { + heartbeats.lock().await.insert(device_id, Some(heartbeat)); + } + Err(error) => { + tracing::warn!(key = %entry.key, %error, "device-status: invalid heartbeat") + } } } Operation::Delete | Operation::Purge => { if let Some(id) = entry.key.strip_prefix("heartbeat.") { - heartbeats.lock().await.remove(id); + heartbeats.lock().await.insert(id.to_string(), None); } } } } - Ok(()) + bail!("device-status: heartbeat watch ended") } async fn patch_loop( devices: &Api, - heartbeats: &Mutex>>, + heartbeats: &Mutex>>, ) -> Result<()> { // Last status written per device, to skip no-op patches. - let mut applied: HashMap)> = HashMap::new(); + let mut applied: HashMap = HashMap::new(); let mut ticker = tokio::time::interval(TICK); loop { ticker.tick().await; - let snapshot: Vec<(String, DateTime)> = heartbeats + let snapshot: Vec<(String, Option)> = heartbeats .lock() .await .iter() - .map(|(k, v)| (k.clone(), *v)) + .map(|(k, v)| (k.clone(), v.clone())) .collect(); let now = Utc::now(); - for (id, at) in snapshot { - let reachability = reachability(at, now); - if applied.get(&id) == Some(&(reachability, at)) { + for (id, heartbeat) in snapshot { + let status = heartbeat_status(heartbeat, now); + if applied.get(&id) == Some(&status) { continue; } - if patch_status(devices, &id, reachability, at).await { - applied.insert(id, (reachability, at)); + if patch_status(devices, &id, &status).await { + if status.reachability == Reachability::Unknown { + let mut heartbeats = heartbeats.lock().await; + if matches!(heartbeats.get(&id), Some(None)) { + heartbeats.remove(&id); + applied.remove(&id); + continue; + } + } + applied.insert(id, status); } } } @@ -128,29 +190,19 @@ fn reachability(last_heartbeat: DateTime, now: DateTime) -> Reachabili /// Returns whether the patch succeeded (so we only cache applied state /// on success and retry next tick otherwise). -async fn patch_status( - devices: &Api, - id: &str, - reachability: Reachability, - last_heartbeat: DateTime, -) -> bool { - let status = json!({ - "status": { - "lastHeartbeat": last_heartbeat.to_rfc3339(), - "reachability": reachability, - } - }); +async fn patch_status(devices: &Api, id: &str, status: &DeviceStatus) -> bool { + let patch = json!({ "status": status }); match devices - .patch_status(id, &PatchParams::default(), &Patch::Merge(&status)) + .patch_status(id, &PatchParams::default(), &Patch::Merge(&patch)) .await { Ok(d) => { - tracing::debug!(device = %d.name_any(), ?reachability, "device-status: patched"); + tracing::debug!(device = %d.name_any(), reachability = ?status.reachability, "device-status: patched"); true } - // A heartbeat can outrace the Device CR's creation by the - // device-reconciler; skip this tick and retry on the next. - Err(kube::Error::Api(ae)) if ae.code == 404 => false, + // Retry a heartbeat that outraced CR creation, but a tombstone + // for an absent Device is already converged. + Err(kube::Error::Api(ae)) if ae.code == 404 => status.reachability == Reachability::Unknown, Err(e) => { tracing::warn!(%id, error = %e, "device-status: patch failed"); false @@ -161,6 +213,7 @@ async fn patch_status( #[cfg(test)] mod tests { use super::*; + use harmony_reconciler_contracts::Id; #[test] fn reachable_within_window_stale_after() { @@ -174,4 +227,49 @@ mod tests { Reachability::Stale ); } + + #[test] + fn observation_uses_nats_time_and_authorized_key_identity() { + let device_time = Utc::now() - chrono::Duration::hours(2); + let server_time = Utc::now(); + let payload = serde_json::to_vec(&HeartbeatPayload { + device_id: Id::from("device-1".to_string()), + at: device_time, + agent_version: Some("1.2.3".to_string()), + }) + .unwrap(); + + let (device_id, observed) = + observe_heartbeat("heartbeat.device-1", &payload, server_time).unwrap(); + assert_eq!(device_id, "device-1"); + assert_eq!(observed.received_at, server_time); + assert_eq!(observed.agent_version.as_deref(), Some("1.2.3")); + assert_eq!( + reachability(observed.received_at, server_time), + Reachability::Reachable + ); + + assert!(observe_heartbeat("heartbeat.other-device", &payload, server_time).is_err()); + } + + #[test] + fn unknown_version_clears_previous_status_value() { + let status = DeviceStatus { + last_heartbeat: Some(Utc::now().to_rfc3339()), + reachability: Reachability::Reachable, + current_version: None, + }; + assert!(serde_json::to_value(status).unwrap()["currentVersion"].is_null()); + } + + #[test] + fn missing_heartbeat_clears_status() { + let status = heartbeat_status(None, Utc::now()); + assert_eq!(status.reachability, Reachability::Unknown); + assert_eq!(status.last_heartbeat, None); + assert_eq!(status.current_version, None); + let json = serde_json::to_value(status).unwrap(); + assert!(json["lastHeartbeat"].is_null()); + assert!(json["currentVersion"].is_null()); + } } diff --git a/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs b/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs index 7b42b577..07540192 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/deployments.rs @@ -211,7 +211,7 @@ fn devices_tab(devices: &[DeviceDetail]) -> Markup { td { (badges::device_status(d.status)) } td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } } td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" { - @if let Some(inv) = &d.inventory { (&inv.agent_version) } @else { "\u{2014}" } + @if let Some(version) = &d.current_version { (version) } @else { "\u{2014}" } } } td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } } } diff --git a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs index 4db94bc5..03344404 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs @@ -272,6 +272,7 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark (definition("Device ID", &device.id, true, true)) (definition("Region", &device.region, true, false)) (definition("Last ping", &time_ago(device.minutes_ago), false, false)) + (definition("Agent", agent_version(device), true, false)) @if let Some(inv) = &device.inventory { (definition("Hostname", &inv.hostname, true, false)) (definition("Arch", &inv.arch, true, false)) @@ -279,7 +280,6 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark (definition("Kernel", &inv.kernel, true, false)) (definition("CPU cores", &inv.cpu_cores.to_string(), false, false)) (definition("Memory", &format!("{} MB", inv.memory_mb), false, false)) - (definition("Agent", &inv.agent_version, true, false)) } @else { div class="text-[12px] text-slate-500 mt-2" { "No inventory reported yet" } } @@ -478,13 +478,8 @@ pub fn row(d: &DeviceDetail) -> Markup { // ── Helpers ──────────────────────────────────────────────────────────── -/// Agent version from the device's inventory, or an em-dash placeholder -/// when the agent hasn't reported inventory yet. fn agent_version(d: &DeviceDetail) -> &str { - d.inventory - .as_ref() - .map(|i| i.agent_version.as_str()) - .unwrap_or("\u{2014}") + d.current_version.as_deref().unwrap_or("\u{2014}") } fn time_ago(minutes: i64) -> String { @@ -527,6 +522,7 @@ mod tests { deployment: Some("edge-gateway".into()), region: "eu-paris-1".into(), tags: vec!["prod".into()], + current_version: Some("v1.2.3".into()), inventory: Some(crate::service::InventorySnapshot { hostname: "hf-edge-001".into(), arch: "aarch64".into(), @@ -543,7 +539,7 @@ mod tests { fn overview_shows_device_info_not_removed_mock() { let html = detail(&sample(), Some("v2.14.1")).into_string(); assert!(html.contains("Device info")); - assert!(html.contains("v1.2.3"), "agent version from inventory"); + assert!(html.contains("v1.2.3"), "agent version from heartbeat"); assert!(html.contains("aarch64")); assert!(html.contains("Run command")); // Removed mock surfaces must be gone. diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 99756a64..0b791006 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -6,7 +6,7 @@ mod frontend; mod service; use harmony_fleet_operator::access::StaticDeviceGroups; -use harmony_fleet_operator::{device_reconciler, fleet_aggregator}; +use harmony_fleet_operator::{device_reconciler, device_status, fleet_aggregator}; use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource}; use harmony_secret::OpenBaoDeploymentSecretGrants; use harmony_zitadel_auth::ZitadelDeviceGroups; @@ -344,9 +344,12 @@ async fn run( let ctl_client = client.clone(); let dr_client = client.clone(); let dr_js = js.clone(); + let ds_client = client.clone(); + let ds_js = js.clone(); tokio::select! { r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r, r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r, + r = device_status::run(ds_client, tenant_namespace, ds_js) => r, r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r, } } diff --git a/fleet/harmony-fleet-operator/src/service/mock.rs b/fleet/harmony-fleet-operator/src/service/mock.rs index faa64645..b35135e6 100644 --- a/fleet/harmony-fleet-operator/src/service/mock.rs +++ b/fleet/harmony-fleet-operator/src/service/mock.rs @@ -168,6 +168,7 @@ fn seed_devices() -> Vec { deployment, region, tags, + current_version: inventory.as_ref().map(|i| i.agent_version.clone()), inventory, }); } diff --git a/fleet/harmony-fleet-operator/src/service/mod.rs b/fleet/harmony-fleet-operator/src/service/mod.rs index fb60b2e4..ce96e766 100644 --- a/fleet/harmony-fleet-operator/src/service/mod.rs +++ b/fleet/harmony-fleet-operator/src/service/mod.rs @@ -42,6 +42,8 @@ pub struct DeviceDetail { pub deployment: Option, pub region: String, pub tags: Vec, + /// Running version from the latest heartbeat. + pub current_version: Option, /// Hardware/OS facts from the agent. `None` until the first /// post-enrollment publish (mirrors `DeviceInfo.inventory`). pub inventory: Option, diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index 553b397b..f3fdb11c 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -108,6 +108,7 @@ fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime) - .cloned() .unwrap_or_else(|| "\u{2014}".to_string()), tags: tags_from_labels(&labels), + current_version: cr.status.as_ref().and_then(|s| s.current_version.clone()), inventory: cr.spec.inventory.clone(), } } @@ -398,6 +399,7 @@ mod tests { DeviceLiveness { last_heartbeat: None, reachability: r, + current_version: None, } } @@ -444,6 +446,7 @@ mod tests { deployment: None, region: "\u{2014}".into(), tags: vec![], + current_version: None, inventory: None, }]; let deployments = vec![DeploymentDetail { diff --git a/harmony-reconciler-contracts/src/fleet.rs b/harmony-reconciler-contracts/src/fleet.rs index 92ef773f..43a446ff 100644 --- a/harmony-reconciler-contracts/src/fleet.rs +++ b/harmony-reconciler-contracts/src/fleet.rs @@ -139,6 +139,8 @@ pub struct DeploymentState { pub struct HeartbeatPayload { pub device_id: Id, pub at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_version: Option, } #[cfg(test)] @@ -239,16 +241,24 @@ mod tests { let hb = HeartbeatPayload { device_id: Id::from("pi-01".to_string()), at: ts("2026-04-22T10:00:30Z"), + agent_version: Some("0.1.0".to_string()), }; let bytes = serde_json::to_vec(&hb).unwrap(); assert!( - bytes.len() < 96, + bytes.len() < 128, "heartbeat payload grew to {} bytes: {}", bytes.len(), String::from_utf8_lossy(&bytes), ); } + #[test] + fn heartbeat_from_old_agent_has_unknown_version() { + let heartbeat: HeartbeatPayload = + serde_json::from_str(r#"{"device_id":"pi-01","at":"2026-04-22T10:00:30Z"}"#).unwrap(); + assert_eq!(heartbeat.agent_version, None); + } + #[test] fn device_info_roundtrip() { let original = DeviceInfo { -- 2.39.5 From 929bdb3a6627bbbf62f791def182609fad3ae10b Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 21 Jul 2026 23:22:58 -0400 Subject: [PATCH 14/47] fix: retry fleet deployment convergence --- fleet/harmony-fleet-agent/src/reconciler.rs | 1 + fleet/harmony-fleet-e2e/tests/operator.rs | 36 +- .../src/fleet_aggregator.rs | 610 +++++++++++------- 3 files changed, 421 insertions(+), 226 deletions(-) diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 40455d53..45e713f4 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -341,6 +341,7 @@ impl Reconciler { if let Some(name) = &deployment { self.drop_phase(name).await; } + tracing::info!(key, "deployment removed"); } Ok(()) } diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index aa1ab51e..110a9aa9 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -11,7 +11,7 @@ use k8s_openapi::api::authorization::v1::{ use k8s_openapi::api::core::v1::Namespace; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use kube::Client; -use kube::api::{Api, DeleteParams, ObjectMeta, PostParams}; +use kube::api::{Api, DeleteParams, ObjectMeta, Patch, PatchParams, PostParams}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -85,6 +85,40 @@ async fn operator_deletes_desired_state_when_deployment_is_deleted() -> anyhow:: Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn operator_deletes_desired_state_when_selector_stops_matching() -> anyhow::Result<()> { + if !e2e_enabled() { + skip_e2e(); + return Ok(()); + } + + let stack = operator_stack().await?; + let client = Client::try_default().await?; + let devices: Api = Api::namespaced(client.clone(), &stack.namespace); + let deployments: Api = Api::namespaced(client, &stack.namespace); + + create_device(&devices, "retarget-device").await?; + create_fleet_deployment(&deployments, "retarget-test").await?; + wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", true).await?; + + deployments + .patch( + "retarget-test", + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "spec": { + "targetSelector": { + "matchLabels": { "device-id": "missing" } + } + } + })), + ) + .await?; + + wait_for_desired_state_entry(&stack, "retarget-device", "retarget-test", false).await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn operator_ignores_other_tenant_namespaces() -> anyhow::Result<()> { if !e2e_enabled() { diff --git a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs index 1beb9e56..6cdbbd76 100644 --- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs +++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs @@ -74,7 +74,7 @@ pub struct DevicePair { /// Thin projection of a Deployment CR — everything we need for /// selector evaluation + desired-state writes + status aggregation, /// without borrowing the full kube object. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct CachedDeployment { key: DeploymentKey, deployment_name: DeploymentName, @@ -101,14 +101,20 @@ pub struct FleetState { device_groups: HashMap>, /// Latest DeploymentState per (device, deployment) pair. states: HashMap, - /// Which devices have we pushed desired-state for, per deployment? + /// NATS-acknowledged desired-state payloads per device and deployment. /// Diff against recomputed targets on any change. Keyed by /// `DeploymentName` (not `DeploymentKey`) because the /// `desired-state` KV key space doesn't carry namespace — /// deployment names are globally unique at the NATS level. This /// lets cold-start seeding from the KV populate the map /// correctly without having to guess namespaces. - owned_targets: HashMap>, + owned_targets: HashMap>>, + /// Deployments whose NATS desired state differs or may differ from + /// the current device, group, or Deployment caches. + desired_dirty: HashSet, + deployment_watch_ready: bool, + device_watch_ready: bool, + group_source_ready: bool, /// Per-deployment latest-failure surface for the CR status. last_error: HashMap, /// CR keys whose status needs re-patching on the next tick. @@ -183,6 +189,88 @@ fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet .collect() } +fn upsert_deployment_state( + state: &mut FleetState, + cached: CachedDeployment, +) -> Option<&'static str> { + if state.deployments.get(&cached.key) == Some(&cached) { + return None; + } + let change = match state.deployments.get(&cached.key) { + None => "new", + Some(previous) if previous.score_json != cached.score_json => "upgrade", + Some(_) => "update", + }; + state.desired_dirty.insert(cached.deployment_name.clone()); + state.dirty.insert(cached.key.clone()); + state.deployments.insert(cached.key.clone(), cached); + Some(change) +} + +#[derive(Debug, PartialEq, Eq)] +struct DesiredStateDiff { + puts: Vec, + deletes: Vec, + targeted: usize, +} + +fn desired_state_diff(state: &FleetState, deployment_name: &DeploymentName) -> DesiredStateDiff { + let deployment = state + .deployments + .values() + .find(|deployment| &deployment.deployment_name == deployment_name); + let owned = state.owned_targets.get(deployment_name); + let desired = deployment + .map(|deployment| matched_devices(deployment, state)) + .unwrap_or_default(); + let mut puts: Vec<_> = deployment + .map(|deployment| { + desired + .iter() + .filter(|device| { + owned.and_then(|owned| owned.get(*device)) != Some(&deployment.score_json) + }) + .cloned() + .collect() + }) + .unwrap_or_default(); + let mut deletes: Vec<_> = owned + .into_iter() + .flat_map(|owned| owned.keys()) + .filter(|device| !desired.contains(*device)) + .cloned() + .collect(); + puts.sort(); + deletes.sort(); + DesiredStateDiff { + puts, + deletes, + targeted: desired.len(), + } +} + +fn mark_deployments_dirty(state: &mut FleetState) { + state.desired_dirty.extend( + state + .deployments + .values() + .map(|deployment| deployment.deployment_name.clone()), + ); +} + +fn reconciliation_ready(state: &FleetState) -> bool { + state.deployment_watch_ready && state.device_watch_ready && state.group_source_ready +} + +fn enable_reconciliation_if_ready(state: &mut FleetState) { + if reconciliation_ready(state) { + mark_deployments_dirty(state); + state + .desired_dirty + .extend(state.owned_targets.keys().cloned()); + } +} + // --------------------------------------------------------------------------- // Top-level run // --------------------------------------------------------------------------- @@ -210,7 +298,10 @@ pub async fn run( // Cold-start: initialize owned_targets from the current contents // of the desired-state bucket so we don't orphan entries written // by a previous operator run. - let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default())); + let state: SharedFleetState = Arc::new(Mutex::new(FleetState { + group_source_ready: group_source.is_none(), + ..Default::default() + })); seed_owned_targets(&desired_bucket, &state).await?; let deployments_api: Api = Api::namespaced(client.clone(), namespace); @@ -240,12 +331,9 @@ pub async fn run( let deployment_watcher_handle = { let state = state.clone(); - let desired = desired_bucket.clone(); let grants = secret_grants.clone(); tokio::spawn(async move { - if let Err(e) = - run_deployment_watcher(deployments_api.clone(), state, desired, grants).await - { + if let Err(e) = run_deployment_watcher(deployments_api.clone(), state, grants).await { tracing::warn!(error = %e, "aggregator: deployment watcher exited"); } }) @@ -253,9 +341,8 @@ pub async fn run( let device_watcher_handle = { let state = state.clone(); - let desired = desired_bucket.clone(); tokio::spawn(async move { - if let Err(e) = run_device_watcher(devices_api, state, desired).await { + if let Err(e) = run_device_watcher(devices_api, state).await { tracing::warn!(error = %e, "aggregator: device watcher exited"); } }) @@ -266,7 +353,6 @@ pub async fn run( // the poll only moves *scheduling* in line with membership. let group_poll_handle = { let state = state.clone(); - let desired = desired_bucket.clone(); tokio::spawn(async move { let Some(source) = group_source else { // No source configured: park forever — returning would @@ -279,7 +365,7 @@ pub async fn run( loop { ticker.tick().await; match source.device_groups().await { - Ok(snapshot) => apply_device_groups(&state, &desired, snapshot).await, + Ok(snapshot) => apply_device_groups(&state, snapshot).await, Err(e) => warn!(error = %e, "aggregator: group source fetch failed"), } } @@ -287,11 +373,13 @@ pub async fn run( }; let patch_state = state.clone(); + let patch_desired = desired_bucket.clone(); let patch_loop = async move { let mut ticker = tokio::time::interval(PATCH_TICK); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { ticker.tick().await; + reconcile_dirty_desired_state(&patch_desired, &patch_state).await; if let Err(e) = patch_tick(&patch_api, &patch_state).await { tracing::warn!(error = %e, "aggregator: patch tick failed"); } @@ -448,19 +536,23 @@ fn matching_deployment_keys(state: &FleetState, deployment: &DeploymentName) -> async fn run_deployment_watcher( api: Api, state: SharedFleetState, - desired: Store, grants: Option>, ) -> anyhow::Result<()> { let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed(); while let Some(event) = stream.try_next().await? { match event { Event::Apply(cr) | Event::InitApply(cr) => { - on_deployment_upsert(&state, &desired, grants.as_ref(), cr).await; + on_deployment_upsert(&state, grants.as_ref(), cr).await; } Event::Delete(cr) => { - on_deployment_delete(&state, &desired, grants.as_ref(), cr).await; + on_deployment_delete(&state, grants.as_ref(), cr).await; + } + Event::Init => {} + Event::InitDone => { + let mut state = state.lock().await; + state.deployment_watch_ready = true; + enable_reconciliation_if_ready(&mut state); } - Event::Init | Event::InitDone => {} } } Ok(()) @@ -468,7 +560,6 @@ async fn run_deployment_watcher( async fn on_deployment_upsert( state: &SharedFleetState, - desired: &Store, grants: Option<&Arc>, cr: Deployment, ) { @@ -494,6 +585,10 @@ async fn on_deployment_upsert( score_json: score_json.clone(), }; + if state.lock().await.deployments.get(&key) == Some(&cached) { + return; + } + // Keep the deployment out of schedulable state until its secret grant // exists; device and group watchers share the cache below. if let Some(grants) = grants { @@ -507,34 +602,24 @@ async fn on_deployment_upsert( .await; } - let (new_targets, previous_targets) = { + let selector = cached.selector.clone(); + let change = { let mut guard = state.lock().await; - let new_targets = matched_devices(&cached, &guard); - guard.deployments.insert(key.clone(), cached); - let previous = guard - .owned_targets - .remove(&deployment_name) - .unwrap_or_default(); - guard - .owned_targets - .insert(deployment_name.clone(), new_targets.clone()); - guard.dirty.insert(key.clone()); - (new_targets, previous) + upsert_deployment_state(&mut guard, cached) }; - - reconcile_kv( - desired, - &deployment_name, - &new_targets, - &previous_targets, - &score_json, - ) - .await; + if let Some(change) = change { + tracing::info!( + namespace = %key.namespace, + deployment = %deployment_name, + change, + selector = ?selector, + "aggregator: deployment accepted" + ); + } } async fn on_deployment_delete( state: &SharedFleetState, - desired: &Store, grants: Option<&Arc>, cr: Deployment, ) { @@ -549,26 +634,14 @@ async fn on_deployment_delete( sync_grant_batch(grants, &[(deployment_name.clone(), Vec::new())]).await; } - let previous = { + { let mut guard = state.lock().await; guard.deployments.remove(&key); guard.last_error.remove(&key); guard.dirty.remove(&key); - guard - .owned_targets - .remove(&deployment_name) - .unwrap_or_default() - }; - - // Every previously-owned target becomes a KV delete. Controller - // finalizer does a belt-and-suspenders scan, but we pull our own - // entries here too so agents react immediately. - for device in &previous { - let k = desired_state_key(device, &deployment_name); - if let Err(e) = desired.delete(&k).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed"); - } + guard.desired_dirty.insert(deployment_name.clone()); } + tracing::info!(deployment = %deployment_name, "aggregator: deployment deletion accepted"); } // --------------------------------------------------------------------------- @@ -579,180 +652,175 @@ async fn on_deployment_delete( /// /// For example, if a device adds or deletes a label, its desired state will contain deployments matching /// the device's new set of labels. -async fn run_device_watcher( - api: Api, - state: SharedFleetState, - desired: Store, -) -> anyhow::Result<()> { +async fn run_device_watcher(api: Api, state: SharedFleetState) -> anyhow::Result<()> { let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed(); while let Some(event) = stream.try_next().await? { match event { Event::Apply(dev) | Event::InitApply(dev) => { - on_device_upsert(&state, &desired, dev).await; + on_device_upsert(&state, dev).await; } Event::Delete(dev) => { - on_device_delete(&state, &desired, dev).await; + on_device_delete(&state, dev).await; + } + Event::Init => {} + Event::InitDone => { + let mut state = state.lock().await; + state.device_watch_ready = true; + enable_reconciliation_if_ready(&mut state); } - Event::Init | Event::InitDone => {} } } Ok(()) } -/// Re-evaluate every deployment's ownership of `name` from the current -/// labels + groups caches; mutates `owned_targets`/`dirty` and returns -/// the (deployment, was, now) transitions for KV reconciliation. -fn reevaluate_device(guard: &mut FleetState, name: &str) -> Vec<(CachedDeployment, bool, bool)> { - let labels = guard.devices.get(name).cloned().unwrap_or_default(); - let groups = guard.device_groups.get(name).cloned(); - let snapshot: Vec = guard.deployments.values().cloned().collect(); - - let mut out = Vec::with_capacity(snapshot.len()); - for d in snapshot { - let was = guard - .owned_targets - .get(&d.deployment_name) - .is_some_and(|set| set.contains(name)); - let now = device_eligible(&d, &labels, groups.as_ref()); - if was != now { - let targets = guard - .owned_targets - .entry(d.deployment_name.clone()) - .or_default(); - if now { - targets.insert(name.to_string()); - } else { - targets.remove(name); - } - guard.dirty.insert(d.key.clone()); - } - out.push((d, was, now)); - } - out -} - -/// Apply the KV writes/deletes a [`reevaluate_device`] diff calls for. -async fn reconcile_device_kv( - desired: &Store, - name: &str, - transitions: Vec<(CachedDeployment, bool, bool)>, -) { - for (cached, was, now) in transitions { - match (was, now) { - (false, true) => { - let k = desired_state_key(name, &cached.deployment_name); - if let Err(e) = desired.put(&k, cached.score_json.clone().into()).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed"); - } - } - (true, false) => { - let k = desired_state_key(name, &cached.deployment_name); - if let Err(e) = desired.delete(&k).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed"); - } - } - _ => {} - } - } -} - -async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) { +async fn on_device_upsert(state: &SharedFleetState, dev: Device) { let name = dev.name_any(); let labels: BTreeMap = dev.metadata.labels.clone().unwrap_or_default(); - - let transitions = { - let mut guard = state.lock().await; - guard.devices.insert(name.clone(), labels); - reevaluate_device(&mut guard, &name) - }; - reconcile_device_kv(desired, &name, transitions).await; + let mut state = state.lock().await; + if state.devices.get(&name) != Some(&labels) { + state.devices.insert(name, labels); + mark_deployments_dirty(&mut state); + } } /// Swap in a fresh device → groups snapshot and re-evaluate scheduling /// for every device whose membership changed. Secret grants are /// untouched — they bind deployments to groups, not to devices. -async fn apply_device_groups( - state: &SharedFleetState, +async fn apply_device_groups(state: &SharedFleetState, snapshot: HashMap>) { + let mut state = state.lock().await; + if state.device_groups != snapshot { + state.device_groups = snapshot; + mark_deployments_dirty(&mut state); + } + if !state.group_source_ready { + state.group_source_ready = true; + enable_reconciliation_if_ready(&mut state); + } +} + +async fn on_device_delete(state: &SharedFleetState, dev: Device) { + let name = dev.name_any(); + let mut state = state.lock().await; + if state.devices.remove(&name).is_some() { + mark_deployments_dirty(&mut state); + } +} + +// --------------------------------------------------------------------------- +// Reconcile acknowledged NATS state toward current desired state +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct ReconcileCounts { + targeted: usize, + written: usize, + cleaned: usize, + errors: usize, +} + +async fn reconcile_desired_state( desired: &Store, - snapshot: HashMap>, -) { - let changed: Vec<(String, Vec<(CachedDeployment, bool, bool)>)> = { - let mut guard = state.lock().await; - if guard.device_groups == snapshot { + state: &SharedFleetState, + deployment_name: &DeploymentName, +) -> ReconcileCounts { + let (diff, score_json) = { + let state = state.lock().await; + let diff = desired_state_diff(&state, deployment_name); + let score = state + .deployments + .values() + .find(|deployment| &deployment.deployment_name == deployment_name) + .map(|deployment| deployment.score_json.clone()); + (diff, score) + }; + let mut counts = ReconcileCounts { + targeted: diff.targeted, + ..Default::default() + }; + let mut acknowledged_puts = Vec::new(); + let mut acknowledged_deletes = Vec::new(); + + for device in diff.puts { + let k = desired_state_key(&device, deployment_name); + let payload = score_json + .as_ref() + .expect("puts only exist for a cached deployment"); + if let Err(e) = desired.put(&k, payload.clone().into()).await { + counts.errors += 1; + tracing::warn!(key = %k, error = %e, "aggregator: desired-state put failed"); + } else { + counts.written += 1; + acknowledged_puts.push((device, payload.clone())); + } + } + + for device in diff.deletes { + let k = desired_state_key(&device, deployment_name); + if let Err(e) = desired.delete(&k).await { + counts.errors += 1; + tracing::warn!(key = %k, error = %e, "aggregator: desired-state delete failed"); + } else { + counts.cleaned += 1; + acknowledged_deletes.push(device); + } + } + + let mut state = state.lock().await; + if !acknowledged_puts.is_empty() || !acknowledged_deletes.is_empty() { + let owned = state + .owned_targets + .entry(deployment_name.clone()) + .or_default(); + for (device, payload) in acknowledged_puts { + owned.insert(device, payload); + } + for device in acknowledged_deletes { + owned.remove(&device); + } + if let Some(key) = state + .deployments + .values() + .find(|deployment| &deployment.deployment_name == deployment_name) + .map(|deployment| deployment.key.clone()) + { + state.dirty.insert(key); + } + } + if state + .owned_targets + .get(deployment_name) + .is_some_and(HashMap::is_empty) + { + state.owned_targets.remove(deployment_name); + } + if counts.errors > 0 { + state.desired_dirty.insert(deployment_name.clone()); + } + drop(state); + + if counts.written > 0 || counts.cleaned > 0 || counts.errors > 0 { + tracing::info!( + deployment = %deployment_name, + targeted = counts.targeted, + written = counts.written, + cleaned = counts.cleaned, + errors = counts.errors, + "aggregator: deployment desired state reconciled" + ); + } + counts +} + +async fn reconcile_dirty_desired_state(desired: &Store, state: &SharedFleetState) { + let dirty = { + let mut state = state.lock().await; + if !reconciliation_ready(&state) { return; } - let affected: HashSet = guard - .device_groups - .keys() - .chain(snapshot.keys()) - .filter(|d| guard.device_groups.get(*d) != snapshot.get(*d)) - .cloned() - .collect(); - guard.device_groups = snapshot; - affected - .into_iter() - .map(|name| { - let transitions = reevaluate_device(&mut guard, &name); - (name, transitions) - }) - .collect() + state.desired_dirty.drain().collect::>() }; - for (name, transitions) in changed { - reconcile_device_kv(desired, &name, transitions).await; - } -} - -async fn on_device_delete(state: &SharedFleetState, desired: &Store, dev: Device) { - let name = dev.name_any(); - let removed_from: Vec = { - let mut guard = state.lock().await; - guard.devices.remove(&name); - let mut out = Vec::new(); - let deployments_snapshot: Vec = - guard.deployments.values().cloned().collect(); - for cached in deployments_snapshot { - if let Some(set) = guard.owned_targets.get_mut(&cached.deployment_name) { - if set.remove(&name) { - out.push(cached.deployment_name.clone()); - guard.dirty.insert(cached.key.clone()); - } - } - } - out - }; - for deployment_name in removed_from { - let k = desired_state_key(&name, &deployment_name); - if let Err(e) = desired.delete(&k).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on device delete failed"); - } - } -} - -// --------------------------------------------------------------------------- -// Diff helper: write/delete desired-state entries for one deployment -// --------------------------------------------------------------------------- - -async fn reconcile_kv( - desired: &Store, - deployment_name: &DeploymentName, - new_targets: &HashSet, - previous_targets: &HashSet, - score_json: &[u8], -) { - // Writes: new_targets, unconditionally — idempotent put; agents - // byte-compare and no-op on unchanged content. - for device in new_targets { - let k = desired_state_key(device, deployment_name); - if let Err(e) = desired.put(&k, score_json.to_vec().into()).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed"); - } - } - // Deletes: anything we owned previously but no longer target. - for device in previous_targets.difference(new_targets) { - let k = desired_state_key(device, deployment_name); - if let Err(e) = desired.delete(&k).await { - tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed"); - } + for deployment_name in dirty { + reconcile_desired_state(desired, state, &deployment_name).await; } } @@ -762,7 +830,7 @@ async fn reconcile_kv( /// watch-driven reconcile (otherwise we'd leak orphans when a /// selector change causes a deployment to stop targeting a device). async fn seed_owned_targets(bucket: &Store, state: &SharedFleetState) -> anyhow::Result<()> { - let mut guard = state.lock().await; + let mut seeded = Vec::new(); let mut keys = bucket.keys().await?; while let Some(key_res) = keys.next().await { let key = key_res?; @@ -777,11 +845,18 @@ async fn seed_owned_targets(bucket: &Store, state: &SharedFleetState) -> anyhow: warn!("Invalid deployment name for key {key}"); continue; }; - guard + let Some(value) = bucket.get(&key).await? else { + continue; + }; + seeded.push((deployment_name, device.to_string(), value.to_vec())); + } + let mut state = state.lock().await; + for (deployment_name, device, payload) in seeded { + state .owned_targets .entry(deployment_name) .or_default() - .insert(device.to_string()); + .insert(device, payload); } Ok(()) } @@ -809,6 +884,7 @@ async fn patch_tick(api: &Api, state: &SharedFleetState) -> anyhow:: .patch_status(&key.name, &PatchParams::default(), &Patch::Merge(&status)) .await { + state.lock().await.dirty.insert(key.clone()); tracing::warn!( namespace = %key.namespace, name = %key.name, @@ -831,10 +907,9 @@ async fn patch_tick(api: &Api, state: &SharedFleetState) -> anyhow:: } /// Compute the aggregate for one Deployment from current caches. -/// `owned_targets` is the authoritative "currently selector-matched" -/// set for the deployment, as maintained by the watchers. +/// `owned_targets` is the authoritative NATS-acknowledged target set. pub fn compute_aggregate(state: &FleetState, cached: &CachedDeployment) -> DeploymentAggregate { - let empty = HashSet::new(); + let empty = HashMap::new(); let targets = state .owned_targets .get(&cached.deployment_name) @@ -845,7 +920,7 @@ pub fn compute_aggregate(state: &FleetState, cached: &CachedDeployment) -> Deplo ..Default::default() }; - for device_id in targets { + for device_id in targets.keys() { let pair = DevicePair { device_id: device_id.clone(), deployment: cached.deployment_name.clone(), @@ -992,6 +1067,96 @@ mod tests { assert!(selector_matches(&sel, &BTreeMap::new())); } + #[test] + fn deployment_selector_change_withdraws_previous_target() { + let mut state = FleetState::default(); + state.devices.insert( + "pi-01".to_string(), + BTreeMap::from([("device-id".to_string(), "pi-01".to_string())]), + ); + + let deployment = cached("fleet", "web", "device-id", "pi-01"); + assert_eq!(upsert_deployment_state(&mut state, deployment), Some("new")); + assert_eq!(desired_state_diff(&state, &dn("web")).puts, vec!["pi-01"]); + state + .owned_targets + .entry(dn("web")) + .or_default() + .insert("pi-01".into(), b"{}".to_vec()); + + let retargeted = cached("fleet", "web", "device-id", "missing"); + assert_eq!( + upsert_deployment_state(&mut state, retargeted), + Some("update") + ); + assert_eq!( + desired_state_diff(&state, &dn("web")).deletes, + vec!["pi-01"] + ); + + let mut upgraded = cached("fleet", "web", "device-id", "missing"); + upgraded.score_json = br#"{"revision":2}"#.to_vec(); + assert_eq!( + upsert_deployment_state(&mut state, upgraded), + Some("upgrade") + ); + } + + #[test] + fn desired_state_diff_uses_acknowledged_payloads() { + let mut state = FleetState::default(); + for device in ["pi-01", "pi-02"] { + state.devices.insert( + device.to_string(), + BTreeMap::from([("zone".to_string(), "lab".to_string())]), + ); + } + let deployment = cached("fleet", "web", "zone", "lab"); + state + .deployments + .insert(deployment.key.clone(), deployment.clone()); + + assert_eq!( + desired_state_diff(&state, &dn("web")).puts, + vec!["pi-01", "pi-02"] + ); + + state.owned_targets.insert( + dn("web"), + HashMap::from([("pi-01".to_string(), deployment.score_json.clone())]), + ); + assert_eq!(desired_state_diff(&state, &dn("web")).puts, vec!["pi-02"]); + + let mut upgraded = deployment; + upgraded.score_json = br#"{"revision":2}"#.to_vec(); + state.deployments.insert(upgraded.key.clone(), upgraded); + assert_eq!( + desired_state_diff(&state, &dn("web")).puts, + vec!["pi-01", "pi-02"] + ); + + state.deployments.clear(); + assert_eq!( + desired_state_diff(&state, &dn("web")).deletes, + vec!["pi-01"] + ); + } + + #[test] + fn reconciliation_waits_for_all_initial_snapshots() { + let mut state = FleetState::default(); + state.owned_targets.insert(dn("web"), HashMap::new()); + state.deployment_watch_ready = true; + state.device_watch_ready = true; + + enable_reconciliation_if_ready(&mut state); + assert!(state.desired_dirty.is_empty()); + + state.group_source_ready = true; + enable_reconciliation_if_ready(&mut state); + assert_eq!(state.desired_dirty, HashSet::from([dn("web")])); + } + #[test] fn compute_aggregate_counts_matched_devices() { let cached = cached("fleet-demo", "hello", "zone", "lab"); @@ -1005,7 +1170,7 @@ mod tests { cached.deployment_name.clone(), ["pi-01", "pi-02", "pi-03"] .iter() - .map(|s| s.to_string()) + .map(|device| (device.to_string(), b"{}".to_vec())) .collect(), ); s.states.insert( @@ -1101,22 +1266,17 @@ mod tests { let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]); s.deployments.insert(web.key.clone(), web); - // Not a group member: re-evaluation yields no ownership. - let transitions = reevaluate_device(&mut s, "pi-01"); - assert_eq!(transitions.len(), 1); - assert_eq!((transitions[0].1, transitions[0].2), (false, false)); + assert!(desired_state_diff(&s, &dn("web")).puts.is_empty()); - // Membership lands (as the group poll would apply it) → owned. s.device_groups .insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()])); - let transitions = reevaluate_device(&mut s, "pi-01"); - assert_eq!((transitions[0].1, transitions[0].2), (false, true)); - assert!(s.owned_targets[&dn("web")].contains("pi-01")); + assert_eq!(desired_state_diff(&s, &dn("web")).puts, vec!["pi-01"]); + s.owned_targets + .entry(dn("web")) + .or_default() + .insert("pi-01".into(), b"{}".to_vec()); - // Membership revoked → ownership withdrawn. s.device_groups.remove("pi-01"); - let transitions = reevaluate_device(&mut s, "pi-01"); - assert_eq!((transitions[0].1, transitions[0].2), (true, false)); - assert!(!s.owned_targets[&dn("web")].contains("pi-01")); + assert_eq!(desired_state_diff(&s, &dn("web")).deletes, vec!["pi-01"]); } } -- 2.39.5 From 4d1656b9e7ef81291761ffd8acc687ec76312a9d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 21 Jul 2026 23:42:52 -0400 Subject: [PATCH 15/47] fix: hide unsupported fleet dashboard actions --- fleet/harmony-fleet-operator/README.md | 39 +-- .../src/frontend/assets.rs | 5 +- .../src/frontend/layout.rs | 38 ++- .../src/frontend/server.rs | 137 +-------- .../src/frontend/views/alerts.rs | 2 - .../src/frontend/views/devices.rs | 285 ++--------------- .../src/frontend/views/mod.rs | 1 - .../src/frontend/views/settings.rs | 74 ----- .../src/service/mock.rs | 8 - .../harmony-fleet-operator/src/service/mod.rs | 4 - .../src/service/real.rs | 8 - fleet/harmony-fleet-operator/vendor/app.js | 24 -- .../vendor/htmx-ext-sse.js | 290 ------------------ 13 files changed, 67 insertions(+), 848 deletions(-) delete mode 100644 fleet/harmony-fleet-operator/src/frontend/views/settings.rs delete mode 100644 fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js diff --git a/fleet/harmony-fleet-operator/README.md b/fleet/harmony-fleet-operator/README.md index 9ddd7005..1aabe54f 100644 --- a/fleet/harmony-fleet-operator/README.md +++ b/fleet/harmony-fleet-operator/README.md @@ -33,17 +33,11 @@ SSR + HTMX), so the runtime/macro footprint was dead weight. Maud is a compile-time HTML macro that produces a `Markup` value — smaller dep tree, faster compiles, same Rust-flavored ergonomics. -**Why HTMX + xterm.js for interactivity?** A real terminal needs xterm.js in -the browser regardless; once that JS exists, HTMX (~14 KB) is a rounding -error and lets every other interaction stay declarative in markup -(`hx-post`, `hx-target`, `hx-swap`). - **Why everything bundled?** The operator already ships as a single -container. Tailwind CSS, HTMX, and the HTMX SSE extension are all embedded -via `include_bytes!` so air-gapped clusters get the dashboard with nothing -extra to mount. The only build-time external is the standalone `tailwindcss` -v4 CLI — missing-CLI degrades gracefully (warning + empty embedded CSS); the -dev workflow uses `--css-from` instead anyway. +container. Tailwind CSS, HTMX, and the small CSRF helper are embedded so +air-gapped clusters need nothing extra to mount. The only build-time external +is the standalone `tailwindcss` v4 CLI. A missing CLI produces a warning and +empty embedded CSS; local development uses `--css-from` instead. ### Running it locally (mock data, no NATS, no kube) @@ -76,9 +70,8 @@ Open . `--mock` uses [`MockFleetService`](src/service/mock.rs), an in-memory seeded dataset (10 fake devices in mixed states, 4 deployments). You can -click "Blacklist" on a row and the row will swap in place to reflect the -new status — this exercises the same `FleetService` API the real impl -will satisfy. No NATS, no Kubernetes cluster needed. +blacklist a device and see its updated detail page. This exercises the same +`FleetService` API as production without NATS or a Kubernetes cluster. #### Iteration cost @@ -120,21 +113,21 @@ fleet/harmony-fleet-operator/ │ ├── assets.rs ← embedded Tailwind/HTMX bytes │ └── views/ │ ├── dashboard.rs -│ ├── devices.rs ← also exposes `row()` for HTMX swaps +│ ├── devices.rs │ └── deployments.rs ├── style/ │ └── input.css ← Tailwind v4 entry point └── vendor/ - ├── htmx.min.js ← HTMX v2.0.9 - └── htmx-ext-sse.js ← SSE extension (used by future log-tail views) + ├── app.js ← CSRF header helper + └── htmx.min.js ← HTMX v2.0.9 ``` ### What's deferred -- **Real `FleetService` impl** (wraps the kube client + NATS KV the - reconcilers already use). `serve-web` without `--mock` currently errors - out. -- **Zitadel SSO + admin-role check.** v1 assumes an oauth2-proxy fronts the - dashboard at the cluster edge. -- **Live log tail** (SSE-based, HTMX `sse-swap`) — the wiring is in place. -- **Interactive shell** (xterm.js + axum WS + portable-pty) — separate design. +- **`fleet-admin` authorization.** Zitadel login is implemented, but any + authenticated tenant user can currently reach privileged dashboard routes. +- **Live log tail.** Add a typed, authorized agent transport before exposing it. +- **Device commands.** Define a bounded typed protocol and authorization before + adding command controls. +- **Persistent alert state and receivers.** Per-alert acknowledgement currently + lives in operator memory; receiver configuration remains deployment-time. diff --git a/fleet/harmony-fleet-operator/src/frontend/assets.rs b/fleet/harmony-fleet-operator/src/frontend/assets.rs index 1b0be691..f0d023e0 100644 --- a/fleet/harmony-fleet-operator/src/frontend/assets.rs +++ b/fleet/harmony-fleet-operator/src/frontend/assets.rs @@ -2,10 +2,9 @@ //! //! Tailwind CSS is built by `build.rs` into `$OUT_DIR/tailwind.css` //! (empty if the CLI was unavailable — dev uses `--css-from` instead). -//! HTMX and its SSE extension are vendored under `vendor/` so the -//! container ships with no external script dependencies. +//! HTMX is vendored under `vendor/` so the container ships with no +//! external script dependencies. pub const TAILWIND_CSS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/tailwind.css")); pub const HTMX_JS: &[u8] = include_bytes!("../../vendor/htmx.min.js"); -pub const HTMX_SSE_JS: &[u8] = include_bytes!("../../vendor/htmx-ext-sse.js"); pub const APP_JS: &[u8] = include_bytes!("../../vendor/app.js"); diff --git a/fleet/harmony-fleet-operator/src/frontend/layout.rs b/fleet/harmony-fleet-operator/src/frontend/layout.rs index 7e039d58..65b415e7 100644 --- a/fleet/harmony-fleet-operator/src/frontend/layout.rs +++ b/fleet/harmony-fleet-operator/src/frontend/layout.rs @@ -8,7 +8,6 @@ const ICON_DASHBOARD: &str = r#""#; const ICON_DEPLOY: &str = r#""#; const ICON_BELL: &str = r#""#; -const ICON_COG: &str = r#""#; const ICON_LOGOUT: &str = r#""#; const ICON_BRAND: &str = r#""#; @@ -30,13 +29,12 @@ pub fn page( title { (title) " — Harmony Fleet" } link rel="stylesheet" href="/static/tailwind.css"; script src="/static/htmx.min.js" defer {} - script src="/static/htmx-ext-sse.js" defer {} script src="/static/app.js" defer {} @if live_reload { script { (PreEscaped(LIVE_RELOAD_JS)) } } } - body class="min-h-screen" hx-ext="sse" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" { + body class="min-h-screen" style="background:var(--bg); color:#e2e8f0; font-family:'Inter',sans-serif" { div class="flex h-screen overflow-hidden" style="background:var(--bg)" { (sidebar(current_path, session, unacked_alerts)) main class="flex-1 min-w-0 flex flex-col overflow-hidden" { @@ -44,7 +42,6 @@ pub fn page( div class="flex-1 overflow-y-auto grid-bg" { (content) } } } - div id="modal-root" {} } } } @@ -55,22 +52,21 @@ fn sidebar( session: Option<&DashboardSession>, unacked_alerts: usize, ) -> Markup { - let nav_items: [(&str, &str, &str, usize); 5] = [ + let nav_items: [(&str, &str, &str, usize); 4] = [ ("/", ICON_DASHBOARD, "Dashboard", 0), ("/devices", ICON_DEVICES, "Devices", 0), ("/deployments", ICON_DEPLOY, "Deployments", 0), ("/alerts", ICON_BELL, "Alerts", unacked_alerts), - ("/settings", ICON_COG, "Settings", 0), ]; html! { - aside class="shrink-0 flex flex-col border-r w-[224px]" style="border-color:var(--border); background:var(--bg)" { - div class="flex items-center justify-between px-4 py-4 border-b" style="border-color:var(--border)" { + aside class="shrink-0 flex flex-col border-r w-14 sm:w-[224px]" style="border-color:var(--border); background:var(--bg)" { + div class="flex items-center justify-center sm:justify-between px-2 sm:px-4 py-4 border-b" style="border-color:var(--border)" { div class="flex items-center gap-2" { div class="relative w-6 h-6 rounded-md flex items-center justify-center" style="background:var(--accent); color:#0c0c0c" { (PreEscaped(ICON_BRAND)) } - span class="text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" } + span class="hidden sm:inline text-sm font-semibold tracking-tight text-slate-100" { "Harmony Fleet" } } } @@ -79,7 +75,9 @@ fn sidebar( @let active = is_active(current_path, href); a href=(*href) - class={"group w-full flex items-center gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative " + title=(*label) + aria-label=(*label) + class={"group w-full flex items-center justify-center sm:justify-start gap-2.5 px-2.5 h-9 rounded-md text-[13px] transition-colors duration-150 relative " (if active { "text-slate-100 font-medium" } else { "text-slate-400 hover:text-slate-100" })} style={(if active { "background:rgba(148,163,184,0.06)" } else { "background:transparent" })} { @@ -89,9 +87,9 @@ fn sidebar( span class={(if active { "text-slate-100" } else { "text-slate-500 group-hover:text-slate-300" })} { (PreEscaped(icon)) } - span class="flex-1 text-left" { (label) } + span class="hidden sm:inline flex-1 text-left" { (label) } @if *badge > 0 { - span class="inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" { + span class="hidden sm:inline-flex items-center justify-center min-w-[18px] h-[18px] rounded-full text-[10px] font-semibold px-1" style="background:var(--bad); color:#0c0c0c" { (badge) } } @@ -100,9 +98,12 @@ fn sidebar( } @if let Some(s) = session { - div class="border-t p-3" style="border-color:var(--border)" { + div class="hidden sm:block border-t p-3" style="border-color:var(--border)" { (user_footer(s)) } + a href="/logout" class="sm:hidden border-t py-3 flex justify-center text-slate-500 hover:text-rose-400" style="border-color:var(--border)" title="Log out" { + (PreEscaped(ICON_LOGOUT)) + } } } } @@ -170,14 +171,14 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup { div class="flex items-center gap-2" { div class="relative" { input - class="input w-64" + class="hidden sm:block input w-40 lg:w-64" type="text" name="search" placeholder="Search devices, deployments\u{2026}" - hx-get="/devices/search" + hx-get="/devices" hx-trigger="keyup changed delay:300ms" - hx-target="#device-table-wrapper" - hx-swap="innerHTML"; + hx-target="body" + hx-push-url="true"; } a href="/alerts" class="relative btn btn-ghost py-1.5" { (PreEscaped(ICON_BELL)) @@ -188,9 +189,6 @@ fn topbar(title: &str, unacked_alerts: usize) -> Markup { } } } - a href="/settings" class="btn btn-ghost py-1.5" title="Settings" { - (PreEscaped(ICON_COG)) - } } } } diff --git a/fleet/harmony-fleet-operator/src/frontend/server.rs b/fleet/harmony-fleet-operator/src/frontend/server.rs index c1c46d76..967e0197 100644 --- a/fleet/harmony-fleet-operator/src/frontend/server.rs +++ b/fleet/harmony-fleet-operator/src/frontend/server.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::Result; use axum::Router; use axum::body::Body; -use axum::extract::{Extension, Form, FromRef, Path, Query, State}; +use axum::extract::{Extension, FromRef, Path, Query, State}; use axum::http::Request; use axum::http::{HeaderValue, Method, StatusCode, header}; use axum::middleware::{self, Next}; @@ -19,11 +19,11 @@ use maud::Markup; use serde::Deserialize; use tokio_stream::StreamExt; -use super::assets::{APP_JS, HTMX_JS, HTMX_SSE_JS, TAILWIND_CSS}; +use super::assets::{APP_JS, HTMX_JS, TAILWIND_CSS}; use super::layout::page; use super::views::{ alerts as alerts_view, dashboard as dashboard_view, deployments as deployments_view, - devices as devices_view, settings as settings_view, + devices as devices_view, }; use crate::frontend::auth::{self, DASHBOARD_SESSION_COOKIE, DashboardSession, JwksCache}; use crate::service::FleetService; @@ -91,7 +91,6 @@ pub fn router(state: AppState) -> Router { .route("/auth/callback", get(auth::callback_handler)) .route("/static/tailwind.css", get(tailwind_css)) .route("/static/htmx.min.js", get(htmx_js)) - .route("/static/htmx-ext-sse.js", get(htmx_sse_js)) .route("/static/app.js", get(app_js)); let private_routes = Router::new() @@ -99,11 +98,7 @@ pub fn router(state: AppState) -> Router { .route("/", get(dashboard_handler)) // Devices .route("/devices", get(devices_handler)) - .route("/devices/search", get(devices_search_handler)) .route("/devices/{id}/blacklist", post(blacklist_handler)) - .route("/devices/{id}/logs", get(device_logs_handler)) - .route("/devices/{id}/logs/stream", get(device_logs_stream_handler)) - .route("/devices/{id}/exec", post(device_exec_handler)) // Device detail .route("/device/{id}", get(device_detail_handler)) // Deployments @@ -112,9 +107,6 @@ pub fn router(state: AppState) -> Router { // Alerts .route("/alerts", get(alerts_handler)) .route("/alerts/{id}/ack", post(ack_alert_handler)) - // Settings - .route("/settings", get(settings_handler)) - .route("/settings/toggle/{key}", post(settings_toggle_handler)) // Logout .route("/logout", get(auth::logout_handler)) .route_layer(middleware::from_fn_with_state(state.clone(), csrf_protect)) @@ -371,44 +363,11 @@ async fn devices_handler( )) } -async fn devices_search_handler( - State(s): State, - Query(q): Query, -) -> Result { - let status = q.status.as_deref().and_then(parse_device_status); - - let devices = s - .fleet - .filtered_devices( - status, - q.deployment.clone(), - q.region.clone(), - q.search.clone(), - ) - .await?; - - Ok(devices_view::page( - &devices, - &[], - &[], - status, - q.deployment.as_deref(), - q.region.as_deref(), - q.search.as_deref(), - )) -} - // ── Device detail ────────────────────────────────────────────────────── -#[derive(Deserialize, Default)] -struct DeviceDetailQuery { - tab: Option, -} - async fn device_detail_handler( State(s): State, Path(id): Path, - Query(q): Query, session: Option>, ) -> Result { let device = s @@ -423,18 +382,6 @@ async fn device_detail_handler( None }; - let tab = q.tab.as_deref().unwrap_or("overview"); - - // Tab click (HTMX): return the tabs block (bar + content) so the - // active highlight re-renders with the content. - if q.tab.is_some() { - return Ok(devices_view::device_tabs( - &device, - deployment_version.as_deref(), - tab, - )); - } - let unacked = s .fleet .list_alerts() @@ -561,79 +508,14 @@ async fn ack_alert_handler( Ok(alerts_view::alert_row(alert)) } -// ── Settings ─────────────────────────────────────────────────────────── - -async fn settings_handler( - State(s): State, - session: Option>, -) -> Result { - let unacked = s - .fleet - .list_alerts() - .await? - .iter() - .filter(|a| !a.acked) - .count(); - - Ok(page( - "Settings", - s.live_reload, - "/settings", - session.as_ref().map(|e| &e.0), - unacked, - settings_view::page(), - )) -} - -async fn settings_toggle_handler(Path(_key): Path) -> Result { - // In a real app this would toggle a notification channel. - // For the mock, we return the same static content. - Ok(settings_view::page()) -} - -// ── Device logs ──────────────────────────────────────────────────────── - -async fn device_logs_handler(Path(id): Path) -> Result { - Ok(devices_view::logs_modal(&id)) -} - -async fn device_logs_stream_handler( - Path(_id): Path, -) -> Sse>> { - // One honest notice, then keep-alive. Real agent-log streaming (over - // NATS) is pending — don't fabricate log lines. - let html = r#"
— live agent log streaming is not implemented yet —
"#; - let stream = futures_util::stream::once(async move { - Ok::<_, Infallible>(Event::default().event("log").data(html)) - }); - - Sse::new(stream).keep_alive(KeepAlive::new().interval(Duration::from_secs(15))) -} - -// ── Run command ──────────────────────────────────────────────────────── - -#[derive(Deserialize)] -struct ExecForm { - command: String, -} - -async fn device_exec_handler( - State(s): State, - Path(id): Path, - Form(form): Form, -) -> Result { - let output = s.fleet.run_command(&id, &form.command).await?; - Ok(devices_view::command_output(&form.command, &output)) -} - // ── Blacklist ────────────────────────────────────────────────────────── async fn blacklist_handler( State(s): State, Path(id): Path, -) -> Result { - let updated = s.fleet.blacklist_device(&id).await?; - Ok(devices_view::row(&updated)) +) -> Result { + s.fleet.blacklist_device(&id).await?; + Ok(Redirect::to(&format!("/device/{id}")).into_response()) } // ── Helpers ──────────────────────────────────────────────────────────── @@ -670,13 +552,6 @@ async fn htmx_js() -> Response { static_response(HTMX_JS.to_vec(), "application/javascript; charset=utf-8") } -async fn htmx_sse_js() -> Response { - static_response( - HTMX_SSE_JS.to_vec(), - "application/javascript; charset=utf-8", - ) -} - async fn app_js() -> Response { static_response(APP_JS.to_vec(), "application/javascript; charset=utf-8") } diff --git a/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs b/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs index d3db66be..e750daa2 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/alerts.rs @@ -10,8 +10,6 @@ pub fn page(alerts: &[Alert]) -> Markup { div class="flex items-center gap-2" { h2 class="text-[15px] font-semibold text-slate-200" { "Alerts" } span class="text-[11px] text-slate-500" { "\u{b7} " (unacked) " unacked" } - div class="flex-1" {} - button class="btn btn-ghost" { "Ack all" } } div class="card card-flush" { table class="tbl" { diff --git a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs index 03344404..e257df53 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs @@ -6,13 +6,8 @@ use crate::service::{DeviceDetail, DeviceStatus}; // ── Inline icons ──────────────────────────────────────────────────────── const ICON_SEARCH: &str = r#""#; const ICON_CHEVRON_DOWN: &str = r#""#; -const ICON_POWER: &str = r#""#; -const ICON_PAUSE: &str = r#""#; const ICON_BAN: &str = r#""#; -const ICON_EXPAND: &str = r#""#; const ICON_EXTERNAL: &str = r#""#; -const ICON_REFRESH: &str = r#""#; -const ICON_COPY: &str = r#""#; // ── Devices list page ────────────────────────────────────────────────── @@ -172,7 +167,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup div class="p-6 space-y-4" { // Header div class="card p-5" { - div class="flex items-start justify-between gap-6" { + div class="flex flex-col sm:flex-row sm:items-start justify-between gap-4 sm:gap-6" { div class="min-w-0" { div class="flex items-center gap-3 flex-wrap" { h1 class="text-[22px] font-semibold font-mono text-slate-50 truncate whitespace-nowrap" { (&device.id) } @@ -189,10 +184,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup span { span class="text-slate-600" { "Last ping" } " " span class="text-slate-300 tabular-nums" { (time_ago(device.minutes_ago)) } } } } - div class="flex items-center gap-2 shrink-0" { - button class="btn btn-ghost" { (PreEscaped(ICON_REFRESH)) " Reconcile" } - button class="btn btn-ghost" { (PreEscaped(ICON_POWER)) " Restart" } - button class="btn btn-ghost" { (PreEscaped(ICON_PAUSE)) " Suspend" } + div class="flex items-center gap-2 shrink-0 self-start" { @if device.status != DeviceStatus::Blacklisted { button class="btn btn-danger" @@ -206,80 +198,27 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup } } - // The whole tabs block re-renders on switch so the active - // highlight follows (only the content swapping would leave it - // stuck on Overview). - div id="device-tabs" { - (device_tabs(device, deployment_version, "overview")) - } - } - } -} - -/// Tab bar (active highlighted) + the active tab's content. Swapped as a -/// unit into `#device-tabs`. -pub fn device_tabs( - device: &DeviceDetail, - deployment_version: Option<&str>, - active: &str, -) -> Markup { - let content = match active { - "logs" => logs_tab(device), - "command" => command_tab(&device.id), - _ => overview_tab(device, deployment_version), - }; - html! { - div class="flex items-center gap-1 border-b" style="border-color:var(--border)" { - (tab_button(&device.id, "Overview", "overview", active == "overview")) - (tab_button(&device.id, "Logs", "logs", active == "logs")) - (tab_button(&device.id, "Run command", "command", active == "command")) - div class="flex-1" {} - button - class="btn btn-ghost mb-1" - hx-get={"/devices/" (device.id) "/logs"} - hx-target="#modal-root" - hx-swap="innerHTML" { - (PreEscaped(ICON_EXPAND)) " Pop-out logs" - } - } - div { (content) } - } -} - -fn tab_button(device_id: &str, label: &str, tab: &str, active: bool) -> Markup { - html! { - button - class={"px-3 py-2 text-[13px] font-medium relative " - (if active { "text-slate-100" } else { "text-slate-500 hover:text-slate-300" })} - hx-get={"/device/" (device_id) "?tab=" (tab)} - hx-target="#device-tabs" - hx-swap="innerHTML" { - (label) - @if active { - span class="absolute left-0 right-0 -bottom-px h-0.5" style="background:var(--accent)" {} - } + (overview_tab(device, deployment_version)) } } } fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup { html! { - div class="grid grid-cols-12 gap-4" { - // Device info + current deployment - div class="col-span-12 lg:col-span-5 space-y-4" { + div class="space-y-4" { div class="card p-5" { div class="section-title mb-3" { "Device info" } - (definition("Device ID", &device.id, true, true)) - (definition("Region", &device.region, true, false)) - (definition("Last ping", &time_ago(device.minutes_ago), false, false)) - (definition("Agent", agent_version(device), true, false)) + (definition("Device ID", &device.id, true)) + (definition("Region", &device.region, true)) + (definition("Last ping", &time_ago(device.minutes_ago), false)) + (definition("Agent", agent_version(device), true)) @if let Some(inv) = &device.inventory { - (definition("Hostname", &inv.hostname, true, false)) - (definition("Arch", &inv.arch, true, false)) - (definition("OS", &inv.os, false, false)) - (definition("Kernel", &inv.kernel, true, false)) - (definition("CPU cores", &inv.cpu_cores.to_string(), false, false)) - (definition("Memory", &format!("{} MB", inv.memory_mb), false, false)) + (definition("Hostname", &inv.hostname, true)) + (definition("Arch", &inv.arch, true)) + (definition("OS", &inv.os, false)) + (definition("Kernel", &inv.kernel, true)) + (definition("CPU cores", &inv.cpu_cores.to_string(), false)) + (definition("Memory", &format!("{} MB", inv.memory_mb), false)) } @else { div class="text-[12px] text-slate-500 mt-2" { "No inventory reported yet" } } @@ -310,168 +249,6 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark div class="text-[12px] text-slate-500" { "No deployment assigned" } } } - } - - // Recent logs (live stream) - div class="col-span-12 lg:col-span-7" { - div class="card overflow-hidden" { - div class="flex items-center justify-between px-4 py-3 border-b" style="border-color:var(--border)" { - span class="section-title" { "Recent logs" } - button - class="text-[11px] text-slate-400 hover:text-slate-100 flex items-center gap-1" - hx-get={"/devices/" (device.id) "/logs"} - hx-target="#modal-root" - hx-swap="innerHTML" { - "Pop out " (PreEscaped(ICON_EXTERNAL)) - } - } - (log_stream(&device.id, "260px")) - } - } - } - } -} - -fn logs_tab(device: &DeviceDetail) -> Markup { - html! { - div class="card overflow-hidden mt-4" { - div class="flex items-center gap-2 px-4 py-2.5 border-b" style="border-color:var(--border)" { - span class="relative flex w-1.5 h-1.5" { - span class="absolute inline-flex h-full w-full animate-ping rounded-full opacity-60" style="background:var(--accent)" {} - span class="relative inline-flex w-1.5 h-1.5 rounded-full" style="background:var(--accent)" {} - } - span class="text-[11px] font-mono text-slate-400" { "streaming" } - span class="text-[11px] text-slate-600 font-mono" { "\u{b7} live" } - } - (log_stream(&device.id, "520px")) - } - } -} - -/// Shared SSE log console. The stream endpoint is the seam the real -/// agent-log transport plugs into; the markup is identical wherever a -/// device's logs are shown (overview card, logs tab, pop-out modal). -fn log_stream(device_id: &str, height: &str) -> Markup { - html! { - div - class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto" - style={"background:#050608; height:" (height)} - hx-ext="sse" - sse-connect={"/devices/" (device_id) "/logs/stream"} - sse-swap="log" - hx-swap="beforeend" { - div class="px-0 py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" } - } - } -} - -/// One-shot "run a shell command on the device" panel. Submits the -/// command to the [`run_command`](crate::service::FleetService::run_command) -/// seam and appends the response to the console. Live streaming and a -/// full TTY are later refinements. -fn command_tab(device_id: &str) -> Markup { - html! { - div class="card overflow-hidden mt-4" { - form - class="flex items-center gap-2 px-4 py-3 border-b" - style="border-color:var(--border)" - hx-post={"/devices/" (device_id) "/exec"} - hx-target="#exec-output" - hx-swap="beforeend" - "hx-on::after-request"="this.reset()" { - span class="font-mono text-slate-500 text-[13px]" { "$" } - input - class="input flex-1 font-mono text-[12px]" - type="text" - name="command" - placeholder="e.g. systemctl status harmony-agent" - autocomplete="off" - required; - button class="btn btn-primary" type="submit" { "Run" } - } - div - id="exec-output" - class="font-mono text-[11.5px] leading-6 px-4 py-2 overflow-auto" - style="background:#050608; height:440px" { - div class="px-0 py-px italic text-slate-700" { - "\u{2014} commands run here are sent to the device; output appears below \u{2014}" - } - } - } - } -} - -/// Markup for one command's response, appended to the exec console. -pub fn command_output(command: &str, output: &str) -> Markup { - html! { - div class="py-1 border-t" style="border-color:var(--border)" { - div class="text-slate-300" { span class="text-slate-500" { "$ " } (command) } - pre class="text-slate-400 whitespace-pre-wrap mt-0.5" { (output) } - } - } -} - -// ── Logs modal (SSE streaming) ───────────────────────────────────────── - -pub fn logs_modal(device_id: &str) -> Markup { - html! { - dialog - id="device-logs-modal" - class="m-auto grid grid-rows-[auto_1fr] h-[88vh] w-[min(96vw,82rem)] overflow-hidden rounded-none border-t-2 border-x-0 border-b-0 p-0 text-slate-100 shadow-[0_32px_64px_rgba(0,0,0,0.9),0_0_0_1px_rgba(148,163,184,0.06)] backdrop:bg-black/85" - style="border-color:var(--accent); background:#080a0c" - { - div class="flex items-center justify-between border-b px-5 py-3" style="background:#0c1018; border-color:var(--border)" { - div class="flex items-center gap-3" { - span class="relative flex h-1.5 w-1.5 shrink-0" { - span class="absolute inline-flex h-full w-full animate-ping rounded-full bg-orange-400 opacity-60" {} - span class="relative inline-flex h-1.5 w-1.5 rounded-full" style="background:var(--accent)" {} - } - code class="text-sm font-medium text-slate-100" { (device_id) } - span class="text-[10px] font-semibold uppercase tracking-[0.15em] text-orange-500/60" { "\u{b7} logs" } - } - form method="dialog" { - button - type="submit" - class="flex items-center gap-1.5 text-slate-500 transition-colors hover:text-slate-200" - aria-label="Close" - { - kbd class="rounded border bg-slate-800/60 px-1.5 py-0.5 font-mono text-[10px] text-slate-400" style="border-color:var(--border-strong)" { "esc" } - span class="text-xs" { "close" } - } - } - } - - div - class="overflow-y-auto py-3 font-mono text-[11.5px] leading-6 px-5" - style="background:#050608" - hx-ext="sse" - sse-connect={"/devices/" (device_id) "/logs/stream"} - sse-swap="log" - hx-swap="beforeend" { - div class="py-px italic text-slate-700" { "\u{2014} connecting \u{2014}" } - } - } - } -} - -// ── Row (for blacklist response) ─────────────────────────────────────── - -pub fn row(d: &DeviceDetail) -> Markup { - html! { - tr id={"device-" (d.id)} hx-get={"/device/" (d.id)} hx-target="body" hx-push-url="true" class="cursor-pointer" { - td { - span class="font-mono text-slate-100 hover:text-(--accent-fg) hover:underline underline-offset-2 whitespace-nowrap" { - (&d.id) - } - } - td { (badges::device_status(d.status)) } - td { - @if let Some(dep) = &d.deployment { span class="font-mono text-[12px] text-slate-300 whitespace-nowrap" { (dep) } } - @else { span class="text-slate-700" { "\u{2014}" } } - } - td { span class="text-[12px] text-slate-400 font-mono whitespace-nowrap" { (&d.region) } } - td { span class="font-mono text-[11px] text-slate-500 whitespace-nowrap" { (agent_version(d)) } } - td { span class="text-[12px] text-slate-500 tabular-nums" { (time_ago(d.minutes_ago)) } } } } } @@ -494,15 +271,12 @@ fn time_ago(minutes: i64) -> String { } } -fn definition(label: &str, value: &str, mono: bool, copyable: bool) -> Markup { +fn definition(label: &str, value: &str, mono: bool) -> Markup { html! { div class="flex items-center justify-between py-1.5 text-[12px] border-b last:border-b-0" style="border-color:var(--border)" { span class="text-slate-500" { (label) } span class={(if mono { "font-mono whitespace-nowrap" } else { "" }) " text-slate-200 flex items-center gap-1.5"} { (value) - @if copyable { - button class="text-slate-600 hover:text-slate-300" title="Copy" { (PreEscaped(ICON_COPY)) } - } } } } @@ -536,28 +310,19 @@ mod tests { } #[test] - fn overview_shows_device_info_not_removed_mock() { + fn detail_only_shows_implemented_device_features() { let html = detail(&sample(), Some("v2.14.1")).into_string(); assert!(html.contains("Device info")); assert!(html.contains("v1.2.3"), "agent version from heartbeat"); assert!(html.contains("aarch64")); - assert!(html.contains("Run command")); - // Removed mock surfaces must be gone. - assert!(!html.contains("MAC")); - assert!(!html.contains("triggered reconcile")); - assert!(!html.contains("Deployment history")); - } - - #[test] - fn command_tab_posts_to_exec_seam() { - let html = device_tabs(&sample(), None, "command").into_string(); - assert!(html.contains("/devices/hf-edge-001/exec")); - assert!(html.contains(r#"name="command""#)); - } - - #[test] - fn logs_tab_connects_to_stream() { - let html = device_tabs(&sample(), None, "logs").into_string(); - assert!(html.contains("/devices/hf-edge-001/logs/stream")); + for unsupported in [ + "Reconcile", + "Restart", + "Suspend", + "Recent logs", + "Run command", + ] { + assert!(!html.contains(unsupported)); + } } } diff --git a/fleet/harmony-fleet-operator/src/frontend/views/mod.rs b/fleet/harmony-fleet-operator/src/frontend/views/mod.rs index 7cabd8a0..847a2a3e 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/mod.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/mod.rs @@ -3,4 +3,3 @@ pub mod badges; pub mod dashboard; pub mod deployments; pub mod devices; -pub mod settings; diff --git a/fleet/harmony-fleet-operator/src/frontend/views/settings.rs b/fleet/harmony-fleet-operator/src/frontend/views/settings.rs deleted file mode 100644 index 4142072a..00000000 --- a/fleet/harmony-fleet-operator/src/frontend/views/settings.rs +++ /dev/null @@ -1,74 +0,0 @@ -use maud::{Markup, PreEscaped, html}; - -pub fn page() -> Markup { - html! { - div class="p-6 max-w-3xl space-y-4" { - div { - h2 class="text-[15px] font-semibold text-slate-200" { "Notification channels" } - p class="text-[12px] text-slate-500 mt-1" { "Where alerts get delivered when something needs your attention." } - } - (channel_row("Email", "email", "alerts@example.com", true)) - (channel_row("Slack", "slack", "#fleet-alerts", true)) - (channel_row("Discord", "discord", "https://discord.com/api/webhooks/\u{2026}", false)) - (channel_row("SMS", "sms", "+1 555 010 0001", true)) - } - } -} - -fn channel_row(name: &str, key: &str, placeholder: &str, enabled: bool) -> Markup { - let enabled_val = if enabled { - "var(--ok)" - } else { - "rgba(148,163,184,0.2)" - }; - let translate = if enabled { "18px" } else { "2px" }; - let display_val = if enabled { placeholder } else { "disabled" }; - - html! { - div class="card p-5" { - div class="flex items-center justify-between" { - div class="flex items-center gap-3" { - span class="inline-flex items-center justify-center w-9 h-9 rounded-md" style="background:var(--bg-elev-2); color:var(--accent-fg)" { - (PreEscaped(channel_icon(key))) - } - div { - div class="text-[14px] text-slate-100 font-medium" { (name) } - div class="text-[11px] text-slate-500" { (display_val) } - } - } - button - class="relative w-9 h-5 rounded-full transition-colors" - style={"background:" (enabled_val)} - hx-post={"/settings/toggle/" (key)} - hx-target="closest .card" - hx-swap="outerHTML" { - span class="absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform" style={"transform:translateX(" (translate) ")"} {} - } - } - div class={(if enabled { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3" } else { "mt-3 grid grid-cols-1 md:grid-cols-2 gap-3 max-h-0 opacity-0 overflow-hidden" })} { - div { - label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Destination" } - input class="input mt-1 w-full" style="padding-left:10px" type="text" placeholder=(placeholder) value=(placeholder) {} - } - div { - label class="text-[11px] text-slate-500 uppercase tracking-wider" { "Notify on" } - div class="mt-1 flex gap-1.5" { - span class="chip active" { "critical" } - span class="chip active" { "warning" } - span class="chip" { "info" } - } - } - } - } - } -} - -fn channel_icon(key: &str) -> String { - match key { - "email" => r#""#.to_string(), - "slack" => r#""#.to_string(), - "discord" => r#""#.to_string(), - "sms" => r#""#.to_string(), - _ => r#""#.to_string(), - } -} diff --git a/fleet/harmony-fleet-operator/src/service/mock.rs b/fleet/harmony-fleet-operator/src/service/mock.rs index b35135e6..eece7b85 100644 --- a/fleet/harmony-fleet-operator/src/service/mock.rs +++ b/fleet/harmony-fleet-operator/src/service/mock.rs @@ -444,14 +444,6 @@ impl FleetService for MockFleetService { } } - async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result { - // Seam only: the real impl publishes the command to the device - // and streams stdout/stderr back. - Ok(format!( - "$ {command}\n[{device_id}] command transport not yet implemented" - )) - } - async fn filtered_devices( &self, status: Option, diff --git a/fleet/harmony-fleet-operator/src/service/mod.rs b/fleet/harmony-fleet-operator/src/service/mod.rs index ce96e766..db6fef88 100644 --- a/fleet/harmony-fleet-operator/src/service/mod.rs +++ b/fleet/harmony-fleet-operator/src/service/mod.rs @@ -25,10 +25,6 @@ pub trait FleetService: Send + Sync + 'static { region: Option, search: Option, ) -> anyhow::Result>; - /// Send a one-shot shell command to a device for administrative - /// access. Returns the (eventual) output; streaming back live is a - /// later refinement. The device round-trip is not wired yet. - async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result; } // ── Device ───────────────────────────────────────────────────────────── diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index f3fdb11c..be536885 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -352,14 +352,6 @@ impl FleetService for RealFleetService { Ok(self.acked_alerts.lock().unwrap().insert(id.to_string())) } - async fn run_command(&self, _device_id: &str, command: &str) -> anyhow::Result { - // Seam only: the device round-trip (publish to the agent over - // NATS, stream stdout/stderr back) needs agent-side support. - Ok(format!( - "$ {command}\n[device command transport not implemented yet]" - )) - } - async fn filtered_devices( &self, status: Option, diff --git a/fleet/harmony-fleet-operator/vendor/app.js b/fleet/harmony-fleet-operator/vendor/app.js index bb382ab7..218b0e6f 100644 --- a/fleet/harmony-fleet-operator/vendor/app.js +++ b/fleet/harmony-fleet-operator/vendor/app.js @@ -1,27 +1,3 @@ document.body.addEventListener('htmx:configRequest', (event) => { event.detail.headers['x-csrf-token'] = '1'; }); - -// Open a modal dialog swapped into #modal-root. Lives here (not inline) -// because the production CSP forbids inline scripts/handlers. -document.body.addEventListener('htmx:afterSwap', (event) => { - if (!event.target || event.target.id !== 'modal-root') return; - const dialog = event.target.querySelector('dialog'); - if (!dialog || typeof dialog.showModal !== 'function') return; - - dialog.showModal(); - // Backdrop click closes; closing clears the root so it can re-open. - dialog.addEventListener('click', (e) => { - if (e.target === dialog) dialog.close(); - }); - dialog.addEventListener('close', () => { - event.target.innerHTML = ''; - }); - // Keep a streaming log body scrolled to the latest line. - const body = dialog.querySelector('[sse-connect]'); - if (body) { - new MutationObserver(() => { - body.scrollTop = body.scrollHeight; - }).observe(body, { childList: true }); - } -}); diff --git a/fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js b/fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js deleted file mode 100644 index 9f5af5c5..00000000 --- a/fleet/harmony-fleet-operator/vendor/htmx-ext-sse.js +++ /dev/null @@ -1,290 +0,0 @@ -/* -Server Sent Events Extension -============================ -This extension adds support for Server Sent Events to htmx. See /www/extensions/sse.md for usage instructions. - -*/ - -(function() { - /** @type {import("../htmx").HtmxInternalApi} */ - var api - - htmx.defineExtension('sse', { - - /** - * Init saves the provided reference to the internal HTMX API. - * - * @param {import("../htmx").HtmxInternalApi} api - * @returns void - */ - init: function(apiRef) { - // store a reference to the internal API. - api = apiRef - - // set a function in the public API for creating new EventSource objects - if (htmx.createEventSource == undefined) { - htmx.createEventSource = createEventSource - } - }, - - getSelectors: function() { - return ['[sse-connect]', '[data-sse-connect]', '[sse-swap]', '[data-sse-swap]'] - }, - - /** - * onEvent handles all events passed to this extension. - * - * @param {string} name - * @param {Event} evt - * @returns void - */ - onEvent: function(name, evt) { - var parent = evt.target || evt.detail.elt - switch (name) { - case 'htmx:beforeCleanupElement': - var internalData = api.getInternalData(parent) - // Try to remove remove an EventSource when elements are removed - var source = internalData.sseEventSource - if (source) { - api.triggerEvent(parent, 'htmx:sseClose', { - source, - type: 'nodeReplaced', - }) - internalData.sseEventSource.close() - } - - return - - // Try to create EventSources when elements are processed - case 'htmx:afterProcessNode': - ensureEventSourceOnElement(parent) - } - } - }) - - /// //////////////////////////////////////////// - // HELPER FUNCTIONS - /// //////////////////////////////////////////// - - /** - * createEventSource is the default method for creating new EventSource objects. - * it is hoisted into htmx.config.createEventSource to be overridden by the user, if needed. - * - * @param {string} url - * @returns EventSource - */ - function createEventSource(url) { - return new EventSource(url, { withCredentials: true }) - } - - /** - * registerSSE looks for attributes that can contain sse events, right - * now hx-trigger and sse-swap and adds listeners based on these attributes too - * the closest event source - * - * @param {HTMLElement} elt - */ - function registerSSE(elt) { - // Add message handlers for every `sse-swap` attribute - if (api.getAttributeValue(elt, 'sse-swap')) { - // Find closest existing event source - var sourceElement = api.getClosestMatch(elt, hasEventSource) - if (sourceElement == null) { - // api.triggerErrorEvent(elt, "htmx:noSSESourceError") - return null // no eventsource in parentage, orphaned element - } - - // Set internalData and source - var internalData = api.getInternalData(sourceElement) - var source = internalData.sseEventSource - - var sseSwapAttr = api.getAttributeValue(elt, 'sse-swap') - var sseEventNames = sseSwapAttr.split(',') - - for (var i = 0; i < sseEventNames.length; i++) { - const sseEventName = sseEventNames[i].trim() - const listener = function(event) { - // If the source is missing then close SSE - if (maybeCloseSSESource(sourceElement)) { - return - } - - // If the body no longer contains the element, remove the listener - if (!api.bodyContains(elt)) { - source.removeEventListener(sseEventName, listener) - return - } - - // swap the response into the DOM and trigger a notification - if (!api.triggerEvent(elt, 'htmx:sseBeforeMessage', event)) { - return - } - swap(elt, event.data) - api.triggerEvent(elt, 'htmx:sseMessage', event) - } - - // Register the new listener - api.getInternalData(elt).sseEventListener = listener - source.addEventListener(sseEventName, listener) - } - } - - // Add message handlers for every `hx-trigger="sse:*"` attribute - if (api.getAttributeValue(elt, 'hx-trigger')) { - // Find closest existing event source - var sourceElement = api.getClosestMatch(elt, hasEventSource) - if (sourceElement == null) { - // api.triggerErrorEvent(elt, "htmx:noSSESourceError") - return null // no eventsource in parentage, orphaned element - } - - // Set internalData and source - var internalData = api.getInternalData(sourceElement) - var source = internalData.sseEventSource - - var triggerSpecs = api.getTriggerSpecs(elt) - triggerSpecs.forEach(function(ts) { - if (ts.trigger.slice(0, 4) !== 'sse:') { - return - } - - var listener = function (event) { - if (maybeCloseSSESource(sourceElement)) { - return - } - if (!api.bodyContains(elt)) { - source.removeEventListener(ts.trigger.slice(4), listener) - } - // Trigger events to be handled by the rest of htmx - htmx.trigger(elt, ts.trigger, event) - htmx.trigger(elt, 'htmx:sseMessage', event) - } - - // Register the new listener - api.getInternalData(elt).sseEventListener = listener - source.addEventListener(ts.trigger.slice(4), listener) - }) - } - } - - /** - * ensureEventSourceOnElement creates a new EventSource connection on the provided element. - * If a usable EventSource already exists, then it is returned. If not, then a new EventSource - * is created and stored in the element's internalData. - * @param {HTMLElement} elt - * @param {number} retryCount - * @returns {EventSource | null} - */ - function ensureEventSourceOnElement(elt, retryCount) { - if (elt == null) { - return null - } - - // handle extension source creation attribute - if (api.getAttributeValue(elt, 'sse-connect')) { - var sseURL = api.getAttributeValue(elt, 'sse-connect') - if (sseURL == null) { - return - } - - ensureEventSource(elt, sseURL, retryCount) - } - - registerSSE(elt) - } - - function ensureEventSource(elt, url, retryCount) { - var source = htmx.createEventSource(url) - - source.onerror = function(err) { - // Log an error event - api.triggerErrorEvent(elt, 'htmx:sseError', { error: err, source }) - - // If parent no longer exists in the document, then clean up this EventSource - if (maybeCloseSSESource(elt)) { - return - } - - // Otherwise, try to reconnect the EventSource - if (source.readyState === EventSource.CLOSED) { - retryCount = retryCount || 0 - retryCount = Math.max(Math.min(retryCount * 2, 128), 1) - var timeout = retryCount * 500 - window.setTimeout(function() { - ensureEventSourceOnElement(elt, retryCount) - }, timeout) - } - } - - source.onopen = function(evt) { - api.triggerEvent(elt, 'htmx:sseOpen', { source }) - - if (retryCount && retryCount > 0) { - const childrenToFix = elt.querySelectorAll("[sse-swap], [data-sse-swap], [hx-trigger], [data-hx-trigger]") - for (let i = 0; i < childrenToFix.length; i++) { - registerSSE(childrenToFix[i]) - } - // We want to increase the reconnection delay for consecutive failed attempts only - retryCount = 0 - } - } - - api.getInternalData(elt).sseEventSource = source - - - var closeAttribute = api.getAttributeValue(elt, "sse-close"); - if (closeAttribute) { - // close eventsource when this message is received - source.addEventListener(closeAttribute, function() { - api.triggerEvent(elt, 'htmx:sseClose', { - source, - type: 'message', - }) - source.close() - }); - } - } - - /** - * maybeCloseSSESource confirms that the parent element still exists. - * If not, then any associated SSE source is closed and the function returns true. - * - * @param {HTMLElement} elt - * @returns boolean - */ - function maybeCloseSSESource(elt) { - if (!api.bodyContains(elt)) { - var source = api.getInternalData(elt).sseEventSource - if (source != undefined) { - api.triggerEvent(elt, 'htmx:sseClose', { - source, - type: 'nodeMissing', - }) - source.close() - // source = null - return true - } - } - return false - } - - - /** - * @param {HTMLElement} elt - * @param {string} content - */ - function swap(elt, content) { - api.withExtensions(elt, function(extension) { - content = extension.transformResponse(content, null, elt) - }) - - var swapSpec = api.getSwapSpecification(elt) - var target = api.getTarget(elt) - api.swap(target, content, swapSpec) - } - - - function hasEventSource(node) { - return api.getInternalData(node).sseEventSource != null - } -})() -- 2.39.5 From dded0344eb7b208a38664550da040aba2d18843d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 08:47:24 -0400 Subject: [PATCH 16/47] docs: plan fleet agent convergence and upgrades --- .../agent-reconciliation-and-upgrade-plan.md | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 fleet/agent-reconciliation-and-upgrade-plan.md diff --git a/fleet/agent-reconciliation-and-upgrade-plan.md b/fleet/agent-reconciliation-and-upgrade-plan.md new file mode 100644 index 00000000..45348be4 --- /dev/null +++ b/fleet/agent-reconciliation-and-upgrade-plan.md @@ -0,0 +1,221 @@ +# Fleet agent reconciliation and upgrade plan + +## Decisions + +- One agent instance owns workload reconciliation for a device at a time. +- NATS desired state is authoritative. Podman labels are the durable observed + state. The agent rebuilds its in-memory state from both after every restart. +- Desired entries carry their JetStream revision. Older watch or snapshot data + never replaces a newer revision. +- KV watches accelerate convergence. A periodic full snapshot repairs missed + events and drives orphan cleanup. +- One worker serializes runtime mutations. New events replace older intent + before the next reconciliation pass. +- A deployment upgrade follows Kubernetes replacement semantics: validate and + pull first, send the old container its normal stop signal, wait up to the + agent's fixed 30-second termination grace period, force it only after that + deadline, remove it, then start the replacement. A score-level override is + deferred until a real deployment needs one. +- An existing container owned by another deployment or by a human is a + conflict. The agent reports it and does not delete the container. +- Agent upgrade never runs two workload reconcilers. The candidate runs only a + non-mutating probe. A root-owned updater then switches the single permanent + systemd service and rolls back if the new agent does not become ready. + +## Deployment reconciliation + +### Inputs and reconstruction + +The reconciler owns one map from `DeploymentName` to the latest desired value. +A value is a valid unresolved score or an invalid payload with its error. At +startup and every periodic resync it replaces that map from the current +`desired-state..>` KV snapshot. Live puts and deletes update the same +map and wake the worker. A failed or incomplete snapshot cannot authorize +deletion. Because KV key enumeration and value reads are not atomic, a managed +deployment must be absent from two consecutive complete snapshots before it is +removed as an orphan. Any newer watch revision resets that absence count. +An explicit revisioned delete schedules removal immediately; the two-snapshot +rule applies only when reconstructing an absent key. + +No local deployment database is needed. After a reboot or power loss, current +KV values reconstruct intent and Harmony-managed Podman labels reconstruct +observed state. + +### Reconciliation pass + +For one complete desired snapshot: + +1. Validate every deployment. A duplicate service name or explicit host port + fails the conflicting deployments but does not block unrelated deployments. +2. Resolve secrets for the current score revision. +3. Ask the runtime to converge each valid deployment. +4. List Harmony-managed runtime deployments and remove deployments absent from + two complete snapshots. Cleanup runs before retrying a desired deployment + that needs the same service name. +5. Publish coarse `Phase` plus bounded `lastError`. Only an acknowledged phase + and error pair is deduplicated; failed status writes remain dirty. + +The runtime convergence operation preflights the full score before stopping +anything. It validates ports and service names, pulls missing images, and checks +container ownership. Preflight prevents known destructive failures; it cannot +make Podman operations transactional. For each changed service the runtime +performs stop, remove, create, start, and inspect in that order. Successfully +changed services remain changed if a later service fails, and the next pass +resumes from observed labels. Unchanged running services are untouched. Missing +or stopped desired services are recreated or restarted. Services no longer in +the score are removed with the same graceful-stop behavior. + +A malformed current payload remains present for orphan accounting, reports +`Failed`, and causes no workload mutation. The last valid workload is preserved +but no longer reported healthy until the payload is corrected or deleted. +If a newer revision arrives during a runtime call, that call may finish but its +result is not published. The worker immediately reconciles the latest revision; +it never interrupts one Podman operation halfway. + +### Edge cases + +The first implementation and its tests cover: + +- new deployment; +- unchanged deployment as a no-op; +- upgrade with stop-before-replace ordering; +- preflight failure without stopping the old version; +- exited desired container reported as failed with its exit details; Podman + restart count is included for diagnosis but no time-window crashloop + classifier is introduced; +- online deletion; +- offline deletion followed by orphan cleanup when the device returns; +- agent restart and host reboot; +- power loss before stop, after stop, after remove, after create, and after + start, each repaired by the next full pass; +- service-name conflict between deployments; +- conflict with an unmanaged or differently owned container; +- desired container manually stopped or deleted; +- malformed desired payload replacing, rather than preserving, old intent; +- partial failure in a multi-service deployment; +- Podman unavailability and retry; +- failed state publication and retry; +- watch interruption or missed events repaired by full resync; +- failed and incomplete full snapshots; +- puts and deletes before, during, and after snapshot enumeration; +- invalid desired keys and failed state deletion; +- a newer desired revision arriving during reconciliation. + +Secret values remain cached for one desired revision. Secret-only rotation is +out of scope until the secret source exposes a revision or watch contract. + +## Agent upgrade + +This section replaces ADR-022's dual-active cutover and its guarantee that the +old agent remains active until the operator observes a full new agent. That +guarantee conflicts with the single-active-agent decision. The replacement +accepts a bounded interval with no reconciler while systemd starts the new +binary; workloads continue under Podman's restart policy, and the updater rolls +back on failed readiness. ADR-022 must be updated in the same change. + +### Components + +- The unprivileged fleet agent drains workload mutations, stages upgrade intent, + and reports attempt-scoped status. +- A candidate binary runs `--self-test`. It parses configuration, authenticates, + connects to NATS, reads its permitted desired-state snapshot, checks Podman, + reports probe success for the attempt, and exits. It never publishes a normal + heartbeat, subscribes as an active reconciler, or mutates workloads. +- A narrow root-owned helper owns versioned binaries, signature and digest + verification, the active symlink, systemd restart, rollback, and one durable + transaction file. `FleetDeviceSetupScore` installs it as a root systemd + service with a Unix socket owned by root and writable only by the + `fleet-agent` group. Its fixed request protocol accepts only stage and switch + for paths derived under one compiled artifact root. Finalization, recovery, + and rollback are internal. It accepts no shell command or caller-selected + path. +- The operator owns desired version and attempt identity. Every intent, status, + probe, and switch authorization carries the same attempt ID. + +### Control-plane contract + +- `agent-upgrade-intent.` contains the latest immutable attempt. The + operator may write it; that device may read it. +- `agent-upgrade-authorize..` authorizes the exclusive switch + after the operator observes probe success. The operator may write it; that + device may read it. +- `agent-upgrade-status.` contains the agent's latest attempt-scoped + phase and bounded error. The device may write it; the operator may read it. + +These are JetStream KV entries. Attempts use UUIDs and revisions are applied +monotonically. Replayed intent for an attempt whose durable status is terminal +is a no-op. `authorize-switch` is the operator transition; `finalize` is the +helper's local transaction completion. Callout permissions expose only the +device's own keys. + +### Upgrade transaction + +1. The operator publishes an immutable attempt containing device ID, source and + target versions, architecture, artifact URL, size limit, SHA-256 digest, + signature, signing key ID, and creation time. +2. The active agent rejects stale, duplicate, wrong-device, wrong-source, + wrong-architecture, or unsupported-version attempts. +3. The agent enters draining. Existing workloads continue running; the current + runtime mutation finishes, and newer desired changes remain queued. +4. The helper downloads to a same-filesystem temporary file, enforces HTTPS and + size limits, verifies architecture, SHA-256, and an Ed25519 signature against + `/etc/fleet-agent/trusted-upgrade-keys/.pub`, fsyncs, and atomically + installs the immutable versioned binary. The candidate always runs as + `fleet-agent`. +5. The candidate runs the non-mutating probe. Failure leaves the active agent + and symlink unchanged. +6. The operator observes probe success for the exact attempt and publishes + `authorize-switch`. If authorization does not arrive within five minutes, + the agent leaves draining, resumes queued workload reconciliation, and + retains the staged binary for inspection or a new attempt. +7. The updater records the previous target, switches the symlink atomically, + and restarts the one permanent `fleet-agent.service`. +8. The new agent acquires an OS advisory process lock, loads configuration, + authenticates, reaches Podman when enabled, initializes a complete desired + snapshot, starts the reconciliation worker, sends systemd readiness, and + reports ready for the attempt. Workload health is not an agent-readiness + predicate. +9. The helper allows 60 seconds for systemd readiness, then finalizes after a + further 60-second probation in which the service remains active and systemd + reports no restart. If startup, readiness, or probation fails, it restores + the previous symlink and restarts the previous version. + +The root-owned transaction file records `staged`, `switching`, `probation`, +`committed`, `rolling-back`, or `failed`, with attempt ID and previous and target +paths. File and parent-directory fsync happen before and after binary rename, +symlink rename, and state transitions. On helper startup, pre-switch states are +safe to resume; uncommitted post-switch states attempt rollback. Rollback failure +is durably reported rather than claimed as recovery. A failed attempt is +terminal until a new attempt ID arrives. + +### Upgrade edge cases + +Tests cover successful upgrade, duplicate and stale attempts, downgrade, +download failure, oversized artifact, digest mismatch, bad signature, wrong +architecture, probe failure, commit before probe, cancellation, operator outage +before commit, queued workload recovery after timeout, stale commit, helper +unavailability, missing previous binary, readiness arriving after timeout, +power loss at every persisted transition, new-agent startup failure, readiness +timeout, helper restart, rollback failure reporting, repeated failed intent, and +reboot after commit. At every pre-switch failure the old agent remains active. +At every post-switch failure the helper automatically attempts rollback. + +The first updater-capable release is bootstrapped by one final +`FleetDeviceSetupScore` run, which installs the versioned binary layout, helper, +socket permissions, and revised systemd unit. The operator remains compatible +with old agents whose heartbeat has no version. Automatic upgrades begin only +after this bootstrap release is observed healthy. + +## Delivery checkpoints + +1. Rework deployment reconciliation around a complete snapshot and a fakeable + runtime boundary. Add the full unit matrix and retain the existing VM E2E. +2. Update Podman convergence for ownership checks, preflight, graceful + stop-before-replace, inspection, and orphan inventory. +3. Amend ADR-022 and add attempt-scoped upgrade contracts. +4. Add the updater, probe mode, exclusive agent lock, durable transaction, and + operator coordination. +5. Run unit tests, compile environment-gated E2E, run focused local VM tests if + available without production credentials, and complete independent reviews. +6. Production QA is manual and stepwise: deployment recovery first, then one + agent upgrade attempt with rollback rehearsed before any canary rollout. -- 2.39.5 From 63a91a222218c4dcd3f8d65e125784af69ea49e8 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 10:14:10 -0400 Subject: [PATCH 17/47] fix: reconcile fleet workloads from durable state --- .../src/fleet_publisher.rs | 43 +- fleet/harmony-fleet-agent/src/main.rs | 153 +- fleet/harmony-fleet-agent/src/podman.rs | 346 ++++- fleet/harmony-fleet-agent/src/reconciler.rs | 1251 ++++++++++------- 4 files changed, 1179 insertions(+), 614 deletions(-) diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index 20f0ff80..5ea04148 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -3,10 +3,6 @@ //! Thin wrapper around three KV buckets: [`BUCKET_DEVICE_INFO`], //! [`BUCKET_DEVICE_STATE`], [`BUCKET_DEVICE_HEARTBEAT`]. //! -//! Failure mode: log and swallow. The KV is the source of truth — -//! a dropped put gets corrected on the next reconcile transition -//! or operator watch reconnection. - use async_nats::jetstream::{self, kv}; use harmony_reconciler_contracts::{ BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, @@ -27,6 +23,12 @@ pub struct FleetPublisher { heartbeat_bucket: kv::Store, } +#[async_trait::async_trait] +pub trait DeploymentStatePublisher: Send + Sync { + async fn write(&self, state: &DeploymentState) -> anyhow::Result<()>; + async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()>; +} + impl FleetPublisher { /// Open every bucket the agent needs, creating those that don't /// exist yet. Idempotent with operator-side creation. @@ -110,17 +112,12 @@ impl FleetPublisher { /// bucket picks up this put and updates CR status counters. /// Also fans out the same payload on `device-state.` /// for live observers that don't want to consume the KV stream. - pub async fn write_deployment_state(&self, state: &DeploymentState) { + pub async fn write_deployment_state(&self, state: &DeploymentState) -> anyhow::Result<()> { let key = device_state_key(&self.device_id.to_string(), &state.deployment); - match serde_json::to_vec(state) { - Ok(payload) => { - if let Err(e) = self.state_bucket.put(&key, payload.clone().into()).await { - tracing::warn!(%key, error = %e, "write_deployment_state: kv put failed"); - } - self.publish_direct_state(payload).await; - } - Err(e) => tracing::warn!(error = %e, "write_deployment_state: serialize failed"), - } + let payload = serde_json::to_vec(state)?; + self.state_bucket.put(&key, payload.clone().into()).await?; + self.publish_direct_state(payload).await; + Ok(()) } /// Emit a tiny presence pulse on `device-state.` so live @@ -150,10 +147,20 @@ impl FleetPublisher { /// Delete the authoritative current-phase entry, e.g. when the /// Deployment CR is removed and the agent has torn down the /// container. - pub async fn delete_deployment_state(&self, deployment: &DeploymentName) { + pub async fn delete_deployment_state(&self, deployment: &DeploymentName) -> anyhow::Result<()> { let key = device_state_key(&self.device_id.to_string(), deployment); - if let Err(e) = self.state_bucket.delete(&key).await { - tracing::debug!(%key, error = %e, "delete_deployment_state: kv delete failed"); - } + self.state_bucket.delete(&key).await?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl DeploymentStatePublisher for FleetPublisher { + async fn write(&self, state: &DeploymentState) -> anyhow::Result<()> { + self.write_deployment_state(state).await + } + + async fn delete(&self, deployment: &DeploymentName) -> anyhow::Result<()> { + self.delete_deployment_state(deployment).await } } diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 62fdf5d1..1be6bbb5 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -24,7 +24,7 @@ use harmony_reconciler_contracts::{ use crate::command_server::CommandServer; use crate::fleet_publisher::FleetPublisher; use crate::podman::PodmanRuntime; -use crate::reconciler::Reconciler; +use crate::reconciler::{Reconciler, SnapshotEntry}; /// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch /// events are accelerators. @@ -77,49 +77,104 @@ async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result, -) -> Result<()> { - let jetstream = async_nats::jetstream::new(client); - let bucket = jetstream +) -> Result { + Ok(async_nats::jetstream::new(client) .create_key_value(async_nats::jetstream::kv::Config { bucket: BUCKET_DESIRED_STATE.to_string(), ..Default::default() }) - .await?; + .await?) +} +async fn load_desired_snapshot( + bucket: &async_nats::jetstream::kv::Store, + device_id: &Id, +) -> Result> { + let prefix = format!("{device_id}."); + let mut entries = Vec::new(); + let mut keys = bucket.keys().await?; + while let Some(key) = keys.next().await { + let key = key?; + if !key.starts_with(&prefix) { + continue; + } + if let Some(entry) = bucket.entry(&key).await? + && entry.operation == async_nats::jetstream::kv::Operation::Put + { + entries.push(SnapshotEntry { + key, + revision: entry.revision, + value: entry.value.to_vec(), + }); + } + } + Ok(entries) +} + +async fn watch_desired_state( + bucket: async_nats::jetstream::kv::Store, + device_id: Id, + reconciler: Arc, +) -> Result<()> { let key_filter = desired_state_watch_filter(&device_id.to_string()); tracing::info!(filter = %key_filter, "watching KV keys"); - - let mut watch = bucket.watch(&key_filter).await?; - while let Some(result) = watch.next().await { - let entry = match result { - Ok(e) => e, - Err(e) => { - tracing::warn!(error = %e, "watch error"); + loop { + let mut watch = match bucket.watch(&key_filter).await { + Ok(watch) => watch, + Err(error) => { + tracing::warn!(%error, "desired-state watch start failed"); + tokio::time::sleep(Duration::from_secs(1)).await; continue; } }; - - tracing::debug!(key = %entry.key, "bucket watch new value {entry:?}"); - - match entry.operation { - async_nats::jetstream::kv::Operation::Put => { - if let Err(e) = reconciler.apply(&entry.key, &entry.value).await { - tracing::warn!(key = %entry.key, error = %e, "apply failed"); + while let Some(result) = watch.next().await { + let entry = match result { + Ok(entry) => entry, + Err(error) => { + tracing::warn!(%error, "desired-state watch failed; restarting"); + break; } - } - async_nats::jetstream::kv::Operation::Delete - | async_nats::jetstream::kv::Operation::Purge => { - if let Err(e) = reconciler.remove(&entry.key).await { - tracing::warn!(key = %entry.key, error = %e, "remove failed"); + }; + let result = match entry.operation { + async_nats::jetstream::kv::Operation::Put => { + reconciler + .put(&entry.key, entry.revision, &entry.value) + .await } + async_nats::jetstream::kv::Operation::Delete + | async_nats::jetstream::kv::Operation::Purge => { + reconciler.delete(&entry.key, entry.revision).await + } + }; + if let Err(error) = result { + tracing::warn!(key = %entry.key, %error, "desired-state event rejected"); } } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +async fn snapshot_loop( + bucket: async_nats::jetstream::kv::Store, + device_id: Id, + reconciler: Arc, +) { + let mut interval = tokio::time::interval(RECONCILE_INTERVAL); + interval.tick().await; + loop { + interval.tick().await; + let generation = reconciler.generation().await; + match load_desired_snapshot(&bucket, &device_id).await { + Ok(snapshot) => { + if let Err(error) = reconciler.replace_snapshot(snapshot, generation).await { + tracing::warn!(%error, "desired-state snapshot rejected"); + } + } + Err(error) => tracing::warn!(%error, "desired-state snapshot failed"), + } } - Ok(()) } /// Tiny liveness-only loop: push a `HeartbeatPayload` into the @@ -282,8 +337,6 @@ async fn main() -> Result<()> { .publish_device_info(startup_labels, Some(inventory_snapshot.clone())) .await .context("publishing device registration")?; - sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) - .context("notifying systemd that registration completed")?; // Reconciler exists only when a podman topology is available. // Without it, the desired-state watch + periodic reconcile arms @@ -298,6 +351,20 @@ async fn main() -> Result<()> { )) }); + let desired_bucket = if let Some(reconciler) = &reconciler { + let bucket = desired_state_store(client.clone()).await?; + let generation = reconciler.generation().await; + let snapshot = load_desired_snapshot(&bucket, &device_id).await?; + reconciler.replace_snapshot(snapshot, generation).await?; + reconciler.reconcile_once().await?; + Some(bucket) + } else { + None + }; + + sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) + .context("notifying systemd that initialization completed")?; + let command_server = Arc::new(CommandServer::new(device_id.clone(), client.clone())); let ctrlc = async { @@ -315,24 +382,31 @@ async fn main() -> Result<()> { let _ = inventory_snapshot; // consumed by the DeviceInfo publish above let watch: std::pin::Pin> + Send>> = - match reconciler.as_ref() { - Some(r) => Box::pin(watch_desired_state( - client.clone(), + match (reconciler.as_ref(), desired_bucket.as_ref()) { + (Some(r), Some(bucket)) => Box::pin(watch_desired_state( + bucket.clone(), device_id.clone(), r.clone(), )), - None => Box::pin(async { + _ => Box::pin(async { std::future::pending::<()>().await; Ok(()) }), }; - let reconcile: std::pin::Pin + Send>> = - match reconciler.as_ref() { - Some(r) => Box::pin(r.clone().run_periodic(RECONCILE_INTERVAL)), - None => Box::pin(std::future::pending::<()>()), + let snapshots: std::pin::Pin + Send>> = + match (reconciler.as_ref(), desired_bucket) { + (Some(reconciler), Some(bucket)) => { + Box::pin(snapshot_loop(bucket, device_id.clone(), reconciler.clone())) + } + _ => Box::pin(std::future::pending()), }; let heartbeat = publish_heartbeat_loop(fleet); let commands = command_server.run(); + let worker: std::pin::Pin + Send>> = + match reconciler.as_ref() { + Some(reconciler) => Box::pin(reconciler.clone().run()), + None => Box::pin(std::future::pending()), + }; tokio::select! { // Waiting on ctrlc in a select will automatically terminate other branches when @@ -340,7 +414,8 @@ async fn main() -> Result<()> { _ = ctrlc => {}, r = sigterm => { r?; } r = watch => { r?; } - _ = reconcile => {} + _ = snapshots => {} + _ = worker => {} _ = heartbeat => {} r = commands => { r?; } } diff --git a/fleet/harmony-fleet-agent/src/podman.rs b/fleet/harmony-fleet-agent/src/podman.rs index de0efc29..98bef031 100644 --- a/fleet/harmony-fleet-agent/src/podman.rs +++ b/fleet/harmony-fleet-agent/src/podman.rs @@ -17,7 +17,14 @@ const DEPLOYMENT_LABEL: &str = "io.nationtech.harmony.deployment"; const SPEC_LABEL: &str = "io.nationtech.harmony.spec-sha256"; const MANAGED_BY_LABEL: &str = "io.nationtech.harmony.managed-by"; const MANAGED_BY_VALUE: &str = "harmony"; -const STOP_TIMEOUT: Duration = Duration::from_secs(300); +const STOP_TIMEOUT: Duration = Duration::from_secs(30); + +#[async_trait::async_trait] +pub trait WorkloadRuntime: Send + Sync { + async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()>; + async fn remove_deployment(&self, deployment: &str) -> Result<()>; + async fn managed_deployments(&self) -> Result>; +} pub struct PodmanRuntime { podman: Podman, @@ -43,39 +50,124 @@ impl PodmanRuntime { } pub async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> { - for service in &score.services { - self.ensure_service_running(service, deployment).await?; - } let desired = score .services .iter() .map(|service| service.name.as_str()) .collect::>(); - for container in self.deployment_containers(deployment).await? { - let name = container_name(&container); - if !desired.contains(name.as_str()) { - self.remove_service(&name).await?; - } + let stale = self.preflight(deployment, score, &desired).await?; + for name in stale { + self.remove_service(&name, deployment).await?; + } + for service in &score.services { + self.ensure_service_running(service, deployment).await?; + } + for service in &score.services { + self.ensure_service_healthy(service).await?; } Ok(()) } + async fn preflight( + &self, + deployment: &str, + score: &PodmanV0Score, + desired: &HashSet<&str>, + ) -> Result> { + let mut names = HashSet::new(); + let mut host_ports = HashSet::new(); + let containers = self.all_containers().await?; + for service in &score.services { + validate_service_name(&service.name)?; + if !names.insert(&service.name) { + bail!("duplicate service name '{}'", service.name); + } + for port in &service.ports { + let requested = parse_port_mapping(port)?; + let host_port = requested.host_port.unwrap_or_default(); + if !host_ports.insert(host_port) { + bail!("host port {host_port} is requested more than once"); + } + if containers.iter().any(|container| { + let name = container_name(container); + let removable_stale = + is_owned_by(container, deployment) && !desired.contains(name.as_str()); + name != service.name + && !removable_stale + && container.ports.as_ref().is_some_and(|ports| { + ports + .iter() + .any(|port| port.host_port == requested.host_port) + }) + }) { + bail!( + "host port {} is already in use", + requested.host_port.unwrap_or_default() + ); + } + } + if let Some(existing) = self.get_by_name(&service.name).await? { + ensure_owned_by(&existing, deployment)?; + } + self.ensure_image_present(&service.image).await?; + } + Ok(containers + .iter() + .filter(|container| { + is_owned_by(container, deployment) + && !desired.contains(container_name(container).as_str()) + }) + .map(container_name) + .collect()) + } + pub async fn remove_deployment(&self, deployment: &str) -> Result<()> { for container in self.deployment_containers(deployment).await? { - self.remove_service(&container_name(&container)).await?; + self.remove_service(&container_name(&container), deployment) + .await?; } Ok(()) } - pub async fn remove_service(&self, name: &str) -> Result<()> { + pub async fn managed_deployments(&self) -> Result> { + let opts = ContainerListOpts::builder() + .all(true) + .filter([ContainerListFilter::LabelKeyVal( + MANAGED_BY_LABEL.to_string(), + MANAGED_BY_VALUE.to_string(), + )]) + .build(); + Ok(self + .podman + .containers() + .list(&opts) + .await? + .into_iter() + .filter_map(|container| { + container + .labels + .and_then(|labels| labels.get(DEPLOYMENT_LABEL).cloned()) + }) + .collect()) + } + + async fn remove_service(&self, name: &str, deployment: &str) -> Result<()> { + let Some(existing) = self.get_by_name(name).await? else { + return Ok(()); + }; + ensure_owned_by(&existing, deployment)?; + let id = existing + .id + .clone() + .ok_or_else(|| anyhow!("container '{name}' has no id"))?; let opts = ContainerStopOpts::builder() .timeout(STOP_TIMEOUT.as_secs() as usize) .build(); - let container = self.podman.containers().get(name); - if container.exists().await.unwrap_or(false) { - let _ = container.stop(&opts).await; + let container = self.podman.containers().get(&id); + if existing.state.as_deref() == Some("running") { + container.stop(&opts).await?; } - self.remove_container(name).await + self.remove_container(&id).await } async fn ensure_service_running( @@ -84,16 +176,20 @@ impl PodmanRuntime { deployment: &str, ) -> Result<()> { let existing = self.get_by_name(&service.name).await?; - if let Some(existing) = existing.as_ref() { - if matches_spec(existing, service) { - if existing.state.as_deref() == Some("running") { - return Ok(()); - } + match service_action(existing.as_ref(), service) { + ServiceAction::Keep => return Ok(()), + ServiceAction::Start => { + let existing = existing + .as_ref() + .expect("start requires an existing container"); let id = existing.id.clone().unwrap_or_else(|| service.name.clone()); self.podman.containers().get(id).start(None).await?; return Ok(()); } - self.remove_container(&service.name).await?; + ServiceAction::Replace => { + self.remove_service(&service.name, deployment).await?; + } + ServiceAction::Create => {} } self.ensure_image_present(&service.image).await?; @@ -132,6 +228,28 @@ impl PodmanRuntime { Ok(()) } + async fn ensure_service_healthy(&self, service: &PodmanService) -> Result<()> { + let inspected = self + .podman + .containers() + .get(&service.name) + .inspect() + .await?; + let state = inspected + .state + .ok_or_else(|| anyhow!("service '{}' has no runtime state", service.name))?; + let restarts = inspected.restart_count.unwrap_or_default(); + if state.running != Some(true) { + bail!( + "service '{}' is not healthy: status={}, exit_code={}, restarts={restarts}", + service.name, + state.status.as_deref().unwrap_or("unknown"), + state.exit_code.unwrap_or_default(), + ); + } + Ok(()) + } + async fn get_by_name(&self, name: &str) -> Result> { let opts = ContainerListOpts::builder() .all(true) @@ -143,7 +261,12 @@ impl PodmanRuntime { .list(&opts) .await? .into_iter() - .next()) + .find(|container| container_name(container) == name)) + } + + async fn all_containers(&self) -> Result> { + let opts = ContainerListOpts::builder().all(true).build(); + Ok(self.podman.containers().list(&opts).await?) } async fn deployment_containers( @@ -152,10 +275,16 @@ impl PodmanRuntime { ) -> Result> { let opts = ContainerListOpts::builder() .all(true) - .filter([ContainerListFilter::LabelKeyVal( - DEPLOYMENT_LABEL.to_string(), - deployment.to_string(), - )]) + .filter([ + ContainerListFilter::LabelKeyVal( + MANAGED_BY_LABEL.to_string(), + MANAGED_BY_VALUE.to_string(), + ), + ContainerListFilter::LabelKeyVal( + DEPLOYMENT_LABEL.to_string(), + deployment.to_string(), + ), + ]) .build(); Ok(self.podman.containers().list(&opts).await?) } @@ -184,6 +313,21 @@ impl PodmanRuntime { } } +#[async_trait::async_trait] +impl WorkloadRuntime for PodmanRuntime { + async fn reconcile(&self, deployment: &str, score: &PodmanV0Score) -> Result<()> { + self.reconcile(deployment, score).await + } + + async fn remove_deployment(&self, deployment: &str) -> Result<()> { + self.remove_deployment(deployment).await + } + + async fn managed_deployments(&self) -> Result> { + self.managed_deployments().await + } +} + fn default_user_socket() -> PathBuf { if let Ok(dir) = std::env::var("XDG_RUNTIME_DIR") { return PathBuf::from(format!("{dir}/podman/podman.sock")); @@ -229,6 +373,69 @@ fn matches_spec(observed: &podman_api::models::ListContainer, service: &PodmanSe .is_some_and(|hash| spec_hash(service).is_ok_and(|expected| hash == &expected)) } +#[derive(Debug, PartialEq, Eq)] +enum ServiceAction { + Keep, + Start, + Replace, + Create, +} + +fn service_action( + observed: Option<&podman_api::models::ListContainer>, + service: &PodmanService, +) -> ServiceAction { + match observed { + None => ServiceAction::Create, + Some(observed) if !matches_spec(observed, service) => ServiceAction::Replace, + Some(observed) if observed.state.as_deref() == Some("running") => ServiceAction::Keep, + Some(_) => ServiceAction::Start, + } +} + +fn ensure_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> Result<()> { + let labels = observed.labels.as_ref(); + let managed = labels.and_then(|labels| labels.get(MANAGED_BY_LABEL)); + let owner = labels.and_then(|labels| labels.get(DEPLOYMENT_LABEL)); + if managed.map(String::as_str) != Some(MANAGED_BY_VALUE) + || owner.map(String::as_str) != Some(deployment) + { + bail!( + "container '{}' conflicts with deployment '{deployment}' (owner: {})", + container_name(observed), + owner.map(String::as_str).unwrap_or("unmanaged") + ); + } + Ok(()) +} + +fn is_owned_by(observed: &podman_api::models::ListContainer, deployment: &str) -> bool { + let labels = observed.labels.as_ref(); + labels + .and_then(|labels| labels.get(MANAGED_BY_LABEL)) + .map(String::as_str) + == Some(MANAGED_BY_VALUE) + && labels + .and_then(|labels| labels.get(DEPLOYMENT_LABEL)) + .map(String::as_str) + == Some(deployment) +} + +fn validate_service_name(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|character| character.is_ascii_alphanumeric() || "_.-".contains(character)) + || !name + .chars() + .next() + .is_some_and(|character| character.is_ascii_alphanumeric()) + { + bail!("invalid service name '{name}'"); + } + Ok(()) +} + fn spec_hash(service: &PodmanService) -> Result { Ok(format!( "{:x}", @@ -290,6 +497,19 @@ mod tests { } } + fn observed( + name: &str, + state: &str, + labels: HashMap, + ) -> podman_api::models::ListContainer { + serde_json::from_value(serde_json::json!({ + "Names": [name], + "State": state, + "Labels": labels, + })) + .unwrap() + } + #[test] fn spec_hash_is_stable_and_covers_desired_state() { let original = service(); @@ -299,4 +519,78 @@ mod tests { changed.env[0].value = "dev".into(); assert_ne!(spec_hash(&changed).unwrap(), spec_hash(&service()).unwrap()); } + + #[test] + fn replacement_is_selected_before_create_for_changed_spec() { + let original = service(); + let observed = observed( + &original.name, + "running", + HashMap::from([ + (MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()), + (DEPLOYMENT_LABEL.into(), "deployment-a".into()), + (SPEC_LABEL.into(), spec_hash(&original).unwrap()), + ]), + ); + let mut changed = original; + changed.image = "nginx:new".into(); + + assert_eq!( + service_action(Some(&observed), &changed), + ServiceAction::Replace + ); + assert_eq!(STOP_TIMEOUT, Duration::from_secs(30)); + } + + #[test] + fn ownership_requires_both_harmony_and_deployment_labels() { + let mut observed = observed("web", "running", HashMap::new()); + assert!(ensure_owned_by(&observed, "deployment-a").is_err()); + + observed.labels = Some(HashMap::from([ + (MANAGED_BY_LABEL.into(), MANAGED_BY_VALUE.into()), + (DEPLOYMENT_LABEL.into(), "deployment-a".into()), + ])); + assert!(ensure_owned_by(&observed, "deployment-a").is_ok()); + assert!(ensure_owned_by(&observed, "deployment-b").is_err()); + } + + #[test] + fn non_running_service_is_restarted_regardless_of_exit_code() { + let service = service(); + let observed: podman_api::models::ListContainer = + serde_json::from_value(serde_json::json!({ + "Names": [service.name], + "State": "exited", + "Exited": true, + "ExitCode": 42, + "Labels": { SPEC_LABEL: spec_hash(&service).unwrap() }, + })) + .unwrap(); + assert_eq!( + service_action(Some(&observed), &service), + ServiceAction::Start + ); + + let stopped: podman_api::models::ListContainer = + serde_json::from_value(serde_json::json!({ + "Names": [service.name], + "State": "exited", + "Exited": true, + "ExitCode": 0, + "Labels": { SPEC_LABEL: spec_hash(&service).unwrap() }, + })) + .unwrap(); + assert_eq!( + service_action(Some(&stopped), &service), + ServiceAction::Start + ); + } + + #[test] + fn invalid_service_names_are_rejected() { + assert!(validate_service_name("web-1").is_ok()); + assert!(validate_service_name("").is_err()); + assert!(validate_service_name("/web").is_err()); + } } diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 45e713f4..b75f47aa 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -1,379 +1,503 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use std::time::Duration; -use anyhow::Result; +use anyhow::{Result, anyhow}; use chrono::Utc; use harmony_reconciler_contracts::{ DeploymentName, DeploymentState, EnvVar, Id, Phase, PodmanV0Score, ReconcileScore, }; +use harmony_secret::SecretStore; use tokio::sync::Mutex; -use harmony_secret::SecretStore; +use crate::fleet_publisher::DeploymentStatePublisher; +use crate::podman::WorkloadRuntime; -use crate::fleet_publisher::FleetPublisher; -use crate::podman::PodmanRuntime; - -/// Where deployment secrets come from (ADR-025). `prefix` scopes the -/// KV reads to `//` — it must match the -/// prefix the grant writer scopes `deployment-` policies to, -/// or every fetch is denied. pub struct SecretSource { pub store: Arc, pub prefix: String, } -/// Cache key → last-seen state, populated by `apply` and consulted by the -/// 30-second periodic tick and the delete path. #[derive(Clone)] -enum CachedEntry { - Apply { - serialized: String, +enum DesiredValue { + Valid { score: PodmanV0Score, resolved: bool, }, - Remove { - score: Option, - }, + Invalid(String), +} + +#[derive(Clone)] +struct DesiredEntry { + revision: u64, + value: DesiredValue, +} + +#[derive(Default)] +struct State { + desired: HashMap, + revisions: HashMap, + explicit_removals: HashMap, + orphan_misses: HashMap, + acknowledged: HashMap)>, + complete_snapshot_pending: bool, + generation: u64, +} + +pub struct SnapshotEntry { + pub key: String, + pub revision: u64, + pub value: Vec, } pub struct Reconciler { device_id: Id, - runtime: Arc, - /// Keyed by NATS KV key (`.`). A single entry per - /// KV key — in v0 there is no fan-out from one key to many scores. - state: Mutex>, - /// Current phase per deployment, used to decide whether a new - /// write to the `device-state` KV is needed. - /// - /// NOTE : this feels dangerous, conflict on deployment name could be a problem - /// We must explore this and clarify it in the design and decide if it is a constraint - deployments: Mutex>, - /// Publish surface. Optional so unit tests without a live NATS - /// client still work; always populated in the real agent runtime. - fleet: Option>, - /// Deployment secret store. `None` = any score referencing - /// `secret_env` fails to apply (loudly, via `Phase::Failed`). + runtime: Arc, + state: Mutex, + runtime_gate: Mutex<()>, + wake: tokio::sync::Notify, + publisher: Option>, secrets: Option, } impl Reconciler { pub fn new( device_id: Id, - runtime: Arc, - fleet: Option>, + runtime: Arc, + publisher: Option>, secrets: Option, ) -> Self { Self { device_id, runtime, - state: Mutex::new(HashMap::new()), - deployments: Mutex::new(HashMap::new()), - fleet, + state: Mutex::new(State::default()), + runtime_gate: Mutex::new(()), + wake: tokio::sync::Notify::new(), + publisher, secrets, } } - /// Materialize every `secret_env` reference into a plain env var. - /// Resolution is retried until it succeeds, then the resolved score is - /// reused until the spec changes. This keeps the secret store off the - /// periodic hot path for converged deployments. + pub async fn put(&self, key: &str, revision: u64, value: &[u8]) -> Result<()> { + let deployment = self.deployment_from_key(key)?; + self.update_desired(deployment, revision, value).await; + self.wake.notify_one(); + Ok(()) + } + + pub async fn delete(&self, key: &str, revision: u64) -> Result<()> { + let deployment = self.deployment_from_key(key)?; + { + let mut state = self.state.lock().await; + if state + .revisions + .get(&deployment) + .is_some_and(|seen| *seen >= revision) + { + return Ok(()); + } + state.revisions.insert(deployment.clone(), revision); + state.generation += 1; + state.desired.remove(&deployment); + state.orphan_misses.remove(&deployment); + state.explicit_removals.insert(deployment, revision); + } + self.wake.notify_one(); + Ok(()) + } + + pub async fn generation(&self) -> u64 { + self.state.lock().await.generation + } + + pub async fn replace_snapshot( + &self, + entries: Vec, + started_at_generation: u64, + ) -> Result<()> { + let mut snapshot = HashMap::new(); + for entry in entries { + let deployment = self.deployment_from_key(&entry.key)?; + snapshot.insert( + deployment, + DesiredEntry { + revision: entry.revision, + value: parse_desired(&entry.value), + }, + ); + } + { + let mut state = self.state.lock().await; + let absence_is_current = state.generation == started_at_generation; + if absence_is_current { + state + .desired + .retain(|deployment, _| snapshot.contains_key(deployment)); + } + for (deployment, entry) in snapshot { + let seen = state + .revisions + .get(&deployment) + .copied() + .unwrap_or_default(); + if entry.revision < seen + || (entry.revision == seen && state.desired.contains_key(&deployment)) + { + continue; + } + state.revisions.insert(deployment.clone(), entry.revision); + state.desired.insert(deployment.clone(), entry); + state.explicit_removals.remove(&deployment); + state.orphan_misses.remove(&deployment); + } + state.complete_snapshot_pending |= absence_is_current; + } + self.wake.notify_one(); + Ok(()) + } + + pub async fn reconcile_once(&self) -> Result<()> { + self.reconcile().await + } + + pub async fn run(self: Arc) { + loop { + self.wake.notified().await; + if let Err(error) = self.reconcile().await { + tracing::warn!(%error, "deployment reconciliation failed"); + } + } + } + + async fn update_desired(&self, deployment: DeploymentName, revision: u64, value: &[u8]) { + let mut state = self.state.lock().await; + if state + .revisions + .get(&deployment) + .is_some_and(|seen| *seen >= revision) + { + return; + } + let value = parse_desired(value); + state.revisions.insert(deployment.clone(), revision); + state.generation += 1; + state + .desired + .insert(deployment.clone(), DesiredEntry { revision, value }); + state.explicit_removals.remove(&deployment); + state.orphan_misses.remove(&deployment); + } + + async fn reconcile(&self) -> Result<()> { + let _gate = self.runtime_gate.lock().await; + let (desired, removals, complete_snapshot) = { + let mut state = self.state.lock().await; + let complete = std::mem::take(&mut state.complete_snapshot_pending); + ( + state.desired.clone(), + state.explicit_removals.clone(), + complete, + ) + }; + + for (deployment, revision) in removals { + if let Err(error) = self.remove(&deployment, Some(revision)).await { + tracing::warn!(%deployment, %error, "deployment removal failed"); + } + } + + let managed = self.runtime.managed_deployments().await?; + let desired_names = desired + .keys() + .map(ToString::to_string) + .collect::>(); + for orphan in managed.difference(&desired_names) { + let should_remove = { + let mut state = self.state.lock().await; + let Ok(deployment) = DeploymentName::try_new(orphan) else { + tracing::warn!( + deployment = orphan, + "ignoring malformed managed deployment label" + ); + continue; + }; + if state.desired.contains_key(&deployment) { + state.orphan_misses.remove(&deployment); + continue; + } + let misses = state.orphan_misses.entry(deployment).or_default(); + if complete_snapshot { + *misses = misses.saturating_add(1); + } + *misses >= 2 + }; + if should_remove { + let deployment = DeploymentName::try_new(orphan).map_err(|e| anyhow!(e))?; + if let Err(error) = self.remove(&deployment, None).await { + tracing::warn!(%deployment, %error, "orphan removal failed"); + } + } + } + + let conflicts = snapshot_conflicts(&desired); + let mut ordered = desired.into_iter().collect::>(); + ordered.sort_by(|(a, _), (b, _)| a.as_str().cmp(b.as_str())); + for (deployment, entry) in ordered { + if let Some(error) = conflicts.get(&deployment) { + self.publish_if_current( + &deployment, + entry.revision, + Phase::Failed, + Some(error.clone()), + ) + .await; + continue; + } + match entry.value { + DesiredValue::Invalid(error) => { + self.publish_if_current( + &deployment, + entry.revision, + Phase::Failed, + Some(error), + ) + .await; + } + DesiredValue::Valid { score, resolved } => { + self.publish_if_current(&deployment, entry.revision, Phase::Pending, None) + .await; + let score = if resolved { + score + } else { + match self.resolve_secrets(&deployment, score).await { + Ok(score) => { + let mut state = self.state.lock().await; + if let Some(current) = state.desired.get_mut(&deployment) + && current.revision == entry.revision + { + current.value = DesiredValue::Valid { + score: score.clone(), + resolved: true, + }; + } + score + } + Err(error) => { + self.publish_if_current( + &deployment, + entry.revision, + Phase::Failed, + Some(short(&error.to_string())), + ) + .await; + continue; + } + } + }; + let result = self.runtime.reconcile(deployment.as_str(), &score).await; + let (phase, error) = match result { + Ok(()) => (Phase::Running, None), + Err(error) => (Phase::Failed, Some(short(&error.to_string()))), + }; + self.publish_if_current(&deployment, entry.revision, phase, error) + .await; + } + } + } + Ok(()) + } + + async fn remove(&self, deployment: &DeploymentName, revision: Option) -> Result<()> { + if let Some(revision) = revision { + let state = self.state.lock().await; + if state.explicit_removals.get(deployment) != Some(&revision) { + return Ok(()); + } + } else { + let state = self.state.lock().await; + if state.desired.contains_key(deployment) + || state + .orphan_misses + .get(deployment) + .copied() + .unwrap_or_default() + < 2 + { + return Ok(()); + } + } + self.runtime.remove_deployment(deployment.as_str()).await?; + if let Some(revision) = revision { + let state = self.state.lock().await; + if state.explicit_removals.get(deployment) != Some(&revision) { + return Ok(()); + } + } else { + let state = self.state.lock().await; + if state.desired.contains_key(deployment) { + return Ok(()); + } + } + if let Some(publisher) = &self.publisher { + publisher.delete(deployment).await?; + } + let mut state = self.state.lock().await; + state.explicit_removals.remove(deployment); + state.orphan_misses.remove(deployment); + state.acknowledged.remove(deployment); + tracing::info!(%deployment, "deployment removed"); + Ok(()) + } + + async fn publish_if_current( + &self, + deployment: &DeploymentName, + revision: u64, + phase: Phase, + error: Option, + ) { + { + let state = self.state.lock().await; + if state.desired.get(deployment).map(|entry| entry.revision) != Some(revision) + || state.acknowledged.get(deployment) == Some(&(phase, error.clone())) + { + return; + } + } + let published = if let Some(publisher) = &self.publisher { + publisher + .write(&DeploymentState { + device_id: self.device_id.clone(), + deployment: deployment.clone(), + phase, + last_event_at: Utc::now(), + last_error: error.clone(), + }) + .await + } else { + Ok(()) + }; + match published { + Ok(()) => { + let mut state = self.state.lock().await; + if state.desired.get(deployment).map(|entry| entry.revision) == Some(revision) { + state + .acknowledged + .insert(deployment.clone(), (phase, error)); + } + } + Err(error) => tracing::warn!(%deployment, %error, "deployment state publish failed"), + } + } + async fn resolve_secrets( &self, - deployment: Option<&DeploymentName>, + deployment: &DeploymentName, mut score: PodmanV0Score, ) -> Result { for service in &mut score.services { if service.secret_env.is_empty() { continue; } - let src = self.secrets.as_ref().ok_or_else(|| { - anyhow::anyhow!( + let source = self.secrets.as_ref().ok_or_else(|| { + anyhow!( "service '{}' references secrets but this device has no secret store configured", service.name ) })?; - let dep = deployment - .ok_or_else(|| anyhow::anyhow!("secret_env requires a deployment-scoped key"))?; - let namespace = format!("{}/{}", src.prefix, dep); - for sref in &service.secret_env { - let bytes = src + let namespace = format!("{}/{deployment}", source.prefix); + for reference in &service.secret_env { + let bytes = source .store - .get_raw(&namespace, &sref.secret) + .get_raw(&namespace, &reference.secret) .await - .map_err(|e| { - anyhow::anyhow!("fetching secret '{}/{}': {e}", namespace, sref.secret) + .map_err(|error| { + anyhow!( + "fetching secret '{}/{}': {error}", + namespace, + reference.secret + ) })?; let value = String::from_utf8(bytes).map_err(|_| { - anyhow::anyhow!("secret '{}/{}' is not valid UTF-8", namespace, sref.secret) + anyhow!( + "secret '{}/{}' is not valid UTF-8", + namespace, + reference.secret + ) })?; - service.env.push(EnvVar::new(&sref.name, &value)); + service.env.push(EnvVar::new(&reference.name, value)); } } Ok(score) } - /// Record a new phase for a deployment and, if it changed, write - /// the updated [`DeploymentState`] to the KV. Same-phase - /// re-confirmations are no-ops so the periodic reconcile tick - /// doesn't churn the bucket. - async fn apply_phase( - &self, - deployment: &DeploymentName, - phase: Phase, - last_error: Option, - ) { - { - let mut phases = self.deployments.lock().await; - // performance nitpick : we don't need a write lock here, we could check before acquiring the write - // lock - if phases.get(deployment).copied() == Some(phase) { - return; - } - phases.insert(deployment.clone(), phase); - } - - if let Some(publisher) = &self.fleet { - let state = DeploymentState { - device_id: self.device_id.clone(), - deployment: deployment.clone(), - phase, - last_event_at: Utc::now(), - last_error, - }; - publisher.write_deployment_state(&state).await; - } + fn deployment_from_key(&self, key: &str) -> Result { + let prefix = format!("{}.", self.device_id); + let name = key + .strip_prefix(&prefix) + .ok_or_else(|| anyhow!("desired-state key '{key}' does not belong to this device"))?; + DeploymentName::try_new(name).map_err(|error| anyhow!(error)) } +} - /// Clear the in-memory phase for a deployment and delete its KV - /// entry. Idempotent: a delete for a never-applied deployment is - /// a no-op in memory and a harmless tombstone write on the wire. - async fn drop_phase(&self, deployment: &DeploymentName) { - self.deployments.lock().await.remove(deployment); - if let Some(publisher) = &self.fleet { - publisher.delete_deployment_state(deployment).await; - } +fn parse_desired(value: &[u8]) -> DesiredValue { + match serde_json::from_slice::(value) { + Ok(ReconcileScore::PodmanV0(score)) => DesiredValue::Valid { + score, + resolved: false, + }, + Err(error) => DesiredValue::Invalid(short(&format!("bad payload: {error}"))), } +} - /// Handle a Put event (new or updated score on NATS KV). No-ops if the - /// serialized score is byte-identical to the last-seen value for this - /// key. - pub async fn apply(&self, key: &str, value: &[u8]) -> Result<()> { - let deployment = deployment_from_key(key); - let incoming = match serde_json::from_slice::(value) { - Ok(ReconcileScore::PodmanV0(s)) => s, - Err(e) => { - tracing::warn!(key, error = %e, "failed to deserialize score"); - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Failed, Some(format!("bad payload: {e}"))) - .await; - } - return Ok(()); - } +fn snapshot_conflicts( + desired: &HashMap, +) -> HashMap { + let mut service_owners: HashMap<&str, &DeploymentName> = HashMap::new(); + let mut port_owners: HashMap<&str, &DeploymentName> = HashMap::new(); + let mut conflicts = HashMap::new(); + for (deployment, entry) in desired { + let DesiredValue::Valid { score, .. } = &entry.value else { + continue; }; - let serialized = String::from_utf8_lossy(value).into_owned(); - - { - let state = self.state.lock().await; - if let Some(CachedEntry::Apply { - serialized: existing, - .. - }) = state.get(key) + for service in &score.services { + if let Some(previous) = service_owners.insert(&service.name, deployment) + && previous != deployment { - if existing == &serialized { - tracing::debug!(key, "score unchanged — noop"); - return Ok(()); + let error = format!( + "service '{}' is also desired by deployment '{}'", + service.name, previous + ); + conflicts.insert(deployment.clone(), error.clone()); + conflicts.insert(previous.clone(), error); + } + for port in &service.ports { + let Some((host, _)) = port.split_once(':') else { + continue; + }; + if let Some(previous) = port_owners.insert(host, deployment) + && previous != deployment + { + let error = + format!("host port {host} is also desired by deployment '{previous}'"); + conflicts.insert(deployment.clone(), error.clone()); + conflicts.insert(previous.clone(), error); } } } - - self.state.lock().await.insert( - key.to_string(), - CachedEntry::Apply { - serialized, - score: incoming, - resolved: false, - }, - ); - - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Pending, None).await; - } - - match self.reconcile_cached(key).await { - Ok(()) => { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Running, None).await; - } - } - Err(e) => { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) - .await; - } - return Err(e); - } - } - Ok(()) - } - - /// Handle a Delete/Purge event. The tombstone remains cached until all - /// deployment containers are removed, including when no score survived a - /// restart. - pub async fn remove(&self, key: &str) -> Result<()> { - let mut state = self.state.lock().await; - let score = match state.get(key) { - Some(CachedEntry::Apply { score, .. }) => Some(score.clone()), - Some(CachedEntry::Remove { score }) => score.clone(), - None => None, - }; - state.insert(key.to_string(), CachedEntry::Remove { score }); - drop(state); - self.remove_cached(key).await - } - - /// Periodic ground-truth reconcile. ROADMAP §5.6 — "polling instead of - /// event-driven PLEG. Agent polls podman every 30s as ground truth; - /// KV watch events are accelerators." Retries cached applies and removals - /// against podman-api. - pub async fn tick(&self) -> Result<()> { - let keys: Vec = self.state.lock().await.keys().cloned().collect(); - for key in keys { - let deployment = deployment_from_key(&key); - let removing = matches!( - self.state.lock().await.get(&key), - Some(CachedEntry::Remove { .. }) - ); - let result = if removing { - self.remove_cached(&key).await - } else { - self.reconcile_cached(&key).await - }; - match result { - Ok(()) => { - if !removing { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Running, None).await; - } - } - } - Err(e) => { - tracing::warn!(key, error = %e, "periodic reconcile failed"); - if !removing { - if let Some(name) = &deployment { - self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) - .await; - } - } - } - } - } - Ok(()) - } - - pub async fn run_periodic(self: Arc, interval: Duration) { - let mut ticker = tokio::time::interval(interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - if let Err(e) = self.tick().await { - tracing::warn!(error = %e, "reconcile tick error"); - } - } - } - - async fn reconcile_cached(&self, key: &str) -> Result<()> { - let Some(CachedEntry::Apply { - serialized, - mut score, - resolved, - }) = self.state.lock().await.get(key).cloned() - else { - return Ok(()); - }; - - if !resolved { - score = self - .resolve_secrets(deployment_from_key(key).as_ref(), score) - .await?; - let mut state = self.state.lock().await; - let Some(CachedEntry::Apply { - serialized: current, - score: cached_score, - resolved, - }) = state.get_mut(key) - else { - return Ok(()); - }; - if current != &serialized { - return Ok(()); - } - *cached_score = score.clone(); - *resolved = true; - } - - self.run_score(key, &score).await - } - - async fn remove_cached(&self, key: &str) -> Result<()> { - let Some(CachedEntry::Remove { score }) = self.state.lock().await.get(key).cloned() else { - return Ok(()); - }; - let deployment = deployment_from_key(key); - if let Some(name) = &deployment { - self.runtime.remove_deployment(name.as_str()).await?; - } else if let Some(score) = score { - for service in &score.services { - self.runtime.remove_service(&service.name).await?; - } - } - - let removed = { - let mut state = self.state.lock().await; - if matches!(state.get(key), Some(CachedEntry::Remove { .. })) { - state.remove(key); - true - } else { - false - } - }; - if removed { - if let Some(name) = &deployment { - self.drop_phase(name).await; - } - tracing::info!(key, "deployment removed"); - } - Ok(()) - } - - async fn run_score(&self, key: &str, score: &PodmanV0Score) -> Result<()> { - let deployment = deployment_from_key(key) - .map(|name| name.to_string()) - .unwrap_or_else(|| key.to_string()); - self.runtime - .reconcile(&deployment, score) - .await - .map_err(|e| anyhow::anyhow!("PodmanV0Score reconcile failed for {key}: {e}"))?; - tracing::info!(key, services = score.services.len(), "reconciled"); - Ok(()) } + conflicts } -/// Extract the deployment name from a NATS KV key of the form -/// `.`. -fn deployment_from_key(key: &str) -> Option { - let (_, rest) = key.split_once('.')?; - DeploymentName::try_new(rest).ok() -} - -/// Truncate a long error message so the DeploymentState payload stays -/// comfortably below NATS JetStream's per-message limit. -fn short(s: &str) -> String { +fn short(message: &str) -> String { const MAX: usize = 512; - if s.len() <= MAX { - s.to_string() + if message.len() <= MAX { + message.to_string() } else { - let mut cut = s[..MAX].to_string(); + let mut end = MAX; + while !message.is_char_boundary(end) { + end -= 1; + } + let mut cut = message[..end].to_string(); cut.push('…'); cut } @@ -381,283 +505,348 @@ fn short(s: &str) -> String { #[cfg(test)] mod tests { - //! Focused tests for transition detection. Drive `apply_phase` / - //! `drop_phase` directly with an inert topology (no real podman - //! socket) and a `None` FleetPublisher. use super::*; - use std::path::PathBuf; + use harmony_reconciler_contracts::{PodmanService, RestartPolicy}; use std::sync::Mutex as StdMutex; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, Ordering}; - fn reconciler() -> Reconciler { - reconciler_with_secrets(None) - } - - fn reconciler_with_secrets(secrets: Option) -> Reconciler { - let runtime = Arc::new( - PodmanRuntime::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(), - ); - Reconciler::new(Id::from("test-device".to_string()), runtime, None, secrets) - } - - fn dn(s: &str) -> DeploymentName { - DeploymentName::try_new(s).expect("valid test name") - } - - /// In-memory secret store keyed by `(namespace, key)`. - #[derive(Debug, Default)] - struct MapStore(HashMap<(String, String), String>); - - #[async_trait::async_trait] - impl SecretStore for MapStore { - async fn get_raw( - &self, - namespace: &str, - key: &str, - ) -> Result, harmony_secret::SecretStoreError> { - self.0 - .get(&(namespace.to_string(), key.to_string())) - .map(|v| v.as_bytes().to_vec()) - .ok_or_else(|| harmony_secret::SecretStoreError::Store("permission denied".into())) - } - - async fn set_raw( - &self, - _namespace: &str, - _key: &str, - _value: &[u8], - ) -> Result<(), harmony_secret::SecretStoreError> { - unimplemented!("tests never write") - } - } - - #[derive(Debug, Default)] - struct FlakyStore { - calls: AtomicUsize, - value: StdMutex>, + #[derive(Default)] + struct FakeRuntime { + managed: StdMutex>, + actions: StdMutex>, + fail: AtomicBool, } #[async_trait::async_trait] - impl SecretStore for FlakyStore { - async fn get_raw( - &self, - _namespace: &str, - _key: &str, - ) -> Result, harmony_secret::SecretStoreError> { - self.calls.fetch_add(1, Ordering::Relaxed); - self.value + impl WorkloadRuntime for FakeRuntime { + async fn reconcile(&self, deployment: &str, _score: &PodmanV0Score) -> Result<()> { + self.actions .lock() .unwrap() - .as_ref() - .map(|value| value.as_bytes().to_vec()) - .ok_or_else(|| harmony_secret::SecretStoreError::Store("temporarily denied".into())) + .push(format!("apply:{deployment}")); + if self.fail.load(Ordering::Relaxed) { + anyhow::bail!("runtime failed"); + } + self.managed.lock().unwrap().insert(deployment.to_string()); + Ok(()) } - async fn set_raw( - &self, - _namespace: &str, - _key: &str, - _value: &[u8], - ) -> Result<(), harmony_secret::SecretStoreError> { - unimplemented!("tests never write") + async fn remove_deployment(&self, deployment: &str) -> Result<()> { + self.actions + .lock() + .unwrap() + .push(format!("remove:{deployment}")); + if self.fail.load(Ordering::Relaxed) { + anyhow::bail!("runtime failed"); + } + self.managed.lock().unwrap().remove(deployment); + Ok(()) + } + + async fn managed_deployments(&self) -> Result> { + Ok(self.managed.lock().unwrap().clone()) } } - fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score { - use harmony_reconciler_contracts::{PodmanService, SecretEnvVar}; - PodmanV0Score { + #[derive(Default)] + struct FakePublisher { + writes: StdMutex)>>, + deletes: StdMutex>, + fail: AtomicBool, + } + + #[async_trait::async_trait] + impl DeploymentStatePublisher for FakePublisher { + async fn write(&self, state: &DeploymentState) -> Result<()> { + if self.fail.load(Ordering::Relaxed) { + anyhow::bail!("publish failed"); + } + self.writes.lock().unwrap().push(( + state.deployment.clone(), + state.phase, + state.last_error.clone(), + )); + Ok(()) + } + + async fn delete(&self, deployment: &DeploymentName) -> Result<()> { + if self.fail.load(Ordering::Relaxed) { + anyhow::bail!("delete failed"); + } + self.deletes.lock().unwrap().push(deployment.clone()); + Ok(()) + } + } + + fn score(service: &str, port: &str) -> Vec { + serde_json::to_vec(&ReconcileScore::PodmanV0(PodmanV0Score { services: vec![PodmanService { - name: "web-svc".to_string(), - image: "nginx".to_string(), - ports: vec![], + name: service.into(), + image: "image:v1".into(), + ports: (!port.is_empty()) + .then(|| port.into()) + .into_iter() + .collect(), env: vec![], - secret_env: vec![SecretEnvVar { - name: env_name.to_string(), - secret: secret.to_string(), - }], + secret_env: vec![], volumes: vec![], - restart_policy: Default::default(), + restart_policy: RestartPolicy::UnlessStopped, }], + })) + .unwrap() + } + + fn entry(deployment: &str, revision: u64) -> SnapshotEntry { + SnapshotEntry { + key: format!("device-1.{deployment}"), + revision, + value: score(deployment, ""), } } - fn map_source(entries: &[(&str, &str, &str)]) -> SecretSource { - let map = entries - .iter() - .map(|(ns, k, v)| ((ns.to_string(), k.to_string()), v.to_string())) - .collect(); - SecretSource { - store: Arc::new(MapStore(map)), - prefix: "fleet-test".to_string(), - } + fn reconciler() -> (Reconciler, Arc, Arc) { + let runtime = Arc::new(FakeRuntime::default()); + let publisher = Arc::new(FakePublisher::default()); + ( + Reconciler::new( + Id::from("device-1".to_string()), + runtime.clone(), + Some(publisher.clone()), + None, + ), + runtime, + publisher, + ) } #[tokio::test] - async fn resolve_secrets_injects_env_value() { - let r = reconciler_with_secrets(Some(map_source(&[( - "fleet-test/web", - "db_password", - "hunter2", - )]))); - let resolved = r - .resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password")) + async fn creates_and_repairs_desired_deployment() { + let (reconciler, runtime, publisher) = reconciler(); + reconciler + .replace_snapshot(vec![entry("web", 1)], 0) .await - .expect("resolution succeeds"); - let env = &resolved.services[0].env; - assert_eq!(env.len(), 1); - assert_eq!(env[0], EnvVar::new("DB_PASSWORD", "hunter2")); - } + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + runtime.managed.lock().unwrap().clear(); + reconciler.reconcile_once().await.unwrap(); - #[tokio::test] - async fn resolve_secrets_without_store_fails() { - let r = reconciler(); - let err = r - .resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password")) - .await - .expect_err("must fail without a store"); - assert!(err.to_string().contains("no secret store"), "{err}"); - } - - #[tokio::test] - async fn apply_marks_failed_when_secret_denied() { - // Store has no entry → MapStore answers "permission denied", - // the same observable a revoked OpenBao policy produces. The - // failure must surface as Phase::Failed *before* any podman - // call (the test topology has no socket). - let r = reconciler_with_secrets(Some(map_source(&[]))); - let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score( - "DB_PASSWORD", - "db_password", - ))) - .unwrap(); - let err = r - .apply("test-device.web", &payload) - .await - .expect_err("denied fetch fails the apply"); - assert!(err.to_string().contains("permission denied"), "{err}"); - let phases = r.deployments.lock().await; - assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed)); - } - - #[tokio::test] - async fn tick_retries_secret_resolution_then_reuses_resolved_score() { - let store = Arc::new(FlakyStore::default()); - let r = reconciler_with_secrets(Some(SecretSource { - store: store.clone(), - prefix: "fleet-test".to_string(), - })); - let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score( - "DB_PASSWORD", - "db_password", - ))) - .unwrap(); - - r.apply("test-device.web", &payload) - .await - .expect_err("first secret read fails"); - *store.value.lock().unwrap() = Some("hunter2".to_string()); - - r.tick().await.unwrap(); - r.tick().await.unwrap(); - - assert_eq!(store.calls.load(Ordering::Relaxed), 2); - let state = r.state.lock().await; - let Some(CachedEntry::Apply { - score, resolved, .. - }) = state.get("test-device.web") - else { - panic!("apply retry state missing"); - }; - assert!(*resolved); assert_eq!( - score.services[0].env[0], - EnvVar::new("DB_PASSWORD", "hunter2") + runtime.actions.lock().unwrap().as_slice(), + ["apply:web", "apply:web"] + ); + assert_eq!( + publisher.writes.lock().unwrap().last().unwrap().1, + Phase::Running ); } #[tokio::test] - async fn failed_removal_is_retried_and_later_put_cancels_it() { - let r = reconciler(); - - r.remove("test-device.web") + async fn explicit_delete_is_immediate_and_retried() { + let (reconciler, runtime, publisher) = reconciler(); + reconciler + .replace_snapshot(vec![entry("web", 1)], 0) .await - .expect_err("unreachable Podman keeps the tombstone pending"); - assert!(matches!( - r.state.lock().await.get("test-device.web"), - Some(CachedEntry::Remove { score: None }) - )); + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + runtime.fail.store(true, Ordering::Relaxed); + reconciler.delete("device-1.web", 2).await.unwrap(); + reconciler.reconcile_once().await.unwrap(); + assert!(runtime.managed.lock().unwrap().contains("web")); - r.tick().await.unwrap(); - assert!(matches!( - r.state.lock().await.get("test-device.web"), - Some(CachedEntry::Remove { .. }) - )); + runtime.fail.store(false, Ordering::Relaxed); + reconciler.reconcile_once().await.unwrap(); + assert!(!runtime.managed.lock().unwrap().contains("web")); + assert_eq!(publisher.deletes.lock().unwrap().as_slice(), [dn("web")]); + } - let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(PodmanV0Score { - services: vec![], - })) - .unwrap(); - r.apply("test-device.web", &payload) + #[tokio::test] + async fn offline_delete_requires_two_complete_absence_observations() { + let (reconciler, runtime, publisher) = reconciler(); + runtime.managed.lock().unwrap().insert("web".into()); + + reconciler.replace_snapshot(vec![], 0).await.unwrap(); + reconciler.reconcile_once().await.unwrap(); + assert!(runtime.managed.lock().unwrap().contains("web")); + reconciler.replace_snapshot(vec![], 0).await.unwrap(); + reconciler.reconcile_once().await.unwrap(); + assert!(!runtime.managed.lock().unwrap().contains("web")); + assert_eq!(publisher.deletes.lock().unwrap().as_slice(), [dn("web")]); + } + + #[tokio::test] + async fn same_revision_reappearing_cancels_orphan_cleanup() { + let (reconciler, runtime, _) = reconciler(); + reconciler + .replace_snapshot(vec![entry("web", 1)], 0) .await - .expect_err("unreachable Podman keeps the new apply pending"); - assert!(matches!( - r.state.lock().await.get("test-device.web"), - Some(CachedEntry::Apply { .. }) - )); - } - - #[tokio::test] - async fn plain_scores_resolve_without_store() { - // No secret_env → no store needed; resolution is a no-op. - let r = reconciler(); - let score = PodmanV0Score { services: vec![] }; - r.resolve_secrets(Some(&dn("web")), score.clone()) + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + reconciler.replace_snapshot(vec![], 0).await.unwrap(); + reconciler.reconcile_once().await.unwrap(); + reconciler + .replace_snapshot(vec![entry("web", 1)], 0) .await - .expect("no-op resolution"); + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + + assert!(runtime.managed.lock().unwrap().contains("web")); + assert!( + !runtime + .actions + .lock() + .unwrap() + .iter() + .any(|action| action == "remove:web") + ); } #[tokio::test] - async fn apply_phase_records_new_phase() { - let r = reconciler(); - r.apply_phase(&dn("hello"), Phase::Running, None).await; - let phases = r.deployments.lock().await; - assert_eq!(phases.get(&dn("hello")), Some(&Phase::Running)); + async fn newer_put_cancels_pending_delete() { + let (reconciler, runtime, _) = reconciler(); + reconciler.delete("device-1.web", 2).await.unwrap(); + reconciler + .put("device-1.web", 3, &score("web", "")) + .await + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + + assert_eq!(runtime.actions.lock().unwrap().as_slice(), ["apply:web"]); } #[tokio::test] - async fn apply_phase_idempotent_for_same_phase() { - let r = reconciler(); - r.apply_phase(&dn("hello"), Phase::Running, None).await; - r.apply_phase(&dn("hello"), Phase::Running, None).await; - let phases = r.deployments.lock().await; - assert_eq!(phases.len(), 1); + async fn snapshot_absence_cannot_override_concurrent_put() { + let (reconciler, runtime, _) = reconciler(); + let snapshot_generation = reconciler.generation().await; + reconciler + .put("device-1.web", 1, &score("web", "")) + .await + .unwrap(); + reconciler + .replace_snapshot(vec![], snapshot_generation) + .await + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + + assert_eq!(runtime.actions.lock().unwrap().as_slice(), ["apply:web"]); } #[tokio::test] - async fn apply_phase_transitions_update_phase() { - let r = reconciler(); - r.apply_phase(&dn("hello"), Phase::Pending, None).await; - r.apply_phase(&dn("hello"), Phase::Running, None).await; - r.apply_phase(&dn("hello"), Phase::Failed, Some("oom".to_string())) - .await; - let phases = r.deployments.lock().await; - assert_eq!(phases.get(&dn("hello")), Some(&Phase::Failed)); + async fn orphan_removal_rechecks_current_desired_state() { + let (reconciler, runtime, _) = reconciler(); + runtime.managed.lock().unwrap().insert("web".into()); + reconciler + .put("device-1.web", 1, &score("web", "")) + .await + .unwrap(); + reconciler + .state + .lock() + .await + .orphan_misses + .insert(dn("web"), 2); + + reconciler.remove(&dn("web"), None).await.unwrap(); + assert!(runtime.managed.lock().unwrap().contains("web")); } #[tokio::test] - async fn drop_phase_clears_known_deployment() { - let r = reconciler(); - r.apply_phase(&dn("hello"), Phase::Running, None).await; - r.drop_phase(&dn("hello")).await; - let phases = r.deployments.lock().await; - assert!(!phases.contains_key(&dn("hello"))); + async fn newer_revision_wins_and_malformed_intent_does_not_mutate_runtime() { + let (reconciler, runtime, publisher) = reconciler(); + reconciler.put("device-1.web", 2, b"invalid").await.unwrap(); + reconciler + .put("device-1.web", 1, &score("web", "")) + .await + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + + assert!(runtime.actions.lock().unwrap().is_empty()); + let writes = publisher.writes.lock().unwrap(); + assert_eq!(writes.last().unwrap().1, Phase::Failed); + assert!( + writes + .last() + .unwrap() + .2 + .as_deref() + .unwrap() + .contains("bad payload") + ); } #[tokio::test] - async fn drop_phase_on_unknown_deployment_is_noop() { - let r = reconciler(); - r.drop_phase(&dn("never-existed")).await; - let phases = r.deployments.lock().await; - assert!(phases.is_empty()); + async fn conflicts_fail_only_conflicting_deployments() { + let (reconciler, runtime, publisher) = reconciler(); + reconciler + .replace_snapshot( + vec![ + SnapshotEntry { + key: "device-1.a".into(), + revision: 1, + value: score("shared", "8080:80"), + }, + SnapshotEntry { + key: "device-1.b".into(), + revision: 2, + value: score("shared", "8081:80"), + }, + entry("independent", 3), + ], + 0, + ) + .await + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + + assert_eq!( + runtime.actions.lock().unwrap().as_slice(), + ["apply:independent"] + ); + let failed = publisher + .writes + .lock() + .unwrap() + .iter() + .filter(|(_, phase, _)| *phase == Phase::Failed) + .count(); + assert_eq!(failed, 2); + } + + #[tokio::test] + async fn failed_runtime_and_status_publication_retry() { + let (reconciler, runtime, publisher) = reconciler(); + runtime.fail.store(true, Ordering::Relaxed); + publisher.fail.store(true, Ordering::Relaxed); + reconciler + .replace_snapshot(vec![entry("web", 1)], 0) + .await + .unwrap(); + reconciler.reconcile_once().await.unwrap(); + assert!(publisher.writes.lock().unwrap().is_empty()); + + publisher.fail.store(false, Ordering::Relaxed); + reconciler.reconcile_once().await.unwrap(); + assert_eq!( + publisher.writes.lock().unwrap().last().unwrap().1, + Phase::Failed + ); + + runtime.fail.store(false, Ordering::Relaxed); + reconciler.reconcile_once().await.unwrap(); + assert_eq!( + publisher.writes.lock().unwrap().last().unwrap().1, + Phase::Running + ); + } + + #[test] + fn error_truncation_preserves_utf8_boundaries() { + let message = "x".repeat(511) + "é"; + let shortened = short(&message); + assert!(shortened.ends_with('…')); + assert!(shortened.is_char_boundary(shortened.len())); + } + + fn dn(name: &str) -> DeploymentName { + DeploymentName::try_new(name).unwrap() } } -- 2.39.5 From 09248a543978e67c01926c1b4318ff4ea43a5f16 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 12:29:20 -0400 Subject: [PATCH 18/47] feat(fleet): Agent upgrade first implementation and operator reconciliation algorithm faetures --- Cargo.lock | 18 + Cargo.toml | 2 + book.toml | 2 +- docs/SUMMARY.md | 1 + docs/adr/022-fleet-agent-upgrade.md | 423 +++-------- docs/design/fleet-agent-upgrades.md | 389 ++++++++++ .../fleet-agent-upgrade-architecture.svg | 62 ++ .../diagrams/fleet-agent-upgrade-recovery.svg | 45 ++ .../diagrams/fleet-agent-upgrade-sequence.svg | 41 ++ examples/fleet_device_enroll/src/main.rs | 1 + examples/fleet_rpi_setup/src/main.rs | 1 + examples/fleet_vm_setup/src/main.rs | 1 + .../agent-reconciliation-and-upgrade-plan.md | 16 +- fleet/harmony-fleet-agent/Cargo.toml | 7 +- fleet/harmony-fleet-agent/src/main.rs | 110 ++- fleet/harmony-fleet-agent/src/reconciler.rs | 19 + fleet/harmony-fleet-agent/src/updater.rs | 682 ++++++++++++++++++ fleet/harmony-fleet-agent/src/upgrade.rs | 644 +++++++++++++++++ .../harmony-fleet-deploy/src/device_setup.rs | 147 +++- fleet/harmony-fleet-deploy/src/lib.rs | 2 +- .../src/operator/chart.rs | 44 ++ .../src/operator/score.rs | 18 + fleet/harmony-fleet-e2e/src/vm/device.rs | 1 + fleet/harmony-fleet-e2e/tests/operator.rs | 8 +- fleet/harmony-fleet-operator/Cargo.toml | 2 + .../src/agent_upgrade.rs | 275 +++++++ fleet/harmony-fleet-operator/src/crd.rs | 26 + .../src/device_reconciler.rs | 1 + .../src/device_status.rs | 3 + fleet/harmony-fleet-operator/src/lib.rs | 6 +- fleet/harmony-fleet-operator/src/main.rs | 26 +- .../src/service/real.rs | 1 + harmony-reconciler-contracts/Cargo.toml | 1 + harmony-reconciler-contracts/src/kv.rs | 25 + harmony-reconciler-contracts/src/lib.rs | 6 + harmony-reconciler-contracts/src/upgrade.rs | 136 ++++ nats/callout/src/permissions.rs | 46 +- 37 files changed, 2879 insertions(+), 359 deletions(-) create mode 100644 docs/design/fleet-agent-upgrades.md create mode 100644 docs/diagrams/fleet-agent-upgrade-architecture.svg create mode 100644 docs/diagrams/fleet-agent-upgrade-recovery.svg create mode 100644 docs/diagrams/fleet-agent-upgrade-sequence.svg create mode 100644 fleet/harmony-fleet-agent/src/updater.rs create mode 100644 fleet/harmony-fleet-agent/src/upgrade.rs create mode 100644 fleet/harmony-fleet-operator/src/agent_upgrade.rs create mode 100644 harmony-reconciler-contracts/src/upgrade.rs diff --git a/Cargo.lock b/Cargo.lock index a1d9f778..6b6ef48d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3642,6 +3642,16 @@ dependencies = [ "serde", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4025,13 +4035,17 @@ dependencies = [ "anyhow", "async-nats", "async-trait", + "base64 0.22.1", "chrono", "clap", + "ed25519-dalek", + "fs2", "futures-util", "harmony-fleet-auth", "harmony-reconciler-contracts", "harmony_secret", "podman-api", + "reqwest 0.12.28", "sd-notify", "serde", "serde_json", @@ -4041,6 +4055,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -4143,6 +4158,7 @@ dependencies = [ "chrono", "clap", "dotenvy", + "ed25519-dalek", "futures-util", "harmony-fleet-auth", "harmony-reconciler-contracts", @@ -4163,6 +4179,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid", ] [[package]] @@ -4231,6 +4248,7 @@ dependencies = [ "schemars 0.8.22", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.18", ] diff --git a/Cargo.toml b/Cargo.toml index 5254ec6e..85004771 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,8 @@ convert_case = "0.8" chrono = "0.4" similar = "2" uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] } +ed25519-dalek = "2" +fs2 = "0.4" pretty_assertions = "1.4.1" tempfile = "3.20.0" bollard = "0.19.1" diff --git a/book.toml b/book.toml index 806229b8..e923182d 100644 --- a/book.toml +++ b/book.toml @@ -2,7 +2,7 @@ title = "Harmony" description = "Infrastructure orchestration that treats your platform like first-class code" src = "docs" -build-dir = "book" +# build-dir = "book" authors = ["NationTech"] [output.html] diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 208aa2d0..33c87f77 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -38,6 +38,7 @@ ## Reference Designs - [Fleet Score References](./reference/fleet-score-references.md) +- [Fleet Agent Upgrades](./design/fleet-agent-upgrades.md) ## Architecture Decision Records diff --git a/docs/adr/022-fleet-agent-upgrade.md b/docs/adr/022-fleet-agent-upgrade.md index d33068d4..c59e299a 100644 --- a/docs/adr/022-fleet-agent-upgrade.md +++ b/docs/adr/022-fleet-agent-upgrade.md @@ -4,353 +4,138 @@ Initial Author: Jean-Gabriel Gill-Couture Initial Date: 2026-05-06 -Last Updated Date: 2026-05-06 +Last Updated Date: 2026-07-22 ## Status -Accepted (design); implementation deferred — see roadmap -`ROADMAP/fleet_platform/v0_2_plan.md`. +Accepted. This revision replaces the original dual-active cutover design. ## Context -The v0.1 fleet agent ships as a single static aarch64-musl binary -sitting at `/usr/local/bin/fleet-agent`, started by a systemd -unit dropped at install time by `FleetDeviceSetupScore`. Every -managed device runs one. Today the only "upgrade procedure" is -`scp` + `systemctl restart` — fine for the bring-up phase, not -fine once paying customers run real workloads on the fleet. +Fleet devices need unattended agent upgrades without restarting managed +workloads. A failed candidate must return the device to the last working agent, +including after process crashes or power loss. -Without a defined upgrade story we cannot ship a v0.1 agent into -the field. The contract a customer needs is: - -1. New agent versions can be rolled out without operator-side - manual intervention per device. -2. Workloads currently reconciled on the device do not flap - (start/stop/start) during the upgrade. -3. A failed new version automatically reverts to the last - known-good version, on its own, without page. -4. The operator (the central one in the cluster, not the human) - sees what version each device is running, can drive a target - version per device, and observes upgrade progress. - -The agent itself is the only process on-device with full context -on what's reconciling and what's healthy. Anything centralized -(Ansible-pushed, OS-package-managed) doesn't have that signal. -The agent must be the one driving its own swap, with the -operator coordinating but not executing. +The workload reconciler must have one active owner. Running old and new agents +at the same time would allow both processes to mutate Podman state and publish +competing observations. A candidate therefore runs only a non-mutating probe. +Cutover accepts a bounded interval with no reconciler while systemd starts the +new process. Existing workloads continue under their Podman restart policies. ## Decision -We adopt a **K8s rolling-update–shape upgrade**, single-host, -agent-driven, operator-coordinated. Old version stays alive until -new is verified healthy from the operator's vantage point; only -then does the operator signal old to exit. **No version is ever -erased from disk.** Symlinks select the active binary. +The central operator creates an attempt for a device. The active agent drains +new workload mutations and asks a root-owned updater to stage the candidate. +After the candidate passes `--self-test`, the operator publishes a signed switch +authorization for that exact attempt. The updater atomically changes the active +symlink and restarts the one permanent `fleet-agent.service`. -### On-disk layout +The updater rolls back when startup readiness or the 60-second probation period +fails. No second reconciler runs during staging or cutover. -``` -/usr/bin/fleet-agent-v0.1.1 ← versioned binary, immutable -/usr/bin/fleet-agent-v0.1.2 ← versioned binary, immutable -/usr/bin/fleet-agent-v0.1.3 ← versioned binary, immutable -/usr/local/bin/fleet-agent → symlink to current versioned binary +### On-device layout + +```text +/usr/lib/harmony-fleet/fleet-agent-bootstrap +/usr/lib/harmony-fleet/fleet-agent-v0.2.0 +/usr/lib/harmony-fleet/fleet-agent-v0.3.0 +/usr/local/bin/fleet-agent -> /usr/lib/harmony-fleet/fleet-agent-v0.3.0 +/var/lib/harmony-fleet-updater/transaction.json +/etc/fleet-agent/trusted-upgrade-keys/.pub +/etc/fleet-agent/trusted-upgrade-key-id ``` -- Versioned binaries are the source of truth. They live forever - (history-preserving, no GC). Disk use is bounded by humans - cleaning up explicitly, not by the upgrade procedure. -- The systemd unit installed by `FleetDeviceSetupScore` references - `/usr/local/bin/fleet-agent`. Symlink swap is the cutover - primitive — atomic on POSIX (`renameat2`). -- Naming convention: exact crate version string, `v..`, - no build metadata in the path. Build metadata lives in the agent's - reported version string but not in the file path (otherwise you - can't predict the path from a version pin). +`FleetDeviceSetupScore` installs the bootstrap binary, trusted Ed25519 public +key and active key ID, `harmony-fleet-updater.service`, and +`fleet-agent.service`. The updater is +root-owned and listens on a Unix socket writable by the `fleet-agent` group. +The agent service remains unprivileged. -### State machine on the agent side +The bootstrap binary supplies the privileged updater mode. Downloaded binaries +run only as the unprivileged agent, including their candidate probe. Versioned +binaries are immutable and retained for rollback. -``` -Running ──[operator publishes desired_version != current]──▶ Draining - ▲ │ - │ │ - │ ▼ - │ Staging - │ │ - │ ▼ - │ Verifying - │ │ - │ ▼ - │ ┌──────────────────────────────[smoke fails]────────┤ - │ │ │ - │ [revert: symlink → previous, ▼ - │ stay at current] Cutover-Ready - │ │ - │ [Cutover-Ready persists ≥ T_OPERATOR_OBSERVE │ - │ until operator publishes stop_signal] │ - │ ▼ - └────────────────────────────────────────────────────── Stopping - │ - ▼ - (exit) -``` +### Control-plane contract -States in detail: +The upgrade uses three JetStream KV buckets: -- **Running** — normal reconcile loop. -- **Draining** — refuses to start new podman services for new - desired-state writes. In-flight reconciles complete and report - their final state to the operator. Existing services stay - running. Heartbeat continues. State is published as part of the - agent's heartbeat (`agent_state: "draining"`). -- **Staging** — fetch new versioned binary URL (signed, - hash-pinned), verify, place at `/usr/bin/fleet-agent-v`. - Set chmod, ownership. No other state mutation. -- **Verifying** — invoke the staged binary with `--self-test`. New - binary parses its config, opens NATS connection, validates JWT, - prints version + "ok", exits 0. **No state mutation.** Catches - obvious breakage (missing dependency, wrong arch, corrupt - download, broken config-schema migration) before swap. -- **Cutover-Ready** — staged binary is healthy. Old agent updates - the symlink atomically: - ``` - ln -sfn /usr/bin/fleet-agent-v0.1.2 /usr/local/bin/fleet-agent.new - mv -T /usr/local/bin/fleet-agent.new /usr/local/bin/fleet-agent - ``` - Old agent then `systemctl start fleet-agent-v0.1.2.service` (a - parallel transient service, not `systemctl restart` of itself). - Both old and new are now running. New publishes its first - heartbeat with `version=v0.1.2`. Operator sees two heartbeats - per device for a brief window. -- **Stopping** — operator publishes a stop signal to the old - agent's NATS subject. Old agent receives, gracefully exits. - systemd's `Restart=on-failure` does *not* trigger because the - exit is `success` (rc=0, code-path-driven). New agent is now - the only one running. systemd unit is reconfigured to point at - the *current* symlink target on its next restart, but that's - cosmetic — the symlink already does the job. +- `agent-upgrade-intent.` stores the latest immutable attempt. +- `agent-upgrade-authorize..` stores the signed switch authorization. +- `agent-upgrade-status.` stores the latest attempt phase and bounded error. -### Operator-side coordination +An attempt includes its UUID, device ID, source and target versions, +architecture, HTTPS artifact URL, maximum size, SHA-256 digest, Ed25519 +signature, signing key ID, and creation time. The operator writes intent and +authorization. A device can read only its own entries and can write only its +own status. -The operator is the only source of truth for "what version should -this device run". One new field per device, two new subjects. +`Device.spec.agentUpgrade` holds the desired artifact metadata. +`Device.status.agentUpgrade` reflects the attempt ID, target version, phase, +timestamp, and last error. The operator does not retry a failed matching +attempt automatically; changing the desired release metadata or removing the +old intent creates a new attempt. -**New on `Device` CR / KV `device-info`:** -- `current_version` — what the agent is running right now. - Reported in heartbeat; reflected to the CR. -- `desired_version` — what the operator wants the agent to run. - Set by operator-side logic (default: latest published; eventually - canary / %-based). +### Transaction -**New NATS subjects (per-device, scoped by callout permissions):** -- `device-cmd..upgrade-stop` — operator → old agent. - Payload: `{"reason": "...", "deadline_ms": ...}`. Sent only after - operator has observed a heartbeat from the new version with - `current_version == desired_version` AND `agent_state == "running"`. -- `device-state..upgrade` — agent → operator. Status - events: `staging`, `verifying`, `cutover-ready`, `failed`, `done`. - Drives `Device.status.upgrade.{phase, last_error, ...}`. +1. The operator writes an attempt when the desired and reported versions differ. +2. The active agent rejects wrong-device, wrong-source, wrong-architecture, + stale, future, malformed, or mutated attempts. +3. The agent pauses new workload mutations and publishes `draining`. +4. The updater downloads over HTTPS with fixed time and size limits, verifies + the digest and Ed25519 signature, fsyncs the binary, and installs it by atomic + rename. +5. The updater runs the candidate as `fleet-agent --self-test`. The probe loads + configuration, reaches Podman when enabled, authenticates to NATS, reads the + permitted desired-state snapshot, and exits. It does not heartbeat, watch, + reconcile, or mutate workloads. +6. The agent publishes `awaiting-authorization`. If no authorization arrives + within five minutes, it cancels the staged transaction and resumes workload + reconciliation. +7. After observing that phase, the operator signs and writes an authorization + for the exact attempt. +8. The updater verifies the authorization, records `switching`, atomically + changes `/usr/local/bin/fleet-agent`, and restarts `fleet-agent.service`. +9. The new agent acquires the exclusive process lock, restores upgrade state, + loads a complete desired-state snapshot, starts its worker, and sends systemd + readiness. +10. The updater records `probation` and verifies that the systemd invocation ID + does not change for 60 seconds. It then records `committed`. -The operator only emits `upgrade-stop` after it has independently -verified the new agent is up. **Old agent does not stop itself -based on its own observations.** This is the load-bearing -property: the same operator that disagreed with the upgrade -("haven't seen new version's heartbeat") would never have sent -the stop signal. Single-source-of-truth handoff. +The transaction journal records `staged`, `switching`, `probation`, +`committed`, `rolling-back`, `failed`, or `rollback-failed`, together with the +attempt digest and previous and target paths. Journal writes, binary installs, +and symlink changes use atomic rename and parent-directory fsync. -### Failure modes and rollback +On updater startup, `switching`, `probation`, and `rolling-back` transactions +resume rollback. The updater starts its socket before restarting the previous +agent so that agent startup can inspect the transaction. Corrupt or unreadable +journals fail closed. Rollback failure is persisted as `rollback-failed`. -- **Staging fails (download / hash mismatch):** Agent stays in - `Running`. Reports `phase: "failed"`, `last_error`. Operator - sees the failure; can fix the artifact + retry by re-publishing - the same desired_version (any change to desired_version - re-triggers the state machine). -- **Verifying fails (smoke test rc != 0):** Agent stays in - `Running`. Reports failure. Staged binary stays on disk for - inspection. Operator can collect, debug, ship a fixed version. -- **Cutover-ready, but new agent never publishes a heartbeat - with the new version within T_HEARTBEAT_TIMEOUT (suggested - 60s):** Old agent reverts the symlink, stops the parallel - systemd transient service, transitions back to Running with - the old version. Reports `failed`. Same recovery path. -- **Operator never sends stop signal (e.g., operator-side - outage):** Old agent stays in Cutover-Ready indefinitely. Both - agents are running; only the new one is publishing as the - active one (the old one's writes are gated on its state). This - is expensive (2× resource use) but safe — the operator is the - authoritative coordinator and any other behavior would risk - losing both agents at once. -- **Both agents alive but new agent crashes:** systemd's - `Restart=on-failure` on the new agent's transient unit retries. - If it can't come back, the operator never sends the stop signal, - the old agent stays Cutover-Ready, and a human investigates. - The fleet keeps working on the old version — the rollback is - implicit. -- **Operator publishes an older `desired_version`:** Reverse - rollout. Same mechanism, just with old/new swapped. The "new" - binary is older, but the procedure is identical. The fact that - no version is ever GC'd is what makes this work. +### Failure behavior -### What this isn't - -- **Not fleet-wide.** Per-device. Fleet-wide canary / %-based - rollouts are operator-side orchestration **on top of** this - primitive. The operator would publish `desired_version` to a - rolling subset of devices and watch heartbeats. Out of scope - for v0.2 — single-device upgrade is sufficient for a 100-Pi - fleet which is more than the 12-month customer roadmap. -- **Not blue/green of the entire OS.** We swap one userspace - binary. The OS, podman, the systemd unit text, the kernel — all - unchanged. Out of scope. -- **Not a package manager.** Versioned binaries land at fixed - paths because we control them. apt / dpkg / OSTree are - orthogonal and not in the loop. - -## Rationale - -- **No version ever erased.** Trivializes rollback (the previous - binary is a `ln -sfn` away). Simplifies the failure tree: - every "what if" branch resolves to "old still on disk". Disk - cost on aarch64-musl is ~5–10 MB per version — at 12 versions - / year, that's 100 MB after a decade of upgrades. Negligible - compared to Pi storage. -- **Symlink swap as cutover.** POSIX-atomic. No daemon state. - Cheap to revert. Compatible with systemd unit references that - point at a stable path. -- **Old verifies new, then reports up.** This is the load-bearing - property: it places the verification at the agent (which has - the only complete view of its own runtime state) but the - *commitment* at the operator (which is the only thing safe to - trust as the cluster-wide source of truth). Either side alone - can fail safe; only consensus advances the upgrade. -- **Operator-driven stop, not agent self-stop.** A self-stopping - agent could decide to exit before the operator agrees, leaving - the cluster blind. Forcing the stop through the operator means - any disagreement keeps the old agent alive — which is the - desired bias. -- **Drains in-flight work first.** Mirrors K8s pod-shutdown - semantics. A workload reconciling at the moment of swap - finishes its current step, reports state, then queues. New - agent picks up the queue once it's the active version. No - observable flap on the workload. -- **Heartbeat-driven version reporting.** The agent already - publishes heartbeats; adding the version field is one line. - No new transport. +- Download, size, digest, signature, architecture, or probe failure leaves the + active symlink unchanged and resumes the old reconciler. +- Operator outage before authorization leaves workloads running and returns the + old agent to normal reconciliation after five minutes. +- New-agent startup failure, readiness timeout, restart during probation, or + updater restart before commit restores the previous symlink and restarts the + previous agent. +- A failed rollback remains visible as `rollback-failed`; it is never reported + as successful recovery. +- Replayed terminal attempts are no-ops. Reusing an attempt UUID with different + content is rejected. ## Consequences -**Pros:** +Only one process can reconcile workloads. Cutover has a bounded period with no +reconciler, but running workloads are not stopped. The root helper is a small, +fixed protocol rather than a command executor, and it accepts no caller-chosen +filesystem path. -- Bounded blast radius per upgrade (one device). -- Rollback is the same code path as upgrade — no special-case - bug class. -- Operator's view is monotonic: heartbeats with versions are - immutable history; there's no "did the upgrade really happen" - state. -- Old agent never decides to exit on its own. The most dangerous - failure mode in self-upgrading software (premature exit) is - designed out. -- Compatible with eventual fleet-wide rollouts (canary, %-based) - which become operator-side orchestration on top of this - primitive. +The first updater-capable release requires one final `FleetDeviceSetupScore` +run to install the bootstrap layout, helper service, socket permissions, +trusted keys, and revised agent unit. Older agents remain visible through their +heartbeat version but cannot use automatic upgrades until bootstrapped. -**Cons:** - -- Briefly runs two agents per device (Cutover-Ready window). - Memory and connection-count both ~2× during that window. - Acceptable for the upgrade duration (typically <60s). -- Requires reliable connectivity between agent and operator to - complete the handoff. A device whose NATS link fails mid- - upgrade stays in Cutover-Ready until link recovers. -- Disk grows monotonically with version count. Bounded by human - cleanup. We do not GC. -- New NATS subjects, new heartbeat fields, new `Device.status` - fields. Schema bump that operators-in-the-field need to handle - (the operator must understand "old agent reporting no version - field" as `version: unknown`, not crash). - -## Alternatives considered - -1. **OS-package upgrade (apt / dpkg / OSTree).** *Pros:* zero - custom code, standard toolchain, GPG-signed. - *Cons:* Loses the "agent verifies the new agent before swap" - property. apt's restart hook flips the symlink and `systemctl - restart`s; if the new binary is broken, the device is bricked - until human intervention. Doesn't drain in-flight work. Doesn't - know about NATS-managed pause states. Couples the upgrade - schedule to the distro's repo, not to the cluster operator's - intent. Rejected. - -2. **Pull-from-OCI-registry on each agent restart.** *Pros:* same - primitive as podman / kube node-image-rotation. - *Cons:* Coupling to a registry the device must reach — many - customer fleets are on private subnets without registry - access. Would mean shipping a registry mirror per fleet. Adds - a dependency for a problem we can solve with a signed binary - on a CDN. - -3. **Two systemd units, blue/green at the unit level.** - `fleet-agent-v0.1.1.service` and `fleet-agent-v0.1.2.service`, - ratchet via systemctl enable/disable. *Pros:* no symlink dance. - *Cons:* duplicates a lot of unit-file content; harder to - reason about what the "active" unit is (you have to ask - systemd, not `readlink`); doesn't compose well with the - `ExecStart=/usr/local/bin/fleet-agent` line we already ship. - Symlink swap is the lighter primitive. - -4. **Self-stopping agent (no operator stop signal).** New agent - tells old agent "I'm up, you can go" via NATS. *Pros:* one - fewer subject. - *Cons:* The new agent is also the agent we're least sure of - — putting it in charge of the old one's lifecycle inverts the - trust model. If the new agent has a bug that causes it to - announce ready prematurely, the cluster goes blind. The - operator path is the conservative choice. - -5. **Operator-pushed binary (instead of agent-pulled).** The - operator sshes / executes a one-off command per device. - *Pros:* operator controls timing precisely. - *Cons:* Reintroduces SSH as a control plane (we just spent a - month getting rid of it for the enrollment flow). Doesn't - scale to fleets where most devices are NATted away from the - operator. - -## Implementation milestones - -(For a future implementer; not committed to a date here. Lives -in the v0.2+ backlog.) - -1. **M1** — Versioned binary layout: builds produce - `fleet-agent-v` artifacts; install Score writes them - to `/usr/bin/fleet-agent-v` + creates - `/usr/local/bin/fleet-agent` symlink. Existing tests cover the - rest. -2. **M2** — Version field in heartbeat + `Device.status.current_version` - reflection on the operator side. No upgrade behavior yet. -3. **M3** — `desired_version` field on the device-info KV + - operator setter. No agent-side action yet. -4. **M4** — Agent state machine, end to end, gated by a feature - flag. Operator publishes desired_version → agent does the - dance → operator sends stop signal → done. Includes failure- - mode tests (download fail, smoke fail, heartbeat-timeout - revert). -5. **M5** — Remove the feature flag. Default-on. -6. **M6** — Operator-side rollout strategies (canary, %-based) — - only after M5 has been in production for 30 days against a - real fleet. - -## Additional Notes - -- Binary signing + signature verification is in scope for the - `Staging` step but the *which* signing scheme (cosign / Rekor - / minisign) is deferred until the M1 implementation. Whatever - we pick must work on aarch64-musl Pi devices without - additional system dependencies. -- The N-versions-on-disk policy is "all of them, forever" per - the constraint above. If disk pressure becomes real on some - customer fleet, a manual GC tool can prune `/usr/bin/fleet-agent-v*` - by date — never automatic, never as part of the upgrade - itself. -- See JG's *Pour l'amour des compilateurs* talk (Botpress - Meetup, 2026-04-30) for the framing applied here: - cardinality-matched types and operator-as-coordinator are the - same idea, applied to one function and to one platform. +Fleet-wide canary and percentage rollout policy remains outside this ADR. This +decision defines one attempt on one device. diff --git a/docs/design/fleet-agent-upgrades.md b/docs/design/fleet-agent-upgrades.md new file mode 100644 index 00000000..0387a2e0 --- /dev/null +++ b/docs/design/fleet-agent-upgrades.md @@ -0,0 +1,389 @@ +# Fleet Agent Upgrades + +## Why this exists + +Harmony places a reconciler inside each decentralized micro datacenter. That +agent turns durable desired state from NATS into local Podman operations. Once a +device carries real workloads, replacing the agent is no longer a file-copy +problem: the updater must preserve a single owner for workload mutation, survive +loss of power, reject unauthorized code, and recover without a technician at the +device. + +The fleet upgrade protocol solves that narrow problem. It updates one Harmony +agent binary on one Linux device. It does not update the operating system, +kernel, Podman, or workload containers. + +The design follows Harmony's normal separation of concerns: + +- `Device.spec.agentUpgrade` declares the target release. +- The fleet operator converts that declaration into durable NATS intent. +- The unprivileged agent drains its Score reconciliation loop. +- A root-owned helper performs the filesystem and systemd transaction. +- `FleetDeviceSetupScore` installs the helper, trust key, and systemd units. + +![Harmony fleet upgrade architecture](../diagrams/fleet-agent-upgrade-architecture.svg) + +The important architectural decision is **single active ownership**. Harmony +does not run old and new reconcilers together. Existing containers continue to +run while systemd replaces the agent, but desired-state convergence pauses for +the cutover. + +## Design contract + +The protocol is built around five guarantees. + +| Guarantee | Mechanism | +|---|---| +| One workload owner | An advisory process lock plus stop-before-start systemd restart | +| No root candidate | Downloaded binaries and `--self-test` run as `fleet-agent` | +| Exact authorization | Ed25519 authorization covers the complete attempt digest and identity | +| Atomic selection | A temporary symlink is renamed over the active symlink, then its directory is synced | +| Conservative recovery | Every uncommitted post-switch state attempts rollback; failure is durable and explicit | + +Running workloads are not part of the transaction. Their continuity depends on +Podman process independence and restart policies. The guarantee is that an agent +upgrade does not deliberately stop them. + +## Components and trust boundaries + +### Fleet operator + +The operator watches `Device` resources and creates one immutable attempt when +the reported and desired versions differ. Attempt creation uses JetStream +compare-and-set, so overlapping operator pods cannot silently replace each +other's decision. + +After the old agent reports a successful candidate probe, the operator signs a +switch authorization for that exact attempt. The signing key is injected from a +Kubernetes Secret and is never sent to a device. + +Code: + +- [`AgentUpgradeTarget` and reflected status](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/crd.rs#L106-L151) +- [`reconcile_device_inner`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/agent_upgrade.rs#L115-L227) +- [`AuthorizationSigner`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/agent_upgrade.rs#L18-L54) + +### Active fleet agent + +The ordinary agent runs as `fleet-agent`. It validates attempts, waits for any +current Podman mutation to finish, pauses new mutations, asks the helper to +stage the candidate, and publishes progress. + +Desired-state notifications still enter memory while paused. Reconciliation +resumes after cancellation or pre-switch failure. On successful cutover, the +old process is stopped by systemd and never resumes. + +Code: + +- [`UpgradeController`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L74-L256) +- [`Reconciler::pause` and `resume`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/reconciler.rs#L161-L173) +- [Paused mutation check](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/reconciler.rs#L206-L210) + +### Root updater + +The updater is deliberately small. Its Unix-socket protocol accepts `stage`, +`switch`, `cancel`, and read-only `status`; it accepts no shell command or +caller-selected filesystem path. Mutating requests are serialized. + +The updater owns: + +- artifact download and verification; +- immutable versioned binaries; +- the active symlink; +- the transaction journal; +- systemd restart and probation; +- rollback. + +The updater itself comes from `fleet-agent-bootstrap`, not from the downloaded +candidate. Fixing the privileged updater therefore requires a new +`FleetDeviceSetupScore` run. + +Code: + +- [Updater protocol and journal types](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L16-L67) +- [`run_server`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L121-L185) +- [`FleetDeviceSetupConfig::render_updater_systemd_unit`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-deploy/src/device_setup.rs#L325-L346) + +## Upgrade algorithm + +![Fleet agent upgrade sequence](../diagrams/fleet-agent-upgrade-sequence.svg) + +### 1. Create immutable intent + +The operator reads the current version from `Device.status`, compares it with +`Device.spec.agentUpgrade.version`, and writes an `AgentUpgradeAttempt` under +the device's intent key. The attempt includes: + +- UUID and device ID; +- source and target versions; +- architecture; +- HTTPS artifact URL and maximum size; +- SHA-256 and Ed25519 signature; +- signing key ID and creation time. + +The canonical wire types and attempt digest live in +[`harmony-reconciler-contracts/src/upgrade.rs`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs#L7-L89). + +### 2. Validate and drain + +The active agent rejects an attempt for another device, a different source +version or architecture, an equal target version, an invalid UUID, or a creation +time outside the accepted window. Reusing a UUID with different content is also +rejected. + +The agent then acquires the reconciler's runtime gate. An operation already in +progress may finish; no new Podman mutation can begin afterward. + +Implementation: [`UpgradeController::accept`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L90-L136). + +### 3. Stage and probe + +The helper derives the destination from the validated version under +`/usr/lib/harmony-fleet`. It allows HTTPS only, disables redirects, applies a +15-second connection timeout, a 300-second request timeout, the attempt's size +limit, and a compiled 100 MiB ceiling. + +The helper verifies SHA-256 and Ed25519 before installation. It sets executable +permissions, syncs the temporary file, renames it atomically, and syncs the +directory. If that version already exists, its bytes must still match the +attempt. + +The candidate then runs as: + +```text +runuser -u fleet-agent -- --self-test +``` + +The probe loads configuration, checks Podman when enabled, authenticates to +NATS, and consumes a complete server-filtered desired-state snapshot. It does +not acquire the active-agent lock, publish heartbeat, watch desired state, or +mutate workloads. + +Implementation: + +- [`stage`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L234-L323) +- [`--self-test` startup path](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L294-L374) +- [`load_desired_snapshot`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L109-L162) + +### 4. Authorize the exact attempt + +After probe success, the old agent publishes `awaiting-authorization`. The +operator signs a payload containing the attempt digest, device ID, source and +target versions, artifact key ID, and authorization time. + +The helper verifies the signature and compares every bound field with its local +staged transaction. A local process cannot stage one signed artifact and reuse +authorization issued for another. + +If authorization does not arrive within five minutes, the old agent cancels the +staged transaction and resumes reconciliation. + +Implementation: + +- [`AgentUpgradeAuthorization::signing_payload`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs#L62-L89) +- [`verify_authorization`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L475-L491) +- [`UpgradeController::check_timeout`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L171-L183) + +### 5. Switch and prove readiness + +The helper writes `switching` to the journal before changing the symlink. It +then atomically replaces `/usr/local/bin/fleet-agent` and asks systemd to restart +the permanent service. + +The new process must acquire the exclusive agent lock, load configuration, +reach Podman and NATS, restore upgrade state, consume a complete desired-state +snapshot, and initialize reconciliation before sending `READY=1`. + +Implementation: + +- [`switch`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L325-L374) +- [`switch_link`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L515-L522) +- [Agent initialization and readiness](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L413-L440) + +### 6. Probation and commit + +The helper records `probation` and watches the systemd unit for 60 seconds. The +unit must remain active and its `InvocationID` must remain unchanged. A candidate +that crashes and is restarted therefore fails probation even if it happens to +be active at the final check. + +After a stable probation, the helper records `committed`. The new agent observes +that journal transition and publishes `complete`. + +## Crash consistency + +![Fleet agent durable transaction states](../diagrams/fleet-agent-upgrade-recovery.svg) + +The journal uses write-to-temporary, file sync, atomic rename, and parent +directory sync. Binary installation and symlink changes use the same durability +pattern. + +| Last durable phase | Selected binary | Recovery action | +|---|---|---| +| No journal | Existing active target | Nothing | +| `staged` | Previous target | Old agent reconstructs the attempt or times out | +| `switching` | Unknown | Restore previous symlink, then restart | +| `probation` | Candidate | Restore previous symlink, then restart | +| `rolling-back` | Previous target intended | Repeat rollback idempotently | +| `committed` | Candidate | Keep candidate and republish completion | +| `failed` | Previous target | Keep previous target | +| `rollback-failed` | Unknown | Report manual intervention required | + +On updater startup, the previous symlink is restored before the updater reports +systemd readiness. This prevents boot ordering from starting a candidate already +marked for rollback. The updater then serves transaction status while it +restarts the previous agent. + +Corrupt or unreadable journal data fails closed. The helper does not infer state +from an incomplete record. + +Implementation: + +- [Startup recovery](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L121-L185) +- [`prepare_rollback` and `finish_rollback`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L407-L423) +- [`write_transaction`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L584-L602) + +## Security model + +The normal agent is assumed capable of requesting staging but not of authorizing +cutover. Filesystem permissions restrict the root updater socket to root and the +`fleet-agent` group. Cryptographic authorization protects `switch` even if the +unprivileged agent process is compromised. + +The current design uses one active Ed25519 key for two roles: + +1. signing artifact bytes; +2. signing rollout authorization. + +That is adequate for a controlled first-party release pipeline, but it is not a +TUF or Uptane trust model. A long-lived system should separate offline artifact +signing from online rollout authorization and eventually support multiple keys, +explicit revocation epochs, and threshold policy. + +The device trusts one active key ID. Key rotation therefore needs coordinated +deployment of the new public key to devices and the corresponding private key to +the operator. Mixed-key fleets cannot currently authorize upgrades from one +operator instance. + +## What this design does not guarantee + +### No dual-agent continuity + +Cutover has an interval with no reconciler, bounded in practice by systemd's +service-start timeout. Running containers continue, but new desired state does +not converge until the new or rolled-back agent starts. + +The HTTP request is bounded to five minutes, candidate `--self-test` to 60 +seconds, and the post-probe authorization wait to another five minutes. Desired +changes accumulate but do not converge while paused. + +### No semantic health gate + +The probe catches configuration, dependency, credential, architecture, and +control-plane access failures. Probation catches process crashes. Neither proves +that the candidate continues to reconcile workloads correctly while remaining +alive. + +### No operating-system rollback + +The protocol cannot recover a broken kernel, filesystem, systemd installation, +Podman package, or bootloader. RAUC, Mender, and SWUpdate solve that larger A/B +system problem. + +### No irreversible local migrations + +Rollback changes the executable only. Agent releases must keep configuration, +local databases, Podman labels, and journal formats readable by the previous +binary until commit. Any future persistent migration needs its own reversible +transaction. + +### No automatic garbage collection + +Versioned binaries remain on disk. A safe future collector must retain the +active target, previous rollback target, bootstrap binary, and every target +named by a non-terminal journal. + +### No automatic retry of identical failure + +A failed matching attempt is terminal. Retrying requires changing release +metadata or removing the old intent so the operator creates a new UUID. Removing +`Device.spec.agentUpgrade` while the agent waits does not cancel immediately; +the five-minute timeout still applies. + +### Wall-clock dependency + +Attempt and authorization freshness use UTC. Devices with a bad RTC or delayed +NTP can reject valid upgrades. Local probation correctly uses monotonic time. + +### Runtime coupling + +The upgrade controller is currently created only when `runtime_enabled=true`. +An agent running without the Podman reconciler cannot upgrade automatically. + +### Root helper lifecycle + +The bootstrap updater is intentionally outside OTA. This limits privilege +escalation risk, but helper fixes require device setup. Running +`FleetDeviceSetupScore` during an active upgrade is not yet a supported +operation. + +## Comparison with established tooling + +| Tool or pattern | Similarity | What Harmony lacks | +|---|---|---| +| Nix profiles / OSTree | Immutable generations and atomic active pointer | Content-addressed closure, generations database, GC | +| systemd service rollout | Notification readiness, restart policy, invocation identity | A fully tested cross-version update framework | +| Kubernetes `Recreate` | One active owner and bounded unavailability | Rollout policy, canaries, progress budgets, history | +| TUF / Uptane | Signed targets and freshness checks | Role separation, thresholds, delegated metadata, rollback/freeze protection | +| RAUC / Mender / SWUpdate | Staged candidate and automatic rollback | Bootloader and full OS A/B recovery | +| rpm / dpkg | Installed version tracking | Package dependency database and script transaction model | + +The closest description is: **a small, single-binary OSTree generation switch +with systemd readiness and an operator-authorized cutover**. + +## Operational constraints + +| Constraint | Current value | +|---|---:| +| Artifact connect timeout | 15 seconds | +| Artifact request timeout | 300 seconds | +| Candidate self-test timeout | 60 seconds | +| Compiled artifact ceiling | 100 MiB | +| Attempt accepted age | 24 hours | +| Authorization wait | 5 minutes | +| Authorization accepted age | 10 minutes | +| Future clock tolerance | 5 minutes | +| systemd readiness check | 60 seconds after restart returns | +| Probation | 60 seconds | +| Error text retained | 1,024 characters | +| Operator scan interval | 2 seconds | + +These are compiled policy, not Score configuration. Change them only with a +clear operational requirement; each additional knob expands the compatibility +surface. + +## Ownership priorities + +Before broad production rollout: + +1. Make `rollback-failed` a hard quarantine that blocks new attempts until an + explicit repair clears it. +2. Run VM fault injection at every journal write, binary rename, symlink rename, + readiness transition, and rollback transition. +3. Separate artifact-signing and rollout-authorization keys. +4. Add explicit retry generation and immediate cancellation to the Device API. +5. Gate commit on attempt-bound behavioral health, not process stability alone. +6. Define bootstrap updater compatibility and update procedures. +7. Add attempt history and key-rotation support before scaling fleet rollout. + +The local transaction is intentionally conservative and has clear failure +boundaries. Its long-term risk lies outside the atomic symlink operation: key +lifecycle, semantic health, bootstrap maintenance, migration discipline, and +real systemd and power-loss testing. + +## Related decisions + +- [ADR-016: Harmony agent and global mesh](../adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md) +- [ADR-022: Fleet agent upgrade procedure](../adr/022-fleet-agent-upgrade.md) +- [ADR-023: Deploy architecture](../adr/023-deploy-architecture.md) +- [Agent reconciliation and upgrade implementation plan](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/agent-reconciliation-and-upgrade-plan.md) diff --git a/docs/diagrams/fleet-agent-upgrade-architecture.svg b/docs/diagrams/fleet-agent-upgrade-architecture.svg new file mode 100644 index 00000000..380f6e77 --- /dev/null +++ b/docs/diagrams/fleet-agent-upgrade-architecture.svg @@ -0,0 +1,62 @@ + + Harmony fleet agent upgrade architecture + A Device desired release flows through the fleet operator and NATS to an unprivileged device agent. A root updater controls binaries, the active symlink, and systemd. Podman workloads remain independent. + + + + + + + + Upgrade control crosses two trust boundaries + Kubernetes declares intent. NATS carries durable state. Only the root helper mutates the active binary. + + + CONTROL PLANE + + DESIRED STATE + Device.spec + release metadata + + FLEET OPERATOR + attempt + authorization + CAS · status reflection + + + + DURABLE MESH + + NATS KV + upgrade-intent + upgrade-authorize + upgrade-status + per-device permissions + + + + + DEVICE + + UNPRIVILEGED + Fleet agent + drain · probe · report + + ROOT + Updater + verify · switch · rollback + + + Unix socket + + + Podman workloads + continue independently + + systemd + symlink + exclusive cutover + + + + The agent owns workload intent; the updater owns only the binary transaction. + + diff --git a/docs/diagrams/fleet-agent-upgrade-recovery.svg b/docs/diagrams/fleet-agent-upgrade-recovery.svg new file mode 100644 index 00000000..73a77b3f --- /dev/null +++ b/docs/diagrams/fleet-agent-upgrade-recovery.svg @@ -0,0 +1,45 @@ + + Fleet agent upgrade durable transaction and recovery + Staged is safe before switch. Switching and probation are uncommitted post-switch states that attempt rollback. Committed keeps the candidate. Rollback failure leaves selection uncertain and requires manual repair. + + + + + + + + + The journal decides recovery; process memory does not + Any crash before commit attempts rollback; failure is recorded instead of hidden. + + + PRE-SWITCH · PREVIOUS AGENT IS ACTIVE + no journalnothing staged + stagedcandidate verified + + probe ok + + + POST-SWITCH · NOT COMMITTED + switchingpointer may change + probationcandidate is active + authorized + + + committedcandidate retained + 60 s stable + + rolling-backrestore previous first + + + crash, timeout, restart, or boot before commit + + failedprevious restored + rollback-failedselection is uncertain + restart succeeds + restore or restart fails + + Durability boundary + journal fsync → atomic rename → directory fsync + + diff --git a/docs/diagrams/fleet-agent-upgrade-sequence.svg b/docs/diagrams/fleet-agent-upgrade-sequence.svg new file mode 100644 index 00000000..bf7ebc13 --- /dev/null +++ b/docs/diagrams/fleet-agent-upgrade-sequence.svg @@ -0,0 +1,41 @@ + + Fleet agent upgrade sequence + The operator publishes immutable intent, the active agent drains, the updater verifies and probes a candidate, the operator signs authorization, and systemd starts exactly one new agent before probation commits. + + + + + + + + One attempt, two signatures, one active process + Artifact trust permits execution as a probe. Operator authorization permits the exclusive switch. + + + Fleet operator + NATS KV + Active agent + Root updater + systemd / new agent + + + + + 1CAS immutable intent + 2attempt delivery + drain mutationscontainers keep running + 3stage signed artifact + HTTPS + SHA-256Ed25519 + fsyncprobe as fleet-agent + 4staged + awaiting-authorization + probe status + 5signed exact authorization + + 6journal · symlink · restart + snapshot + READY=1one process lock + 7stable InvocationID + committed journal + new agent publishes complete + + + diff --git a/examples/fleet_device_enroll/src/main.rs b/examples/fleet_device_enroll/src/main.rs index b01f8251..4472031e 100644 --- a/examples/fleet_device_enroll/src/main.rs +++ b/examples/fleet_device_enroll/src/main.rs @@ -301,6 +301,7 @@ async fn main() -> Result<()> { nats_urls: vec![nats_url], auth, agent_binary_path: agent_binary, + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }; diff --git a/examples/fleet_rpi_setup/src/main.rs b/examples/fleet_rpi_setup/src/main.rs index 5e572dd9..3d3d0df6 100644 --- a/examples/fleet_rpi_setup/src/main.rs +++ b/examples/fleet_rpi_setup/src/main.rs @@ -167,6 +167,7 @@ async fn main() -> Result<()> { nats_urls: vec![cli.nats_url.clone()], auth, agent_binary_path: cli.agent_binary.clone(), + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }); diff --git a/examples/fleet_vm_setup/src/main.rs b/examples/fleet_vm_setup/src/main.rs index 5ef1f31a..27217335 100644 --- a/examples/fleet_vm_setup/src/main.rs +++ b/examples/fleet_vm_setup/src/main.rs @@ -218,6 +218,7 @@ async fn main() -> Result<()> { nats_pass: cli.nats_pass.clone(), }, agent_binary_path: agent_binary, + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }); diff --git a/fleet/agent-reconciliation-and-upgrade-plan.md b/fleet/agent-reconciliation-and-upgrade-plan.md index 45348be4..10d65b06 100644 --- a/fleet/agent-reconciliation-and-upgrade-plan.md +++ b/fleet/agent-reconciliation-and-upgrade-plan.md @@ -125,10 +125,10 @@ back on failed readiness. ADR-022 must be updated in the same change. verification, the active symlink, systemd restart, rollback, and one durable transaction file. `FleetDeviceSetupScore` installs it as a root systemd service with a Unix socket owned by root and writable only by the - `fleet-agent` group. Its fixed request protocol accepts only stage and switch - for paths derived under one compiled artifact root. Finalization, recovery, - and rollback are internal. It accepts no shell command or caller-selected - path. + `fleet-agent` group. Its fixed request protocol accepts stage, signed switch, + cancellation, and read-only transaction status for paths derived under one + compiled artifact root. Finalization, recovery, and rollback are internal. It + accepts no shell command or caller-selected path. - The operator owns desired version and attempt identity. Every intent, status, probe, and switch authorization carries the same attempt ID. @@ -136,9 +136,9 @@ back on failed readiness. ADR-022 must be updated in the same change. - `agent-upgrade-intent.` contains the latest immutable attempt. The operator may write it; that device may read it. -- `agent-upgrade-authorize..` authorizes the exclusive switch - after the operator observes probe success. The operator may write it; that - device may read it. +- `agent-upgrade-authorize..` carries a signed authorization + for the exclusive switch after the operator observes probe success. The + operator may write it; that device may read it. - `agent-upgrade-status.` contains the agent's latest attempt-scoped phase and bounded error. The device may write it; the operator may read it. @@ -190,7 +190,7 @@ terminal until a new attempt ID arrives. ### Upgrade edge cases -Tests cover successful upgrade, duplicate and stale attempts, downgrade, +The required test matrix covers successful upgrade, duplicate and stale attempts, downgrade, download failure, oversized artifact, digest mismatch, bad signature, wrong architecture, probe failure, commit before probe, cancellation, operator outage before commit, queued workload recovery after timeout, stale commit, helper diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 64c88ebe..29d27cdb 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -15,7 +15,7 @@ futures-util = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2.workspace = true -tokio = { workspace = true } +tokio = { workspace = true, features = ["process"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } anyhow = { workspace = true } @@ -24,3 +24,8 @@ toml = { workspace = true } thiserror = { workspace = true } podman-api = "0.9" sd-notify = "0.4" +base64.workspace = true +reqwest.workspace = true +uuid.workspace = true +fs2.workspace = true +ed25519-dalek.workspace = true diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 1be6bbb5..e4d830a1 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -3,6 +3,8 @@ mod config; mod fleet_publisher; mod podman; mod reconciler; +mod updater; +mod upgrade; use std::sync::Arc; use std::time::Duration; @@ -41,6 +43,25 @@ struct Cli { default_value = "/etc/fleet-agent/config.toml" )] config: std::path::PathBuf, + #[arg(long)] + self_test: bool, + #[arg(long)] + updater: bool, + #[arg(long, default_value = updater::DEFAULT_SOCKET)] + updater_socket: std::path::PathBuf, +} + +fn acquire_process_lock() -> Result { + use fs2::FileExt; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open("/run/harmony-fleet-agent/agent.lock")?; + file.try_lock_exclusive() + .context("another fleet agent instance is already active")?; + Ok(file) } async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result { @@ -81,10 +102,7 @@ async fn desired_state_store( client: async_nats::Client, ) -> Result { Ok(async_nats::jetstream::new(client) - .create_key_value(async_nats::jetstream::kv::Config { - bucket: BUCKET_DESIRED_STATE.to_string(), - ..Default::default() - }) + .get_key_value(BUCKET_DESIRED_STATE) .await?) } @@ -93,20 +111,50 @@ async fn load_desired_snapshot( device_id: &Id, ) -> Result> { let prefix = format!("{device_id}."); + let filter = format!( + "{}{}", + bucket.prefix, + desired_state_watch_filter(&device_id.to_string()) + ); + let mut consumer = bucket + .stream + .create_consumer(async_nats::jetstream::consumer::pull::OrderedConfig { + filter_subject: filter, + deliver_policy: async_nats::jetstream::consumer::DeliverPolicy::LastPerSubject, + ..Default::default() + }) + .await?; + let pending = consumer.info().await?.num_pending; + if pending == 0 { + return Ok(Vec::new()); + } let mut entries = Vec::new(); - let mut keys = bucket.keys().await?; - while let Some(key) = keys.next().await { - let key = key?; - if !key.starts_with(&prefix) { - continue; - } - if let Some(entry) = bucket.entry(&key).await? - && entry.operation == async_nats::jetstream::kv::Operation::Put - { + let mut snapshot = consumer.messages().await?; + for _ in 0..pending { + let message = tokio::time::timeout(Duration::from_secs(10), snapshot.next()) + .await + .context("timed out before desired-state snapshot completed")? + .context("desired-state snapshot ended before completion")??; + let info = message + .info() + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let key = message + .message + .subject + .strip_prefix(&bucket.prefix) + .context("desired-state subject has the wrong bucket prefix")? + .to_string(); + let deleted = message.message.headers.as_ref().is_some_and(|headers| { + headers + .get("KV-Operation") + .is_some_and(|operation| matches!(operation.as_str(), "DEL" | "PURGE")) + || headers.get("Nats-Marker-Reason").is_some() + }); + if !deleted && key.starts_with(&prefix) { entries.push(SnapshotEntry { key, - revision: entry.revision, - value: entry.value.to_vec(), + revision: info.stream_sequence, + value: message.message.payload.to_vec(), }); } } @@ -244,6 +292,10 @@ async fn main() -> Result<()> { tracing_subscriber::fmt().with_env_filter(filter).init(); let cli = Cli::parse(); + if cli.updater { + return updater::run_server(&cli.updater_socket).await; + } + let _process_lock = (!cli.self_test).then(acquire_process_lock).transpose()?; let cfg = config::load_config(&cli.config)?; tracing::info!( device_id = %cfg.agent.device_id, @@ -314,6 +366,13 @@ async fn main() -> Result<()> { Error::msg(msg) })?; + if cli.self_test { + let bucket = desired_state_store(client).await?; + load_desired_snapshot(&bucket, &device_id).await?; + tracing::info!(version = env!("CARGO_PKG_VERSION"), "self-test ok"); + return Ok(()); + } + // Publish surface. Opens the three KV buckets (idempotent // creates). Must be live before the reconciler starts so // writes on the first desired-state KV watch land on the wire. @@ -351,6 +410,21 @@ async fn main() -> Result<()> { )) }); + let upgrade_service = match reconciler.as_ref() { + Some(reconciler) => Some( + upgrade::UpgradeService::connect( + client.clone(), + device_id.clone(), + reconciler.clone(), + cli.updater_socket + .to_str() + .context("non-UTF-8 updater socket")?, + ) + .await?, + ), + None => None, + }; + let desired_bucket = if let Some(reconciler) = &reconciler { let bucket = desired_state_store(client.clone()).await?; let generation = reconciler.generation().await; @@ -407,6 +481,11 @@ async fn main() -> Result<()> { Some(reconciler) => Box::pin(reconciler.clone().run()), None => Box::pin(std::future::pending()), }; + let upgrades: std::pin::Pin> + Send>> = + match upgrade_service { + Some(service) => Box::pin(service.run()), + None => Box::pin(std::future::pending()), + }; tokio::select! { // Waiting on ctrlc in a select will automatically terminate other branches when @@ -416,6 +495,7 @@ async fn main() -> Result<()> { r = watch => { r?; } _ = snapshots => {} _ = worker => {} + r = upgrades => { r?; } _ = heartbeat => {} r = commands => { r?; } } diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index b75f47aa..432035d0 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use anyhow::{Result, anyhow}; use chrono::Utc; @@ -55,6 +56,7 @@ pub struct Reconciler { state: Mutex, runtime_gate: Mutex<()>, wake: tokio::sync::Notify, + paused: AtomicBool, publisher: Option>, secrets: Option, } @@ -72,6 +74,7 @@ impl Reconciler { state: Mutex::new(State::default()), runtime_gate: Mutex::new(()), wake: tokio::sync::Notify::new(), + paused: AtomicBool::new(false), publisher, secrets, } @@ -159,9 +162,22 @@ impl Reconciler { self.reconcile().await } + pub async fn pause(&self) { + self.paused.store(true, Ordering::Release); + let _gate = self.runtime_gate.lock().await; + } + + pub fn resume(&self) { + self.paused.store(false, Ordering::Release); + self.wake.notify_one(); + } + pub async fn run(self: Arc) { loop { self.wake.notified().await; + if self.paused.load(Ordering::Acquire) { + continue; + } if let Err(error) = self.reconcile().await { tracing::warn!(%error, "deployment reconciliation failed"); } @@ -189,6 +205,9 @@ impl Reconciler { async fn reconcile(&self) -> Result<()> { let _gate = self.runtime_gate.lock().await; + if self.paused.load(Ordering::Acquire) { + return Ok(()); + } let (desired, removals, complete_snapshot) = { let mut state = self.state.lock().await; let complete = std::mem::take(&mut state.complete_snapshot_pending); diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs new file mode 100644 index 00000000..c644da4f --- /dev/null +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -0,0 +1,682 @@ +use std::os::unix::fs::{PermissionsExt, symlink}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::Engine; +use ed25519_dalek::{Signature, Verifier, VerifyingKey}; +use futures_util::StreamExt; +use harmony_reconciler_contracts::{AgentUpgradeAttempt, AgentUpgradeAuthorization}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; + +pub const DEFAULT_SOCKET: &str = "/run/harmony-fleet-updater/updater.sock"; +const ROOT: &str = "/usr/lib/harmony-fleet"; +const BOOTSTRAP_BINARY: &str = "/usr/lib/harmony-fleet/fleet-agent-bootstrap"; +const ACTIVE_LINK: &str = "/usr/local/bin/fleet-agent"; +const JOURNAL: &str = "/var/lib/harmony-fleet-updater/transaction.json"; +const TRUSTED_KEYS: &str = "/etc/fleet-agent/trusted-upgrade-keys"; +const TRUSTED_KEY_ID: &str = "/etc/fleet-agent/trusted-upgrade-key-id"; +const SELF_TEST_TIMEOUT: Duration = Duration::from_secs(60); +const READINESS_TIMEOUT: Duration = Duration::from_secs(60); +const PROBATION: Duration = Duration::from_secs(60); +const MAX_ARTIFACT_BYTES: u64 = 100 * 1024 * 1024; +const MAX_REQUEST_BYTES: u64 = 1024 * 1024; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "operation", content = "data", rename_all = "kebab-case")] +enum Request { + Stage(AgentUpgradeAttempt), + Switch(AgentUpgradeAuthorization), + Cancel { attempt_id: String, error: String }, + Status, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Response { + ok: bool, + error: Option, + transaction: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum TransactionPhase { + Staged, + Switching, + Probation, + Committed, + RollingBack, + Failed, + RollbackFailed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Transaction { + pub attempt_id: String, + pub device_id: harmony_reconciler_contracts::Id, + pub from_version: String, + pub phase: TransactionPhase, + pub previous: PathBuf, + pub target: PathBuf, + pub target_version: String, + pub attempt_digest: String, + pub error: Option, +} + +#[derive(Clone)] +pub struct UpdaterClient { + socket: PathBuf, +} + +impl UpdaterClient { + pub fn new(socket: impl Into) -> Self { + Self { + socket: socket.into(), + } + } + + pub async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.request(Request::Stage(attempt.clone())) + .await + .map(|_| ()) + } + + pub async fn switch(&self, authorization: &AgentUpgradeAuthorization) -> Result<()> { + self.request(Request::Switch(authorization.clone())) + .await + .map(|_| ()) + } + + pub async fn status(&self) -> Result> { + self.request(Request::Status).await + } + + pub async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()> { + self.request(Request::Cancel { + attempt_id: attempt_id.into(), + error: error.into(), + }) + .await + .map(|_| ()) + } + + async fn request(&self, request: Request) -> Result> { + let mut stream = UnixStream::connect(&self.socket).await?; + let mut payload = serde_json::to_vec(&request)?; + payload.push(b'\n'); + stream.write_all(&payload).await?; + let mut response = String::new(); + BufReader::new(stream).read_line(&mut response).await?; + let response: Response = serde_json::from_str(&response)?; + if !response.ok { + bail!(response.error.unwrap_or_else(|| "updater failed".into())); + } + Ok(response.transaction) + } +} + +pub async fn run_server(socket: &Path) -> Result<()> { + let _lock = acquire_process_lock()?; + initialize_layout()?; + let recovery = match read_transaction().await { + Ok(transaction) + if matches!( + transaction.phase, + TransactionPhase::Switching + | TransactionPhase::Probation + | TransactionPhase::RollingBack + ) => + { + match prepare_rollback(transaction.clone()).await { + Ok(transaction) => Some(transaction), + Err(error) => { + let mut failed = transaction; + failed.phase = TransactionPhase::RollbackFailed; + failed.error = Some(format!("startup rollback preparation failed: {error}")); + write_transaction(&failed).await?; + return Err(error.context("preparing interrupted upgrade rollback")); + } + } + } + Ok(_) => None, + Err(error) if is_not_found(&error) => None, + Err(error) => return Err(error.context("reading updater transaction during recovery")), + }; + if socket.exists() { + tokio::fs::remove_file(socket).await?; + } + let listener = UnixListener::bind(socket)?; + std::fs::set_permissions(socket, std::fs::Permissions::from_mode(0o660))?; + let transaction_lock = std::sync::Arc::new(tokio::sync::Mutex::new(())); + if let Some(transaction) = recovery { + let lock = transaction_lock.clone(); + tokio::spawn(async move { + let _guard = lock.lock().await; + if let Err(error) = recover_interrupted(transaction).await { + tracing::error!(%error, "interrupted upgrade rollback failed"); + } + }); + } + sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) + .context("notifying systemd that updater socket is ready")?; + loop { + let (stream, _) = listener.accept().await?; + let transaction_lock = transaction_lock.clone(); + tokio::spawn(async move { + if let Err(error) = handle(stream, transaction_lock).await { + tracing::warn!(%error, "updater request failed"); + } + }); + } +} + +async fn recover_interrupted(transaction: Transaction) -> Result<()> { + if let Err(error) = finish_rollback(transaction.clone()).await { + let mut failed = transaction; + failed.phase = TransactionPhase::RollbackFailed; + failed.error = Some(format!("startup rollback failed: {error}")); + write_transaction(&failed).await?; + return Err(error.context("recovering interrupted upgrade")); + } + Ok(()) +} + +async fn handle(stream: UnixStream, transaction_lock: Arc>) -> Result<()> { + let (reader, mut writer) = stream.into_split(); + let mut line = String::new(); + BufReader::new(reader) + .take(MAX_REQUEST_BYTES) + .read_line(&mut line) + .await?; + if line.len() as u64 == MAX_REQUEST_BYTES { + bail!("updater request exceeds {MAX_REQUEST_BYTES} bytes"); + } + let request: Request = serde_json::from_str(&line)?; + let result = match request { + Request::Stage(attempt) => { + let _guard = transaction_lock.lock().await; + stage(&attempt).await.map(|_| None) + } + Request::Switch(authorization) => { + let _guard = transaction_lock.lock().await; + switch(&authorization).await.map(Some) + } + Request::Cancel { attempt_id, error } => { + let _guard = transaction_lock.lock().await; + cancel(&attempt_id, error).await.map(Some) + } + Request::Status => match read_transaction().await { + Ok(transaction) => Ok(Some(transaction)), + Err(error) if is_not_found(&error) => Ok(None), + Err(error) => Err(error), + }, + }; + let response = match result { + Ok(transaction) => Response { + ok: true, + error: None, + transaction, + }, + Err(error) => Response { + ok: false, + error: Some(error.to_string()), + transaction: None, + }, + }; + writer.write_all(&serde_json::to_vec(&response)?).await?; + writer.write_all(b"\n").await?; + Ok(()) +} + +async fn stage(attempt: &AgentUpgradeAttempt) -> Result<()> { + validate_attempt(attempt)?; + let attempt_digest = attempt.digest(); + match read_transaction().await { + Ok(transaction) if transaction.attempt_id == attempt.attempt_id => { + if transaction.attempt_digest != attempt_digest { + bail!("upgrade attempt id was reused with different content"); + } + if transaction.phase == TransactionPhase::Staged { + return Ok(()); + } + bail!("upgrade attempt is already {:?}", transaction.phase); + } + Ok(transaction) + if !matches!( + transaction.phase, + TransactionPhase::Failed + | TransactionPhase::RollbackFailed + | TransactionPhase::Committed + ) => + { + bail!( + "upgrade attempt '{}' is already active", + transaction.attempt_id + ); + } + Ok(_) => {} + Err(error) if is_not_found(&error) => {} + Err(error) => return Err(error.context("reading existing updater transaction")), + } + let target = target_path(&attempt.target_version)?; + let previous = std::fs::read_link(ACTIVE_LINK).context("reading active agent symlink")?; + if target.exists() { + verify_file(&target, attempt).await?; + } else { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(300)) + .redirect(reqwest::redirect::Policy::none()) + .build()?; + let response = client + .get(&attempt.artifact_url) + .send() + .await? + .error_for_status()?; + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if bytes.len() as u64 + chunk.len() as u64 > attempt.max_bytes { + bail!("artifact exceeds {} bytes", attempt.max_bytes); + } + bytes.extend_from_slice(&chunk); + } + verify_bytes(&bytes, attempt).await?; + tokio::fs::create_dir_all(ROOT).await?; + let temporary = target.with_extension("tmp"); + let mut file = tokio::fs::File::create(&temporary).await?; + file.write_all(&bytes).await?; + tokio::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o755)).await?; + file.sync_all().await?; + tokio::fs::rename(&temporary, &target).await?; + sync_directory(Path::new(ROOT))?; + } + let mut command = tokio::process::Command::new("runuser"); + command.kill_on_drop(true).args([ + "-u", + "fleet-agent", + "--", + target.to_str().context("non-UTF-8 target path")?, + "--self-test", + ]); + let status = tokio::time::timeout(SELF_TEST_TIMEOUT, command.status()) + .await + .map_err(|_| anyhow!("candidate self-test timed out after {SELF_TEST_TIMEOUT:?}"))??; + if !status.success() { + bail!("candidate self-test failed: {status}"); + } + write_transaction(&Transaction { + attempt_id: attempt.attempt_id.clone(), + device_id: attempt.device_id.clone(), + from_version: attempt.from_version.clone(), + phase: TransactionPhase::Staged, + previous, + target, + target_version: attempt.target_version.clone(), + attempt_digest, + error: None, + }) + .await +} + +async fn switch(authorization: &AgentUpgradeAuthorization) -> Result { + verify_authorization(authorization).await?; + let mut transaction = read_transaction().await?; + if transaction.attempt_id != authorization.attempt_id + || transaction.attempt_digest != authorization.attempt_digest + || transaction.device_id != authorization.device_id + || transaction.from_version != authorization.from_version + || transaction.target_version != authorization.target_version + || authorization.artifact_signing_key_id != authorization.signing_key_id + || transaction.phase != TransactionPhase::Staged + { + bail!("attempt '{}' is not staged", authorization.attempt_id); + } + transaction.phase = TransactionPhase::Switching; + write_transaction(&transaction).await?; + switch_link(&transaction.target)?; + if let Err(error) = systemctl(&["restart", "fleet-agent.service"]).await { + return rollback_failed(transaction, error).await; + } + if let Err(error) = wait_active(READINESS_TIMEOUT).await { + return rollback_failed(transaction, error).await; + } + transaction.phase = TransactionPhase::Probation; + write_transaction(&transaction).await?; + let invocation = match systemctl_property("InvocationID").await { + Ok(invocation) => invocation, + Err(error) => return rollback_failed(transaction, error).await, + }; + let deadline = tokio::time::Instant::now() + PROBATION; + while tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_secs(1)).await; + if let Err(error) = systemctl(&["is-active", "--quiet", "fleet-agent.service"]).await { + return rollback_failed(transaction, error).await; + } + let current_invocation = match systemctl_property("InvocationID").await { + Ok(invocation) => invocation, + Err(error) => return rollback_failed(transaction, error).await, + }; + if current_invocation != invocation { + return rollback_failed( + transaction, + anyhow!("fleet-agent restarted during probation"), + ) + .await; + } + } + transaction.phase = TransactionPhase::Committed; + write_transaction(&transaction).await?; + Ok(transaction) +} + +async fn cancel(attempt_id: &str, error: String) -> Result { + let mut transaction = read_transaction().await?; + if transaction.attempt_id != attempt_id || transaction.phase != TransactionPhase::Staged { + bail!("attempt '{attempt_id}' is not staged"); + } + transaction.phase = TransactionPhase::Failed; + transaction.error = Some(bounded_error(&error)); + write_transaction(&transaction).await?; + Ok(transaction) +} + +async fn rollback_failed(transaction: Transaction, cause: anyhow::Error) -> Result { + match rollback(transaction.clone()).await { + Ok(mut transaction) => { + transaction.error = Some(bounded_error(&cause.to_string())); + transaction.phase = TransactionPhase::Failed; + write_transaction(&transaction).await?; + Err(cause) + } + Err(rollback_error) => { + let mut transaction = read_transaction().await.unwrap_or(transaction); + transaction.phase = TransactionPhase::RollbackFailed; + transaction.error = Some(bounded_error(&format!( + "{cause}; rollback failed: {rollback_error}" + ))); + write_transaction(&transaction).await?; + Err(anyhow!(transaction.error.unwrap())) + } + } +} + +async fn rollback(mut transaction: Transaction) -> Result { + transaction = prepare_rollback(transaction).await?; + finish_rollback(transaction).await +} + +async fn prepare_rollback(mut transaction: Transaction) -> Result { + transaction.phase = TransactionPhase::RollingBack; + write_transaction(&transaction).await?; + switch_link(&transaction.previous)?; + Ok(transaction) +} + +async fn finish_rollback(mut transaction: Transaction) -> Result { + systemctl(&["restart", "fleet-agent.service"]).await?; + transaction.phase = TransactionPhase::Failed; + write_transaction(&transaction).await?; + Ok(transaction) +} + +fn validate_attempt(attempt: &AgentUpgradeAttempt) -> Result<()> { + uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid attempt id")?; + if attempt.target_version.is_empty() + || !attempt + .target_version + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_') + { + bail!("invalid target version"); + } + if attempt.architecture != std::env::consts::ARCH { + bail!("artifact architecture does not match this device"); + } + if !attempt.artifact_url.starts_with("https://") { + bail!("artifact URL must use HTTPS"); + } + if attempt.max_bytes == 0 || attempt.max_bytes > MAX_ARTIFACT_BYTES { + bail!("artifact size limit must be between 1 and {MAX_ARTIFACT_BYTES} bytes"); + } + if !safe_token(&attempt.signing_key_id) { + bail!("invalid signing key id"); + } + Ok(()) +} + +fn target_path(version: &str) -> Result { + if version.contains('/') || version.contains("..") { + bail!("invalid target version path"); + } + Ok(Path::new(ROOT).join(format!("fleet-agent-v{version}"))) +} + +async fn verify_file(path: &Path, attempt: &AgentUpgradeAttempt) -> Result<()> { + verify_bytes(&tokio::fs::read(path).await?, attempt).await +} + +async fn verify_bytes(bytes: &[u8], attempt: &AgentUpgradeAttempt) -> Result<()> { + let digest = format!("{:x}", Sha256::digest(bytes)); + if digest != attempt.sha256.to_ascii_lowercase() { + bail!("artifact SHA-256 mismatch"); + } + let key = read_verifying_key(&attempt.signing_key_id).await?; + let signature = Signature::from_slice( + &base64::engine::general_purpose::STANDARD.decode(&attempt.signature)?, + )?; + key.verify(bytes, &signature)?; + Ok(()) +} + +async fn verify_authorization(authorization: &AgentUpgradeAuthorization) -> Result<()> { + uuid::Uuid::parse_str(&authorization.attempt_id).context("invalid authorization attempt id")?; + if !safe_token(&authorization.signing_key_id) { + bail!("invalid authorization signing key id"); + } + let now = chrono::Utc::now(); + if authorization.authorized_at < now - chrono::Duration::minutes(10) + || authorization.authorized_at > now + chrono::Duration::minutes(5) + { + bail!("authorization timestamp is outside the accepted window"); + } + let key = read_verifying_key(&authorization.signing_key_id).await?; + let signature = Signature::from_slice( + &base64::engine::general_purpose::STANDARD.decode(&authorization.signature)?, + )?; + key.verify(authorization.signing_payload().as_bytes(), &signature)?; + Ok(()) +} + +async fn read_verifying_key(key_id: &str) -> Result { + let trusted_key_id = tokio::fs::read_to_string(TRUSTED_KEY_ID).await?; + if trusted_key_id.trim() != key_id { + bail!("signing key is not currently trusted"); + } + let key = + tokio::fs::read_to_string(Path::new(TRUSTED_KEYS).join(format!("{key_id}.pub"))).await?; + let key: [u8; 32] = base64::engine::general_purpose::STANDARD + .decode(key.trim())? + .try_into() + .map_err(|_| anyhow!("invalid Ed25519 public key length"))?; + Ok(VerifyingKey::from_bytes(&key)?) +} + +fn safe_token(value: &str) -> bool { + !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +fn switch_link(target: &Path) -> Result<()> { + let link = Path::new(ACTIVE_LINK); + let temporary = link.with_extension("new"); + let _ = std::fs::remove_file(&temporary); + symlink(target, &temporary)?; + std::fs::rename(&temporary, link)?; + sync_directory(link.parent().context("active link has no parent")?) +} + +fn initialize_layout() -> Result<()> { + std::fs::create_dir_all(ROOT)?; + let link = Path::new(ACTIVE_LINK); + match std::fs::symlink_metadata(link) { + Ok(metadata) if metadata.file_type().is_symlink() => return Ok(()), + Ok(_) => std::fs::remove_file(link)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + switch_link(Path::new(BOOTSTRAP_BINARY)) +} + +async fn wait_active(timeout: Duration) -> Result<()> { + tokio::time::timeout(timeout, async { + loop { + if systemctl(&["is-active", "--quiet", "fleet-agent.service"]) + .await + .is_ok() + { + return; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }) + .await + .map_err(|_| anyhow!("fleet-agent readiness timed out"))?; + Ok(()) +} + +async fn systemctl(arguments: &[&str]) -> Result<()> { + let status = tokio::process::Command::new("systemctl") + .args(arguments) + .status() + .await?; + if !status.success() { + bail!("systemctl {} failed: {status}", arguments.join(" ")); + } + Ok(()) +} + +async fn systemctl_property(property: &str) -> Result { + let output = tokio::process::Command::new("systemctl") + .args([ + "show", + "fleet-agent.service", + "--property", + property, + "--value", + ]) + .output() + .await?; + if !output.status.success() { + bail!( + "reading fleet-agent.service {property} failed: {}", + output.status + ); + } + Ok(String::from_utf8(output.stdout)?.trim().to_string()) +} + +async fn read_transaction() -> Result { + Ok(serde_json::from_slice(&tokio::fs::read(JOURNAL).await?)?) +} + +async fn write_transaction(transaction: &Transaction) -> Result<()> { + let path = Path::new(JOURNAL); + let parent = path.parent().context("journal has no parent")?; + tokio::fs::create_dir_all(parent).await?; + let temporary = path.with_extension("tmp"); + let mut file = tokio::fs::File::create(&temporary).await?; + file.write_all(&serde_json::to_vec(transaction)?).await?; + file.sync_all().await?; + tokio::fs::rename(&temporary, path).await?; + sync_directory(parent) +} + +fn sync_directory(path: &Path) -> Result<()> { + std::fs::File::open(path)?.sync_all()?; + Ok(()) +} + +fn acquire_process_lock() -> Result { + use fs2::FileExt; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open("/run/harmony-fleet-updater/updater.lock")?; + file.try_lock_exclusive() + .context("another updater instance is already active")?; + Ok(file) +} + +fn is_not_found(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) +} + +fn bounded_error(error: &str) -> String { + error.chars().take(1024).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use harmony_reconciler_contracts::Id; + + fn attempt() -> AgentUpgradeAttempt { + AgentUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + device_id: Id::from("device-1".to_string()), + from_version: "0.1.0".into(), + target_version: "0.2.0".into(), + architecture: std::env::consts::ARCH.into(), + artifact_url: "https://example.invalid/fleet-agent".into(), + max_bytes: 20_000_000, + sha256: "a".repeat(64), + signature: "signature".into(), + signing_key_id: "production-1".into(), + created_at: Utc::now(), + } + } + + #[test] + fn attempt_validation_rejects_unsafe_inputs() { + assert!(validate_attempt(&attempt()).is_ok()); + let mut invalid = attempt(); + invalid.artifact_url = "http://example.invalid/agent".into(); + assert!(validate_attempt(&invalid).is_err()); + invalid = attempt(); + invalid.target_version = "../../bin/sh".into(); + assert!(validate_attempt(&invalid).is_err()); + } + + #[test] + fn transaction_wire_format_is_stable() { + let transaction = Transaction { + attempt_id: attempt().attempt_id, + device_id: Id::from("device-1"), + from_version: "0.1.0".into(), + phase: TransactionPhase::Probation, + previous: "/usr/lib/harmony-fleet/fleet-agent-v0.1.0".into(), + target: "/usr/lib/harmony-fleet/fleet-agent-v0.2.0".into(), + target_version: "0.2.0".into(), + attempt_digest: "digest".into(), + error: None, + }; + let encoded = serde_json::to_vec(&transaction).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + transaction + ); + } +} diff --git a/fleet/harmony-fleet-agent/src/upgrade.rs b/fleet/harmony-fleet-agent/src/upgrade.rs new file mode 100644 index 00000000..ac804a70 --- /dev/null +++ b/fleet/harmony-fleet-agent/src/upgrade.rs @@ -0,0 +1,644 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use async_nats::jetstream::kv::{Operation, Store}; +use chrono::{DateTime, Utc}; +use futures_util::StreamExt; +use harmony_reconciler_contracts::{ + AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, + BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, Id, + agent_upgrade_intent_key, agent_upgrade_status_key, +}; +use tokio::sync::Mutex; + +use crate::reconciler::Reconciler; +use crate::updater::{Transaction, TransactionPhase, UpdaterClient}; + +const AUTHORIZATION_TIMEOUT: Duration = Duration::from_secs(300); + +#[async_trait::async_trait] +trait UpgradeBackend: Send + Sync { + async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()>; + async fn switch( + &self, + authorization: &AgentUpgradeAuthorization, + ) -> Result>; + async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()>; + async fn status(&self) -> Result>; +} + +#[async_trait::async_trait] +impl UpgradeBackend for UpdaterClient { + async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.stage(attempt).await + } + + async fn switch( + &self, + authorization: &AgentUpgradeAuthorization, + ) -> Result> { + self.switch(authorization).await?; + self.status().await + } + + async fn status(&self) -> Result> { + self.status().await + } + + async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()> { + self.cancel(attempt_id, error).await + } +} + +#[async_trait::async_trait] +trait StatusPublisher: Send + Sync { + async fn publish(&self, status: &AgentUpgradeStatus) -> Result<()>; +} + +struct KvStatusPublisher { + key: String, + bucket: Store, +} + +#[async_trait::async_trait] +impl StatusPublisher for KvStatusPublisher { + async fn publish(&self, status: &AgentUpgradeStatus) -> Result<()> { + self.bucket + .put(&self.key, serde_json::to_vec(status)?.into()) + .await?; + Ok(()) + } +} + +struct ActiveAttempt { + attempt: AgentUpgradeAttempt, + authorization_deadline: DateTime, +} + +struct UpgradeController { + device_id: Id, + reconciler: Arc, + backend: Arc, + publisher: Arc, + active: Mutex>, + terminal_attempt: Mutex>, + observed_transaction: Mutex>, +} + +impl UpgradeController { + async fn accept(&self, attempt: AgentUpgradeAttempt) -> Result<()> { + validate_attempt(&attempt, &self.device_id)?; + if self.terminal_attempt.lock().await.as_deref() == Some(&attempt.attempt_id) { + return Ok(()); + } + { + let active = self.active.lock().await; + if let Some(active) = active.as_ref() { + if active.attempt.attempt_id == attempt.attempt_id { + if active.attempt == attempt { + return Ok(()); + } + bail!("upgrade attempt id was reused with different content"); + } + bail!( + "upgrade attempt '{}' is already active", + active.attempt.attempt_id + ); + } + } + + *self.active.lock().await = Some(ActiveAttempt { + attempt: attempt.clone(), + authorization_deadline: Utc::now() + + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(), + }); + self.reconciler.pause().await; + let result = async { + self.publish(&attempt, AgentUpgradePhase::Draining, None) + .await?; + self.publish(&attempt, AgentUpgradePhase::Staging, None) + .await?; + self.backend.stage(&attempt).await?; + self.publish(&attempt, AgentUpgradePhase::AwaitingAuthorization, None) + .await + } + .await; + if let Err(error) = result { + self.fail(&attempt, error.to_string()).await; + return Err(error); + } + if let Some(active) = self.active.lock().await.as_mut() { + active.authorization_deadline = + Utc::now() + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(); + } + Ok(()) + } + + async fn authorize(&self, authorization: AgentUpgradeAuthorization) -> Result<()> { + let attempt = { + let active = self.active.lock().await; + let active = active + .as_ref() + .context("no upgrade is awaiting authorization")?; + if active.attempt.attempt_id != authorization.attempt_id { + bail!("authorization does not match the active attempt"); + } + active.attempt.clone() + }; + if let Err(error) = self + .publish(&attempt, AgentUpgradePhase::Switching, None) + .await + { + self.fail(&attempt, error.to_string()).await; + return Err(error); + } + match self.backend.switch(&authorization).await { + Ok(Some(transaction)) if transaction.phase == TransactionPhase::Committed => { + self.publish(&attempt, AgentUpgradePhase::Complete, None) + .await?; + self.finish(&attempt.attempt_id).await; + Ok(()) + } + Ok(_) => Ok(()), + Err(error) => { + self.fail(&attempt, error.to_string()).await; + Err(error) + } + } + } + + async fn check_timeout(&self, now: DateTime) { + let timed_out = { + let active = self.active.lock().await; + active + .as_ref() + .filter(|active| now >= active.authorization_deadline) + .map(|active| active.attempt.clone()) + }; + if let Some(attempt) = timed_out { + self.fail(&attempt, "switch authorization timed out".into()) + .await; + } + } + + async fn recover(&self) -> Result<()> { + let Some(transaction) = self.backend.status().await? else { + return Ok(()); + }; + if self.observed_transaction.lock().await.as_ref() + == Some(&(transaction.attempt_id.clone(), transaction.phase)) + { + return Ok(()); + } + let phase = match transaction.phase { + TransactionPhase::Probation => AgentUpgradePhase::Ready, + TransactionPhase::Committed => AgentUpgradePhase::Complete, + TransactionPhase::Failed => AgentUpgradePhase::Failed, + TransactionPhase::RollbackFailed => AgentUpgradePhase::RollbackFailed, + TransactionPhase::RollingBack => return Ok(()), + TransactionPhase::Staged | TransactionPhase::Switching => return Ok(()), + }; + self.publisher + .publish(&AgentUpgradeStatus { + attempt_id: transaction.attempt_id.clone(), + current_version: env!("CARGO_PKG_VERSION").into(), + target_version: transaction.target_version, + phase, + updated_at: Utc::now(), + last_error: transaction.error, + }) + .await?; + if phase.is_terminal() { + *self.terminal_attempt.lock().await = Some(transaction.attempt_id.clone()); + } + *self.observed_transaction.lock().await = Some((transaction.attempt_id, transaction.phase)); + Ok(()) + } + + async fn fail(&self, attempt: &AgentUpgradeAttempt, error: String) { + let error = bounded_error(&error); + if let Err(cancel_error) = self.backend.cancel(&attempt.attempt_id, &error).await { + tracing::debug!(%cancel_error, "staged upgrade cancellation was not needed"); + } + if let Err(publish_error) = self + .publish(attempt, AgentUpgradePhase::Failed, Some(error)) + .await + { + tracing::warn!(%publish_error, "upgrade failure status publish failed"); + } + self.finish(&attempt.attempt_id).await; + } + + async fn finish(&self, attempt_id: &str) { + *self.terminal_attempt.lock().await = Some(attempt_id.to_string()); + *self.active.lock().await = None; + self.reconciler.resume(); + } + + async fn publish( + &self, + attempt: &AgentUpgradeAttempt, + phase: AgentUpgradePhase, + last_error: Option, + ) -> Result<()> { + self.publisher + .publish(&AgentUpgradeStatus { + attempt_id: attempt.attempt_id.clone(), + current_version: env!("CARGO_PKG_VERSION").into(), + target_version: attempt.target_version.clone(), + phase, + updated_at: Utc::now(), + last_error, + }) + .await + } +} + +pub struct UpgradeService { + controller: Arc, + intent: Store, + authorize: Store, + intent_key: String, + authorization_prefix: String, +} + +impl UpgradeService { + pub async fn connect( + client: async_nats::Client, + device_id: Id, + reconciler: Arc, + updater_socket: &str, + ) -> Result { + let jetstream = async_nats::jetstream::new(client); + let intent = jetstream.get_key_value(BUCKET_AGENT_UPGRADE_INTENT).await?; + let authorize = jetstream + .get_key_value(BUCKET_AGENT_UPGRADE_AUTHORIZE) + .await?; + let status = jetstream.get_key_value(BUCKET_AGENT_UPGRADE_STATUS).await?; + let current_status = status + .get(agent_upgrade_status_key(&device_id.to_string())) + .await? + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let controller = Arc::new(UpgradeController { + device_id: device_id.clone(), + reconciler, + backend: Arc::new(UpdaterClient::new(updater_socket)), + publisher: Arc::new(KvStatusPublisher { + key: agent_upgrade_status_key(&device_id.to_string()), + bucket: status.clone(), + }), + active: Mutex::new(None), + terminal_attempt: Mutex::new( + current_status + .as_ref() + .filter(|status| status.phase.is_terminal()) + .map(|status| status.attempt_id.clone()), + ), + observed_transaction: Mutex::new(None), + }); + let intent_key = agent_upgrade_intent_key(&device_id.to_string()); + let transaction = controller.backend.status().await?; + if let Some(transaction) = transaction.as_ref() + && transaction.phase == TransactionPhase::Staged + { + let bytes = intent + .get(&intent_key) + .await? + .context("staged upgrade has no matching durable intent")?; + let attempt: AgentUpgradeAttempt = serde_json::from_slice(&bytes)?; + validate_attempt(&attempt, &device_id)?; + if attempt.attempt_id != transaction.attempt_id + || attempt.target_version != transaction.target_version + { + bail!("staged updater transaction does not match current intent"); + } + controller.reconciler.pause().await; + let updated_at = current_status + .as_ref() + .filter(|status| { + status.attempt_id == attempt.attempt_id + && status.phase == AgentUpgradePhase::AwaitingAuthorization + }) + .map(|status| status.updated_at) + .unwrap_or_else(Utc::now); + *controller.active.lock().await = Some(ActiveAttempt { + attempt, + authorization_deadline: updated_at + + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(), + }); + } + controller.recover().await?; + let switch_in_progress = transaction.as_ref().is_some_and(|transaction| { + matches!( + transaction.phase, + TransactionPhase::Switching + | TransactionPhase::Probation + | TransactionPhase::RollingBack + ) + }); + if !switch_in_progress && let Some(bytes) = intent.get(&intent_key).await? { + let attempt = serde_json::from_slice(&bytes)?; + if let Err(error) = controller.accept(attempt).await { + tracing::warn!(%error, "current upgrade attempt rejected"); + } + } + Ok(Self { + controller, + intent, + authorize, + intent_key, + authorization_prefix: format!("{}.", device_id), + }) + } + + pub async fn run(self) -> Result<()> { + loop { + let mut intents = self.intent.watch_with_history(&self.intent_key).await?; + let mut authorizations = self + .authorize + .watch_with_history(format!("{}>", self.authorization_prefix)) + .await?; + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + entry = intents.next() => match entry { + Some(entry) => { + let entry = entry?; + if entry.operation == Operation::Put { + let attempt = serde_json::from_slice(&entry.value)?; + if let Err(error) = self.controller.accept(attempt).await { + tracing::warn!(%error, "upgrade attempt rejected"); + } + } + } + None => break, + }, + entry = authorizations.next() => match entry { + Some(entry) => { + let entry = entry?; + if entry.operation == Operation::Put + && entry.key.starts_with(&self.authorization_prefix) + { + let authorization = serde_json::from_slice(&entry.value)?; + if let Err(error) = self.controller.authorize(authorization).await { + tracing::warn!(%error, "upgrade authorization rejected"); + } + } + } + None => break, + }, + _ = ticker.tick() => { + self.controller.check_timeout(Utc::now()).await; + if let Err(error) = self.controller.recover().await { + tracing::warn!(%error, "upgrade recovery status failed"); + } + } + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + } +} + +fn validate_attempt(attempt: &AgentUpgradeAttempt, device_id: &Id) -> Result<()> { + uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid upgrade attempt id")?; + if &attempt.device_id != device_id { + bail!("upgrade attempt targets another device"); + } + if attempt.from_version != env!("CARGO_PKG_VERSION") { + bail!("upgrade source version does not match the running agent"); + } + if attempt.target_version == attempt.from_version { + bail!("upgrade target already runs on this device"); + } + if attempt.architecture != std::env::consts::ARCH { + bail!("upgrade architecture does not match this device"); + } + let now = Utc::now(); + if attempt.created_at < now - chrono::Duration::hours(24) + || attempt.created_at > now + chrono::Duration::minutes(5) + { + bail!("upgrade attempt timestamp is outside the accepted window"); + } + Ok(()) +} + +fn bounded_error(error: &str) -> String { + error.chars().take(1024).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fleet_publisher::DeploymentStatePublisher; + use crate::podman::WorkloadRuntime; + use harmony_reconciler_contracts::{DeploymentName, DeploymentState, PodmanV0Score}; + use std::collections::HashSet; + use std::sync::Mutex as StdMutex; + + #[derive(Default)] + struct Runtime; + + #[async_trait::async_trait] + impl WorkloadRuntime for Runtime { + async fn reconcile(&self, _: &str, _: &PodmanV0Score) -> Result<()> { + Ok(()) + } + async fn remove_deployment(&self, _: &str) -> Result<()> { + Ok(()) + } + async fn managed_deployments(&self) -> Result> { + Ok(HashSet::new()) + } + } + + #[derive(Default)] + struct DeploymentPublisher; + + #[async_trait::async_trait] + impl DeploymentStatePublisher for DeploymentPublisher { + async fn write(&self, _: &DeploymentState) -> Result<()> { + Ok(()) + } + async fn delete(&self, _: &DeploymentName) -> Result<()> { + Ok(()) + } + } + + #[derive(Default)] + struct Backend { + fail_stage: bool, + calls: StdMutex>, + } + + #[async_trait::async_trait] + impl UpgradeBackend for Backend { + async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.calls + .lock() + .unwrap() + .push(format!("stage:{}", attempt.attempt_id)); + if self.fail_stage { + bail!("stage failed") + } else { + Ok(()) + } + } + async fn switch( + &self, + authorization: &AgentUpgradeAuthorization, + ) -> Result> { + let attempt_id = &authorization.attempt_id; + self.calls + .lock() + .unwrap() + .push(format!("switch:{attempt_id}")); + Ok(Some(Transaction { + attempt_id: attempt_id.into(), + device_id: Id::from("device-1"), + from_version: env!("CARGO_PKG_VERSION").into(), + phase: TransactionPhase::Committed, + previous: "old".into(), + target: "fleet-agent-v0.2.0".into(), + target_version: "0.2.0".into(), + attempt_digest: "digest".into(), + error: None, + })) + } + async fn status(&self) -> Result> { + Ok(None) + } + async fn cancel(&self, attempt_id: &str, _: &str) -> Result<()> { + self.calls + .lock() + .unwrap() + .push(format!("cancel:{attempt_id}")); + Ok(()) + } + } + + #[derive(Default)] + struct Publisher(StdMutex>); + + #[async_trait::async_trait] + impl StatusPublisher for Publisher { + async fn publish(&self, status: &AgentUpgradeStatus) -> Result<()> { + self.0.lock().unwrap().push(status.clone()); + Ok(()) + } + } + + fn attempt() -> AgentUpgradeAttempt { + AgentUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + device_id: Id::from("device-1".to_string()), + from_version: env!("CARGO_PKG_VERSION").into(), + target_version: "0.2.0".into(), + architecture: std::env::consts::ARCH.into(), + artifact_url: "https://example.invalid/agent".into(), + max_bytes: 1, + sha256: "a".repeat(64), + signature: "signature".into(), + signing_key_id: "key".into(), + created_at: Utc::now(), + } + } + + fn authorization(attempt: &AgentUpgradeAttempt) -> AgentUpgradeAuthorization { + AgentUpgradeAuthorization { + attempt_id: attempt.attempt_id.clone(), + attempt_digest: attempt.digest(), + device_id: attempt.device_id.clone(), + from_version: attempt.from_version.clone(), + target_version: attempt.target_version.clone(), + artifact_signing_key_id: attempt.signing_key_id.clone(), + authorized_at: Utc::now(), + signing_key_id: "key".into(), + signature: "signature".into(), + } + } + + fn controller(backend: Arc, publisher: Arc) -> UpgradeController { + UpgradeController { + device_id: Id::from("device-1".to_string()), + reconciler: Arc::new(Reconciler::new( + Id::from("device-1".to_string()), + Arc::new(Runtime), + Some(Arc::new(DeploymentPublisher)), + None, + )), + backend, + publisher, + active: Mutex::new(None), + terminal_attempt: Mutex::new(None), + observed_transaction: Mutex::new(None), + } + } + + #[tokio::test] + async fn successful_attempt_requires_matching_authorization() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend.clone(), publisher.clone()); + let attempt = attempt(); + controller.accept(attempt.clone()).await.unwrap(); + let mut wrong = authorization(&attempt); + wrong.attempt_id = "wrong".into(); + assert!(controller.authorize(wrong).await.is_err()); + controller.authorize(authorization(&attempt)).await.unwrap(); + + assert_eq!( + backend.calls.lock().unwrap().as_slice(), + [ + format!("stage:{}", attempt.attempt_id), + format!("switch:{}", attempt.attempt_id) + ] + ); + assert_eq!( + publisher.0.lock().unwrap().last().unwrap().phase, + AgentUpgradePhase::Complete + ); + } + + #[tokio::test] + async fn stage_failure_and_authorization_timeout_resume_reconciliation() { + let publisher = Arc::new(Publisher::default()); + let failed_controller = controller( + Arc::new(Backend { + fail_stage: true, + ..Default::default() + }), + publisher.clone(), + ); + assert!(failed_controller.accept(attempt()).await.is_err()); + assert_eq!( + publisher.0.lock().unwrap().last().unwrap().phase, + AgentUpgradePhase::Failed + ); + + let controller = controller(Arc::new(Backend::default()), publisher.clone()); + let attempt = attempt(); + controller.accept(attempt).await.unwrap(); + controller + .check_timeout(Utc::now() + chrono::Duration::minutes(6)) + .await; + assert_eq!( + publisher.0.lock().unwrap().last().unwrap().phase, + AgentUpgradePhase::Failed + ); + } + + #[test] + fn wrong_device_source_and_architecture_are_rejected() { + let mut value = attempt(); + value.device_id = Id::from("other".to_string()); + assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); + value = attempt(); + value.from_version = "old".into(); + assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); + value = attempt(); + value.architecture = "wrong".into(); + assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); + } +} diff --git a/fleet/harmony-fleet-deploy/src/device_setup.rs b/fleet/harmony-fleet-deploy/src/device_setup.rs index 42504706..f79b4c5a 100644 --- a/fleet/harmony-fleet-deploy/src/device_setup.rs +++ b/fleet/harmony-fleet-deploy/src/device_setup.rs @@ -64,6 +64,10 @@ pub struct FleetDeviceSetupConfig { /// `/usr/local/bin/fleet-agent`. Future v0.1: this becomes a /// `DownloadableAsset` pointing at CI-published artifacts. pub agent_binary_path: PathBuf, + /// Ed25519 public key trusted to sign agent artifacts and switch + /// authorizations. + #[serde(default)] + pub upgrade_signing_key: Option, /// `/etc/hosts` entries to add on the device. The fleet rehearsal /// harness uses this so VMs on a libvirt NAT resolve /// `sso.fleet.local` to the host's gateway IP — without it the @@ -90,6 +94,12 @@ pub struct DeviceOpenbao { pub secret_prefix: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpgradeSigningKey { + pub id: String, + pub public_key: String, +} + /// One line in `/etc/hosts`. Order doesn't matter (the file ends up /// being a sorted dedup'd merge of these and any pre-existing /// non-managed entries). @@ -290,11 +300,15 @@ impl FleetDeviceSetupConfig { Description=IoT Agent (Harmony) After=network-online.target Wants=network-online.target +Requires=harmony-fleet-updater.service +After=harmony-fleet-updater.service [Service] Type=notify NotifyAccess=main User=fleet-agent +RuntimeDirectory=harmony-fleet-agent +RuntimeDirectoryMode=0700 Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml Environment=RUST_LOG=info ExecStart=/usr/local/bin/fleet-agent @@ -303,6 +317,29 @@ RestartSec=5 StandardOutput=journal StandardError=journal +[Install] +WantedBy=multi-user.target +"# + } + + pub fn render_updater_systemd_unit(&self) -> &'static str { + r#"[Unit] +Description=Harmony Fleet Agent Updater +Before=fleet-agent.service + +[Service] +Type=notify +NotifyAccess=main +User=root +Group=fleet-agent +RuntimeDirectory=harmony-fleet-updater +RuntimeDirectoryMode=0750 +StateDirectory=harmony-fleet-updater +StateDirectoryMode=0700 +ExecStart=/usr/lib/harmony-fleet/fleet-agent-bootstrap --updater +Restart=on-failure +RestartSec=5 + [Install] WantedBy=multi-user.target "# @@ -652,19 +689,20 @@ impl Interpret for FleetDeviceSetupInte change_count += 1; } - // 4. Binary. Ship via ansible's native copy-from-local-file + // 4. Bootstrap binary. The root updater owns the active symlink; + // normal agent upgrades never replace this privileged helper. // path (`FileSource::LocalPath`). Ansible handles binary // content over SFTP and reports `changed: true` only when the // remote file actually differs from the local one — so // re-running this Score without a new binary is a true NOOP. info!( - "[{tag}] Step 5/7 — uploading agent binary {} -> /usr/local/bin/fleet-agent", + "[{tag}] Step 5/8 — uploading agent bootstrap {}", cfg.agent_binary_path.display() ); let binary_r = FileDelivery::ensure_file( topology, &FileSpec { - path: "/usr/local/bin/fleet-agent".to_string(), + path: "/usr/lib/harmony-fleet/fleet-agent-bootstrap".to_string(), source: FileSource::LocalPath(cfg.agent_binary_path.clone()), owner: Some("root".to_string()), group: Some("root".to_string()), @@ -677,6 +715,58 @@ impl Interpret for FleetDeviceSetupInte change_count += 1; } + let mut signing_keys_changed = false; + if let Some(signing_key) = &cfg.upgrade_signing_key { + let key_id = &signing_key.id; + if key_id.is_empty() + || !key_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + return Err(InterpretError::new(format!( + "Invalid upgrade signing key id '{key_id}'" + ))); + } + let key = FileDelivery::ensure_file( + topology, + &FileSpec { + path: format!("/etc/fleet-agent/trusted-upgrade-keys/{key_id}.pub"), + source: FileSource::Content(format!("{}\n", signing_key.public_key.trim())), + owner: Some("root".to_string()), + group: Some("root".to_string()), + mode: Some(0o644), + }, + ) + .await + .map_err(wrap)?; + signing_keys_changed |= key.changed; + if key.changed { + change_count += 1; + } + } + let active_key = FileDelivery::ensure_file( + topology, + &FileSpec { + path: "/etc/fleet-agent/trusted-upgrade-key-id".to_string(), + source: FileSource::Content(format!( + "{}\n", + cfg.upgrade_signing_key + .as_ref() + .map(|key| key.id.as_str()) + .unwrap_or_default() + )), + owner: Some("root".to_string()), + group: Some("root".to_string()), + mode: Some(0o644), + }, + ) + .await + .map_err(wrap)?; + signing_keys_changed |= active_key.changed; + if active_key.changed { + change_count += 1; + } + // 5a. Drop the Zitadel machine keyfile when using JWT auth. // Order: keyfile first, then config.toml — if both are new the // agent's first systemd start finds the key already in place. @@ -730,8 +820,33 @@ impl Interpret for FleetDeviceSetupInte change_count += 1; } - // 6. systemd unit for the agent itself. - info!("[{tag}] Step 7/7 — installing fleet-agent.service"); + // 6. Root updater must be active before the agent. On first + // install it creates /usr/local/bin/fleet-agent as an atomic + // symlink to the bootstrap binary. + info!("[{tag}] Step 7/8 — installing harmony-fleet-updater.service"); + let updater_unit = SystemdUnitSpec { + name: "harmony-fleet-updater".to_string(), + unit_content: cfg.render_updater_systemd_unit().to_string(), + scope: SystemdScope::System, + start_immediately: true, + }; + let updater_unit_r = SystemdManager::ensure_systemd_unit(topology, &updater_unit) + .await + .map_err(wrap)?; + if updater_unit_r.changed { + change_count += 1; + } + if binary_r.changed || updater_unit_r.changed || signing_keys_changed { + SystemdManager::restart_service( + topology, + "harmony-fleet-updater", + SystemdScope::System, + ) + .await + .map_err(wrap)?; + } + + info!("[{tag}] Step 8/8 — installing fleet-agent.service"); let unit = SystemdUnitSpec { name: "fleet-agent".to_string(), unit_content: cfg.render_systemd_unit().to_string(), @@ -746,7 +861,12 @@ impl Interpret for FleetDeviceSetupInte } // 7. Restart the agent iff anything that affects it changed. - let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed || key_r; + let needs_restart = toml_r.changed + || unit_r.changed + || updater_unit_r.changed + || binary_r.changed + || key_r + || signing_keys_changed; let service_state = if needs_restart { info!("[{tag}] 🔄 Restarting fleet-agent (config/binary/unit changed)"); SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System) @@ -906,6 +1026,7 @@ mod tests { nats_pass: "pw".to_string(), }, agent_binary_path: PathBuf::from("/dev/null"), + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, } @@ -924,6 +1045,7 @@ mod tests { danger_accept_invalid_certs: false, }, agent_binary_path: PathBuf::from("/dev/null"), + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, } @@ -1028,6 +1150,7 @@ mod tests { danger_accept_invalid_certs: false, }, agent_binary_path: PathBuf::from("/dev/null"), + upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }; @@ -1113,8 +1236,18 @@ mod tests { #[test] fn systemd_service_is_ready_only_after_agent_notification() { - let unit = base_config(BTreeMap::new()).render_systemd_unit(); + let config = base_config(BTreeMap::new()); + let unit = config.render_systemd_unit(); assert!(unit.contains("Type=notify\n")); assert!(unit.contains("NotifyAccess=main\n")); + assert!(unit.contains("Requires=harmony-fleet-updater.service\n")); + assert!(unit.contains("RuntimeDirectory=harmony-fleet-agent\n")); + + let updater = config.render_updater_systemd_unit(); + assert!(updater.contains("User=root\n")); + assert!(updater.contains("Type=notify\n")); + assert!(updater.contains("Group=fleet-agent\n")); + assert!(updater.contains("RuntimeDirectoryMode=0750\n")); + assert!(updater.contains("fleet-agent-bootstrap --updater\n")); } } diff --git a/fleet/harmony-fleet-deploy/src/lib.rs b/fleet/harmony-fleet-deploy/src/lib.rs index 4dfdfd7a..8432c7ed 100644 --- a/fleet/harmony-fleet-deploy/src/lib.rs +++ b/fleet/harmony-fleet-deploy/src/lib.rs @@ -19,7 +19,7 @@ pub use agent::{FleetAgentScore, PodTarget}; pub use app::{FleetApp, FleetCrdsApp, FleetTenantProvisionApp}; pub use device_setup::{ AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, - HostsEntry, merge_hosts_file, + HostsEntry, UpgradeSigningKey, merge_hosts_file, }; pub use operator::{FleetCrdsScore, FleetOperatorScore, OperatorCredentials}; diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 272a9686..3244df5f 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -75,6 +75,8 @@ pub struct ChartOptions { pub identity: Option, pub identity_version: Option, pub image_pull_secret: Option, + pub upgrade_signing_key: Option, + pub upgrade_signing_key_id: Option, } #[derive(Debug, Clone, Serialize)] @@ -109,6 +111,8 @@ impl Default for ChartOptions { identity: None, identity_version: None, image_pull_secret: None, + upgrade_signing_key: None, + upgrade_signing_key_id: None, } } } @@ -141,6 +145,7 @@ pub const ENV_WEB_COOKIE_KEY: &str = "HARMONY_CONFIG_OperatorCookieKey"; /// path to the generated chart directory (which is what `helm /// install ` wants). pub fn build_chart(opts: &ChartOptions) -> Result { + validate_options(opts)?; std::fs::create_dir_all(&opts.output_dir) .with_context(|| format!("creating {:?}", opts.output_dir))?; @@ -178,6 +183,13 @@ pub fn build_chart(opts: &ChartOptions) -> Result { Ok(written) } +pub fn validate_options(opts: &ChartOptions) -> Result<()> { + if opts.upgrade_signing_key.is_some() != opts.upgrade_signing_key_id.is_some() { + anyhow::bail!("upgrade signing key and key id must be configured together"); + } + Ok(()) +} + /// Build the operator's Secret holding the `[credentials]` TOML /// (with the JSON keyfile inlined under `key_json`). Returns `None` /// when no credentials are configured (no-auth dev mode). @@ -185,6 +197,7 @@ pub fn operator_secret(opts: &ChartOptions) -> Option { if opts.credentials.is_none() && opts.web_auth_config_json.is_none() && opts.web_cookie_key_json.is_none() + && opts.upgrade_signing_key.is_none() { return None; } @@ -210,6 +223,18 @@ pub fn operator_secret(opts: &ChartOptions) -> Option { ByteString(json.as_bytes().to_vec()), ); } + if let Some(key) = &opts.upgrade_signing_key { + data.insert( + "FLEET_UPGRADE_SIGNING_KEY".into(), + ByteString(key.as_bytes().to_vec()), + ); + } + if let Some(key_id) = &opts.upgrade_signing_key_id { + data.insert( + "FLEET_UPGRADE_SIGNING_KEY_ID".into(), + ByteString(key_id.as_bytes().to_vec()), + ); + } // Namespace deliberately omitted — the caller passes the target // namespace to `K8sResourceScore::single`, which injects it at // apply time. Keeps the Secret manifest reusable across @@ -356,6 +381,8 @@ pub(crate) fn config_hash(opts: &ChartOptions) -> String { .hash(&mut secret_hash); opts.web_auth_config_json.hash(&mut secret_hash); opts.web_cookie_key_json.hash(&mut secret_hash); + opts.upgrade_signing_key.hash(&mut secret_hash); + opts.upgrade_signing_key_id.hash(&mut secret_hash); opts.identity .as_ref() .map(|identity| identity.machine.secret_name()) @@ -461,6 +488,8 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { }); env.push(secret_env(ENV_WEB_AUTH_CONFIG)); env.push(secret_env(ENV_WEB_COOKIE_KEY)); + env.push(secret_env("FLEET_UPGRADE_SIGNING_KEY")); + env.push(secret_env("FLEET_UPGRADE_SIGNING_KEY_ID")); // Secret-grant sync (OpenBao) + the device-group scheduling gate // (Zitadel role grants) — ADR-025. All optional: absent, the // operator logs and runs ungated/without grant sync. @@ -736,4 +765,19 @@ mod tests { assert_eq!(data[ENV_WEB_COOKIE_KEY].0, b"cookie"); assert!(!data.contains_key(SECRET_KEY_CREDENTIALS_TOML)); } + + #[test] + fn upgrade_signer_is_atomic_and_restarts_the_operator() { + let base_hash = config_hash(&ChartOptions::default()); + let mut configured = ChartOptions::default(); + configured.upgrade_signing_key = Some("private-key".into()); + assert!(validate_options(&configured).is_err()); + + configured.upgrade_signing_key_id = Some("production".into()); + assert!(validate_options(&configured).is_ok()); + assert_ne!(base_hash, config_hash(&configured)); + let secret = operator_secret(&configured).unwrap().data.unwrap(); + assert_eq!(secret["FLEET_UPGRADE_SIGNING_KEY"].0, b"private-key"); + assert_eq!(secret["FLEET_UPGRADE_SIGNING_KEY_ID"].0, b"production"); + } } diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index 57833e2e..8bdabfb1 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -167,6 +167,8 @@ pub struct FleetOperatorScore { pub web_auth: Option, pub identity: Option, pub image_pull_secret: Option, + pub upgrade_signing_key: Option, + pub upgrade_signing_key_id: Option, } impl FleetOperatorScore { @@ -189,6 +191,8 @@ impl FleetOperatorScore { web_auth: None, identity: None, image_pull_secret: None, + upgrade_signing_key: None, + upgrade_signing_key_id: None, } } @@ -260,6 +264,16 @@ impl FleetOperatorScore { self.log_level = level.into(); self } + + pub fn upgrade_signer( + mut self, + key_id: impl Into, + private_key: impl Into, + ) -> Self { + self.upgrade_signing_key_id = Some(key_id.into()); + self.upgrade_signing_key = Some(private_key.into()); + self + } } impl Score for FleetOperatorScore { @@ -520,7 +534,11 @@ impl Interpret for FleetOperatorInterp identity: self.score.identity.clone(), identity_version, image_pull_secret: self.score.image_pull_secret.clone(), + upgrade_signing_key: self.score.upgrade_signing_key.clone(), + upgrade_signing_key_id: self.score.upgrade_signing_key_id.clone(), }; + chart::validate_options(&chart_options) + .map_err(|e| InterpretError::new(format!("operator chart options: {e}")))?; let expected_config_hash = chart::config_hash(&chart_options); if let Some(secret) = operator_secret(&chart_options) { info!( diff --git a/fleet/harmony-fleet-e2e/src/vm/device.rs b/fleet/harmony-fleet-e2e/src/vm/device.rs index ca2c4564..f938449d 100644 --- a/fleet/harmony-fleet-e2e/src/vm/device.rs +++ b/fleet/harmony-fleet-e2e/src/vm/device.rs @@ -227,6 +227,7 @@ impl VmDevice { nats_urls: vec![opts.nats_url.clone()], auth: opts.auth.clone(), agent_binary_path: opts.agent_binary.clone(), + upgrade_signing_key: None, hosts_entries: opts.hosts_entries.clone(), openbao: opts.openbao.clone(), }); diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index 110a9aa9..76ac5505 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -253,7 +253,13 @@ async fn fleet_deployments(namespace: &str) -> anyhow::Result> { } async fn create_device(devices: &Api, name: &str) -> anyhow::Result<()> { - let device = Device::new(name, DeviceSpec { inventory: None }); + let device = Device::new( + name, + DeviceSpec { + inventory: None, + agent_upgrade: None, + }, + ); devices.create(&PostParams::default(), &device).await?; Ok(()) } diff --git a/fleet/harmony-fleet-operator/Cargo.toml b/fleet/harmony-fleet-operator/Cargo.toml index a6d98e45..cf2d7699 100644 --- a/fleet/harmony-fleet-operator/Cargo.toml +++ b/fleet/harmony-fleet-operator/Cargo.toml @@ -38,6 +38,8 @@ async-trait.workspace = true url.workspace = true base64.workspace = true reqwest.workspace = true +ed25519-dalek.workspace = true +uuid.workspace = true axum = { version = "0.8", optional = true } axum-extra = { version = "0.10", features = ["cookie", "cookie-private"], optional = true } diff --git a/fleet/harmony-fleet-operator/src/agent_upgrade.rs b/fleet/harmony-fleet-operator/src/agent_upgrade.rs new file mode 100644 index 00000000..6819181c --- /dev/null +++ b/fleet/harmony-fleet-operator/src/agent_upgrade.rs @@ -0,0 +1,275 @@ +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use base64::Engine; +use chrono::Utc; +use ed25519_dalek::{Signer, SigningKey}; +use harmony_reconciler_contracts::{ + AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, + BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, Id, + agent_upgrade_authorize_key, agent_upgrade_intent_key, agent_upgrade_status_key, +}; +use kube::api::{Api, ListParams, Patch, PatchParams}; +use kube::{Client, ResourceExt}; +use serde_json::json; + +use crate::crd::{AgentUpgradeTarget, Device, DeviceUpgradeStatus}; + +pub struct AuthorizationSigner { + key_id: String, + key: SigningKey, +} + +impl AuthorizationSigner { + pub fn from_base64(key_id: String, encoded: &str) -> Result { + let bytes = base64::engine::general_purpose::STANDARD.decode(encoded.trim())?; + let key: [u8; 32] = bytes + .try_into() + .map_err(|_| anyhow::anyhow!("upgrade signing key must contain 32 bytes"))?; + Ok(Self { + key_id, + key: SigningKey::from_bytes(&key), + }) + } + + fn authorization(&self, attempt: &AgentUpgradeAttempt) -> AgentUpgradeAuthorization { + let mut authorization = AgentUpgradeAuthorization { + attempt_id: attempt.attempt_id.clone(), + attempt_digest: attempt.digest(), + device_id: attempt.device_id.clone(), + from_version: attempt.from_version.clone(), + target_version: attempt.target_version.clone(), + artifact_signing_key_id: attempt.signing_key_id.clone(), + authorized_at: Utc::now(), + signing_key_id: self.key_id.clone(), + signature: String::new(), + }; + authorization.signature = base64::engine::general_purpose::STANDARD.encode( + self.key + .sign(authorization.signing_payload().as_bytes()) + .to_bytes(), + ); + authorization + } +} + +pub async fn run( + client: Client, + namespace: &str, + jetstream: async_nats::jetstream::Context, + signer: Option, +) -> Result<()> { + let intents = jetstream + .create_key_value(async_nats::jetstream::kv::Config { + bucket: BUCKET_AGENT_UPGRADE_INTENT.into(), + ..Default::default() + }) + .await?; + let authorizations = jetstream + .create_key_value(async_nats::jetstream::kv::Config { + bucket: BUCKET_AGENT_UPGRADE_AUTHORIZE.into(), + ..Default::default() + }) + .await?; + let statuses = jetstream + .create_key_value(async_nats::jetstream::kv::Config { + bucket: BUCKET_AGENT_UPGRADE_STATUS.into(), + ..Default::default() + }) + .await?; + let devices: Api = Api::namespaced(client, namespace); + let mut ticker = tokio::time::interval(Duration::from_secs(2)); + loop { + ticker.tick().await; + for device in devices.list(&ListParams::default()).await?.items { + if let Err(error) = reconcile_device( + &devices, + &intents, + &authorizations, + &statuses, + signer.as_ref(), + device, + ) + .await + { + tracing::warn!(device = %error.0, error = %error.1, "agent upgrade reconcile failed"); + } + } + } +} + +async fn reconcile_device( + devices: &Api, + intents: &async_nats::jetstream::kv::Store, + authorizations: &async_nats::jetstream::kv::Store, + statuses: &async_nats::jetstream::kv::Store, + signer: Option<&AuthorizationSigner>, + device: Device, +) -> std::result::Result<(), (String, anyhow::Error)> { + let id = device.name_any(); + reconcile_device_inner(devices, intents, authorizations, statuses, signer, device) + .await + .map_err(|error| (id, error)) +} + +async fn reconcile_device_inner( + devices: &Api, + intents: &async_nats::jetstream::kv::Store, + authorizations: &async_nats::jetstream::kv::Store, + statuses: &async_nats::jetstream::kv::Store, + signer: Option<&AuthorizationSigner>, + device: Device, +) -> Result<()> { + let id = device.name_any(); + let status = statuses + .get(agent_upgrade_status_key(&id)) + .await? + .map(|bytes| serde_json::from_slice::(&bytes)) + .transpose()?; + if let Some(status) = status.as_ref() { + let phase = serde_json::to_value(status.phase)? + .as_str() + .context("upgrade phase did not serialize as a string")? + .to_string(); + let reflected = DeviceUpgradeStatus { + attempt_id: status.attempt_id.clone(), + target_version: status.target_version.clone(), + phase, + updated_at: status.updated_at.to_rfc3339(), + last_error: status.last_error.clone(), + }; + if device + .status + .as_ref() + .and_then(|value| value.agent_upgrade.as_ref()) + != Some(&reflected) + { + devices + .patch_status( + &id, + &PatchParams::default(), + &Patch::Merge(&json!({ "status": { "agentUpgrade": reflected } })), + ) + .await?; + } + } + + let Some(target) = device.spec.agent_upgrade.as_ref() else { + return Ok(()); + }; + let current_version = device + .status + .as_ref() + .and_then(|status| status.current_version.as_deref()) + .context("device has not reported an agent version")?; + if current_version == target.version { + return Ok(()); + } + + let intent_key = agent_upgrade_intent_key(&id); + let existing_entry = intents.entry(&intent_key).await?; + let existing = existing_entry + .as_ref() + .filter(|entry| entry.operation == async_nats::jetstream::kv::Operation::Put) + .map(|entry| serde_json::from_slice::(&entry.value)) + .transpose()?; + let attempt = match existing { + Some(attempt) if attempt_matches(&attempt, target, current_version) => attempt, + Some(attempt) + if status.as_ref().is_some_and(|status| { + status.attempt_id == attempt.attempt_id && !status.phase.is_terminal() + }) => + { + bail!("attempt '{}' is still active", attempt.attempt_id) + } + _ => { + let attempt = AgentUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + device_id: Id::from(id.clone()), + from_version: current_version.into(), + target_version: target.version.clone(), + architecture: target.architecture.clone(), + artifact_url: target.artifact_url.clone(), + max_bytes: target.max_bytes, + sha256: target.sha256.clone(), + signature: target.signature.clone(), + signing_key_id: target.signing_key_id.clone(), + created_at: Utc::now(), + }; + let value = serde_json::to_vec(&attempt)?.into(); + if let Some(entry) = existing_entry { + intents.update(&intent_key, value, entry.revision).await?; + } else { + intents.create(&intent_key, value).await?; + } + attempt + } + }; + + if status.as_ref().is_some_and(|status| { + status.attempt_id == attempt.attempt_id + && status.phase == AgentUpgradePhase::AwaitingAuthorization + }) { + let signer = + signer.context("upgrade is ready but no authorization signer is configured")?; + if signer.key_id != attempt.signing_key_id { + bail!("authorization signer does not match artifact signing key"); + } + let key = agent_upgrade_authorize_key(&id, &attempt.attempt_id); + if authorizations.get(&key).await?.is_none() { + let authorization = signer.authorization(&attempt); + authorizations + .put(&key, serde_json::to_vec(&authorization)?.into()) + .await?; + } + } + Ok(()) +} + +fn attempt_matches( + attempt: &AgentUpgradeAttempt, + target: &AgentUpgradeTarget, + current_version: &str, +) -> bool { + attempt.from_version == current_version + && attempt.target_version == target.version + && attempt.architecture == target.architecture + && attempt.artifact_url == target.artifact_url + && attempt.max_bytes == target.max_bytes + && attempt.sha256 == target.sha256 + && attempt.signature == target.signature + && attempt.signing_key_id == target.signing_key_id +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn release_metadata_must_match_the_existing_attempt() { + let target = AgentUpgradeTarget { + version: "0.2.0".into(), + architecture: "aarch64".into(), + artifact_url: "https://example.invalid/agent".into(), + max_bytes: 10, + sha256: "digest".into(), + signature: "signature".into(), + signing_key_id: "key".into(), + }; + let attempt = AgentUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + device_id: Id::from("device"), + from_version: "0.1.0".into(), + target_version: target.version.clone(), + architecture: target.architecture.clone(), + artifact_url: target.artifact_url.clone(), + max_bytes: target.max_bytes, + sha256: target.sha256.clone(), + signature: target.signature.clone(), + signing_key_id: target.signing_key_id.clone(), + created_at: Utc::now(), + }; + assert!(attempt_matches(&attempt, &target, "0.1.0")); + assert!(!attempt_matches(&attempt, &target, "0.0.9")); + } +} diff --git a/fleet/harmony-fleet-operator/src/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs index 2672b1a6..716f4766 100644 --- a/fleet/harmony-fleet-operator/src/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -108,6 +108,20 @@ pub struct DeviceSpec { /// Rarely changes after first publish. #[serde(skip_serializing_if = "Option::is_none")] pub inventory: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_upgrade: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct AgentUpgradeTarget { + pub version: String, + pub architecture: String, + pub artifact_url: String, + pub max_bytes: u64, + pub sha256: String, + pub signature: String, + pub signing_key_id: String, } /// Operator-maintained liveness reflection of the NATS @@ -122,6 +136,18 @@ pub struct DeviceStatus { pub last_heartbeat: Option, pub reachability: Reachability, pub current_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_upgrade: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct DeviceUpgradeStatus { + pub attempt_id: String, + pub target_version: String, + pub phase: String, + pub updated_at: String, + pub last_error: Option, } /// Coarse liveness derived from heartbeat freshness. Failing/Pending diff --git a/fleet/harmony-fleet-operator/src/device_reconciler.rs b/fleet/harmony-fleet-operator/src/device_reconciler.rs index 454ba170..37c0e8d4 100644 --- a/fleet/harmony-fleet-operator/src/device_reconciler.rs +++ b/fleet/harmony-fleet-operator/src/device_reconciler.rs @@ -89,6 +89,7 @@ async fn upsert_device(api: &Api, namespace: &str, info: &DeviceInfo) -> &name, DeviceSpec { inventory: info.inventory.clone(), + agent_upgrade: None, }, ); device.metadata.namespace = Some(namespace.to_string()); diff --git a/fleet/harmony-fleet-operator/src/device_status.rs b/fleet/harmony-fleet-operator/src/device_status.rs index 81482e5d..2ef3e9f2 100644 --- a/fleet/harmony-fleet-operator/src/device_status.rs +++ b/fleet/harmony-fleet-operator/src/device_status.rs @@ -38,11 +38,13 @@ fn heartbeat_status(heartbeat: Option, now: DateTime) -> last_heartbeat: Some(heartbeat.received_at.to_rfc3339()), reachability: reachability(heartbeat.received_at, now), current_version: heartbeat.agent_version, + agent_upgrade: None, }, None => DeviceStatus { last_heartbeat: None, reachability: Reachability::Unknown, current_version: None, + agent_upgrade: None, }, } } @@ -258,6 +260,7 @@ mod tests { last_heartbeat: Some(Utc::now().to_rfc3339()), reachability: Reachability::Reachable, current_version: None, + agent_upgrade: None, }; assert!(serde_json::to_value(status).unwrap()["currentVersion"].is_null()); } diff --git a/fleet/harmony-fleet-operator/src/lib.rs b/fleet/harmony-fleet-operator/src/lib.rs index ddadb0c2..095cfdad 100644 --- a/fleet/harmony-fleet-operator/src/lib.rs +++ b/fleet/harmony-fleet-operator/src/lib.rs @@ -10,6 +10,7 @@ //! `harmony-fleet-deploy` when installing the operator. pub mod access; +pub mod agent_upgrade; pub mod commands; pub mod crd; pub mod device_reconciler; @@ -17,6 +18,7 @@ pub mod device_status; pub mod fleet_aggregator; pub use crd::{ - AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device, - DeviceSpec, DeviceStatus, Reachability, Rollout, RolloutStrategy, + AgentUpgradeTarget, AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, + DeploymentStatus, Device, DeviceSpec, DeviceStatus, DeviceUpgradeStatus, Reachability, Rollout, + RolloutStrategy, }; diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 0b791006..34173142 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -6,7 +6,7 @@ mod frontend; mod service; use harmony_fleet_operator::access::StaticDeviceGroups; -use harmony_fleet_operator::{device_reconciler, device_status, fleet_aggregator}; +use harmony_fleet_operator::{agent_upgrade, device_reconciler, device_status, fleet_aggregator}; use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource}; use harmony_secret::OpenBaoDeploymentSecretGrants; use harmony_zitadel_auth::ZitadelDeviceGroups; @@ -84,6 +84,12 @@ struct Cli { #[arg(long, env = "OPENBAO_TOKEN", global = true)] openbao_token: Option, + + #[arg(long, env = "FLEET_UPGRADE_SIGNING_KEY", global = true)] + upgrade_signing_key: Option, + + #[arg(long, env = "FLEET_UPGRADE_SIGNING_KEY_ID", global = true)] + upgrade_signing_key_id: Option, } #[derive(Subcommand)] @@ -143,6 +149,19 @@ async fn main() -> Result<()> { }; match cli.command.unwrap_or(Command::Run) { Command::Run => { + let upgrade_signer = match ( + cli.upgrade_signing_key.as_deref(), + cli.upgrade_signing_key_id.as_deref(), + ) { + (Some(key), Some(key_id)) => Some(agent_upgrade::AuthorizationSigner::from_base64( + key_id.into(), + key, + )?), + (None, None) => None, + _ => anyhow::bail!( + "FLEET_UPGRADE_SIGNING_KEY and FLEET_UPGRADE_SIGNING_KEY_ID must be set together" + ), + }; run( &cli.nats_url, &cli.kv_bucket, @@ -150,6 +169,7 @@ async fn main() -> Result<()> { &credentials_toml, &cli.openbao_url, &cli.openbao_token, + upgrade_signer, ) .await } @@ -267,6 +287,7 @@ async fn run( credentials_toml: &str, openbao_url: &Option, openbao_token: &Option, + upgrade_signer: Option, ) -> Result<()> { let nats = connect_with_retry(nats_url, credentials_toml).await?; tracing::info!(url = %nats_url, "connected to NATS"); @@ -346,10 +367,13 @@ async fn run( let dr_js = js.clone(); let ds_client = client.clone(); let ds_js = js.clone(); + let upgrade_client = client.clone(); + let upgrade_js = js.clone(); tokio::select! { r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r, r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r, r = device_status::run(ds_client, tenant_namespace, ds_js) => r, + r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js, upgrade_signer) => r, r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r, } } diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index be536885..cc2b998b 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -392,6 +392,7 @@ mod tests { last_heartbeat: None, reachability: r, current_version: None, + agent_upgrade: None, } } diff --git a/harmony-reconciler-contracts/Cargo.toml b/harmony-reconciler-contracts/Cargo.toml index 54da5ca1..3a998477 100644 --- a/harmony-reconciler-contracts/Cargo.toml +++ b/harmony-reconciler-contracts/Cargo.toml @@ -20,4 +20,5 @@ harmony_types = { path = "../harmony_types" } schemars = "0.8.22" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2.workspace = true thiserror = { workspace = true } diff --git a/harmony-reconciler-contracts/src/kv.rs b/harmony-reconciler-contracts/src/kv.rs index c494c89a..dece6b3f 100644 --- a/harmony-reconciler-contracts/src/kv.rs +++ b/harmony-reconciler-contracts/src/kv.rs @@ -32,6 +32,10 @@ pub const BUCKET_DEVICE_STATE: &str = "device-state"; /// the state bucket. Key format: `heartbeat.`. pub const BUCKET_DEVICE_HEARTBEAT: &str = "device-heartbeat"; +pub const BUCKET_AGENT_UPGRADE_INTENT: &str = "agent-upgrade-intent"; +pub const BUCKET_AGENT_UPGRADE_AUTHORIZE: &str = "agent-upgrade-authorize"; +pub const BUCKET_AGENT_UPGRADE_STATUS: &str = "agent-upgrade-status"; + /// KV key for a `(device, deployment)` pair in [`BUCKET_DESIRED_STATE`]. /// Format: `.`. pub fn desired_state_key(device_id: &str, deployment_name: &DeploymentName) -> String { @@ -66,6 +70,18 @@ pub fn desired_state_watch_filter(device_id: &str) -> String { format!("{device_id}.>") } +pub fn agent_upgrade_intent_key(device_id: &str) -> String { + device_id.to_string() +} + +pub fn agent_upgrade_authorize_key(device_id: &str, attempt_id: &str) -> String { + format!("{device_id}.{attempt_id}") +} + +pub fn agent_upgrade_status_key(device_id: &str) -> String { + device_id.to_string() +} + #[cfg(test)] mod tests { use super::*; @@ -90,6 +106,9 @@ mod tests { assert_eq!(BUCKET_DEVICE_INFO, "device-info"); assert_eq!(BUCKET_DEVICE_STATE, "device-state"); assert_eq!(BUCKET_DEVICE_HEARTBEAT, "device-heartbeat"); + assert_eq!(BUCKET_AGENT_UPGRADE_INTENT, "agent-upgrade-intent"); + assert_eq!(BUCKET_AGENT_UPGRADE_AUTHORIZE, "agent-upgrade-authorize"); + assert_eq!(BUCKET_AGENT_UPGRADE_STATUS, "agent-upgrade-status"); } #[test] @@ -100,6 +119,12 @@ mod tests { "state.pi-01.hello-web" ); assert_eq!(device_heartbeat_key("pi-01"), "heartbeat.pi-01"); + assert_eq!(agent_upgrade_intent_key("pi-01"), "pi-01"); + assert_eq!( + agent_upgrade_authorize_key("pi-01", "attempt-1"), + "pi-01.attempt-1" + ); + assert_eq!(agent_upgrade_status_key("pi-01"), "pi-01"); } #[test] diff --git a/harmony-reconciler-contracts/src/lib.rs b/harmony-reconciler-contracts/src/lib.rs index 65ba8cb2..5c64a826 100644 --- a/harmony-reconciler-contracts/src/lib.rs +++ b/harmony-reconciler-contracts/src/lib.rs @@ -22,6 +22,7 @@ pub mod fleet; pub mod kv; pub mod podman; pub mod status; +pub mod upgrade; pub use access::{DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError}; @@ -34,7 +35,9 @@ pub use fleet::{ DeploymentName, DeploymentState, DeviceInfo, HeartbeatPayload, InvalidDeploymentName, }; pub use kv::{ + BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, + agent_upgrade_authorize_key, agent_upgrade_intent_key, agent_upgrade_status_key, desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key, device_state_key, }; @@ -42,6 +45,9 @@ pub use podman::{ EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount, }; pub use status::{InventorySnapshot, Phase}; +pub use upgrade::{ + AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, +}; // Re-exports so consumers (agent, operator) don't need a direct // harmony_types dependency purely to name the cross-boundary types. diff --git a/harmony-reconciler-contracts/src/upgrade.rs b/harmony-reconciler-contracts/src/upgrade.rs new file mode 100644 index 00000000..7403b11a --- /dev/null +++ b/harmony-reconciler-contracts/src/upgrade.rs @@ -0,0 +1,136 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::Id; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentUpgradeAttempt { + pub attempt_id: String, + pub device_id: Id, + pub from_version: String, + pub target_version: String, + pub architecture: String, + pub artifact_url: String, + pub max_bytes: u64, + pub sha256: String, + pub signature: String, + pub signing_key_id: String, + pub created_at: DateTime, +} + +impl AgentUpgradeAttempt { + pub fn digest(&self) -> String { + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(self).expect("upgrade attempt is serializable")) + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AgentUpgradePhase { + Draining, + Staging, + AwaitingAuthorization, + Switching, + Ready, + Complete, + Failed, + RollbackFailed, +} + +impl AgentUpgradePhase { + pub fn is_terminal(self) -> bool { + matches!(self, Self::Complete | Self::Failed | Self::RollbackFailed) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentUpgradeStatus { + pub attempt_id: String, + pub current_version: String, + pub target_version: String, + pub phase: AgentUpgradePhase, + pub updated_at: DateTime, + pub last_error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentUpgradeAuthorization { + pub attempt_id: String, + pub attempt_digest: String, + pub device_id: Id, + pub from_version: String, + pub target_version: String, + pub artifact_signing_key_id: String, + pub authorized_at: DateTime, + pub signing_key_id: String, + pub signature: String, +} + +impl AgentUpgradeAuthorization { + pub fn signing_payload(&self) -> String { + format!( + "{}\n{}\n{}\n{}\n{}\n{}\n{}", + self.attempt_id, + self.attempt_digest, + self.device_id, + self.from_version, + self.target_version, + self.artifact_signing_key_id, + self.authorized_at.to_rfc3339() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn attempt_and_status_round_trip() { + let attempt = AgentUpgradeAttempt { + attempt_id: "550e8400-e29b-41d4-a716-446655440000".into(), + device_id: Id::from("pi-01".to_string()), + from_version: "0.1.0".into(), + target_version: "0.2.0".into(), + architecture: "aarch64".into(), + artifact_url: "https://example.invalid/fleet-agent-v0.2.0".into(), + max_bytes: 20_000_000, + sha256: "a".repeat(64), + signature: "signature".into(), + signing_key_id: "production-1".into(), + created_at: Utc::now(), + }; + let encoded = serde_json::to_vec(&attempt).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + attempt + ); + assert!(AgentUpgradePhase::Complete.is_terminal()); + assert!(!AgentUpgradePhase::Ready.is_terminal()); + + let digest = attempt.digest(); + let mut changed = attempt.clone(); + changed.target_version = "0.3.0".into(); + assert_ne!(digest, changed.digest()); + + let authorization = AgentUpgradeAuthorization { + attempt_id: attempt.attempt_id.clone(), + attempt_digest: digest.clone(), + device_id: attempt.device_id.clone(), + from_version: attempt.from_version.clone(), + target_version: attempt.target_version.clone(), + artifact_signing_key_id: attempt.signing_key_id.clone(), + authorized_at: Utc::now(), + signing_key_id: attempt.signing_key_id.clone(), + signature: "signature".into(), + }; + assert!(authorization.signing_payload().contains(&digest)); + } +} diff --git a/nats/callout/src/permissions.rs b/nats/callout/src/permissions.rs index 6e8677ce..fcdab203 100644 --- a/nats/callout/src/permissions.rs +++ b/nats/callout/src/permissions.rs @@ -69,6 +69,7 @@ impl PermissionsConfig { "$KV.device-state.state.{device_id}".to_string(), "$KV.device-state.state.{device_id}.>".to_string(), "$KV.device-heartbeat.heartbeat.{device_id}".to_string(), + "$KV.agent-upgrade-status.{device_id}".to_string(), ], deny: vec![], }, @@ -81,6 +82,8 @@ impl PermissionsConfig { // Key format: `.` (see // harmony_reconciler_contracts::kv::desired_state_key). "$KV.desired-state.{device_id}.>".to_string(), + "$KV.agent-upgrade-intent.{device_id}".to_string(), + "$KV.agent-upgrade-authorize.{device_id}.>".to_string(), ], deny: vec![], }, @@ -198,9 +201,11 @@ mod tests { #[test] fn device_role_covers_reconciler_contract_kv_subjects() { use harmony_reconciler_contracts::{ - BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, - DeploymentName, desired_state_key, device_heartbeat_key, device_info_key, - device_state_key, + BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, + BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, + BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, agent_upgrade_authorize_key, + agent_upgrade_intent_key, agent_upgrade_status_key, desired_state_key, + device_heartbeat_key, device_info_key, device_state_key, }; let device = "vm-device-00"; @@ -224,6 +229,21 @@ mod tests { BUCKET_DESIRED_STATE, desired_state_key(device, &dn) ); + let upgrade_intent_subject = format!( + "$KV.{}.{}", + BUCKET_AGENT_UPGRADE_INTENT, + agent_upgrade_intent_key(device) + ); + let upgrade_authorize_subject = format!( + "$KV.{}.{}", + BUCKET_AGENT_UPGRADE_AUTHORIZE, + agent_upgrade_authorize_key(device, "attempt-1") + ); + let upgrade_status_subject = format!( + "$KV.{}.{}", + BUCKET_AGENT_UPGRADE_STATUS, + agent_upgrade_status_key(device) + ); assert!( subject_matches_any(&info_subject, &perms.pub_allow), @@ -249,6 +269,18 @@ mod tests { sub_allow={:?}", perms.sub_allow ); + assert!(subject_matches_any( + &upgrade_intent_subject, + &perms.sub_allow + )); + assert!(subject_matches_any( + &upgrade_authorize_subject, + &perms.sub_allow + )); + assert!(subject_matches_any( + &upgrade_status_subject, + &perms.pub_allow + )); let other_info = format!("$KV.{}.{}", BUCKET_DEVICE_INFO, device_info_key(other)); let other_desired = format!( @@ -264,6 +296,14 @@ mod tests { !subject_matches_any(&other_desired, &perms.sub_allow), "cross-device subscribe to {other_desired} must NOT be allowed under device {device}'s permissions" ); + assert!(!subject_matches_any( + &format!( + "$KV.{}.{}", + BUCKET_AGENT_UPGRADE_STATUS, + agent_upgrade_status_key(other) + ), + &perms.pub_allow + )); } /// NATS-style subject match: `*` is a single token, `>` is one-or-more -- 2.39.5 From 25fc9cd4fb73cce7cd52e5103d5f004a2bbf3cbc Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 20:38:30 -0400 Subject: [PATCH 19/47] feat(fleet): harden agent upgrade transactions --- Cargo.lock | 4 - Cargo.toml | 1 - examples/fleet_device_enroll/src/main.rs | 1 - examples/fleet_rpi_setup/src/main.rs | 1 - examples/fleet_vm_setup/src/main.rs | 1 - fleet/harmony-fleet-agent/Cargo.toml | 2 - fleet/harmony-fleet-agent/src/main.rs | 280 ++-- fleet/harmony-fleet-agent/src/reconciler.rs | 19 +- fleet/harmony-fleet-agent/src/updater.rs | 846 +++++++----- fleet/harmony-fleet-agent/src/upgrade.rs | 1218 +++++++++++------ .../harmony-fleet-deploy/src/device_setup.rs | 79 +- fleet/harmony-fleet-deploy/src/lib.rs | 2 +- .../src/operator/chart.rs | 44 - .../src/operator/score.rs | 18 - fleet/harmony-fleet-e2e/src/vm/device.rs | 1 - fleet/harmony-fleet-operator/Cargo.toml | 2 - .../src/agent_upgrade.rs | 163 +-- fleet/harmony-fleet-operator/src/crd.rs | 28 +- fleet/harmony-fleet-operator/src/main.rs | 23 +- harmony-reconciler-contracts/src/kv.rs | 10 - harmony-reconciler-contracts/src/lib.rs | 12 +- harmony-reconciler-contracts/src/upgrade.rs | 86 +- nats/callout/src/permissions.rs | 15 +- 23 files changed, 1655 insertions(+), 1201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b6ef48d..302ed60b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4035,10 +4035,8 @@ dependencies = [ "anyhow", "async-nats", "async-trait", - "base64 0.22.1", "chrono", "clap", - "ed25519-dalek", "fs2", "futures-util", "harmony-fleet-auth", @@ -4154,11 +4152,9 @@ dependencies = [ "async-trait", "axum", "axum-extra", - "base64 0.22.1", "chrono", "clap", "dotenvy", - "ed25519-dalek", "futures-util", "harmony-fleet-auth", "harmony-reconciler-contracts", diff --git a/Cargo.toml b/Cargo.toml index 85004771..616522a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,7 +88,6 @@ convert_case = "0.8" chrono = "0.4" similar = "2" uuid = { version = "1.11", features = ["v4", "fast-rng", "macro-diagnostics"] } -ed25519-dalek = "2" fs2 = "0.4" pretty_assertions = "1.4.1" tempfile = "3.20.0" diff --git a/examples/fleet_device_enroll/src/main.rs b/examples/fleet_device_enroll/src/main.rs index 4472031e..b01f8251 100644 --- a/examples/fleet_device_enroll/src/main.rs +++ b/examples/fleet_device_enroll/src/main.rs @@ -301,7 +301,6 @@ async fn main() -> Result<()> { nats_urls: vec![nats_url], auth, agent_binary_path: agent_binary, - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }; diff --git a/examples/fleet_rpi_setup/src/main.rs b/examples/fleet_rpi_setup/src/main.rs index 3d3d0df6..5e572dd9 100644 --- a/examples/fleet_rpi_setup/src/main.rs +++ b/examples/fleet_rpi_setup/src/main.rs @@ -167,7 +167,6 @@ async fn main() -> Result<()> { nats_urls: vec![cli.nats_url.clone()], auth, agent_binary_path: cli.agent_binary.clone(), - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }); diff --git a/examples/fleet_vm_setup/src/main.rs b/examples/fleet_vm_setup/src/main.rs index 27217335..5ef1f31a 100644 --- a/examples/fleet_vm_setup/src/main.rs +++ b/examples/fleet_vm_setup/src/main.rs @@ -218,7 +218,6 @@ async fn main() -> Result<()> { nats_pass: cli.nats_pass.clone(), }, agent_binary_path: agent_binary, - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }); diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 29d27cdb..f127d7db 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -24,8 +24,6 @@ toml = { workspace = true } thiserror = { workspace = true } podman-api = "0.9" sd-notify = "0.4" -base64.workspace = true reqwest.workspace = true uuid.workspace = true fs2.workspace = true -ed25519-dalek.workspace = true diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index e4d830a1..6bafddd4 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -9,7 +9,7 @@ mod upgrade; use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Error, Result}; +use anyhow::{Context, Result}; use clap::Parser; use config::AgentConfig; use harmony_fleet_auth::{ @@ -20,7 +20,10 @@ use harmony_fleet_auth::{ type Creds = Arc; use futures_util::StreamExt; use harmony_reconciler_contracts::{ - BUCKET_DESIRED_STATE, Id, InventorySnapshot, desired_state_watch_filter, + BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, + BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, Id, InventorySnapshot, + agent_upgrade_intent_key, agent_upgrade_status_key, desired_state_watch_filter, + device_heartbeat_key, device_info_key, }; use crate::command_server::CommandServer; @@ -31,6 +34,8 @@ use crate::reconciler::{Reconciler, SnapshotEntry}; /// ROADMAP §5.6 — agent polls podman every 30s as ground truth; KV watch /// events are accelerators. const RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +const NATS_CONNECT_WINDOW: Duration = Duration::from_secs(3 * 60); +const NATS_CONNECT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15); #[derive(Parser)] #[command(name = "fleet-agent-v0", about = "IoT agent for Raspberry Pi devices")] @@ -67,35 +72,53 @@ fn acquire_process_lock() -> Result { async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result { let urls = &cfg.nats.urls; tracing::info!(device_id = %cfg.agent.device_id, "connecting to NATS {urls:?}"); - // The auth callback is invoked on every (re)connect, so a fresh - // Zitadel access token is minted automatically when the cached one - // is near-expiry — that's how we hold the "never lose connectivity" - // guarantee even across token rollovers and NATS pod restarts. - let client = connect_options_with_credentials(creds) - .ping_interval(Duration::from_secs(10)) - // Surface async-nats's connection lifecycle in our logs. This - // is load-bearing for ops: a device that quietly disconnects - // is exactly the failure mode we promise won't happen, and - // operators need to see the reconnect attempts to debug. - .event_callback(|event| async move { - use async_nats::Event; - match event { - Event::Connected => tracing::info!("NATS connected"), - Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"), - Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"), - Event::SlowConsumer(sid) => { - tracing::warn!(sid = %sid, "NATS slow consumer") + let started = tokio::time::Instant::now(); + let mut last_error = None; + let mut attempt = 0; + loop { + let remaining = NATS_CONNECT_WINDOW.saturating_sub(started.elapsed()); + if remaining.is_zero() { + break; + } + attempt += 1; + // The callback mints a fresh token on every connect and reconnect. + let connect = connect_options_with_credentials(creds.clone()) + .ping_interval(Duration::from_secs(10)) + .connection_timeout(NATS_CONNECT_ATTEMPT_TIMEOUT) + .event_callback(|event| async move { + use async_nats::Event; + match event { + Event::Connected => tracing::info!("NATS connected"), + Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"), + Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"), + Event::SlowConsumer(sid) => { + tracing::warn!(sid = %sid, "NATS slow consumer") + } + Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"), + Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"), + Event::Closed => tracing::error!("NATS connection closed"), + other => tracing::debug!(?other, "NATS event"), } - Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"), - Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"), - Event::Closed => tracing::error!("NATS connection closed"), - other => tracing::debug!(?other, "NATS event"), + }) + .connect(cfg.nats.urls.as_slice()); + match tokio::time::timeout(NATS_CONNECT_ATTEMPT_TIMEOUT.min(remaining), connect).await { + Ok(Ok(client)) => { + tracing::info!(urls = ?cfg.nats.urls, "connected to NATS"); + return Ok(client); } - }) - .connect(cfg.nats.urls.as_slice()) - .await?; - tracing::info!(urls = ?cfg.nats.urls, "connected to NATS"); - Ok(client) + Ok(Err(error)) => last_error = Some(error.to_string()), + Err(_) => last_error = Some("connection attempt timed out".to_string()), + } + tracing::warn!(attempt, error = %last_error.as_deref().unwrap(), "NATS connection failed; retrying"); + tokio::time::sleep( + Duration::from_secs(5).min(NATS_CONNECT_WINDOW.saturating_sub(started.elapsed())), + ) + .await; + } + anyhow::bail!( + "NATS connection failed after bounded retries: {}", + last_error.as_deref().unwrap_or("connection window expired") + ) } async fn desired_state_store( @@ -106,6 +129,36 @@ async fn desired_state_store( .await?) } +async fn probe_services( + client: &async_nats::Client, + device_id: &Id, + updater_socket: &std::path::Path, +) -> Result> { + let jetstream = async_nats::jetstream::new(client.clone()); + let desired = jetstream.get_key_value(BUCKET_DESIRED_STATE).await?; + let id = device_id.to_string(); + for (bucket, key) in [ + (BUCKET_AGENT_UPGRADE_INTENT, agent_upgrade_intent_key(&id)), + (BUCKET_AGENT_UPGRADE_STATUS, agent_upgrade_status_key(&id)), + (BUCKET_DEVICE_INFO, device_info_key(&id)), + (BUCKET_DEVICE_HEARTBEAT, device_heartbeat_key(&id)), + ] { + jetstream.get_key_value(bucket).await?.get(key).await?; + } + jetstream + .get_key_value(BUCKET_DEVICE_STATE) + .await? + .status() + .await?; + tokio::time::timeout( + Duration::from_secs(15), + updater::UpdaterClient::new(updater_socket).status(), + ) + .await + .context("updater status probe timed out")??; + load_desired_snapshot(&desired, device_id).await +} + async fn load_desired_snapshot( bucket: &async_nats::jetstream::kv::Store, device_id: &Id, @@ -360,15 +413,18 @@ async fn main() -> Result<()> { (None, _) => None, }; - let client = connect_nats(&cfg, creds).await.map_err(|e| { - let msg = format!("Nats connection FAILED : {e}"); - tracing::error!(msg); - Error::msg(msg) + let client = connect_nats(&cfg, creds).await.map_err(|error| { + tracing::error!(%error, "NATS connection failed"); + error })?; if cli.self_test { - let bucket = desired_state_store(client).await?; - load_desired_snapshot(&bucket, &device_id).await?; + let snapshot = probe_services(&client, &device_id, &cli.updater_socket).await?; + if let Some(topology) = topology { + let reconciler = Reconciler::new(device_id.clone(), topology, None, secrets); + let generation = reconciler.generation().await; + reconciler.replace_snapshot(snapshot, generation).await?; + } tracing::info!(version = env!("CARGO_PKG_VERSION"), "self-test ok"); return Ok(()); } @@ -410,32 +466,44 @@ async fn main() -> Result<()> { )) }); - let upgrade_service = match reconciler.as_ref() { - Some(reconciler) => Some( - upgrade::UpgradeService::connect( - client.clone(), - device_id.clone(), - reconciler.clone(), - cli.updater_socket - .to_str() - .context("non-UTF-8 updater socket")?, - ) - .await?, - ), - None => None, - }; - - let desired_bucket = if let Some(reconciler) = &reconciler { - let bucket = desired_state_store(client.clone()).await?; - let generation = reconciler.generation().await; - let snapshot = load_desired_snapshot(&bucket, &device_id).await?; - reconciler.replace_snapshot(snapshot, generation).await?; - reconciler.reconcile_once().await?; - Some(bucket) + let updater_socket = cli + .updater_socket + .to_str() + .context("non-UTF-8 updater socket")?; + let upgrade_service = if cli.updater_socket.exists() { + match upgrade::UpgradeService::connect(client.clone(), device_id.clone(), updater_socket) + .await + { + Ok(service) => Some(service), + Err(error) if !cfg.agent.runtime_enabled => { + tracing::warn!(%error, "updater unavailable; automatic agent upgrades disabled"); + None + } + Err(error) => return Err(error.context("connecting required fleet updater")), + } + } else if cfg.agent.runtime_enabled { + anyhow::bail!( + "required fleet updater socket '{}' is unavailable", + cli.updater_socket.display() + ); } else { + tracing::warn!( + socket = %cli.updater_socket.display(), + "updater unavailable; automatic agent upgrades disabled" + ); None }; + let desired_bucket = desired_state_store(client.clone()).await?; + let snapshot = load_desired_snapshot(&desired_bucket, &device_id).await?; + if let Some(reconciler) = &reconciler { + let generation = reconciler.generation().await; + reconciler.replace_snapshot(snapshot, generation).await?; + } + if let Some(upgrade_service) = &upgrade_service { + upgrade_service.ensure_active_startup().await?; + } + sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) .context("notifying systemd that initialization completed")?; @@ -456,49 +524,91 @@ async fn main() -> Result<()> { let _ = inventory_snapshot; // consumed by the DeviceInfo publish above let watch: std::pin::Pin> + Send>> = - match (reconciler.as_ref(), desired_bucket.as_ref()) { - (Some(r), Some(bucket)) => Box::pin(watch_desired_state( - bucket.clone(), + match reconciler.as_ref() { + Some(reconciler) => Box::pin(watch_desired_state( + desired_bucket.clone(), device_id.clone(), - r.clone(), + reconciler.clone(), )), - _ => Box::pin(async { + None => Box::pin(async { std::future::pending::<()>().await; Ok(()) }), }; let snapshots: std::pin::Pin + Send>> = - match (reconciler.as_ref(), desired_bucket) { - (Some(reconciler), Some(bucket)) => { - Box::pin(snapshot_loop(bucket, device_id.clone(), reconciler.clone())) - } + match reconciler.as_ref() { + Some(reconciler) => Box::pin(snapshot_loop( + desired_bucket, + device_id.clone(), + reconciler.clone(), + )), _ => Box::pin(std::future::pending()), }; let heartbeat = publish_heartbeat_loop(fleet); let commands = command_server.run(); - let worker: std::pin::Pin + Send>> = - match reconciler.as_ref() { - Some(reconciler) => Box::pin(reconciler.clone().run()), - None => Box::pin(std::future::pending()), - }; + let mut worker = reconciler + .as_ref() + .map(|reconciler| tokio::spawn(reconciler.clone().run())); + let worker_finished = async { + match worker.as_mut() { + Some(worker) => worker.await.context("reconciler worker task"), + None => std::future::pending().await, + } + }; let upgrades: std::pin::Pin> + Send>> = - match upgrade_service { - Some(service) => Box::pin(service.run()), - None => Box::pin(std::future::pending()), + match upgrade_service.as_ref() { + Some(service) => Box::pin(service.clone().run()), + None => Box::pin(async { + std::future::pending::<()>().await; + Ok(()) + }), }; - tokio::select! { - // Waiting on ctrlc in a select will automatically terminate other branches when - // ctrlc happens. - _ = ctrlc => {}, - r = sigterm => { r?; } - r = watch => { r?; } - _ = snapshots => {} - _ = worker => {} - r = upgrades => { r?; } - _ = heartbeat => {} - r = commands => { r?; } + enum Shutdown { + Interrupt, + Terminate, } + let signal = tokio::select! { + _ = ctrlc => Shutdown::Interrupt, + r = sigterm => { r?; Shutdown::Terminate } + r = watch => { r?; anyhow::bail!("desired-state watch exited unexpectedly") } + _ = snapshots => anyhow::bail!("desired-state snapshot loop exited unexpectedly"), + r = worker_finished => { r?; anyhow::bail!("reconciler worker exited unexpectedly") } + r = upgrades => { r?; anyhow::bail!("agent upgrade loop exited unexpectedly") } + _ = heartbeat => anyhow::bail!("heartbeat loop exited unexpectedly"), + r = commands => { r?; anyhow::bail!("command server exited unexpectedly") } + }; + + let drain_started = tokio::time::Instant::now(); + if let Some(reconciler) = &reconciler { + reconciler.drain().await; + } + let drain_duration_ms = drain_started + .elapsed() + .as_millis() + .min(u128::from(u64::MAX)) as u64; + if let Some(worker) = &worker { + worker.abort(); + } + let acknowledgement = if matches!(signal, Shutdown::Terminate) + && let Some(upgrade_service) = &upgrade_service + { + upgrade_service + .acknowledge_shutdown(drain_duration_ms) + .await + .map(|_| ()) + } else { + Ok(()) + }; + let flush = client + .flush() + .await + .context("flushing NATS during shutdown"); + let stopping = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]) + .context("notifying systemd that shutdown started"); + acknowledgement?; + flush?; + stopping?; Ok(()) } diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 432035d0..5644b486 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -158,20 +158,16 @@ impl Reconciler { Ok(()) } + #[cfg(test)] pub async fn reconcile_once(&self) -> Result<()> { self.reconcile().await } - pub async fn pause(&self) { + pub async fn drain(&self) { self.paused.store(true, Ordering::Release); let _gate = self.runtime_gate.lock().await; } - pub fn resume(&self) { - self.paused.store(false, Ordering::Release); - self.wake.notify_one(); - } - pub async fn run(self: Arc) { loop { self.wake.notified().await; @@ -204,7 +200,6 @@ impl Reconciler { } async fn reconcile(&self) -> Result<()> { - let _gate = self.runtime_gate.lock().await; if self.paused.load(Ordering::Acquire) { return Ok(()); } @@ -312,7 +307,12 @@ impl Reconciler { } } }; + let _gate = self.runtime_gate.lock().await; + if self.paused.load(Ordering::Acquire) { + return Ok(()); + } let result = self.runtime.reconcile(deployment.as_str(), &score).await; + drop(_gate); let (phase, error) = match result { Ok(()) => (Phase::Running, None), Err(error) => (Phase::Failed, Some(short(&error.to_string()))), @@ -344,7 +344,12 @@ impl Reconciler { return Ok(()); } } + let _gate = self.runtime_gate.lock().await; + if self.paused.load(Ordering::Acquire) { + return Ok(()); + } self.runtime.remove_deployment(deployment.as_str()).await?; + drop(_gate); if let Some(revision) = revision { let state = self.state.lock().await; if state.explicit_removals.get(deployment) != Some(&revision) { diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs index c644da4f..c6accc47 100644 --- a/fleet/harmony-fleet-agent/src/updater.rs +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -4,10 +4,9 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; -use base64::Engine; -use ed25519_dalek::{Signature, Verifier, VerifyingKey}; +use chrono::{DateTime, Utc}; use futures_util::StreamExt; -use harmony_reconciler_contracts::{AgentUpgradeAttempt, AgentUpgradeAuthorization}; +use harmony_reconciler_contracts::AgentUpgradeAttempt; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; @@ -18,20 +17,20 @@ const ROOT: &str = "/usr/lib/harmony-fleet"; const BOOTSTRAP_BINARY: &str = "/usr/lib/harmony-fleet/fleet-agent-bootstrap"; const ACTIVE_LINK: &str = "/usr/local/bin/fleet-agent"; const JOURNAL: &str = "/var/lib/harmony-fleet-updater/transaction.json"; -const TRUSTED_KEYS: &str = "/etc/fleet-agent/trusted-upgrade-keys"; -const TRUSTED_KEY_ID: &str = "/etc/fleet-agent/trusted-upgrade-key-id"; -const SELF_TEST_TIMEOUT: Duration = Duration::from_secs(60); -const READINESS_TIMEOUT: Duration = Duration::from_secs(60); -const PROBATION: Duration = Duration::from_secs(60); +const SELF_TEST_TIMEOUT: Duration = Duration::from_secs(4 * 60); +const STOP_TIMEOUT: Duration = Duration::from_secs(65); +const START_TIMEOUT: Duration = Duration::from_secs(4 * 60 + 15); const MAX_ARTIFACT_BYTES: u64 = 100 * 1024 * 1024; const MAX_REQUEST_BYTES: u64 = 1024 * 1024; +const MAX_TRANSITIONS: usize = 16; +const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10); +const STATUS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(15); +const UPGRADE_RESPONSE_TIMEOUT: Duration = Duration::from_secs(20 * 60); #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "operation", content = "data", rename_all = "kebab-case")] enum Request { - Stage(AgentUpgradeAttempt), - Switch(AgentUpgradeAuthorization), - Cancel { attempt_id: String, error: String }, + Upgrade(AgentUpgradeAttempt), Status, } @@ -45,26 +44,80 @@ struct Response { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum TransactionPhase { - Staged, - Switching, - Probation, + Preparing, + Activating, Committed, RollingBack, Failed, RollbackFailed, } +impl TransactionPhase { + fn recovery_stops_service(self) -> Option { + match self { + Self::Preparing => Some(false), + Self::Activating | Self::RollingBack => Some(true), + Self::Committed | Self::Failed | Self::RollbackFailed => None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct TransactionTransition { + pub phase: TransactionPhase, + pub entered_at: DateTime, + pub exited_at: Option>, + pub duration_ms: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Transaction { pub attempt_id: String, pub device_id: harmony_reconciler_contracts::Id, - pub from_version: String, pub phase: TransactionPhase, pub previous: PathBuf, pub target: PathBuf, pub target_version: String, pub attempt_digest: String, pub error: Option, + pub started_at: DateTime, + pub updated_at: DateTime, + pub transitions: Vec, +} + +impl Transaction { + fn transition(&mut self, phase: TransactionPhase) { + let now = Utc::now(); + if self.transitions.is_empty() { + self.transitions.push(TransactionTransition { + phase: self.phase, + entered_at: self.started_at, + exited_at: None, + duration_ms: None, + }); + } + if self.phase != phase { + let previous = self.transitions.last_mut().expect("transition exists"); + previous.exited_at = Some(now); + previous.duration_ms = Some( + now.signed_duration_since(previous.entered_at) + .num_milliseconds() + .max(0) as u64, + ); + self.phase = phase; + self.transitions.push(TransactionTransition { + phase, + entered_at: now, + exited_at: None, + duration_ms: None, + }); + } + if self.transitions.len() > MAX_TRANSITIONS { + let excess = self.transitions.len() - MAX_TRANSITIONS; + self.transitions.drain(..excess); + } + self.updated_at = now; + } } #[derive(Clone)] @@ -79,43 +132,38 @@ impl UpdaterClient { } } - pub async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { - self.request(Request::Stage(attempt.clone())) - .await - .map(|_| ()) - } - - pub async fn switch(&self, authorization: &AgentUpgradeAuthorization) -> Result<()> { - self.request(Request::Switch(authorization.clone())) + pub async fn upgrade(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.request(Request::Upgrade(attempt.clone()), UPGRADE_RESPONSE_TIMEOUT) .await .map(|_| ()) } pub async fn status(&self) -> Result> { - self.request(Request::Status).await + self.request(Request::Status, STATUS_RESPONSE_TIMEOUT).await } - pub async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()> { - self.request(Request::Cancel { - attempt_id: attempt_id.into(), - error: error.into(), + async fn request(&self, request: Request, timeout: Duration) -> Result> { + tokio::time::timeout(timeout, async { + let mut stream = UnixStream::connect(&self.socket).await?; + let mut payload = serde_json::to_vec(&request)?; + payload.push(b'\n'); + stream.write_all(&payload).await?; + let mut response = String::new(); + BufReader::new(stream) + .take(MAX_REQUEST_BYTES) + .read_line(&mut response) + .await?; + if response.len() as u64 == MAX_REQUEST_BYTES { + bail!("updater response exceeds {MAX_REQUEST_BYTES} bytes"); + } + let response: Response = serde_json::from_str(&response)?; + if !response.ok { + bail!(response.error.unwrap_or_else(|| "updater failed".into())); + } + Ok(response.transaction) }) .await - .map(|_| ()) - } - - async fn request(&self, request: Request) -> Result> { - let mut stream = UnixStream::connect(&self.socket).await?; - let mut payload = serde_json::to_vec(&request)?; - payload.push(b'\n'); - stream.write_all(&payload).await?; - let mut response = String::new(); - BufReader::new(stream).read_line(&mut response).await?; - let response: Response = serde_json::from_str(&response)?; - if !response.ok { - bail!(response.error.unwrap_or_else(|| "updater failed".into())); - } - Ok(response.transaction) + .map_err(|_| anyhow!("updater response timed out after {timeout:?}"))? } } @@ -123,26 +171,7 @@ pub async fn run_server(socket: &Path) -> Result<()> { let _lock = acquire_process_lock()?; initialize_layout()?; let recovery = match read_transaction().await { - Ok(transaction) - if matches!( - transaction.phase, - TransactionPhase::Switching - | TransactionPhase::Probation - | TransactionPhase::RollingBack - ) => - { - match prepare_rollback(transaction.clone()).await { - Ok(transaction) => Some(transaction), - Err(error) => { - let mut failed = transaction; - failed.phase = TransactionPhase::RollbackFailed; - failed.error = Some(format!("startup rollback preparation failed: {error}")); - write_transaction(&failed).await?; - return Err(error.context("preparing interrupted upgrade rollback")); - } - } - } - Ok(_) => None, + Ok(transaction) => prepare_startup_recovery(transaction).await?, Err(error) if is_not_found(&error) => None, Err(error) => return Err(error.context("reading updater transaction during recovery")), }; @@ -152,21 +181,31 @@ pub async fn run_server(socket: &Path) -> Result<()> { let listener = UnixListener::bind(socket)?; std::fs::set_permissions(socket, std::fs::Permissions::from_mode(0o660))?; let transaction_lock = std::sync::Arc::new(tokio::sync::Mutex::new(())); - if let Some(transaction) = recovery { - let lock = transaction_lock.clone(); + let connections = Arc::new(tokio::sync::Semaphore::new(32)); + let recovery_guard = if recovery.is_some() { + Some(transaction_lock.clone().lock_owned().await) + } else { + None + }; + sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) + .context("notifying systemd that updater socket is ready")?; + if let (Some((transaction, stop_first)), Some(guard)) = (recovery, recovery_guard) { tokio::spawn(async move { - let _guard = lock.lock().await; - if let Err(error) = recover_interrupted(transaction).await { - tracing::error!(%error, "interrupted upgrade rollback failed"); + let _guard = guard; + if let Err(error) = finish_previous(transaction, stop_first).await { + tracing::error!(%error, "interrupted upgrade recovery failed"); } }); } - sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) - .context("notifying systemd that updater socket is ready")?; loop { let (stream, _) = listener.accept().await?; let transaction_lock = transaction_lock.clone(); + let Ok(permit) = connections.clone().try_acquire_owned() else { + tracing::warn!("updater connection limit reached"); + continue; + }; tokio::spawn(async move { + let _permit = permit; if let Err(error) = handle(stream, transaction_lock).await { tracing::warn!(%error, "updater request failed"); } @@ -174,41 +213,51 @@ pub async fn run_server(socket: &Path) -> Result<()> { } } -async fn recover_interrupted(transaction: Transaction) -> Result<()> { - if let Err(error) = finish_rollback(transaction.clone()).await { - let mut failed = transaction; - failed.phase = TransactionPhase::RollbackFailed; - failed.error = Some(format!("startup rollback failed: {error}")); - write_transaction(&failed).await?; - return Err(error.context("recovering interrupted upgrade")); +async fn prepare_startup_recovery( + mut transaction: Transaction, +) -> Result> { + let Some(stop_first) = transaction.phase.recovery_stops_service() else { + return Ok(None); + }; + if stop_first { + prepare_rollback(transaction, anyhow!("upgrade interrupted before commit")) + .await + .map(|transaction| Some((transaction, true))) + } else { + if let Err(error) = switch_link(&transaction.previous) { + return Err(quarantine( + transaction, + format!("startup recovery could not restore previous symlink: {error}"), + ) + .await); + } + transaction.transition(TransactionPhase::Failed); + transaction.error = Some("upgrade interrupted during preparation".into()); + write_transaction(&transaction).await?; + Ok(Some((transaction, false))) } - Ok(()) } async fn handle(stream: UnixStream, transaction_lock: Arc>) -> Result<()> { let (reader, mut writer) = stream.into_split(); let mut line = String::new(); - BufReader::new(reader) - .take(MAX_REQUEST_BYTES) - .read_line(&mut line) - .await?; + tokio::time::timeout( + REQUEST_READ_TIMEOUT, + BufReader::new(reader) + .take(MAX_REQUEST_BYTES) + .read_line(&mut line), + ) + .await + .map_err(|_| anyhow!("updater request read timed out after {REQUEST_READ_TIMEOUT:?}"))??; if line.len() as u64 == MAX_REQUEST_BYTES { bail!("updater request exceeds {MAX_REQUEST_BYTES} bytes"); } let request: Request = serde_json::from_str(&line)?; let result = match request { - Request::Stage(attempt) => { - let _guard = transaction_lock.lock().await; - stage(&attempt).await.map(|_| None) - } - Request::Switch(authorization) => { - let _guard = transaction_lock.lock().await; - switch(&authorization).await.map(Some) - } - Request::Cancel { attempt_id, error } => { - let _guard = transaction_lock.lock().await; - cancel(&attempt_id, error).await.map(Some) - } + Request::Upgrade(attempt) => match transaction_lock.try_lock() { + Ok(_guard) => upgrade(&attempt).await.map(Some), + Err(_) => Err(anyhow!("another upgrade is already in progress")), + }, Request::Status => match read_transaction().await { Ok(transaction) => Ok(Some(transaction)), Err(error) if is_not_found(&error) => Ok(None), @@ -232,7 +281,7 @@ async fn handle(stream: UnixStream, transaction_lock: Arc Ok(()) } -async fn stage(attempt: &AgentUpgradeAttempt) -> Result<()> { +async fn upgrade(attempt: &AgentUpgradeAttempt) -> Result { validate_attempt(attempt)?; let attempt_digest = attempt.digest(); match read_transaction().await { @@ -240,212 +289,234 @@ async fn stage(attempt: &AgentUpgradeAttempt) -> Result<()> { if transaction.attempt_digest != attempt_digest { bail!("upgrade attempt id was reused with different content"); } - if transaction.phase == TransactionPhase::Staged { - return Ok(()); + match transaction.phase { + TransactionPhase::Committed => return Ok(transaction), + TransactionPhase::Failed => bail!( + "{}", + transaction + .error + .as_deref() + .unwrap_or("upgrade attempt previously failed") + ), + TransactionPhase::Preparing => {} + TransactionPhase::RollbackFailed => { + bail!("updater requires explicit repair after rollback failure") + } + phase => bail!("upgrade attempt is already {phase:?}"), } - bail!("upgrade attempt is already {:?}", transaction.phase); } - Ok(transaction) - if !matches!( - transaction.phase, - TransactionPhase::Failed - | TransactionPhase::RollbackFailed - | TransactionPhase::Committed - ) => - { - bail!( - "upgrade attempt '{}' is already active", - transaction.attempt_id - ); + Ok(transaction) if transaction.phase == TransactionPhase::RollbackFailed => { + bail!("updater requires explicit repair after rollback failure") } + Ok(transaction) if transaction.phase.recovery_stops_service().is_some() => bail!( + "upgrade attempt '{}' is already active", + transaction.attempt_id + ), Ok(_) => {} Err(error) if is_not_found(&error) => {} Err(error) => return Err(error.context("reading existing updater transaction")), } let target = target_path(&attempt.target_version)?; let previous = std::fs::read_link(ACTIVE_LINK).context("reading active agent symlink")?; - if target.exists() { - verify_file(&target, attempt).await?; - } else { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(15)) - .timeout(Duration::from_secs(300)) - .redirect(reqwest::redirect::Policy::none()) - .build()?; - let response = client - .get(&attempt.artifact_url) - .send() - .await? - .error_for_status()?; - let mut bytes = Vec::new(); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - if bytes.len() as u64 + chunk.len() as u64 > attempt.max_bytes { - bail!("artifact exceeds {} bytes", attempt.max_bytes); - } - bytes.extend_from_slice(&chunk); - } - verify_bytes(&bytes, attempt).await?; - tokio::fs::create_dir_all(ROOT).await?; - let temporary = target.with_extension("tmp"); - let mut file = tokio::fs::File::create(&temporary).await?; - file.write_all(&bytes).await?; - tokio::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o755)).await?; - file.sync_all().await?; - tokio::fs::rename(&temporary, &target).await?; - sync_directory(Path::new(ROOT))?; - } - let mut command = tokio::process::Command::new("runuser"); - command.kill_on_drop(true).args([ - "-u", - "fleet-agent", - "--", - target.to_str().context("non-UTF-8 target path")?, - "--self-test", - ]); - let status = tokio::time::timeout(SELF_TEST_TIMEOUT, command.status()) - .await - .map_err(|_| anyhow!("candidate self-test timed out after {SELF_TEST_TIMEOUT:?}"))??; - if !status.success() { - bail!("candidate self-test failed: {status}"); - } - write_transaction(&Transaction { + validate_binary_path(&previous)?; + let now = Utc::now(); + let mut transaction = Transaction { attempt_id: attempt.attempt_id.clone(), device_id: attempt.device_id.clone(), - from_version: attempt.from_version.clone(), - phase: TransactionPhase::Staged, + phase: TransactionPhase::Preparing, previous, target, target_version: attempt.target_version.clone(), attempt_digest, error: None, - }) - .await -} - -async fn switch(authorization: &AgentUpgradeAuthorization) -> Result { - verify_authorization(authorization).await?; - let mut transaction = read_transaction().await?; - if transaction.attempt_id != authorization.attempt_id - || transaction.attempt_digest != authorization.attempt_digest - || transaction.device_id != authorization.device_id - || transaction.from_version != authorization.from_version - || transaction.target_version != authorization.target_version - || authorization.artifact_signing_key_id != authorization.signing_key_id - || transaction.phase != TransactionPhase::Staged - { - bail!("attempt '{}' is not staged", authorization.attempt_id); - } - transaction.phase = TransactionPhase::Switching; - write_transaction(&transaction).await?; - switch_link(&transaction.target)?; - if let Err(error) = systemctl(&["restart", "fleet-agent.service"]).await { - return rollback_failed(transaction, error).await; - } - if let Err(error) = wait_active(READINESS_TIMEOUT).await { - return rollback_failed(transaction, error).await; - } - transaction.phase = TransactionPhase::Probation; - write_transaction(&transaction).await?; - let invocation = match systemctl_property("InvocationID").await { - Ok(invocation) => invocation, - Err(error) => return rollback_failed(transaction, error).await, + started_at: now, + updated_at: now, + transitions: vec![TransactionTransition { + phase: TransactionPhase::Preparing, + entered_at: now, + exited_at: None, + duration_ms: None, + }], }; - let deadline = tokio::time::Instant::now() + PROBATION; - while tokio::time::Instant::now() < deadline { - tokio::time::sleep(Duration::from_secs(1)).await; - if let Err(error) = systemctl(&["is-active", "--quiet", "fleet-agent.service"]).await { - return rollback_failed(transaction, error).await; - } - let current_invocation = match systemctl_property("InvocationID").await { - Ok(invocation) => invocation, - Err(error) => return rollback_failed(transaction, error).await, - }; - if current_invocation != invocation { - return rollback_failed( - transaction, - anyhow!("fleet-agent restarted during probation"), - ) - .await; - } - } - transaction.phase = TransactionPhase::Committed; write_transaction(&transaction).await?; + + let target = &transaction.target; + let preparation = async { + if target.exists() { + verify_file(target, attempt).await?; + } else { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(300)) + .redirect(reqwest::redirect::Policy::none()) + .build()?; + let response = client + .get(&attempt.artifact_url) + .send() + .await? + .error_for_status()?; + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if bytes.len() as u64 + chunk.len() as u64 > attempt.max_bytes { + bail!("artifact exceeds {} bytes", attempt.max_bytes); + } + bytes.extend_from_slice(&chunk); + } + verify_bytes(&bytes, attempt).await?; + tokio::fs::create_dir_all(ROOT).await?; + let temporary = target.with_extension("tmp"); + let mut file = tokio::fs::File::create(&temporary).await?; + file.write_all(&bytes).await?; + tokio::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o755)).await?; + file.sync_all().await?; + tokio::fs::rename(&temporary, target).await?; + sync_directory(Path::new(ROOT))?; + } + let mut command = tokio::process::Command::new("runuser"); + command.kill_on_drop(true).args([ + "-u", + "fleet-agent", + "--", + target.to_str().context("non-UTF-8 target path")?, + "--self-test", + ]); + let status = tokio::time::timeout(SELF_TEST_TIMEOUT, command.status()) + .await + .map_err(|_| anyhow!("candidate self-test timed out after {SELF_TEST_TIMEOUT:?}"))??; + if !status.success() { + bail!("candidate self-test failed: {status}"); + } + Ok(()) + } + .await; + if let Err(error) = preparation { + transaction.transition(TransactionPhase::Failed); + transaction.error = Some(bounded_error(&error.to_string())); + write_transaction(&transaction).await?; + return Err(error); + } + transaction.transition(TransactionPhase::Activating); + write_transaction(&transaction).await?; + if let Err(error) = activate(&transaction).await { + // TODO: Retry and classify external dependency readiness failures before rollback or quarantine. + let message = error.to_string(); + rollback(transaction, error).await?; + bail!(message); + } + let activating = transaction.clone(); + transaction.transition(TransactionPhase::Committed); + if let Err(error) = write_transaction(&transaction).await { + let error = error.context("persisting committed upgrade"); + let message = error.to_string(); + rollback(activating, error).await?; + bail!(message); + } Ok(transaction) } -async fn cancel(attempt_id: &str, error: String) -> Result { - let mut transaction = read_transaction().await?; - if transaction.attempt_id != attempt_id || transaction.phase != TransactionPhase::Staged { - bail!("attempt '{attempt_id}' is not staged"); +async fn activate(transaction: &Transaction) -> Result<()> { + stop_and_wait_inactive().await?; + switch_link(&transaction.target)?; + start_and_wait_ready().await +} + +async fn rollback(mut transaction: Transaction, cause: anyhow::Error) -> Result { + transaction = prepare_rollback(transaction, cause).await?; + finish_previous(transaction, true).await +} + +async fn prepare_rollback( + mut transaction: Transaction, + cause: anyhow::Error, +) -> Result { + let cause = cause.to_string(); + transaction.transition(TransactionPhase::RollingBack); + transaction.error = Some(bounded_error(&cause)); + let journal_error = write_transaction(&transaction).await.err(); + let switch_error = switch_link(&transaction.previous).err(); + if journal_error.is_some() || switch_error.is_some() { + let error = format!( + "{cause}; rollback preparation failed: journal={}; symlink={}", + journal_error + .as_ref() + .map_or_else(|| "ok".into(), ToString::to_string), + switch_error + .as_ref() + .map_or_else(|| "ok".into(), ToString::to_string) + ); + return Err(quarantine(transaction, error).await); } - transaction.phase = TransactionPhase::Failed; - transaction.error = Some(bounded_error(&error)); - write_transaction(&transaction).await?; Ok(transaction) } -async fn rollback_failed(transaction: Transaction, cause: anyhow::Error) -> Result { - match rollback(transaction.clone()).await { - Ok(mut transaction) => { - transaction.error = Some(bounded_error(&cause.to_string())); - transaction.phase = TransactionPhase::Failed; +async fn finish_previous(mut transaction: Transaction, stop_first: bool) -> Result { + let result = async { + if stop_first { + stop_and_wait_inactive().await?; + } + start_and_wait_ready().await + } + .await; + match result { + Ok(()) => { + transaction.transition(TransactionPhase::Failed); write_transaction(&transaction).await?; - Err(cause) + Ok(transaction) } Err(rollback_error) => { - let mut transaction = read_transaction().await.unwrap_or(transaction); - transaction.phase = TransactionPhase::RollbackFailed; - transaction.error = Some(bounded_error(&format!( - "{cause}; rollback failed: {rollback_error}" - ))); - write_transaction(&transaction).await?; - Err(anyhow!(transaction.error.unwrap())) + let cause = transaction + .error + .clone() + .unwrap_or_else(|| "upgrade failed".into()); + Err(quarantine( + transaction, + format!("{cause}; rollback failed: {rollback_error}"), + ) + .await) } } } -async fn rollback(mut transaction: Transaction) -> Result { - transaction = prepare_rollback(transaction).await?; - finish_rollback(transaction).await -} - -async fn prepare_rollback(mut transaction: Transaction) -> Result { - transaction.phase = TransactionPhase::RollingBack; - write_transaction(&transaction).await?; - switch_link(&transaction.previous)?; - Ok(transaction) -} - -async fn finish_rollback(mut transaction: Transaction) -> Result { - systemctl(&["restart", "fleet-agent.service"]).await?; - transaction.phase = TransactionPhase::Failed; - write_transaction(&transaction).await?; - Ok(transaction) +async fn quarantine(mut transaction: Transaction, error: String) -> anyhow::Error { + transaction.transition(TransactionPhase::RollbackFailed); + transaction.error = Some(bounded_error(&error)); + match write_transaction(&transaction).await { + Ok(()) => anyhow!(transaction.error.unwrap()), + Err(journal_error) => { + anyhow!("{error}; persisting rollback failure failed: {journal_error}") + } + } } fn validate_attempt(attempt: &AgentUpgradeAttempt) -> Result<()> { uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid attempt id")?; + if !safe_token(&attempt.device_id.to_string()) { + bail!("invalid device id"); + } if attempt.target_version.is_empty() || !attempt .target_version .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_') + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) { bail!("invalid target version"); } + target_path(&attempt.target_version)?; if attempt.architecture != std::env::consts::ARCH { bail!("artifact architecture does not match this device"); } - if !attempt.artifact_url.starts_with("https://") { + let artifact_url = + reqwest::Url::parse(&attempt.artifact_url).context("invalid artifact URL")?; + if artifact_url.scheme() != "https" || artifact_url.host_str().is_none() { bail!("artifact URL must use HTTPS"); } if attempt.max_bytes == 0 || attempt.max_bytes > MAX_ARTIFACT_BYTES { bail!("artifact size limit must be between 1 and {MAX_ARTIFACT_BYTES} bytes"); } - if !safe_token(&attempt.signing_key_id) { - bail!("invalid signing key id"); + if attempt.sha256.len() != 64 || !attempt.sha256.chars().all(|c| c.is_ascii_hexdigit()) { + bail!("invalid artifact SHA-256"); } Ok(()) } @@ -458,55 +529,23 @@ fn target_path(version: &str) -> Result { } async fn verify_file(path: &Path, attempt: &AgentUpgradeAttempt) -> Result<()> { + if tokio::fs::metadata(path).await?.len() > attempt.max_bytes { + bail!("artifact exceeds {} bytes", attempt.max_bytes); + } verify_bytes(&tokio::fs::read(path).await?, attempt).await } async fn verify_bytes(bytes: &[u8], attempt: &AgentUpgradeAttempt) -> Result<()> { + if bytes.len() as u64 > attempt.max_bytes { + bail!("artifact exceeds {} bytes", attempt.max_bytes); + } let digest = format!("{:x}", Sha256::digest(bytes)); if digest != attempt.sha256.to_ascii_lowercase() { bail!("artifact SHA-256 mismatch"); } - let key = read_verifying_key(&attempt.signing_key_id).await?; - let signature = Signature::from_slice( - &base64::engine::general_purpose::STANDARD.decode(&attempt.signature)?, - )?; - key.verify(bytes, &signature)?; Ok(()) } -async fn verify_authorization(authorization: &AgentUpgradeAuthorization) -> Result<()> { - uuid::Uuid::parse_str(&authorization.attempt_id).context("invalid authorization attempt id")?; - if !safe_token(&authorization.signing_key_id) { - bail!("invalid authorization signing key id"); - } - let now = chrono::Utc::now(); - if authorization.authorized_at < now - chrono::Duration::minutes(10) - || authorization.authorized_at > now + chrono::Duration::minutes(5) - { - bail!("authorization timestamp is outside the accepted window"); - } - let key = read_verifying_key(&authorization.signing_key_id).await?; - let signature = Signature::from_slice( - &base64::engine::general_purpose::STANDARD.decode(&authorization.signature)?, - )?; - key.verify(authorization.signing_payload().as_bytes(), &signature)?; - Ok(()) -} - -async fn read_verifying_key(key_id: &str) -> Result { - let trusted_key_id = tokio::fs::read_to_string(TRUSTED_KEY_ID).await?; - if trusted_key_id.trim() != key_id { - bail!("signing key is not currently trusted"); - } - let key = - tokio::fs::read_to_string(Path::new(TRUSTED_KEYS).join(format!("{key_id}.pub"))).await?; - let key: [u8; 32] = base64::engine::general_purpose::STANDARD - .decode(key.trim())? - .try_into() - .map_err(|_| anyhow!("invalid Ed25519 public key length"))?; - Ok(VerifyingKey::from_bytes(&key)?) -} - fn safe_token(value: &str) -> bool { !value.is_empty() && value @@ -515,6 +554,7 @@ fn safe_token(value: &str) -> bool { } fn switch_link(target: &Path) -> Result<()> { + validate_binary_path(target)?; let link = Path::new(ACTIVE_LINK); let temporary = link.with_extension("new"); let _ = std::fs::remove_file(&temporary); @@ -523,6 +563,27 @@ fn switch_link(target: &Path) -> Result<()> { sync_directory(link.parent().context("active link has no parent")?) } +fn validate_binary_path(path: &Path) -> Result<()> { + if path == Path::new(BOOTSTRAP_BINARY) { + return Ok(()); + } + let version = path + .strip_prefix(ROOT) + .ok() + .and_then(|path| path.to_str()) + .and_then(|path| path.strip_prefix("fleet-agent-v")); + if version.is_none_or(|version| { + version.is_empty() + || version.contains('/') + || !version + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) + }) { + bail!("agent binary path is outside the managed layout"); + } + Ok(()) +} + fn initialize_layout() -> Result<()> { std::fs::create_dir_all(ROOT)?; let link = Path::new(ACTIVE_LINK); @@ -535,25 +596,55 @@ fn initialize_layout() -> Result<()> { switch_link(Path::new(BOOTSTRAP_BINARY)) } -async fn wait_active(timeout: Duration) -> Result<()> { - tokio::time::timeout(timeout, async { - loop { - if systemctl(&["is-active", "--quiet", "fleet-agent.service"]) - .await - .is_ok() - { - return; - } - tokio::time::sleep(Duration::from_secs(1)).await; - } +async fn stop_and_wait_inactive() -> Result<()> { + tokio::time::timeout(STOP_TIMEOUT, async { + systemctl(&["stop", "fleet-agent.service"]).await?; + wait_systemd_state("inactive").await }) .await - .map_err(|_| anyhow!("fleet-agent readiness timed out"))?; - Ok(()) + .map_err(|_| anyhow!("fleet-agent stop timed out after {STOP_TIMEOUT:?}"))? +} + +async fn start_and_wait_ready() -> Result<()> { + tokio::time::timeout(START_TIMEOUT, async { + systemctl(&["start", "fleet-agent.service"]).await?; + wait_ready().await + }) + .await + .map_err(|_| anyhow!("fleet-agent startup timed out after {START_TIMEOUT:?}"))? +} + +async fn wait_systemd_state(expected: &str) -> Result<()> { + loop { + if matches!( + systemctl_property("ActiveState").await.as_deref(), + Ok(state) if state == expected + ) { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +async fn wait_ready() -> Result<()> { + loop { + if matches!( + systemctl_property("ActiveState").await.as_deref(), + Ok("active") + ) && matches!( + systemctl_property("SubState").await.as_deref(), + Ok("running") + ) && matches!(systemctl_property("Result").await.as_deref(), Ok("success")) + { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } } async fn systemctl(arguments: &[&str]) -> Result<()> { let status = tokio::process::Command::new("systemctl") + .kill_on_drop(true) .args(arguments) .status() .await?; @@ -565,6 +656,7 @@ async fn systemctl(arguments: &[&str]) -> Result<()> { async fn systemctl_property(property: &str) -> Result { let output = tokio::process::Command::new("systemctl") + .kill_on_drop(true) .args([ "show", "fleet-agent.service", @@ -637,46 +729,164 @@ mod tests { AgentUpgradeAttempt { attempt_id: uuid::Uuid::new_v4().to_string(), device_id: Id::from("device-1".to_string()), - from_version: "0.1.0".into(), target_version: "0.2.0".into(), architecture: std::env::consts::ARCH.into(), artifact_url: "https://example.invalid/fleet-agent".into(), max_bytes: 20_000_000, sha256: "a".repeat(64), - signature: "signature".into(), - signing_key_id: "production-1".into(), - created_at: Utc::now(), } } #[test] - fn attempt_validation_rejects_unsafe_inputs() { + fn upgrade_rejects_invalid_identity_artifact_and_path_inputs() { assert!(validate_attempt(&attempt()).is_ok()); - let mut invalid = attempt(); - invalid.artifact_url = "http://example.invalid/agent".into(); - assert!(validate_attempt(&invalid).is_err()); - invalid = attempt(); - invalid.target_version = "../../bin/sh".into(); - assert!(validate_attempt(&invalid).is_err()); + for invalid in [ + { + let mut attempt = attempt(); + attempt.attempt_id = "not-a-uuid".into(); + attempt + }, + { + let mut attempt = attempt(); + attempt.device_id = Id::empty(); + attempt + }, + { + let mut attempt = attempt(); + attempt.target_version = "../../bin/sh".into(); + attempt + }, + { + let mut attempt = attempt(); + attempt.architecture = "wrong-architecture".into(); + attempt + }, + { + let mut attempt = attempt(); + attempt.artifact_url = "http://example.invalid/agent".into(); + attempt + }, + { + let mut attempt = attempt(); + attempt.max_bytes = 0; + attempt + }, + { + let mut attempt = attempt(); + attempt.sha256 = "not-a-digest".into(); + attempt + }, + ] { + assert!(validate_attempt(&invalid).is_err(), "accepted {invalid:?}"); + } } #[test] - fn transaction_wire_format_is_stable() { - let transaction = Transaction { + fn updater_wire_protocol_only_accepts_upgrade_and_status() { + let encoded = serde_json::to_value(Request::Upgrade(attempt())).unwrap(); + assert_eq!(encoded["operation"], "upgrade"); + assert!(matches!( + serde_json::from_value::(encoded).unwrap(), + Request::Upgrade(_) + )); + assert!(matches!( + serde_json::from_value::(serde_json::json!({ "operation": "status" })) + .unwrap(), + Request::Status + )); + for removed in ["stage", "switch", "cancel"] { + assert!( + serde_json::from_value::(serde_json::json!({ "operation": removed })) + .is_err() + ); + } + } + + #[test] + fn journal_phases_and_recovery_match_the_transaction_contract() { + let phases = [ + TransactionPhase::Preparing, + TransactionPhase::Activating, + TransactionPhase::Committed, + TransactionPhase::RollingBack, + TransactionPhase::Failed, + TransactionPhase::RollbackFailed, + ]; + assert_eq!( + serde_json::to_value(phases).unwrap(), + serde_json::json!([ + "preparing", + "activating", + "committed", + "rolling-back", + "failed", + "rollback-failed" + ]) + ); + assert_eq!( + phases.map(TransactionPhase::recovery_stops_service), + [Some(false), Some(true), None, Some(true), None, None] + ); + + let now = Utc::now(); + let mut transaction = Transaction { attempt_id: attempt().attempt_id, device_id: Id::from("device-1"), - from_version: "0.1.0".into(), - phase: TransactionPhase::Probation, + phase: TransactionPhase::Preparing, previous: "/usr/lib/harmony-fleet/fleet-agent-v0.1.0".into(), target: "/usr/lib/harmony-fleet/fleet-agent-v0.2.0".into(), target_version: "0.2.0".into(), attempt_digest: "digest".into(), error: None, + started_at: now, + updated_at: now, + transitions: vec![TransactionTransition { + phase: TransactionPhase::Preparing, + entered_at: now, + exited_at: None, + duration_ms: None, + }], }; - let encoded = serde_json::to_vec(&transaction).unwrap(); + transaction.transition(TransactionPhase::Activating); + transaction.transition(TransactionPhase::RollingBack); + transaction.transition(TransactionPhase::Failed); assert_eq!( - serde_json::from_slice::(&encoded).unwrap(), transaction + .transitions + .iter() + .map(|transition| transition.phase) + .collect::>(), + [ + TransactionPhase::Preparing, + TransactionPhase::Activating, + TransactionPhase::RollingBack, + TransactionPhase::Failed + ] + ); + assert!(transaction.transitions[..3].iter().all(|transition| { + transition.exited_at.is_some() && transition.duration_ms.is_some() + })); + assert_eq!(transaction.transitions[3].exited_at, None); + assert!(transaction.updated_at >= transaction.started_at); + + let encoded = serde_json::to_value(&transaction).unwrap(); + assert!(encoded.get("started_at").is_some()); + assert!(encoded.get("updated_at").is_some()); + assert!(encoded["transitions"][0].get("entered_at").is_some()); + assert!(encoded["transitions"][0].get("exited_at").is_some()); + assert!(encoded["transitions"][0].get("duration_ms").is_some()); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + transaction + ); + + for phase in [TransactionPhase::Preparing, TransactionPhase::Failed].repeat(9) { + transaction.transition(phase); + } + assert_eq!(transaction.transitions.len(), MAX_TRANSITIONS); + assert_eq!( + transaction.transitions.last().unwrap().phase, + transaction.phase ); } } diff --git a/fleet/harmony-fleet-agent/src/upgrade.rs b/fleet/harmony-fleet-agent/src/upgrade.rs index ac804a70..40c0827f 100644 --- a/fleet/harmony-fleet-agent/src/upgrade.rs +++ b/fleet/harmony-fleet-agent/src/upgrade.rs @@ -3,52 +3,37 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use async_nats::jetstream::kv::{Operation, Store}; -use chrono::{DateTime, Utc}; +use chrono::Utc; use futures_util::StreamExt; use harmony_reconciler_contracts::{ - AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, - BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, Id, - agent_upgrade_intent_key, agent_upgrade_status_key, + AgentUpgradeAttempt, AgentUpgradeJournalRef, AgentUpgradePhase, AgentUpgradeReason, + AgentUpgradeStatus, AgentUpgradeTransition, BUCKET_AGENT_UPGRADE_INTENT, + BUCKET_AGENT_UPGRADE_STATUS, Id, agent_upgrade_intent_key, agent_upgrade_status_key, }; use tokio::sync::Mutex; -use crate::reconciler::Reconciler; use crate::updater::{Transaction, TransactionPhase, UpdaterClient}; -const AUTHORIZATION_TIMEOUT: Duration = Duration::from_secs(300); +const TRANSITION_LIMIT: usize = 16; +const TEXT_LIMIT: usize = 1024; +const AGENT_UNIT: &str = "fleet-agent.service"; +const UPDATER_UNIT: &str = "harmony-fleet-updater.service"; #[async_trait::async_trait] trait UpgradeBackend: Send + Sync { - async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()>; - async fn switch( - &self, - authorization: &AgentUpgradeAuthorization, - ) -> Result>; - async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()>; + async fn upgrade(&self, attempt: &AgentUpgradeAttempt) -> Result<()>; async fn status(&self) -> Result>; } #[async_trait::async_trait] impl UpgradeBackend for UpdaterClient { - async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { - self.stage(attempt).await - } - - async fn switch( - &self, - authorization: &AgentUpgradeAuthorization, - ) -> Result> { - self.switch(authorization).await?; - self.status().await + async fn upgrade(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.upgrade(attempt).await } async fn status(&self) -> Result> { self.status().await } - - async fn cancel(&self, attempt_id: &str, error: &str) -> Result<()> { - self.cancel(attempt_id, error).await - } } #[async_trait::async_trait] @@ -71,326 +56,488 @@ impl StatusPublisher for KvStatusPublisher { } } -struct ActiveAttempt { - attempt: AgentUpgradeAttempt, - authorization_deadline: DateTime, +#[derive(Default)] +struct ControllerState { + attempt: Option, + status: Option, + observed_transaction: Option<(String, TransactionPhase)>, } struct UpgradeController { device_id: Id, - reconciler: Arc, backend: Arc, publisher: Arc, - active: Mutex>, - terminal_attempt: Mutex>, - observed_transaction: Mutex>, + state: Mutex, +} + +struct StatusUpdate<'a> { + attempt_id: &'a str, + target_version: &'a str, + phase: AgentUpgradePhase, + reason: AgentUpgradeReason, + error: Option<&'a str>, + drain_duration_ms: Option, + transaction: Option<&'a Transaction>, } impl UpgradeController { async fn accept(&self, attempt: AgentUpgradeAttempt) -> Result<()> { - validate_attempt(&attempt, &self.device_id)?; - if self.terminal_attempt.lock().await.as_deref() == Some(&attempt.attempt_id) { - return Ok(()); - } + validate_intent(&attempt, &self.device_id)?; + if let Some(current) = self.state.lock().await.attempt.as_ref() + && current.attempt_id == attempt.attempt_id { - let active = self.active.lock().await; - if let Some(active) = active.as_ref() { - if active.attempt.attempt_id == attempt.attempt_id { - if active.attempt == attempt { - return Ok(()); - } - bail!("upgrade attempt id was reused with different content"); - } + if current == &attempt { + return Ok(()); + } + bail!("upgrade attempt id was reused with different content"); + } + + if let Some(transaction) = self.backend.status().await? { + if transaction.attempt_id == attempt.attempt_id { + validate_transaction(&transaction, &attempt)?; + self.state.lock().await.attempt = Some(attempt); + return self + .reflect(transaction, AgentUpgradeReason::Recovery) + .await; + } + if matches!( + transaction.phase, + TransactionPhase::Preparing + | TransactionPhase::Activating + | TransactionPhase::RollingBack + ) { bail!( "upgrade attempt '{}' is already active", - active.attempt.attempt_id + transaction.attempt_id ); } } - *self.active.lock().await = Some(ActiveAttempt { - attempt: attempt.clone(), - authorization_deadline: Utc::now() - + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(), - }); - self.reconciler.pause().await; - let result = async { - self.publish(&attempt, AgentUpgradePhase::Draining, None) - .await?; - self.publish(&attempt, AgentUpgradePhase::Staging, None) - .await?; - self.backend.stage(&attempt).await?; - self.publish(&attempt, AgentUpgradePhase::AwaitingAuthorization, None) - .await - } - .await; - if let Err(error) = result { - self.fail(&attempt, error.to_string()).await; + self.publish(StatusUpdate { + attempt_id: &attempt.attempt_id, + target_version: &attempt.target_version, + phase: AgentUpgradePhase::Preparing, + reason: AgentUpgradeReason::Intent, + error: None, + drain_duration_ms: None, + transaction: None, + }) + .await?; + self.state.lock().await.attempt = Some(attempt.clone()); + if let Err(error) = self.backend.upgrade(&attempt).await { + let message = error.to_string(); + self.publish(StatusUpdate { + attempt_id: &attempt.attempt_id, + target_version: &attempt.target_version, + phase: AgentUpgradePhase::Failed, + reason: AgentUpgradeReason::Error, + error: Some(&message), + drain_duration_ms: None, + transaction: None, + }) + .await?; return Err(error); } - if let Some(active) = self.active.lock().await.as_mut() { - active.authorization_deadline = - Utc::now() + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(); + self.recover(AgentUpgradeReason::Updater).await + } + + async fn recover(&self, reason: AgentUpgradeReason) -> Result<()> { + if let Some(transaction) = self.backend.status().await? { + self.reflect(transaction, reason).await?; } Ok(()) } - async fn authorize(&self, authorization: AgentUpgradeAuthorization) -> Result<()> { - let attempt = { - let active = self.active.lock().await; - let active = active - .as_ref() - .context("no upgrade is awaiting authorization")?; - if active.attempt.attempt_id != authorization.attempt_id { - bail!("authorization does not match the active attempt"); - } - active.attempt.clone() - }; - if let Err(error) = self - .publish(&attempt, AgentUpgradePhase::Switching, None) - .await - { - self.fail(&attempt, error.to_string()).await; - return Err(error); + async fn reflect(&self, transaction: Transaction, reason: AgentUpgradeReason) -> Result<()> { + if transaction.device_id != self.device_id { + bail!("updater transaction targets another device"); } - match self.backend.switch(&authorization).await { - Ok(Some(transaction)) if transaction.phase == TransactionPhase::Committed => { - self.publish(&attempt, AgentUpgradePhase::Complete, None) - .await?; - self.finish(&attempt.attempt_id).await; - Ok(()) - } - Ok(_) => Ok(()), - Err(error) => { - self.fail(&attempt, error.to_string()).await; - Err(error) - } - } - } - - async fn check_timeout(&self, now: DateTime) { - let timed_out = { - let active = self.active.lock().await; - active - .as_ref() - .filter(|active| now >= active.authorization_deadline) - .map(|active| active.attempt.clone()) - }; - if let Some(attempt) = timed_out { - self.fail(&attempt, "switch authorization timed out".into()) - .await; - } - } - - async fn recover(&self) -> Result<()> { - let Some(transaction) = self.backend.status().await? else { - return Ok(()); - }; - if self.observed_transaction.lock().await.as_ref() + if self.state.lock().await.observed_transaction.as_ref() == Some(&(transaction.attempt_id.clone(), transaction.phase)) { return Ok(()); } - let phase = match transaction.phase { - TransactionPhase::Probation => AgentUpgradePhase::Ready, - TransactionPhase::Committed => AgentUpgradePhase::Complete, - TransactionPhase::Failed => AgentUpgradePhase::Failed, - TransactionPhase::RollbackFailed => AgentUpgradePhase::RollbackFailed, - TransactionPhase::RollingBack => return Ok(()), - TransactionPhase::Staged | TransactionPhase::Switching => return Ok(()), - }; - self.publisher - .publish(&AgentUpgradeStatus { - attempt_id: transaction.attempt_id.clone(), - current_version: env!("CARGO_PKG_VERSION").into(), - target_version: transaction.target_version, - phase, - updated_at: Utc::now(), - last_error: transaction.error, - }) - .await?; - if phase.is_terminal() { - *self.terminal_attempt.lock().await = Some(transaction.attempt_id.clone()); + if transaction.phase == TransactionPhase::Activating + && transaction.target_version != env!("CARGO_PKG_VERSION") + { + return Ok(()); } - *self.observed_transaction.lock().await = Some((transaction.attempt_id, transaction.phase)); + let phase = public_phase(transaction.phase); + self.publish(StatusUpdate { + attempt_id: &transaction.attempt_id, + target_version: &transaction.target_version, + phase, + reason, + error: transaction.error.as_deref(), + drain_duration_ms: None, + transaction: Some(&transaction), + }) + .await?; + self.state.lock().await.observed_transaction = + Some((transaction.attempt_id, transaction.phase)); Ok(()) } - async fn fail(&self, attempt: &AgentUpgradeAttempt, error: String) { - let error = bounded_error(&error); - if let Err(cancel_error) = self.backend.cancel(&attempt.attempt_id, &error).await { - tracing::debug!(%cancel_error, "staged upgrade cancellation was not needed"); + async fn acknowledge_shutdown(&self, drain_duration_ms: u64) -> Result { + let Some(transaction) = self.backend.status().await? else { + return Ok(false); + }; + if transaction.phase != TransactionPhase::Activating { + return Ok(false); } - if let Err(publish_error) = self - .publish(attempt, AgentUpgradePhase::Failed, Some(error)) - .await + self.publish(StatusUpdate { + attempt_id: &transaction.attempt_id, + target_version: &transaction.target_version, + phase: AgentUpgradePhase::Stopping, + reason: AgentUpgradeReason::Updater, + error: None, + drain_duration_ms: Some(drain_duration_ms), + transaction: None, + }) + .await?; + Ok(true) + } + + async fn ensure_active_startup(&self) -> Result { + let Some(transaction) = self.backend.status().await? else { + return Ok(false); + }; + let phase = match transaction.phase { + TransactionPhase::Activating => { + if transaction.target_version != env!("CARGO_PKG_VERSION") { + bail!( + "activating updater transaction expects version {}, but running agent is {}", + transaction.target_version, + env!("CARGO_PKG_VERSION") + ); + } + AgentUpgradePhase::Starting + } + TransactionPhase::RollingBack => AgentUpgradePhase::RollingBack, + _ => return Ok(false), + }; + self.publish(StatusUpdate { + attempt_id: &transaction.attempt_id, + target_version: &transaction.target_version, + phase, + reason: AgentUpgradeReason::Ready, + error: transaction.error.as_deref(), + drain_duration_ms: None, + transaction: Some(&transaction), + }) + .await?; + self.state.lock().await.observed_transaction = + Some((transaction.attempt_id, transaction.phase)); + Ok(true) + } + + async fn publish(&self, update: StatusUpdate<'_>) -> Result<()> { + let now = Utc::now(); + let mut state = self.state.lock().await; + let mut status = state + .status + .as_ref() + .filter(|status| status.attempt_id == update.attempt_id) + .cloned() + .unwrap_or_else(|| AgentUpgradeStatus { + attempt_id: update.attempt_id.into(), + target_version: update.target_version.into(), + phase: update.phase, + started_at: now, + updated_at: now, + reason: None, + detail: None, + error: None, + drain_duration_ms: None, + boot_id: boot_id(), + invocation_id: std::env::var("INVOCATION_ID") + .ok() + .filter(|id| !id.is_empty()), + journal: None, + transitions: Vec::new(), + }); + if let Some(transaction) = update.transaction { + status.started_at = status.started_at.min(transaction.started_at); + for transition in &transaction.transitions { + // Activation starts before systemd stops the old process. The old and new + // agents report the narrower stopping/starting phases at their actual times. + if transition.phase == TransactionPhase::Activating { + continue; + } + let transition = AgentUpgradeTransition { + phase: public_phase(transition.phase), + entered_at: transition.entered_at, + exited_at: transition.exited_at, + duration_ms: transition.duration_ms, + }; + if let Some(existing) = status + .transitions + .iter_mut() + .find(|existing| existing.phase == transition.phase) + { + existing.entered_at = existing.entered_at.min(transition.entered_at); + existing.exited_at = transition.exited_at.or(existing.exited_at); + existing.duration_ms = existing.exited_at.map(|exited_at| { + (exited_at - existing.entered_at).num_milliseconds().max(0) as u64 + }); + } else { + status.transitions.push(transition); + } + } + status + .transitions + .sort_by_key(|transition| transition.entered_at); + if transaction.phase != TransactionPhase::Activating + && let Some(current) = transaction + .transitions + .iter() + .rev() + .find(|transition| transition.phase == transaction.phase) + && let Some(index) = status + .transitions + .iter() + .position(|transition| transition.phase == update.phase) + && index > 0 + && status.transitions[index - 1].exited_at.is_none() + { + let previous = &mut status.transitions[index - 1]; + previous.exited_at = Some(current.entered_at); + previous.duration_ms = Some( + (current.entered_at - previous.entered_at) + .num_milliseconds() + .max(0) as u64, + ); + } + } + let durable_current = update.transaction.is_some() + && status + .transitions + .iter() + .any(|transition| transition.phase == update.phase); + if !durable_current + && status.transitions.last().map(|transition| transition.phase) != Some(update.phase) { - tracing::warn!(%publish_error, "upgrade failure status publish failed"); + if let Some(previous) = status.transitions.last_mut() + && previous.exited_at.is_none() + { + previous.exited_at = Some(now); + previous.duration_ms = Some( + (now - previous.entered_at) + .num_milliseconds() + .max(0) + .try_into() + .unwrap_or(u64::MAX), + ); + } + status.transitions.push(AgentUpgradeTransition { + phase: update.phase, + entered_at: now, + exited_at: None, + duration_ms: None, + }); } - self.finish(&attempt.attempt_id).await; - } - - async fn finish(&self, attempt_id: &str) { - *self.terminal_attempt.lock().await = Some(attempt_id.to_string()); - *self.active.lock().await = None; - self.reconciler.resume(); - } - - async fn publish( - &self, - attempt: &AgentUpgradeAttempt, - phase: AgentUpgradePhase, - last_error: Option, - ) -> Result<()> { - self.publisher - .publish(&AgentUpgradeStatus { - attempt_id: attempt.attempt_id.clone(), - current_version: env!("CARGO_PKG_VERSION").into(), - target_version: attempt.target_version.clone(), - phase, - updated_at: Utc::now(), - last_error, - }) - .await + if status.transitions.len() > TRANSITION_LIMIT { + status + .transitions + .drain(..status.transitions.len() - TRANSITION_LIMIT); + } + status.target_version = update.target_version.into(); + status.phase = update.phase; + status.updated_at = now; + status.reason = Some(update.reason); + status.detail = Some(bounded_text(match (update.phase, update.reason) { + (AgentUpgradePhase::Preparing, AgentUpgradeReason::Intent) => "upgrade intent accepted", + (AgentUpgradePhase::Stopping, _) => "updater requested service stop for activation", + (AgentUpgradePhase::Starting, AgentUpgradeReason::Ready) => { + "active target initialized before systemd readiness" + } + _ => "status restored from updater transaction", + })); + status.error = update.error.map(bounded_text); + if update.drain_duration_ms.is_some() { + status.drain_duration_ms = update.drain_duration_ms; + } + status.boot_id = boot_id(); + status.invocation_id = std::env::var("INVOCATION_ID") + .ok() + .filter(|id| !id.is_empty()); + status.journal = Some(AgentUpgradeJournalRef { + unit: if matches!( + update.phase, + AgentUpgradePhase::Stopping | AgentUpgradePhase::Starting + ) { + AGENT_UNIT + } else { + UPDATER_UNIT + } + .into(), + since: status + .transitions + .iter() + .rev() + .find(|transition| transition.phase == update.phase) + .map(|transition| transition.entered_at) + .unwrap_or(now), + }); + self.publisher.publish(&status).await?; + state.status = Some(status); + Ok(()) } } +fn public_phase(phase: TransactionPhase) -> AgentUpgradePhase { + match phase { + TransactionPhase::Preparing => AgentUpgradePhase::Preparing, + TransactionPhase::Activating => AgentUpgradePhase::Starting, + TransactionPhase::Committed => AgentUpgradePhase::Complete, + TransactionPhase::RollingBack => AgentUpgradePhase::RollingBack, + TransactionPhase::Failed => AgentUpgradePhase::Failed, + TransactionPhase::RollbackFailed => AgentUpgradePhase::RollbackFailed, + } +} + +#[derive(Clone)] pub struct UpgradeService { controller: Arc, intent: Store, - authorize: Store, intent_key: String, - authorization_prefix: String, + pending_initial_intent: Arc>>, } impl UpgradeService { pub async fn connect( client: async_nats::Client, device_id: Id, - reconciler: Arc, updater_socket: &str, ) -> Result { let jetstream = async_nats::jetstream::new(client); let intent = jetstream.get_key_value(BUCKET_AGENT_UPGRADE_INTENT).await?; - let authorize = jetstream - .get_key_value(BUCKET_AGENT_UPGRADE_AUTHORIZE) - .await?; - let status = jetstream.get_key_value(BUCKET_AGENT_UPGRADE_STATUS).await?; - let current_status = status - .get(agent_upgrade_status_key(&device_id.to_string())) - .await? - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); - let controller = Arc::new(UpgradeController { - device_id: device_id.clone(), - reconciler, - backend: Arc::new(UpdaterClient::new(updater_socket)), - publisher: Arc::new(KvStatusPublisher { - key: agent_upgrade_status_key(&device_id.to_string()), - bucket: status.clone(), - }), - active: Mutex::new(None), - terminal_attempt: Mutex::new( - current_status - .as_ref() - .filter(|status| status.phase.is_terminal()) - .map(|status| status.attempt_id.clone()), - ), - observed_transaction: Mutex::new(None), - }); + let status_bucket = jetstream.get_key_value(BUCKET_AGENT_UPGRADE_STATUS).await?; let intent_key = agent_upgrade_intent_key(&device_id.to_string()); - let transaction = controller.backend.status().await?; - if let Some(transaction) = transaction.as_ref() - && transaction.phase == TransactionPhase::Staged - { - let bytes = intent - .get(&intent_key) - .await? - .context("staged upgrade has no matching durable intent")?; - let attempt: AgentUpgradeAttempt = serde_json::from_slice(&bytes)?; - validate_attempt(&attempt, &device_id)?; - if attempt.attempt_id != transaction.attempt_id - || attempt.target_version != transaction.target_version - { - bail!("staged updater transaction does not match current intent"); - } - controller.reconciler.pause().await; - let updated_at = current_status - .as_ref() - .filter(|status| { - status.attempt_id == attempt.attempt_id - && status.phase == AgentUpgradePhase::AwaitingAuthorization - }) - .map(|status| status.updated_at) - .unwrap_or_else(Utc::now); - *controller.active.lock().await = Some(ActiveAttempt { - attempt, - authorization_deadline: updated_at - + chrono::Duration::from_std(AUTHORIZATION_TIMEOUT).unwrap(), - }); - } - controller.recover().await?; - let switch_in_progress = transaction.as_ref().is_some_and(|transaction| { - matches!( - transaction.phase, - TransactionPhase::Switching - | TransactionPhase::Probation - | TransactionPhase::RollingBack - ) + let status_key = agent_upgrade_status_key(&device_id.to_string()); + let current_status = status_bucket.get(&status_key).await?.and_then(|bytes| { + serde_json::from_slice::(&bytes) + .inspect_err(|error| tracing::warn!(%error, "current upgrade status ignored")) + .ok() }); - if !switch_in_progress && let Some(bytes) = intent.get(&intent_key).await? { - let attempt = serde_json::from_slice(&bytes)?; - if let Err(error) = controller.accept(attempt).await { - tracing::warn!(%error, "current upgrade attempt rejected"); + let current_intent = intent.get(&intent_key).await?.and_then(|bytes| { + serde_json::from_slice::(&bytes) + .inspect_err(|error| tracing::warn!(%error, "current upgrade attempt rejected")) + .ok() + }); + + let backend: Arc = Arc::new(UpdaterClient::new(updater_socket)); + let transaction = backend.status().await?; + let current_intent = match current_intent { + Some(attempt) => { + if let Some(transaction) = transaction + .as_ref() + .filter(|transaction| transaction.attempt_id == attempt.attempt_id) + { + validate_intent(&attempt, &device_id)?; + validate_transaction(transaction, &attempt)?; + Some(attempt) + } else { + match validate_intent(&attempt, &device_id) { + Ok(()) => Some(attempt), + Err(error) => { + tracing::warn!(%error, "current upgrade attempt rejected"); + None + } + } + } } + None => None, + }; + let controller = Arc::new(UpgradeController { + device_id, + backend, + publisher: Arc::new(KvStatusPublisher { + key: status_key, + bucket: status_bucket, + }), + state: Mutex::new(ControllerState { + attempt: current_intent + .as_ref() + .filter(|attempt| { + transaction + .as_ref() + .is_some_and(|transaction| transaction.attempt_id == attempt.attempt_id) + }) + .cloned(), + status: current_status, + observed_transaction: None, + }), + }); + let pending_initial_intent = current_intent.filter(|attempt| { + transaction + .as_ref() + .is_none_or(|transaction| transaction.attempt_id != attempt.attempt_id) + }); + if let Some(transaction) = transaction { + controller + .reflect(transaction, AgentUpgradeReason::Recovery) + .await?; } Ok(Self { controller, intent, - authorize, intent_key, - authorization_prefix: format!("{}.", device_id), + pending_initial_intent: Arc::new(Mutex::new(pending_initial_intent)), }) } + pub async fn acknowledge_shutdown(&self, drain_duration_ms: u64) -> Result { + self.controller + .acknowledge_shutdown(drain_duration_ms) + .await + } + + pub async fn ensure_active_startup(&self) -> Result { + self.controller.ensure_active_startup().await + } + pub async fn run(self) -> Result<()> { + if let Some(attempt) = self.pending_initial_intent.lock().await.take() + && let Err(error) = self.controller.accept(attempt).await + { + tracing::warn!(%error, "current upgrade attempt rejected"); + } loop { - let mut intents = self.intent.watch_with_history(&self.intent_key).await?; - let mut authorizations = self - .authorize - .watch_with_history(format!("{}>", self.authorization_prefix)) - .await?; + let mut intents = match self.intent.watch(&self.intent_key).await { + Ok(intents) => intents, + Err(error) => { + tracing::warn!(%error, "upgrade intent watch start failed"); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + }; let mut ticker = tokio::time::interval(Duration::from_secs(1)); loop { tokio::select! { entry = intents.next() => match entry { - Some(entry) => { - let entry = entry?; + Some(Ok(entry)) => { if entry.operation == Operation::Put { - let attempt = serde_json::from_slice(&entry.value)?; - if let Err(error) = self.controller.accept(attempt).await { - tracing::warn!(%error, "upgrade attempt rejected"); + match serde_json::from_slice(&entry.value) { + Ok(attempt) => { + if let Err(error) = self.controller.accept(attempt).await { + tracing::warn!(%error, "upgrade attempt rejected"); + } + } + Err(error) => { + tracing::warn!(%error, "upgrade attempt rejected"); + } } } } - None => break, - }, - entry = authorizations.next() => match entry { - Some(entry) => { - let entry = entry?; - if entry.operation == Operation::Put - && entry.key.starts_with(&self.authorization_prefix) - { - let authorization = serde_json::from_slice(&entry.value)?; - if let Err(error) = self.controller.authorize(authorization).await { - tracing::warn!(%error, "upgrade authorization rejected"); - } - } + Some(Err(error)) => { + tracing::warn!(%error, "upgrade intent watch failed; restarting"); + break; } None => break, }, _ = ticker.tick() => { - self.controller.check_timeout(Utc::now()).await; - if let Err(error) = self.controller.recover().await { + if let Err(error) = self.controller.recover(AgentUpgradeReason::Updater).await { tracing::warn!(%error, "upgrade recovery status failed"); } } @@ -401,130 +548,79 @@ impl UpgradeService { } } -fn validate_attempt(attempt: &AgentUpgradeAttempt, device_id: &Id) -> Result<()> { +fn validate_intent(attempt: &AgentUpgradeAttempt, device_id: &Id) -> Result<()> { uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid upgrade attempt id")?; if &attempt.device_id != device_id { bail!("upgrade attempt targets another device"); } - if attempt.from_version != env!("CARGO_PKG_VERSION") { - bail!("upgrade source version does not match the running agent"); - } - if attempt.target_version == attempt.from_version { - bail!("upgrade target already runs on this device"); - } if attempt.architecture != std::env::consts::ARCH { bail!("upgrade architecture does not match this device"); } - let now = Utc::now(); - if attempt.created_at < now - chrono::Duration::hours(24) - || attempt.created_at > now + chrono::Duration::minutes(5) - { - bail!("upgrade attempt timestamp is outside the accepted window"); - } Ok(()) } -fn bounded_error(error: &str) -> String { - error.chars().take(1024).collect() +fn validate_transaction(transaction: &Transaction, attempt: &AgentUpgradeAttempt) -> Result<()> { + if transaction.attempt_digest != attempt.digest() + || transaction.device_id != attempt.device_id + || transaction.target_version != attempt.target_version + { + bail!("updater transaction does not match current intent"); + } + Ok(()) +} + +fn bounded_text(value: &str) -> String { + value.chars().take(TEXT_LIMIT).collect() +} + +fn boot_id() -> Option { + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .ok() + .map(|id| id.trim().to_string()) + .filter(|id| !id.is_empty()) } #[cfg(test)] mod tests { use super::*; - use crate::fleet_publisher::DeploymentStatePublisher; - use crate::podman::WorkloadRuntime; - use harmony_reconciler_contracts::{DeploymentName, DeploymentState, PodmanV0Score}; - use std::collections::HashSet; + use std::path::PathBuf; use std::sync::Mutex as StdMutex; - #[derive(Default)] - struct Runtime; - - #[async_trait::async_trait] - impl WorkloadRuntime for Runtime { - async fn reconcile(&self, _: &str, _: &PodmanV0Score) -> Result<()> { - Ok(()) - } - async fn remove_deployment(&self, _: &str) -> Result<()> { - Ok(()) - } - async fn managed_deployments(&self) -> Result> { - Ok(HashSet::new()) - } - } - - #[derive(Default)] - struct DeploymentPublisher; - - #[async_trait::async_trait] - impl DeploymentStatePublisher for DeploymentPublisher { - async fn write(&self, _: &DeploymentState) -> Result<()> { - Ok(()) - } - async fn delete(&self, _: &DeploymentName) -> Result<()> { - Ok(()) - } - } - #[derive(Default)] struct Backend { - fail_stage: bool, - calls: StdMutex>, + calls: StdMutex>, + transaction: StdMutex>, + error: StdMutex>, } #[async_trait::async_trait] impl UpgradeBackend for Backend { - async fn stage(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { - self.calls - .lock() - .unwrap() - .push(format!("stage:{}", attempt.attempt_id)); - if self.fail_stage { - bail!("stage failed") - } else { - Ok(()) + async fn upgrade(&self, attempt: &AgentUpgradeAttempt) -> Result<()> { + self.calls.lock().unwrap().push(attempt.clone()); + if let Some(error) = self.error.lock().unwrap().as_ref() { + bail!(error.clone()); } - } - async fn switch( - &self, - authorization: &AgentUpgradeAuthorization, - ) -> Result> { - let attempt_id = &authorization.attempt_id; - self.calls - .lock() - .unwrap() - .push(format!("switch:{attempt_id}")); - Ok(Some(Transaction { - attempt_id: attempt_id.into(), - device_id: Id::from("device-1"), - from_version: env!("CARGO_PKG_VERSION").into(), - phase: TransactionPhase::Committed, - previous: "old".into(), - target: "fleet-agent-v0.2.0".into(), - target_version: "0.2.0".into(), - attempt_digest: "digest".into(), - error: None, - })) - } - async fn status(&self) -> Result> { - Ok(None) - } - async fn cancel(&self, attempt_id: &str, _: &str) -> Result<()> { - self.calls - .lock() - .unwrap() - .push(format!("cancel:{attempt_id}")); Ok(()) } + + async fn status(&self) -> Result> { + Ok(self.transaction.lock().unwrap().clone()) + } } #[derive(Default)] - struct Publisher(StdMutex>); + struct Publisher { + statuses: StdMutex>, + fail: StdMutex, + } #[async_trait::async_trait] impl StatusPublisher for Publisher { async fn publish(&self, status: &AgentUpgradeStatus) -> Result<()> { - self.0.lock().unwrap().push(status.clone()); + if *self.fail.lock().unwrap() { + bail!("publish failed"); + } + self.statuses.lock().unwrap().push(status.clone()); Ok(()) } } @@ -533,112 +629,380 @@ mod tests { AgentUpgradeAttempt { attempt_id: uuid::Uuid::new_v4().to_string(), device_id: Id::from("device-1".to_string()), - from_version: env!("CARGO_PKG_VERSION").into(), target_version: "0.2.0".into(), architecture: std::env::consts::ARCH.into(), artifact_url: "https://example.invalid/agent".into(), max_bytes: 1, sha256: "a".repeat(64), - signature: "signature".into(), - signing_key_id: "key".into(), - created_at: Utc::now(), } } - fn authorization(attempt: &AgentUpgradeAttempt) -> AgentUpgradeAuthorization { - AgentUpgradeAuthorization { + fn transaction(attempt: &AgentUpgradeAttempt, phase: TransactionPhase) -> Transaction { + let started_at = Utc::now(); + Transaction { attempt_id: attempt.attempt_id.clone(), - attempt_digest: attempt.digest(), device_id: attempt.device_id.clone(), - from_version: attempt.from_version.clone(), + phase, + previous: PathBuf::from("old"), + target: PathBuf::from("new"), target_version: attempt.target_version.clone(), - artifact_signing_key_id: attempt.signing_key_id.clone(), - authorized_at: Utc::now(), - signing_key_id: "key".into(), - signature: "signature".into(), + attempt_digest: attempt.digest(), + error: None, + started_at, + updated_at: started_at, + transitions: vec![crate::updater::TransactionTransition { + phase, + entered_at: started_at, + exited_at: None, + duration_ms: None, + }], } } fn controller(backend: Arc, publisher: Arc) -> UpgradeController { UpgradeController { device_id: Id::from("device-1".to_string()), - reconciler: Arc::new(Reconciler::new( - Id::from("device-1".to_string()), - Arc::new(Runtime), - Some(Arc::new(DeploymentPublisher)), - None, - )), backend, publisher, - active: Mutex::new(None), - terminal_attempt: Mutex::new(None), - observed_transaction: Mutex::new(None), + state: Mutex::new(ControllerState::default()), } } + #[test] + fn intent_validation_rejects_wrong_identity_and_architecture() { + let device_id = Id::from("device-1".to_string()); + for invalid in [ + { + let mut attempt = attempt(); + attempt.attempt_id = "invalid".into(); + attempt + }, + { + let mut attempt = attempt(); + attempt.device_id = Id::from("other".to_string()); + attempt + }, + { + let mut attempt = attempt(); + attempt.architecture = "wrong".into(); + attempt + }, + ] { + assert!(validate_intent(&invalid, &device_id).is_err()); + } + } + + #[test] + fn intent_is_timeless_and_may_target_the_running_version() { + let device_id = Id::from("device-1".to_string()); + let mut attempt = attempt(); + attempt.target_version = env!("CARGO_PKG_VERSION").into(); + let transaction = transaction(&attempt, TransactionPhase::Committed); + + validate_intent(&attempt, &device_id).unwrap(); + validate_transaction(&transaction, &attempt).unwrap(); + } + #[tokio::test] - async fn successful_attempt_requires_matching_authorization() { + async fn one_intent_calls_updater_once_without_a_reconciler() { let backend = Arc::new(Backend::default()); let publisher = Arc::new(Publisher::default()); let controller = controller(backend.clone(), publisher.clone()); let attempt = attempt(); - controller.accept(attempt.clone()).await.unwrap(); - let mut wrong = authorization(&attempt); - wrong.attempt_id = "wrong".into(); - assert!(controller.authorize(wrong).await.is_err()); - controller.authorize(authorization(&attempt)).await.unwrap(); + controller.accept(attempt.clone()).await.unwrap(); + + assert_eq!(backend.calls.lock().unwrap().as_slice(), &[attempt]); + let status = publisher.statuses.lock().unwrap().first().unwrap().clone(); + assert_eq!(status.phase, AgentUpgradePhase::Preparing); + assert_eq!(status.reason, Some(AgentUpgradeReason::Intent)); + } + + #[tokio::test] + async fn identical_duplicate_is_a_noop_and_changed_content_is_rejected() { + let backend = Arc::new(Backend::default()); + let controller = controller(backend.clone(), Arc::new(Publisher::default())); + let attempt = attempt(); + controller.accept(attempt.clone()).await.unwrap(); + controller.accept(attempt.clone()).await.unwrap(); + let mut changed = attempt; + changed.target_version = "0.3.0".into(); + + assert!(controller.accept(changed).await.is_err()); + assert_eq!(backend.calls.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn transitions_are_timed_and_bounded() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend, publisher.clone()); + let attempt = attempt(); + for index in 0..TRANSITION_LIMIT + 4 { + controller + .publish(StatusUpdate { + attempt_id: &attempt.attempt_id, + target_version: &attempt.target_version, + phase: if index % 2 == 0 { + AgentUpgradePhase::Preparing + } else { + AgentUpgradePhase::RollingBack + }, + reason: AgentUpgradeReason::Updater, + error: None, + drain_duration_ms: None, + transaction: None, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + } + + let status = publisher.statuses.lock().unwrap().last().unwrap().clone(); + assert_eq!(status.transitions.len(), TRANSITION_LIMIT); + assert!( + status.transitions[..TRANSITION_LIMIT - 1] + .iter() + .all( + |transition| transition.exited_at.is_some() && transition.duration_ms.is_some() + ) + ); + assert!(status.started_at <= status.updated_at); + } + + #[tokio::test] + async fn durable_timings_preserve_stop_start_order_and_close_starting() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend, publisher.clone()); + let mut attempt = attempt(); + attempt.target_version = env!("CARGO_PKG_VERSION").into(); + let prepared_at = Utc::now() - chrono::Duration::seconds(2); + let activated_at = prepared_at + chrono::Duration::seconds(1); + let mut activating = transaction(&attempt, TransactionPhase::Activating); + activating.started_at = prepared_at; + activating.updated_at = activated_at; + activating.transitions = vec![ + crate::updater::TransactionTransition { + phase: TransactionPhase::Preparing, + entered_at: prepared_at, + exited_at: Some(activated_at), + duration_ms: Some(1_000), + }, + crate::updater::TransactionTransition { + phase: TransactionPhase::Activating, + entered_at: activated_at, + exited_at: None, + duration_ms: None, + }, + ]; + controller + .publish(StatusUpdate { + attempt_id: &attempt.attempt_id, + target_version: &attempt.target_version, + phase: AgentUpgradePhase::Stopping, + reason: AgentUpgradeReason::Updater, + error: None, + drain_duration_ms: Some(42), + transaction: None, + }) + .await + .unwrap(); + + controller + .reflect(activating, AgentUpgradeReason::Recovery) + .await + .unwrap(); + + let starting_at = publisher + .statuses + .lock() + .unwrap() + .last() + .unwrap() + .transitions + .last() + .unwrap() + .entered_at; + let committed_at = starting_at + chrono::Duration::seconds(1); + let mut committed = transaction(&attempt, TransactionPhase::Committed); + committed.started_at = prepared_at; + committed.updated_at = committed_at; + committed.transitions = vec![ + crate::updater::TransactionTransition { + phase: TransactionPhase::Preparing, + entered_at: prepared_at, + exited_at: Some(activated_at), + duration_ms: Some(1_000), + }, + crate::updater::TransactionTransition { + phase: TransactionPhase::Activating, + entered_at: activated_at, + exited_at: Some(committed_at), + duration_ms: Some(2_000), + }, + crate::updater::TransactionTransition { + phase: TransactionPhase::Committed, + entered_at: committed_at, + exited_at: None, + duration_ms: None, + }, + ]; + controller + .reflect(committed, AgentUpgradeReason::Recovery) + .await + .unwrap(); + + let status = publisher.statuses.lock().unwrap().last().unwrap().clone(); + assert_eq!(status.started_at, prepared_at); + assert_eq!(status.phase, AgentUpgradePhase::Complete); assert_eq!( - backend.calls.lock().unwrap().as_slice(), + status + .transitions + .iter() + .map(|transition| transition.phase) + .collect::>(), [ - format!("stage:{}", attempt.attempt_id), - format!("switch:{}", attempt.attempt_id) + AgentUpgradePhase::Preparing, + AgentUpgradePhase::Stopping, + AgentUpgradePhase::Starting, + AgentUpgradePhase::Complete, ] ); + assert_eq!(status.transitions[0].duration_ms, Some(1_000)); + assert!(status.transitions[1].entered_at > activated_at); + assert!(status.transitions[1].duration_ms.is_some()); + assert_eq!(status.transitions[2].exited_at, Some(committed_at)); + assert_eq!(status.transitions[2].duration_ms, Some(1_000)); + assert_eq!(status.drain_duration_ms, Some(42)); + assert_eq!(status.journal.unwrap().since, committed_at); + } + + #[tokio::test] + async fn failed_publication_keeps_the_last_acknowledged_history() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend, publisher.clone()); + let attempt = attempt(); + let update = |phase| StatusUpdate { + attempt_id: &attempt.attempt_id, + target_version: &attempt.target_version, + phase, + reason: AgentUpgradeReason::Updater, + error: None, + drain_duration_ms: None, + transaction: None, + }; + controller + .publish(update(AgentUpgradePhase::Preparing)) + .await + .unwrap(); + *publisher.fail.lock().unwrap() = true; + assert!( + controller + .publish(update(AgentUpgradePhase::Stopping)) + .await + .is_err() + ); + *publisher.fail.lock().unwrap() = false; + controller + .publish(update(AgentUpgradePhase::RollingBack)) + .await + .unwrap(); + assert_eq!( - publisher.0.lock().unwrap().last().unwrap().phase, - AgentUpgradePhase::Complete + publisher + .statuses + .lock() + .unwrap() + .last() + .unwrap() + .transitions + .iter() + .map(|transition| transition.phase) + .collect::>(), + [AgentUpgradePhase::Preparing, AgentUpgradePhase::RollingBack,] ); } #[tokio::test] - async fn stage_failure_and_authorization_timeout_resume_reconciliation() { + async fn updater_phases_recover_to_public_phases() { + let backend = Arc::new(Backend::default()); let publisher = Arc::new(Publisher::default()); - let failed_controller = controller( - Arc::new(Backend { - fail_stage: true, - ..Default::default() - }), - publisher.clone(), - ); - assert!(failed_controller.accept(attempt()).await.is_err()); - assert_eq!( - publisher.0.lock().unwrap().last().unwrap().phase, - AgentUpgradePhase::Failed - ); - - let controller = controller(Arc::new(Backend::default()), publisher.clone()); - let attempt = attempt(); - controller.accept(attempt).await.unwrap(); - controller - .check_timeout(Utc::now() + chrono::Duration::minutes(6)) - .await; - assert_eq!( - publisher.0.lock().unwrap().last().unwrap().phase, - AgentUpgradePhase::Failed - ); + let controller = controller(backend.clone(), publisher.clone()); + let mut attempt = attempt(); + attempt.target_version = env!("CARGO_PKG_VERSION").into(); + let cases = [ + (TransactionPhase::Preparing, AgentUpgradePhase::Preparing), + (TransactionPhase::Activating, AgentUpgradePhase::Starting), + (TransactionPhase::Committed, AgentUpgradePhase::Complete), + ( + TransactionPhase::RollingBack, + AgentUpgradePhase::RollingBack, + ), + (TransactionPhase::Failed, AgentUpgradePhase::Failed), + ( + TransactionPhase::RollbackFailed, + AgentUpgradePhase::RollbackFailed, + ), + ]; + for (local, public) in cases { + *backend.transaction.lock().unwrap() = Some(transaction(&attempt, local)); + controller + .recover(AgentUpgradeReason::Recovery) + .await + .unwrap(); + assert_eq!( + publisher.statuses.lock().unwrap().last().unwrap().phase, + public + ); + } } - #[test] - fn wrong_device_source_and_architecture_are_rejected() { - let mut value = attempt(); - value.device_id = Id::from("other".to_string()); - assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); - value = attempt(); - value.from_version = "old".into(); - assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); - value = attempt(); - value.architecture = "wrong".into(); - assert!(validate_attempt(&value, &Id::from("device-1".to_string())).is_err()); + #[tokio::test] + async fn shutdown_only_acknowledges_an_activating_transaction() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend.clone(), publisher.clone()); + assert!(!controller.acknowledge_shutdown(0).await.unwrap()); + let attempt = attempt(); + *backend.transaction.lock().unwrap() = + Some(transaction(&attempt, TransactionPhase::Activating)); + + assert!(controller.acknowledge_shutdown(73).await.unwrap()); + let status = publisher.statuses.lock().unwrap().last().unwrap().clone(); + assert_eq!(status.phase, AgentUpgradePhase::Stopping); + assert_eq!(status.reason, Some(AgentUpgradeReason::Updater)); + assert_eq!(status.drain_duration_ms, Some(73)); + assert_eq!(status.journal.unwrap().unit, AGENT_UNIT); + } + + #[tokio::test] + async fn active_startup_requires_the_selected_activation_version() { + let backend = Arc::new(Backend::default()); + let publisher = Arc::new(Publisher::default()); + let controller = controller(backend.clone(), publisher.clone()); + let attempt = attempt(); + *backend.transaction.lock().unwrap() = + Some(transaction(&attempt, TransactionPhase::Activating)); + assert!(controller.ensure_active_startup().await.is_err()); + + let mut matching = attempt; + matching.target_version = env!("CARGO_PKG_VERSION").into(); + *backend.transaction.lock().unwrap() = + Some(transaction(&matching, TransactionPhase::Activating)); + assert!(controller.ensure_active_startup().await.unwrap()); + assert_eq!( + publisher.statuses.lock().unwrap().last().unwrap().phase, + AgentUpgradePhase::Starting + ); + + *backend.transaction.lock().unwrap() = + Some(transaction(&matching, TransactionPhase::RollingBack)); + assert!(controller.ensure_active_startup().await.unwrap()); + assert_eq!( + publisher.statuses.lock().unwrap().last().unwrap().phase, + AgentUpgradePhase::RollingBack + ); } } diff --git a/fleet/harmony-fleet-deploy/src/device_setup.rs b/fleet/harmony-fleet-deploy/src/device_setup.rs index f79b4c5a..17f30c3e 100644 --- a/fleet/harmony-fleet-deploy/src/device_setup.rs +++ b/fleet/harmony-fleet-deploy/src/device_setup.rs @@ -64,10 +64,6 @@ pub struct FleetDeviceSetupConfig { /// `/usr/local/bin/fleet-agent`. Future v0.1: this becomes a /// `DownloadableAsset` pointing at CI-published artifacts. pub agent_binary_path: PathBuf, - /// Ed25519 public key trusted to sign agent artifacts and switch - /// authorizations. - #[serde(default)] - pub upgrade_signing_key: Option, /// `/etc/hosts` entries to add on the device. The fleet rehearsal /// harness uses this so VMs on a libvirt NAT resolve /// `sso.fleet.local` to the host's gateway IP — without it the @@ -94,12 +90,6 @@ pub struct DeviceOpenbao { pub secret_prefix: String, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UpgradeSigningKey { - pub id: String, - pub public_key: String, -} - /// One line in `/etc/hosts`. Order doesn't matter (the file ends up /// being a sorted dedup'd merge of these and any pre-existing /// non-managed entries). @@ -312,6 +302,8 @@ RuntimeDirectoryMode=0700 Environment=FLEET_AGENT_CONFIG=/etc/fleet-agent/config.toml Environment=RUST_LOG=info ExecStart=/usr/local/bin/fleet-agent +TimeoutStartSec=4min +TimeoutStopSec=60s Restart=on-failure RestartSec=5 StandardOutput=journal @@ -715,58 +707,6 @@ impl Interpret for FleetDeviceSetupInte change_count += 1; } - let mut signing_keys_changed = false; - if let Some(signing_key) = &cfg.upgrade_signing_key { - let key_id = &signing_key.id; - if key_id.is_empty() - || !key_id - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') - { - return Err(InterpretError::new(format!( - "Invalid upgrade signing key id '{key_id}'" - ))); - } - let key = FileDelivery::ensure_file( - topology, - &FileSpec { - path: format!("/etc/fleet-agent/trusted-upgrade-keys/{key_id}.pub"), - source: FileSource::Content(format!("{}\n", signing_key.public_key.trim())), - owner: Some("root".to_string()), - group: Some("root".to_string()), - mode: Some(0o644), - }, - ) - .await - .map_err(wrap)?; - signing_keys_changed |= key.changed; - if key.changed { - change_count += 1; - } - } - let active_key = FileDelivery::ensure_file( - topology, - &FileSpec { - path: "/etc/fleet-agent/trusted-upgrade-key-id".to_string(), - source: FileSource::Content(format!( - "{}\n", - cfg.upgrade_signing_key - .as_ref() - .map(|key| key.id.as_str()) - .unwrap_or_default() - )), - owner: Some("root".to_string()), - group: Some("root".to_string()), - mode: Some(0o644), - }, - ) - .await - .map_err(wrap)?; - signing_keys_changed |= active_key.changed; - if active_key.changed { - change_count += 1; - } - // 5a. Drop the Zitadel machine keyfile when using JWT auth. // Order: keyfile first, then config.toml — if both are new the // agent's first systemd start finds the key already in place. @@ -836,7 +776,7 @@ impl Interpret for FleetDeviceSetupInte if updater_unit_r.changed { change_count += 1; } - if binary_r.changed || updater_unit_r.changed || signing_keys_changed { + if binary_r.changed || updater_unit_r.changed { SystemdManager::restart_service( topology, "harmony-fleet-updater", @@ -861,12 +801,8 @@ impl Interpret for FleetDeviceSetupInte } // 7. Restart the agent iff anything that affects it changed. - let needs_restart = toml_r.changed - || unit_r.changed - || updater_unit_r.changed - || binary_r.changed - || key_r - || signing_keys_changed; + let needs_restart = + toml_r.changed || unit_r.changed || updater_unit_r.changed || binary_r.changed || key_r; let service_state = if needs_restart { info!("[{tag}] 🔄 Restarting fleet-agent (config/binary/unit changed)"); SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System) @@ -1026,7 +962,6 @@ mod tests { nats_pass: "pw".to_string(), }, agent_binary_path: PathBuf::from("/dev/null"), - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, } @@ -1045,7 +980,6 @@ mod tests { danger_accept_invalid_certs: false, }, agent_binary_path: PathBuf::from("/dev/null"), - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, } @@ -1150,7 +1084,6 @@ mod tests { danger_accept_invalid_certs: false, }, agent_binary_path: PathBuf::from("/dev/null"), - upgrade_signing_key: None, hosts_entries: vec![], openbao: None, }; @@ -1240,6 +1173,8 @@ mod tests { let unit = config.render_systemd_unit(); assert!(unit.contains("Type=notify\n")); assert!(unit.contains("NotifyAccess=main\n")); + assert!(unit.contains("TimeoutStartSec=4min\n")); + assert!(unit.contains("TimeoutStopSec=60s\n")); assert!(unit.contains("Requires=harmony-fleet-updater.service\n")); assert!(unit.contains("RuntimeDirectory=harmony-fleet-agent\n")); diff --git a/fleet/harmony-fleet-deploy/src/lib.rs b/fleet/harmony-fleet-deploy/src/lib.rs index 8432c7ed..4dfdfd7a 100644 --- a/fleet/harmony-fleet-deploy/src/lib.rs +++ b/fleet/harmony-fleet-deploy/src/lib.rs @@ -19,7 +19,7 @@ pub use agent::{FleetAgentScore, PodTarget}; pub use app::{FleetApp, FleetCrdsApp, FleetTenantProvisionApp}; pub use device_setup::{ AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, - HostsEntry, UpgradeSigningKey, merge_hosts_file, + HostsEntry, merge_hosts_file, }; pub use operator::{FleetCrdsScore, FleetOperatorScore, OperatorCredentials}; diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 3244df5f..272a9686 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -75,8 +75,6 @@ pub struct ChartOptions { pub identity: Option, pub identity_version: Option, pub image_pull_secret: Option, - pub upgrade_signing_key: Option, - pub upgrade_signing_key_id: Option, } #[derive(Debug, Clone, Serialize)] @@ -111,8 +109,6 @@ impl Default for ChartOptions { identity: None, identity_version: None, image_pull_secret: None, - upgrade_signing_key: None, - upgrade_signing_key_id: None, } } } @@ -145,7 +141,6 @@ pub const ENV_WEB_COOKIE_KEY: &str = "HARMONY_CONFIG_OperatorCookieKey"; /// path to the generated chart directory (which is what `helm /// install ` wants). pub fn build_chart(opts: &ChartOptions) -> Result { - validate_options(opts)?; std::fs::create_dir_all(&opts.output_dir) .with_context(|| format!("creating {:?}", opts.output_dir))?; @@ -183,13 +178,6 @@ pub fn build_chart(opts: &ChartOptions) -> Result { Ok(written) } -pub fn validate_options(opts: &ChartOptions) -> Result<()> { - if opts.upgrade_signing_key.is_some() != opts.upgrade_signing_key_id.is_some() { - anyhow::bail!("upgrade signing key and key id must be configured together"); - } - Ok(()) -} - /// Build the operator's Secret holding the `[credentials]` TOML /// (with the JSON keyfile inlined under `key_json`). Returns `None` /// when no credentials are configured (no-auth dev mode). @@ -197,7 +185,6 @@ pub fn operator_secret(opts: &ChartOptions) -> Option { if opts.credentials.is_none() && opts.web_auth_config_json.is_none() && opts.web_cookie_key_json.is_none() - && opts.upgrade_signing_key.is_none() { return None; } @@ -223,18 +210,6 @@ pub fn operator_secret(opts: &ChartOptions) -> Option { ByteString(json.as_bytes().to_vec()), ); } - if let Some(key) = &opts.upgrade_signing_key { - data.insert( - "FLEET_UPGRADE_SIGNING_KEY".into(), - ByteString(key.as_bytes().to_vec()), - ); - } - if let Some(key_id) = &opts.upgrade_signing_key_id { - data.insert( - "FLEET_UPGRADE_SIGNING_KEY_ID".into(), - ByteString(key_id.as_bytes().to_vec()), - ); - } // Namespace deliberately omitted — the caller passes the target // namespace to `K8sResourceScore::single`, which injects it at // apply time. Keeps the Secret manifest reusable across @@ -381,8 +356,6 @@ pub(crate) fn config_hash(opts: &ChartOptions) -> String { .hash(&mut secret_hash); opts.web_auth_config_json.hash(&mut secret_hash); opts.web_cookie_key_json.hash(&mut secret_hash); - opts.upgrade_signing_key.hash(&mut secret_hash); - opts.upgrade_signing_key_id.hash(&mut secret_hash); opts.identity .as_ref() .map(|identity| identity.machine.secret_name()) @@ -488,8 +461,6 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { }); env.push(secret_env(ENV_WEB_AUTH_CONFIG)); env.push(secret_env(ENV_WEB_COOKIE_KEY)); - env.push(secret_env("FLEET_UPGRADE_SIGNING_KEY")); - env.push(secret_env("FLEET_UPGRADE_SIGNING_KEY_ID")); // Secret-grant sync (OpenBao) + the device-group scheduling gate // (Zitadel role grants) — ADR-025. All optional: absent, the // operator logs and runs ungated/without grant sync. @@ -765,19 +736,4 @@ mod tests { assert_eq!(data[ENV_WEB_COOKIE_KEY].0, b"cookie"); assert!(!data.contains_key(SECRET_KEY_CREDENTIALS_TOML)); } - - #[test] - fn upgrade_signer_is_atomic_and_restarts_the_operator() { - let base_hash = config_hash(&ChartOptions::default()); - let mut configured = ChartOptions::default(); - configured.upgrade_signing_key = Some("private-key".into()); - assert!(validate_options(&configured).is_err()); - - configured.upgrade_signing_key_id = Some("production".into()); - assert!(validate_options(&configured).is_ok()); - assert_ne!(base_hash, config_hash(&configured)); - let secret = operator_secret(&configured).unwrap().data.unwrap(); - assert_eq!(secret["FLEET_UPGRADE_SIGNING_KEY"].0, b"private-key"); - assert_eq!(secret["FLEET_UPGRADE_SIGNING_KEY_ID"].0, b"production"); - } } diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index 8bdabfb1..57833e2e 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -167,8 +167,6 @@ pub struct FleetOperatorScore { pub web_auth: Option, pub identity: Option, pub image_pull_secret: Option, - pub upgrade_signing_key: Option, - pub upgrade_signing_key_id: Option, } impl FleetOperatorScore { @@ -191,8 +189,6 @@ impl FleetOperatorScore { web_auth: None, identity: None, image_pull_secret: None, - upgrade_signing_key: None, - upgrade_signing_key_id: None, } } @@ -264,16 +260,6 @@ impl FleetOperatorScore { self.log_level = level.into(); self } - - pub fn upgrade_signer( - mut self, - key_id: impl Into, - private_key: impl Into, - ) -> Self { - self.upgrade_signing_key_id = Some(key_id.into()); - self.upgrade_signing_key = Some(private_key.into()); - self - } } impl Score for FleetOperatorScore { @@ -534,11 +520,7 @@ impl Interpret for FleetOperatorInterp identity: self.score.identity.clone(), identity_version, image_pull_secret: self.score.image_pull_secret.clone(), - upgrade_signing_key: self.score.upgrade_signing_key.clone(), - upgrade_signing_key_id: self.score.upgrade_signing_key_id.clone(), }; - chart::validate_options(&chart_options) - .map_err(|e| InterpretError::new(format!("operator chart options: {e}")))?; let expected_config_hash = chart::config_hash(&chart_options); if let Some(secret) = operator_secret(&chart_options) { info!( diff --git a/fleet/harmony-fleet-e2e/src/vm/device.rs b/fleet/harmony-fleet-e2e/src/vm/device.rs index f938449d..ca2c4564 100644 --- a/fleet/harmony-fleet-e2e/src/vm/device.rs +++ b/fleet/harmony-fleet-e2e/src/vm/device.rs @@ -227,7 +227,6 @@ impl VmDevice { nats_urls: vec![opts.nats_url.clone()], auth: opts.auth.clone(), agent_binary_path: opts.agent_binary.clone(), - upgrade_signing_key: None, hosts_entries: opts.hosts_entries.clone(), openbao: opts.openbao.clone(), }); diff --git a/fleet/harmony-fleet-operator/Cargo.toml b/fleet/harmony-fleet-operator/Cargo.toml index cf2d7699..f6f73dd1 100644 --- a/fleet/harmony-fleet-operator/Cargo.toml +++ b/fleet/harmony-fleet-operator/Cargo.toml @@ -36,9 +36,7 @@ futures-util = { workspace = true } thiserror.workspace = true async-trait.workspace = true url.workspace = true -base64.workspace = true reqwest.workspace = true -ed25519-dalek.workspace = true uuid.workspace = true axum = { version = "0.8", optional = true } diff --git a/fleet/harmony-fleet-operator/src/agent_upgrade.rs b/fleet/harmony-fleet-operator/src/agent_upgrade.rs index 6819181c..7978ece9 100644 --- a/fleet/harmony-fleet-operator/src/agent_upgrade.rs +++ b/fleet/harmony-fleet-operator/src/agent_upgrade.rs @@ -1,63 +1,23 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; -use base64::Engine; -use chrono::Utc; -use ed25519_dalek::{Signer, SigningKey}; use harmony_reconciler_contracts::{ - AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, - BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, Id, - agent_upgrade_authorize_key, agent_upgrade_intent_key, agent_upgrade_status_key, + AgentUpgradeAttempt, AgentUpgradeStatus, BUCKET_AGENT_UPGRADE_INTENT, + BUCKET_AGENT_UPGRADE_STATUS, Id, agent_upgrade_intent_key, agent_upgrade_status_key, }; use kube::api::{Api, ListParams, Patch, PatchParams}; use kube::{Client, ResourceExt}; use serde_json::json; -use crate::crd::{AgentUpgradeTarget, Device, DeviceUpgradeStatus}; - -pub struct AuthorizationSigner { - key_id: String, - key: SigningKey, -} - -impl AuthorizationSigner { - pub fn from_base64(key_id: String, encoded: &str) -> Result { - let bytes = base64::engine::general_purpose::STANDARD.decode(encoded.trim())?; - let key: [u8; 32] = bytes - .try_into() - .map_err(|_| anyhow::anyhow!("upgrade signing key must contain 32 bytes"))?; - Ok(Self { - key_id, - key: SigningKey::from_bytes(&key), - }) - } - - fn authorization(&self, attempt: &AgentUpgradeAttempt) -> AgentUpgradeAuthorization { - let mut authorization = AgentUpgradeAuthorization { - attempt_id: attempt.attempt_id.clone(), - attempt_digest: attempt.digest(), - device_id: attempt.device_id.clone(), - from_version: attempt.from_version.clone(), - target_version: attempt.target_version.clone(), - artifact_signing_key_id: attempt.signing_key_id.clone(), - authorized_at: Utc::now(), - signing_key_id: self.key_id.clone(), - signature: String::new(), - }; - authorization.signature = base64::engine::general_purpose::STANDARD.encode( - self.key - .sign(authorization.signing_payload().as_bytes()) - .to_bytes(), - ); - authorization - } -} +use crate::crd::{ + AgentUpgradeTarget, Device, DeviceUpgradeJournalRef, DeviceUpgradeStatus, + DeviceUpgradeTransition, +}; pub async fn run( client: Client, namespace: &str, jetstream: async_nats::jetstream::Context, - signer: Option, ) -> Result<()> { let intents = jetstream .create_key_value(async_nats::jetstream::kv::Config { @@ -65,12 +25,6 @@ pub async fn run( ..Default::default() }) .await?; - let authorizations = jetstream - .create_key_value(async_nats::jetstream::kv::Config { - bucket: BUCKET_AGENT_UPGRADE_AUTHORIZE.into(), - ..Default::default() - }) - .await?; let statuses = jetstream .create_key_value(async_nats::jetstream::kv::Config { bucket: BUCKET_AGENT_UPGRADE_STATUS.into(), @@ -78,20 +32,12 @@ pub async fn run( }) .await?; let devices: Api = Api::namespaced(client, namespace); + // TODO: Replace full-device polling with watch-driven work before fleet scale. let mut ticker = tokio::time::interval(Duration::from_secs(2)); loop { ticker.tick().await; for device in devices.list(&ListParams::default()).await?.items { - if let Err(error) = reconcile_device( - &devices, - &intents, - &authorizations, - &statuses, - signer.as_ref(), - device, - ) - .await - { + if let Err(error) = reconcile_device(&devices, &intents, &statuses, device).await { tracing::warn!(device = %error.0, error = %error.1, "agent upgrade reconcile failed"); } } @@ -101,13 +47,11 @@ pub async fn run( async fn reconcile_device( devices: &Api, intents: &async_nats::jetstream::kv::Store, - authorizations: &async_nats::jetstream::kv::Store, statuses: &async_nats::jetstream::kv::Store, - signer: Option<&AuthorizationSigner>, device: Device, ) -> std::result::Result<(), (String, anyhow::Error)> { let id = device.name_any(); - reconcile_device_inner(devices, intents, authorizations, statuses, signer, device) + reconcile_device_inner(devices, intents, statuses, device) .await .map_err(|error| (id, error)) } @@ -115,9 +59,7 @@ async fn reconcile_device( async fn reconcile_device_inner( devices: &Api, intents: &async_nats::jetstream::kv::Store, - authorizations: &async_nats::jetstream::kv::Store, statuses: &async_nats::jetstream::kv::Store, - signer: Option<&AuthorizationSigner>, device: Device, ) -> Result<()> { let id = device.name_any(); @@ -135,8 +77,41 @@ async fn reconcile_device_inner( attempt_id: status.attempt_id.clone(), target_version: status.target_version.clone(), phase, + started_at: status.started_at.to_rfc3339(), updated_at: status.updated_at.to_rfc3339(), - last_error: status.last_error.clone(), + reason: status.reason.map(|reason| { + serde_json::to_value(reason) + .expect("upgrade reason is serializable") + .as_str() + .expect("upgrade reason serializes as a string") + .to_string() + }), + detail: status.detail.clone(), + error: status.error.clone(), + drain_duration_ms: status.drain_duration_ms, + boot_id: status.boot_id.clone(), + invocation_id: status.invocation_id.clone(), + journal: status + .journal + .as_ref() + .map(|journal| DeviceUpgradeJournalRef { + unit: journal.unit.clone(), + since: journal.since.to_rfc3339(), + }), + transitions: status + .transitions + .iter() + .map(|transition| DeviceUpgradeTransition { + phase: serde_json::to_value(transition.phase) + .expect("upgrade phase is serializable") + .as_str() + .expect("upgrade phase serializes as a string") + .to_string(), + entered_at: transition.entered_at.to_rfc3339(), + exited_at: transition.exited_at.map(|at| at.to_rfc3339()), + duration_ms: transition.duration_ms, + }) + .collect(), }; if device .status @@ -173,8 +148,8 @@ async fn reconcile_device_inner( .filter(|entry| entry.operation == async_nats::jetstream::kv::Operation::Put) .map(|entry| serde_json::from_slice::(&entry.value)) .transpose()?; - let attempt = match existing { - Some(attempt) if attempt_matches(&attempt, target, current_version) => attempt, + match existing { + Some(attempt) if attempt_matches(&attempt, target) => {} Some(attempt) if status.as_ref().is_some_and(|status| { status.attempt_id == attempt.attempt_id && !status.phase.is_terminal() @@ -186,15 +161,11 @@ async fn reconcile_device_inner( let attempt = AgentUpgradeAttempt { attempt_id: uuid::Uuid::new_v4().to_string(), device_id: Id::from(id.clone()), - from_version: current_version.into(), target_version: target.version.clone(), architecture: target.architecture.clone(), artifact_url: target.artifact_url.clone(), max_bytes: target.max_bytes, sha256: target.sha256.clone(), - signature: target.signature.clone(), - signing_key_id: target.signing_key_id.clone(), - created_at: Utc::now(), }; let value = serde_json::to_vec(&attempt)?.into(); if let Some(entry) = existing_entry { @@ -202,43 +173,18 @@ async fn reconcile_device_inner( } else { intents.create(&intent_key, value).await?; } - attempt - } - }; - - if status.as_ref().is_some_and(|status| { - status.attempt_id == attempt.attempt_id - && status.phase == AgentUpgradePhase::AwaitingAuthorization - }) { - let signer = - signer.context("upgrade is ready but no authorization signer is configured")?; - if signer.key_id != attempt.signing_key_id { - bail!("authorization signer does not match artifact signing key"); - } - let key = agent_upgrade_authorize_key(&id, &attempt.attempt_id); - if authorizations.get(&key).await?.is_none() { - let authorization = signer.authorization(&attempt); - authorizations - .put(&key, serde_json::to_vec(&authorization)?.into()) - .await?; } } + Ok(()) } -fn attempt_matches( - attempt: &AgentUpgradeAttempt, - target: &AgentUpgradeTarget, - current_version: &str, -) -> bool { - attempt.from_version == current_version - && attempt.target_version == target.version +fn attempt_matches(attempt: &AgentUpgradeAttempt, target: &AgentUpgradeTarget) -> bool { + attempt.target_version == target.version && attempt.architecture == target.architecture && attempt.artifact_url == target.artifact_url && attempt.max_bytes == target.max_bytes && attempt.sha256 == target.sha256 - && attempt.signature == target.signature - && attempt.signing_key_id == target.signing_key_id } #[cfg(test)] @@ -253,23 +199,20 @@ mod tests { artifact_url: "https://example.invalid/agent".into(), max_bytes: 10, sha256: "digest".into(), - signature: "signature".into(), - signing_key_id: "key".into(), }; let attempt = AgentUpgradeAttempt { attempt_id: uuid::Uuid::new_v4().to_string(), device_id: Id::from("device"), - from_version: "0.1.0".into(), target_version: target.version.clone(), architecture: target.architecture.clone(), artifact_url: target.artifact_url.clone(), max_bytes: target.max_bytes, sha256: target.sha256.clone(), - signature: target.signature.clone(), - signing_key_id: target.signing_key_id.clone(), - created_at: Utc::now(), }; - assert!(attempt_matches(&attempt, &target, "0.1.0")); - assert!(!attempt_matches(&attempt, &target, "0.0.9")); + assert!(attempt_matches(&attempt, &target)); + + let mut changed = target; + changed.artifact_url = "https://example.invalid/repacked-agent".into(); + assert!(!attempt_matches(&attempt, &changed)); } } diff --git a/fleet/harmony-fleet-operator/src/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs index 716f4766..7796d9e5 100644 --- a/fleet/harmony-fleet-operator/src/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -120,8 +120,6 @@ pub struct AgentUpgradeTarget { pub artifact_url: String, pub max_bytes: u64, pub sha256: String, - pub signature: String, - pub signing_key_id: String, } /// Operator-maintained liveness reflection of the NATS @@ -146,8 +144,32 @@ pub struct DeviceUpgradeStatus { pub attempt_id: String, pub target_version: String, pub phase: String, + pub started_at: String, pub updated_at: String, - pub last_error: Option, + pub reason: Option, + pub detail: Option, + pub error: Option, + pub drain_duration_ms: Option, + pub boot_id: Option, + pub invocation_id: Option, + pub journal: Option, + pub transitions: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct DeviceUpgradeTransition { + pub phase: String, + pub entered_at: String, + pub exited_at: Option, + pub duration_ms: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct DeviceUpgradeJournalRef { + pub unit: String, + pub since: String, } /// Coarse liveness derived from heartbeat freshness. Failing/Pending diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 34173142..723a202a 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -84,12 +84,6 @@ struct Cli { #[arg(long, env = "OPENBAO_TOKEN", global = true)] openbao_token: Option, - - #[arg(long, env = "FLEET_UPGRADE_SIGNING_KEY", global = true)] - upgrade_signing_key: Option, - - #[arg(long, env = "FLEET_UPGRADE_SIGNING_KEY_ID", global = true)] - upgrade_signing_key_id: Option, } #[derive(Subcommand)] @@ -149,19 +143,6 @@ async fn main() -> Result<()> { }; match cli.command.unwrap_or(Command::Run) { Command::Run => { - let upgrade_signer = match ( - cli.upgrade_signing_key.as_deref(), - cli.upgrade_signing_key_id.as_deref(), - ) { - (Some(key), Some(key_id)) => Some(agent_upgrade::AuthorizationSigner::from_base64( - key_id.into(), - key, - )?), - (None, None) => None, - _ => anyhow::bail!( - "FLEET_UPGRADE_SIGNING_KEY and FLEET_UPGRADE_SIGNING_KEY_ID must be set together" - ), - }; run( &cli.nats_url, &cli.kv_bucket, @@ -169,7 +150,6 @@ async fn main() -> Result<()> { &credentials_toml, &cli.openbao_url, &cli.openbao_token, - upgrade_signer, ) .await } @@ -287,7 +267,6 @@ async fn run( credentials_toml: &str, openbao_url: &Option, openbao_token: &Option, - upgrade_signer: Option, ) -> Result<()> { let nats = connect_with_retry(nats_url, credentials_toml).await?; tracing::info!(url = %nats_url, "connected to NATS"); @@ -373,7 +352,7 @@ async fn run( r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r, r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r, r = device_status::run(ds_client, tenant_namespace, ds_js) => r, - r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js, upgrade_signer) => r, + r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js) => r, r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r, } } diff --git a/harmony-reconciler-contracts/src/kv.rs b/harmony-reconciler-contracts/src/kv.rs index dece6b3f..ac9c9ce6 100644 --- a/harmony-reconciler-contracts/src/kv.rs +++ b/harmony-reconciler-contracts/src/kv.rs @@ -33,7 +33,6 @@ pub const BUCKET_DEVICE_STATE: &str = "device-state"; pub const BUCKET_DEVICE_HEARTBEAT: &str = "device-heartbeat"; pub const BUCKET_AGENT_UPGRADE_INTENT: &str = "agent-upgrade-intent"; -pub const BUCKET_AGENT_UPGRADE_AUTHORIZE: &str = "agent-upgrade-authorize"; pub const BUCKET_AGENT_UPGRADE_STATUS: &str = "agent-upgrade-status"; /// KV key for a `(device, deployment)` pair in [`BUCKET_DESIRED_STATE`]. @@ -74,10 +73,6 @@ pub fn agent_upgrade_intent_key(device_id: &str) -> String { device_id.to_string() } -pub fn agent_upgrade_authorize_key(device_id: &str, attempt_id: &str) -> String { - format!("{device_id}.{attempt_id}") -} - pub fn agent_upgrade_status_key(device_id: &str) -> String { device_id.to_string() } @@ -107,7 +102,6 @@ mod tests { assert_eq!(BUCKET_DEVICE_STATE, "device-state"); assert_eq!(BUCKET_DEVICE_HEARTBEAT, "device-heartbeat"); assert_eq!(BUCKET_AGENT_UPGRADE_INTENT, "agent-upgrade-intent"); - assert_eq!(BUCKET_AGENT_UPGRADE_AUTHORIZE, "agent-upgrade-authorize"); assert_eq!(BUCKET_AGENT_UPGRADE_STATUS, "agent-upgrade-status"); } @@ -120,10 +114,6 @@ mod tests { ); assert_eq!(device_heartbeat_key("pi-01"), "heartbeat.pi-01"); assert_eq!(agent_upgrade_intent_key("pi-01"), "pi-01"); - assert_eq!( - agent_upgrade_authorize_key("pi-01", "attempt-1"), - "pi-01.attempt-1" - ); assert_eq!(agent_upgrade_status_key("pi-01"), "pi-01"); } diff --git a/harmony-reconciler-contracts/src/lib.rs b/harmony-reconciler-contracts/src/lib.rs index 5c64a826..4f1b3533 100644 --- a/harmony-reconciler-contracts/src/lib.rs +++ b/harmony-reconciler-contracts/src/lib.rs @@ -35,18 +35,18 @@ pub use fleet::{ DeploymentName, DeploymentState, DeviceInfo, HeartbeatPayload, InvalidDeploymentName, }; pub use kv::{ - BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, - BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, - agent_upgrade_authorize_key, agent_upgrade_intent_key, agent_upgrade_status_key, - desired_state_key, desired_state_watch_filter, device_heartbeat_key, device_info_key, - device_state_key, + BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, + BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, agent_upgrade_intent_key, + agent_upgrade_status_key, desired_state_key, desired_state_watch_filter, device_heartbeat_key, + device_info_key, device_state_key, }; pub use podman::{ EnvVar, PodmanService, PodmanV0Score, ReconcileScore, RestartPolicy, SecretEnvVar, VolumeMount, }; pub use status::{InventorySnapshot, Phase}; pub use upgrade::{ - AgentUpgradeAttempt, AgentUpgradeAuthorization, AgentUpgradePhase, AgentUpgradeStatus, + AgentUpgradeAttempt, AgentUpgradeJournalRef, AgentUpgradePhase, AgentUpgradeReason, + AgentUpgradeStatus, AgentUpgradeTransition, }; // Re-exports so consumers (agent, operator) don't need a direct diff --git a/harmony-reconciler-contracts/src/upgrade.rs b/harmony-reconciler-contracts/src/upgrade.rs index 7403b11a..bc46eaef 100644 --- a/harmony-reconciler-contracts/src/upgrade.rs +++ b/harmony-reconciler-contracts/src/upgrade.rs @@ -9,15 +9,11 @@ use crate::Id; pub struct AgentUpgradeAttempt { pub attempt_id: String, pub device_id: Id, - pub from_version: String, pub target_version: String, pub architecture: String, pub artifact_url: String, pub max_bytes: u64, pub sha256: String, - pub signature: String, - pub signing_key_id: String, - pub created_at: DateTime, } impl AgentUpgradeAttempt { @@ -32,12 +28,11 @@ impl AgentUpgradeAttempt { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum AgentUpgradePhase { - Draining, - Staging, - AwaitingAuthorization, - Switching, - Ready, + Preparing, + Stopping, + Starting, Complete, + RollingBack, Failed, RollbackFailed, } @@ -52,40 +47,44 @@ impl AgentUpgradePhase { #[serde(rename_all = "camelCase")] pub struct AgentUpgradeStatus { pub attempt_id: String, - pub current_version: String, pub target_version: String, pub phase: AgentUpgradePhase, + pub started_at: DateTime, pub updated_at: DateTime, - pub last_error: Option, + pub reason: Option, + pub detail: Option, + pub error: Option, + pub drain_duration_ms: Option, + pub boot_id: Option, + pub invocation_id: Option, + pub journal: Option, + pub transitions: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AgentUpgradeReason { + Intent, + Updater, + Ready, + Recovery, + Error, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct AgentUpgradeAuthorization { - pub attempt_id: String, - pub attempt_digest: String, - pub device_id: Id, - pub from_version: String, - pub target_version: String, - pub artifact_signing_key_id: String, - pub authorized_at: DateTime, - pub signing_key_id: String, - pub signature: String, +pub struct AgentUpgradeTransition { + pub phase: AgentUpgradePhase, + pub entered_at: DateTime, + pub exited_at: Option>, + pub duration_ms: Option, } -impl AgentUpgradeAuthorization { - pub fn signing_payload(&self) -> String { - format!( - "{}\n{}\n{}\n{}\n{}\n{}\n{}", - self.attempt_id, - self.attempt_digest, - self.device_id, - self.from_version, - self.target_version, - self.artifact_signing_key_id, - self.authorized_at.to_rfc3339() - ) - } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentUpgradeJournalRef { + pub unit: String, + pub since: DateTime, } #[cfg(test)] @@ -97,15 +96,11 @@ mod tests { let attempt = AgentUpgradeAttempt { attempt_id: "550e8400-e29b-41d4-a716-446655440000".into(), device_id: Id::from("pi-01".to_string()), - from_version: "0.1.0".into(), target_version: "0.2.0".into(), architecture: "aarch64".into(), artifact_url: "https://example.invalid/fleet-agent-v0.2.0".into(), max_bytes: 20_000_000, sha256: "a".repeat(64), - signature: "signature".into(), - signing_key_id: "production-1".into(), - created_at: Utc::now(), }; let encoded = serde_json::to_vec(&attempt).unwrap(); assert_eq!( @@ -113,24 +108,11 @@ mod tests { attempt ); assert!(AgentUpgradePhase::Complete.is_terminal()); - assert!(!AgentUpgradePhase::Ready.is_terminal()); + assert!(!AgentUpgradePhase::Starting.is_terminal()); let digest = attempt.digest(); let mut changed = attempt.clone(); changed.target_version = "0.3.0".into(); assert_ne!(digest, changed.digest()); - - let authorization = AgentUpgradeAuthorization { - attempt_id: attempt.attempt_id.clone(), - attempt_digest: digest.clone(), - device_id: attempt.device_id.clone(), - from_version: attempt.from_version.clone(), - target_version: attempt.target_version.clone(), - artifact_signing_key_id: attempt.signing_key_id.clone(), - authorized_at: Utc::now(), - signing_key_id: attempt.signing_key_id.clone(), - signature: "signature".into(), - }; - assert!(authorization.signing_payload().contains(&digest)); } } diff --git a/nats/callout/src/permissions.rs b/nats/callout/src/permissions.rs index fcdab203..0669df50 100644 --- a/nats/callout/src/permissions.rs +++ b/nats/callout/src/permissions.rs @@ -83,7 +83,6 @@ impl PermissionsConfig { // harmony_reconciler_contracts::kv::desired_state_key). "$KV.desired-state.{device_id}.>".to_string(), "$KV.agent-upgrade-intent.{device_id}".to_string(), - "$KV.agent-upgrade-authorize.{device_id}.>".to_string(), ], deny: vec![], }, @@ -201,9 +200,8 @@ mod tests { #[test] fn device_role_covers_reconciler_contract_kv_subjects() { use harmony_reconciler_contracts::{ - BUCKET_AGENT_UPGRADE_AUTHORIZE, BUCKET_AGENT_UPGRADE_INTENT, - BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, BUCKET_DEVICE_HEARTBEAT, - BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, agent_upgrade_authorize_key, + BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, + BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, agent_upgrade_intent_key, agent_upgrade_status_key, desired_state_key, device_heartbeat_key, device_info_key, device_state_key, }; @@ -234,11 +232,6 @@ mod tests { BUCKET_AGENT_UPGRADE_INTENT, agent_upgrade_intent_key(device) ); - let upgrade_authorize_subject = format!( - "$KV.{}.{}", - BUCKET_AGENT_UPGRADE_AUTHORIZE, - agent_upgrade_authorize_key(device, "attempt-1") - ); let upgrade_status_subject = format!( "$KV.{}.{}", BUCKET_AGENT_UPGRADE_STATUS, @@ -273,10 +266,6 @@ mod tests { &upgrade_intent_subject, &perms.sub_allow )); - assert!(subject_matches_any( - &upgrade_authorize_subject, - &perms.sub_allow - )); assert!(subject_matches_any( &upgrade_status_subject, &perms.pub_allow -- 2.39.5 From 34b64415dd98a8e7cbc48c13cc0d42a6556e47a7 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 20:38:43 -0400 Subject: [PATCH 20/47] docs(fleet): define agent upgrade recovery protocol --- docs/adr/022-fleet-agent-upgrade.md | 163 ++---- docs/design/fleet-agent-upgrades.md | 550 ++++++------------ .../fleet-agent-upgrade-architecture.svg | 49 +- .../diagrams/fleet-agent-upgrade-recovery.svg | 54 +- .../diagrams/fleet-agent-upgrade-sequence.svg | 39 +- .../agent-reconciliation-and-upgrade-plan.md | 109 +--- fleet/agent-upgrade-stop-switch-start-plan.md | 18 + 7 files changed, 341 insertions(+), 641 deletions(-) create mode 100644 fleet/agent-upgrade-stop-switch-start-plan.md diff --git a/docs/adr/022-fleet-agent-upgrade.md b/docs/adr/022-fleet-agent-upgrade.md index c59e299a..d23ce9c6 100644 --- a/docs/adr/022-fleet-agent-upgrade.md +++ b/docs/adr/022-fleet-agent-upgrade.md @@ -8,134 +8,75 @@ Last Updated Date: 2026-07-22 ## Status -Accepted. This revision replaces the original dual-active cutover design. +Accepted. This revision replaces the dual-active design and the later signed +switch-authorization design. ## Context -Fleet devices need unattended agent upgrades without restarting managed -workloads. A failed candidate must return the device to the last working agent, -including after process crashes or power loss. +Harmony Fleet currently manages IoT devices whose agents reconcile Podman +workloads from NATS desired state. Agent replacement must preserve one workload +owner across process or power loss. -The workload reconciler must have one active owner. Running old and new agents -at the same time would allow both processes to mutate Podman state and publish -competing observations. A candidate therefore runs only a non-mutating probe. -Cutover accepts a bounded interval with no reconciler while systemd starts the -new process. Existing workloads continue under their Podman restart policies. +NationTech fleet administrators, the operator, and the NATS control plane share +one administrative trust domain. Device identity comes from a Zitadel JWT whose +signature is validated through Zitadel JWKS and whose registered claims are +checked before its `device_id` is interpolated into device-scoped NATS subjects. +Separate artifact and cutover signatures are omitted because they would remain +inside this trust domain while adding another key lifecycle. + +Future MDC fleets and their OKD control plane are outside this decision. ## Decision -The central operator creates an attempt for a device. The active agent drains -new workload mutations and asks a root-owned updater to stage the candidate. -After the candidate passes `--self-test`, the operator publishes a signed switch -authorization for that exact attempt. The updater atomically changes the active -symlink and restarts the one permanent `fleet-agent.service`. +The operator CAS-writes one per-device attempt. Once accepted, its UUID is bound +to its complete content. The live agent forwards it to a root updater and keeps +reconciling while the updater downloads, verifies, installs, and probes the +candidate. -The updater rolls back when startup readiness or the 60-second probation period -fails. No second reconciler runs during staging or cutover. - -### On-device layout +After probe success, the updater owns one local transaction: ```text -/usr/lib/harmony-fleet/fleet-agent-bootstrap -/usr/lib/harmony-fleet/fleet-agent-v0.2.0 -/usr/lib/harmony-fleet/fleet-agent-v0.3.0 -/usr/local/bin/fleet-agent -> /usr/lib/harmony-fleet/fleet-agent-v0.3.0 -/var/lib/harmony-fleet-updater/transaction.json -/etc/fleet-agent/trusted-upgrade-keys/.pub -/etc/fleet-agent/trusted-upgrade-key-id +stop old service +select candidate atomically +start candidate +commit on strict readiness, otherwise select and start the previous binary ``` -`FleetDeviceSetupScore` installs the bootstrap binary, trusted Ed25519 public -key and active key ID, `harmony-fleet-updater.service`, and -`fleet-agent.service`. The updater is -root-owned and listens on a Unix socket writable by the `fleet-agent` group. -The agent service remains unprivileged. +The old agent attempts to drain its current mutation only after systemd sends +SIGTERM. Shutdown is bounded by systemd, so an operation that exceeds the stop +limit may be killed rather than drained. -The bootstrap binary supplies the privileged updater mode. Downloaded binaries -run only as the unprivileged agent, including their candidate probe. Versioned -binaries are immutable and retained for rollback. +Artifacts use HTTPS and a SHA-256 digest carried by NATS intent. The updater +trusts its `fleet-agent` socket group, derives all managed paths, accepts no +command, and executes candidates without root privileges. It has no NATS, +OpenBao, or operator connection. -### Control-plane contract +There is no artifact signature, second cutover authorization, device trust key, +or extended probation. -The upgrade uses three JetStream KV buckets: +The active service reports ready only after local dependencies, NATS, the +complete desired-state snapshot, activation target-version check, transaction +recovery, and required startup publications succeed. Strict readiness may reject +a valid candidate during a coincident network outage. Retry and failure +classification will be refined separately. -- `agent-upgrade-intent.` stores the latest immutable attempt. -- `agent-upgrade-authorize..` stores the signed switch authorization. -- `agent-upgrade-status.` stores the latest attempt phase and bounded error. - -An attempt includes its UUID, device ID, source and target versions, -architecture, HTTPS artifact URL, maximum size, SHA-256 digest, Ed25519 -signature, signing key ID, and creation time. The operator writes intent and -authorization. A device can read only its own entries and can write only its -own status. - -`Device.spec.agentUpgrade` holds the desired artifact metadata. -`Device.status.agentUpgrade` reflects the attempt ID, target version, phase, -timestamp, and last error. The operator does not retry a failed matching -attempt automatically; changing the desired release metadata or removing the -old intent creates a new attempt. - -### Transaction - -1. The operator writes an attempt when the desired and reported versions differ. -2. The active agent rejects wrong-device, wrong-source, wrong-architecture, - stale, future, malformed, or mutated attempts. -3. The agent pauses new workload mutations and publishes `draining`. -4. The updater downloads over HTTPS with fixed time and size limits, verifies - the digest and Ed25519 signature, fsyncs the binary, and installs it by atomic - rename. -5. The updater runs the candidate as `fleet-agent --self-test`. The probe loads - configuration, reaches Podman when enabled, authenticates to NATS, reads the - permitted desired-state snapshot, and exits. It does not heartbeat, watch, - reconcile, or mutate workloads. -6. The agent publishes `awaiting-authorization`. If no authorization arrives - within five minutes, it cancels the staged transaction and resumes workload - reconciliation. -7. After observing that phase, the operator signs and writes an authorization - for the exact attempt. -8. The updater verifies the authorization, records `switching`, atomically - changes `/usr/local/bin/fleet-agent`, and restarts `fleet-agent.service`. -9. The new agent acquires the exclusive process lock, restores upgrade state, - loads a complete desired-state snapshot, starts its worker, and sends systemd - readiness. -10. The updater records `probation` and verifies that the systemd invocation ID - does not change for 60 seconds. It then records `committed`. - -The transaction journal records `staged`, `switching`, `probation`, -`committed`, `rolling-back`, `failed`, or `rollback-failed`, together with the -attempt digest and previous and target paths. Journal writes, binary installs, -and symlink changes use atomic rename and parent-directory fsync. - -On updater startup, `switching`, `probation`, and `rolling-back` transactions -resume rollback. The updater starts its socket before restarting the previous -agent so that agent startup can inspect the transaction. Corrupt or unreadable -journals fail closed. Rollback failure is persisted as `rollback-failed`. - -### Failure behavior - -- Download, size, digest, signature, architecture, or probe failure leaves the - active symlink unchanged and resumes the old reconciler. -- Operator outage before authorization leaves workloads running and returns the - old agent to normal reconciliation after five minutes. -- New-agent startup failure, readiness timeout, restart during probation, or - updater restart before commit restores the previous symlink and restarts the - previous agent. -- A failed rollback remains visible as `rollback-failed`; it is never reported - as successful recovery. -- Replayed terminal attempts are no-ops. Reusing an attempt UUID with different - content is rejected. +Every pre-commit activation failure biases executable selection toward the +previous binary. `rollback-failed` blocks new attempts until direct repair. +Successful readiness commits immediately. Failures after commit use systemd's +normal restart policy and do not trigger protocol rollback. ## Consequences -Only one process can reconcile workloads. Cutover has a bounded period with no -reconciler, but running workloads are not stopped. The root helper is a small, -fixed protocol rather than a command executor, and it accepts no caller-chosen -filesystem path. +- One process owns workload mutation; there is a bounded interval with no agent + during cutover. +- Preparation does not delay workload reconciliation. +- Rollback changes only the executable. Persistent changes made before commit + must remain readable by the previous release. +- The first updater-capable release requires `FleetDeviceSetupScore`. The root + updater remains outside automatic upgrades. +- NationTech administration, the `fleet-agent` socket, the Zitadel identity + chain, the auth callout, and NATS are trusted control-plane components. -The first updater-capable release requires one final `FleetDeviceSetupScore` -run to install the bootstrap layout, helper service, socket permissions, -trusted keys, and revised agent unit. Older agents remain visible through their -heartbeat version but cannot use automatic upgrades until bootstrapped. - -Fleet-wide canary and percentage rollout policy remains outside this ADR. This -decision defines one attempt on one device. +The [fleet agent upgrade guide](../design/fleet-agent-upgrades.md) owns the +current sequence, recovery behavior, limits, status, and repair constraints. +Wire types remain authoritative in `harmony-reconciler-contracts`. diff --git a/docs/design/fleet-agent-upgrades.md b/docs/design/fleet-agent-upgrades.md index 0387a2e0..5a529845 100644 --- a/docs/design/fleet-agent-upgrades.md +++ b/docs/design/fleet-agent-upgrades.md @@ -1,389 +1,219 @@ -# Fleet Agent Upgrades +# Fleet agent upgrades -## Why this exists +## Scope -Harmony places a reconciler inside each decentralized micro datacenter. That -agent turns durable desired state from NATS into local Podman operations. Once a -device carries real workloads, replacing the agent is no longer a file-copy -problem: the updater must preserve a single owner for workload mutation, survive -loss of power, reject unauthorized code, and recover without a technician at the -device. +This design updates the Harmony agent on IoT devices that reconcile Podman +workloads from NATS desired state. It covers one agent binary on one Linux +device. Operating-system, Podman, workload, and future MDC/OKD upgrades are +outside its scope. -The fleet upgrade protocol solves that narrow problem. It updates one Harmony -agent binary on one Linux device. It does not update the operating system, -kernel, Podman, or workload containers. +![Fleet agent upgrade architecture](../diagrams/fleet-agent-upgrade-architecture.svg) -The design follows Harmony's normal separation of concerns: +## Ownership -- `Device.spec.agentUpgrade` declares the target release. -- The fleet operator converts that declaration into durable NATS intent. -- The unprivileged agent drains its Score reconciliation loop. -- A root-owned helper performs the filesystem and systemd transaction. -- `FleetDeviceSetupScore` installs the helper, trust key, and systemd units. - -![Harmony fleet upgrade architecture](../diagrams/fleet-agent-upgrade-architecture.svg) - -The important architectural decision is **single active ownership**. Harmony -does not run old and new reconcilers together. Existing containers continue to -run while systemd replaces the agent, but desired-state convergence pauses for -the cutover. - -## Design contract - -The protocol is built around five guarantees. - -| Guarantee | Mechanism | +| Component | Responsibility | |---|---| -| One workload owner | An advisory process lock plus stop-before-start systemd restart | -| No root candidate | Downloaded binaries and `--self-test` run as `fleet-agent` | -| Exact authorization | Ed25519 authorization covers the complete attempt digest and identity | -| Atomic selection | A temporary symlink is renamed over the active symlink, then its directory is synced | -| Conservative recovery | Every uncommitted post-switch state attempts rollback; failure is durable and explicit | +| Fleet operator | Select the desired release and CAS-write the current per-device attempt | +| Active agent | Validate and forward intent, reconcile during preparation, drain on SIGTERM, publish status | +| Root updater | Prepare and probe the candidate, stop, switch, start, commit or roll back | +| systemd | Serialize service ownership and enforce startup and shutdown limits | -Running workloads are not part of the transaction. Their continuity depends on -Podman process independence and restart policies. The guarantee is that an agent -upgrade does not deliberately stop them. +The active process holds an exclusive lock, so only one agent can mutate +workloads. Preparation does not pause Podman reconciliation. Running containers +continue under their restart policies during agent cutover. -## Components and trust boundaries +The updater has no NATS, OpenBao, or operator client. Its Unix socket accepts +`upgrade` and read-only `status` requests. It derives every managed path and +accepts no shell command or caller-selected destination. -### Fleet operator +## Trust boundary -The operator watches `Device` resources and creates one immutable attempt when -the reported and desired versions differ. Attempt creation uses JetStream -compare-and-set, so overlapping operator pods cannot silently replace each -other's decision. +The operator CAS-writes `AgentUpgradeAttempt` to the device's +`agent-upgrade-intent` key. CAS orders competing writers; it does not make the KV +key append-only. After local acceptance, the agent and updater bind an attempt +UUID to the digest of its complete content. -After the old agent reports a successful candidate probe, the operator signs a -switch authorization for that exact attempt. The signing key is injected from a -Kubernetes Secret and is never sent to a device. +An attempt contains the device, target version, architecture, artifact URL, byte +limit, SHA-256, and UUID. It has no source version or expiry: a device returning +after days or months can upgrade directly to the current target. Any required +data migration belongs in the binary and must handle the versions it supports. -Code: +HTTPS authenticates transport. SHA-256 binds the downloaded bytes to the NATS +attempt. There is no artifact signature, second cutover authorization, device +upgrade key, or OpenBao dependency. NationTech administrators, the operator, +and NATS share one administrative trust domain, so a separate release-signing +domain is not required today. -- [`AgentUpgradeTarget` and reflected status](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/crd.rs#L106-L151) -- [`reconcile_device_inner`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/agent_upgrade.rs#L115-L227) -- [`AuthorizationSigner`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-operator/src/agent_upgrade.rs#L18-L54) +The updater trusts callers admitted by the `fleet-agent` socket group to forward +NATS intent faithfully. Compromise of that account includes control of agent +upgrades. Installed binaries still run as `fleet-agent`, not root. -### Active fleet agent +The auth callout validates the Zitadel JWT signature through Zitadel JWKS and +checks its registered claims. It then extracts `device_id`, rejects characters +unsafe for NATS subjects, and interpolates the value into device-scoped data +permissions. No unverified client-supplied device ID is used. -The ordinary agent runs as `fleet-agent`. It validates attempts, waits for any -current Podman mutation to finish, pauses new mutations, asks the helper to -stage the candidate, and publishes progress. - -Desired-state notifications still enter memory while paused. Reconciliation -resumes after cancellation or pre-switch failure. On successful cutover, the -old process is stopped by systemd and never resumes. - -Code: - -- [`UpgradeController`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L74-L256) -- [`Reconciler::pause` and `resume`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/reconciler.rs#L161-L173) -- [Paused mutation check](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/reconciler.rs#L206-L210) - -### Root updater - -The updater is deliberately small. Its Unix-socket protocol accepts `stage`, -`switch`, `cancel`, and read-only `status`; it accepts no shell command or -caller-selected filesystem path. Mutating requests are serialized. - -The updater owns: - -- artifact download and verification; -- immutable versioned binaries; -- the active symlink; -- the transaction journal; -- systemd restart and probation; -- rollback. - -The updater itself comes from `fleet-agent-bootstrap`, not from the downloaded -candidate. Fixing the privileged updater therefore requires a new -`FleetDeviceSetupScore` run. - -Code: - -- [Updater protocol and journal types](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L16-L67) -- [`run_server`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L121-L185) -- [`FleetDeviceSetupConfig::render_updater_systemd_unit`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-deploy/src/device_setup.rs#L325-L346) - -## Upgrade algorithm +## Upgrade flow ![Fleet agent upgrade sequence](../diagrams/fleet-agent-upgrade-sequence.svg) -### 1. Create immutable intent - -The operator reads the current version from `Device.status`, compares it with -`Device.spec.agentUpgrade.version`, and writes an `AgentUpgradeAttempt` under -the device's intent key. The attempt includes: - -- UUID and device ID; -- source and target versions; -- architecture; -- HTTPS artifact URL and maximum size; -- SHA-256 and Ed25519 signature; -- signing key ID and creation time. - -The canonical wire types and attempt digest live in -[`harmony-reconciler-contracts/src/upgrade.rs`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs#L7-L89). - -### 2. Validate and drain - -The active agent rejects an attempt for another device, a different source -version or architecture, an equal target version, an invalid UUID, or a creation -time outside the accepted window. Reusing a UUID with different content is also -rejected. - -The agent then acquires the reconciler's runtime gate. An operation already in -progress may finish; no new Podman mutation can begin afterward. - -Implementation: [`UpgradeController::accept`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L90-L136). - -### 3. Stage and probe - -The helper derives the destination from the validated version under -`/usr/lib/harmony-fleet`. It allows HTTPS only, disables redirects, applies a -15-second connection timeout, a 300-second request timeout, the attempt's size -limit, and a compiled 100 MiB ceiling. - -The helper verifies SHA-256 and Ed25519 before installation. It sets executable -permissions, syncs the temporary file, renames it atomically, and syncs the -directory. If that version already exists, its bytes must still match the -attempt. - -The candidate then runs as: - -```text -runuser -u fleet-agent -- --self-test -``` - -The probe loads configuration, checks Podman when enabled, authenticates to -NATS, and consumes a complete server-filtered desired-state snapshot. It does -not acquire the active-agent lock, publish heartbeat, watch desired state, or -mutate workloads. - -Implementation: - -- [`stage`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L234-L323) -- [`--self-test` startup path](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L294-L374) -- [`load_desired_snapshot`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L109-L162) - -### 4. Authorize the exact attempt - -After probe success, the old agent publishes `awaiting-authorization`. The -operator signs a payload containing the attempt digest, device ID, source and -target versions, artifact key ID, and authorization time. - -The helper verifies the signature and compares every bound field with its local -staged transaction. A local process cannot stage one signed artifact and reuse -authorization issued for another. - -If authorization does not arrive within five minutes, the old agent cancels the -staged transaction and resumes reconciliation. - -Implementation: - -- [`AgentUpgradeAuthorization::signing_payload`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs#L62-L89) -- [`verify_authorization`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L475-L491) -- [`UpgradeController::check_timeout`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/upgrade.rs#L171-L183) - -### 5. Switch and prove readiness - -The helper writes `switching` to the journal before changing the symlink. It -then atomically replaces `/usr/local/bin/fleet-agent` and asks systemd to restart -the permanent service. - -The new process must acquire the exclusive agent lock, load configuration, -reach Podman and NATS, restore upgrade state, consume a complete desired-state -snapshot, and initialize reconciliation before sending `READY=1`. - -Implementation: - -- [`switch`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L325-L374) -- [`switch_link`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L515-L522) -- [Agent initialization and readiness](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/main.rs#L413-L440) - -### 6. Probation and commit - -The helper records `probation` and watches the systemd unit for 60 seconds. The -unit must remain active and its `InvocationID` must remain unchanged. A candidate -that crashes and is restarted therefore fails probation even if it happens to -be active at the final check. - -After a stable probation, the helper records `committed`. The new agent observes -that journal transition and publishes `complete`. - -## Crash consistency - -![Fleet agent durable transaction states](../diagrams/fleet-agent-upgrade-recovery.svg) - -The journal uses write-to-temporary, file sync, atomic rename, and parent -directory sync. Binary installation and symlink changes use the same durability -pattern. - -| Last durable phase | Selected binary | Recovery action | -|---|---|---| -| No journal | Existing active target | Nothing | -| `staged` | Previous target | Old agent reconstructs the attempt or times out | -| `switching` | Unknown | Restore previous symlink, then restart | -| `probation` | Candidate | Restore previous symlink, then restart | -| `rolling-back` | Previous target intended | Repeat rollback idempotently | -| `committed` | Candidate | Keep candidate and republish completion | -| `failed` | Previous target | Keep previous target | -| `rollback-failed` | Unknown | Report manual intervention required | - -On updater startup, the previous symlink is restored before the updater reports -systemd readiness. This prevents boot ordering from starting a candidate already -marked for rollback. The updater then serves transaction status while it -restarts the previous agent. - -Corrupt or unreadable journal data fails closed. The helper does not infer state -from an incomplete record. - -Implementation: - -- [Startup recovery](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L121-L185) -- [`prepare_rollback` and `finish_rollback`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L407-L423) -- [`write_transaction`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/harmony-fleet-agent/src/updater.rs#L584-L602) - -## Security model - -The normal agent is assumed capable of requesting staging but not of authorizing -cutover. Filesystem permissions restrict the root updater socket to root and the -`fleet-agent` group. Cryptographic authorization protects `switch` even if the -unprivileged agent process is compromised. - -The current design uses one active Ed25519 key for two roles: - -1. signing artifact bytes; -2. signing rollout authorization. - -That is adequate for a controlled first-party release pipeline, but it is not a -TUF or Uptane trust model. A long-lived system should separate offline artifact -signing from online rollout authorization and eventually support multiple keys, -explicit revocation epochs, and threshold policy. - -The device trusts one active key ID. Key rotation therefore needs coordinated -deployment of the new public key to devices and the corresponding private key to -the operator. Mixed-key fleets cannot currently authorize upgrades from one -operator instance. - -## What this design does not guarantee - -### No dual-agent continuity - -Cutover has an interval with no reconciler, bounded in practice by systemd's -service-start timeout. Running containers continue, but new desired state does -not converge until the new or rolled-back agent starts. - -The HTTP request is bounded to five minutes, candidate `--self-test` to 60 -seconds, and the post-probe authorization wait to another five minutes. Desired -changes accumulate but do not converge while paused. - -### No semantic health gate - -The probe catches configuration, dependency, credential, architecture, and -control-plane access failures. Probation catches process crashes. Neither proves -that the candidate continues to reconcile workloads correctly while remaining -alive. - -### No operating-system rollback - -The protocol cannot recover a broken kernel, filesystem, systemd installation, -Podman package, or bootloader. RAUC, Mender, and SWUpdate solve that larger A/B -system problem. - -### No irreversible local migrations - -Rollback changes the executable only. Agent releases must keep configuration, -local databases, Podman labels, and journal formats readable by the previous -binary until commit. Any future persistent migration needs its own reversible -transaction. - -### No automatic garbage collection - -Versioned binaries remain on disk. A safe future collector must retain the -active target, previous rollback target, bootstrap binary, and every target -named by a non-terminal journal. - -### No automatic retry of identical failure - -A failed matching attempt is terminal. Retrying requires changing release -metadata or removing the old intent so the operator creates a new UUID. Removing -`Device.spec.agentUpgrade` while the agent waits does not cancel immediately; -the five-minute timeout still applies. - -### Wall-clock dependency - -Attempt and authorization freshness use UTC. Devices with a bad RTC or delayed -NTP can reject valid upgrades. Local probation correctly uses monotonic time. - -### Runtime coupling - -The upgrade controller is currently created only when `runtime_enabled=true`. -An agent running without the Podman reconciler cannot upgrade automatically. - -### Root helper lifecycle - -The bootstrap updater is intentionally outside OTA. This limits privilege -escalation risk, but helper fixes require device setup. Running -`FleetDeviceSetupScore` during an active upgrade is not yet a supported -operation. - -## Comparison with established tooling - -| Tool or pattern | Similarity | What Harmony lacks | -|---|---|---| -| Nix profiles / OSTree | Immutable generations and atomic active pointer | Content-addressed closure, generations database, GC | -| systemd service rollout | Notification readiness, restart policy, invocation identity | A fully tested cross-version update framework | -| Kubernetes `Recreate` | One active owner and bounded unavailability | Rollout policy, canaries, progress budgets, history | -| TUF / Uptane | Signed targets and freshness checks | Role separation, thresholds, delegated metadata, rollback/freeze protection | -| RAUC / Mender / SWUpdate | Staged candidate and automatic rollback | Bootloader and full OS A/B recovery | -| rpm / dpkg | Installed version tracking | Package dependency database and script transaction model | - -The closest description is: **a small, single-binary OSTree generation switch -with systemd readiness and an operator-authorized cutover**. - -## Operational constraints - -| Constraint | Current value | +### Intent + +When desired and reported versions differ, the operator writes an attempt with +a new UUID. The agent rejects an attempt for another device, the wrong +architecture, or an invalid UUID. Reuse of a UUID with different content is +rejected. Intent has no age or previous-version gate. + +The per-device intent key can later be replaced by CAS. An unchanged release +whose attempt has failed is reused rather than retried automatically. A new +attempt currently requires changed release metadata or direct control-plane +repair. + +### Preparation and probe + +The agent publishes `preparing` and sends one `upgrade` request. The updater +writes `preparing` to its journal before it: + +1. downloads over HTTPS with redirects disabled; +2. enforces the attempt limit and 100 MiB compiled ceiling; +3. verifies SHA-256; +4. installs or re-verifies `/usr/lib/harmony-fleet/fleet-agent-v`; +5. runs ` --self-test` as `fleet-agent`. + +The live agent continues reconciling during these steps. + +Probe mode parses the real configuration, reaches Podman when enabled, +authenticates to NATS, opens required buckets, reads per-device keys, creates a +filtered desired-state consumer, checks the updater socket, and consumes a +complete snapshot. It starts no loops, publishes no normal status or heartbeat, +and mutates no workload. Creating the ephemeral consumer is a JetStream +management operation. The probe does not prove publish permissions; active +startup exercises its startup writes before readiness. + +NATS connection attempts are limited to 15 seconds over a 3-minute retry +window. The complete probe is limited to 4 minutes. + +### Stop, switch, start + +After probe success, the updater writes `activating` and runs +`systemctl stop fleet-agent.service`. On SIGTERM, the old agent stops admitting +new runtime mutations and waits for the current mutation. It then attempts to +publish acknowledged `stopping` status with `reason=updater`, records drain +duration, flushes NATS, notifies systemd, and exits. + +This drain is bounded by `TimeoutStopSec=60s`. systemd may kill an agent whose +current Podman mutation does not finish in time. The `stopping` status is +diagnostic; the updater does not wait for it. Cutover proceeds when systemd +reports the service inactive. + +The updater atomically replaces `/usr/local/bin/fleet-agent`, syncs its parent +directory, and starts the service. It never deliberately runs old and new +reconcilers together. + +### Readiness and commit + +The new process sends `READY=1` only after it: + +- acquires the process lock and parses configuration; +- reaches Podman when enabled; +- reconnects to NATS and opens required buckets; +- publishes device information; +- reads and validates the recovered updater transaction; +- verifies its compiled version against the activation target; +- consumes a complete desired-state snapshot; +- publishes transaction status when an upgrade is active. + +The local transaction need not match the current NATS intent during recovery. +Its stored digest, target, and exact previous binary path are authoritative after +cutover starts. Rollback restores that path and does not depend on a source +version declared by the operator. + +Successful systemd readiness commits immediately. A network failure after a +successful probe can therefore cause rollback. After `committed`, later crashes +use the normal systemd restart policy; this protocol does not roll back a +candidate that was already ready. + +## Recovery + +![Fleet agent durable transaction and recovery](../diagrams/fleet-agent-upgrade-recovery.svg) + +The updater journal stores attempt identity, previous and target paths, bounded +errors, and up to 16 timed transitions. + +| Durable phase | Recovery | +|---|---| +| `preparing` | Restore the previous link, record `failed`, start the previous service; restore or start failure enters `rollback-failed` | +| `activating` | Record `rolling-back`, restore the previous link, stop, then start the previous service | +| `committed` | Keep the target | +| `rolling-back` | Restore the previous link and repeat stop/start | +| `failed` | Keep the previous target; no automatic retry | +| `rollback-failed` | Refuse every new attempt until direct root repair | + +Before the updater reports systemd readiness, interrupted pre-commit recovery +has selected the previous symlink and persisted its recovery phase. Service +stop/start completion continues asynchronously after updater readiness so the +agent can query local status while starting. Corrupt journal data prevents the +updater socket from starting; state is not inferred from filesystem contents. + +Journal and binary writes use temporary files, file sync, atomic rename, and +parent-directory sync. Symlink selection uses a temporary symlink, atomic +rename, and parent-directory sync. + +`rollback-failed` has no remote clear operation. Repair requires root access to +inspect the journal, active symlink, managed binaries, and both systemd units. +The updater must not be unquarantined until the previous binary is selected and +starts successfully. + +## Status + +Public phases are `preparing`, `stopping`, `starting`, `complete`, +`rolling-back`, `failed`, and `rollback-failed`. The wire contract in +[`harmony-reconciler-contracts/src/upgrade.rs`](https://git.nationtech.io/Nationtech/harmony/src/branch/master/harmony-reconciler-contracts/src/upgrade.rs) +is authoritative for fields and serialization. + +Status includes attempt and target identity, current phase, bounded detail and +error, attempt and transition timing, completed drain duration, typed reason, +boot and systemd invocation IDs when available, one current journald unit +reference, and up to 16 transitions. Log content remains in journald. + +The updater never publishes NATS status. The active agent polls the local +transaction and reflects changes, including `complete`, into NATS and the Device +CR. + +## Limits + +| Limit | Value | |---|---:| -| Artifact connect timeout | 15 seconds | -| Artifact request timeout | 300 seconds | -| Candidate self-test timeout | 60 seconds | -| Compiled artifact ceiling | 100 MiB | -| Attempt accepted age | 24 hours | -| Authorization wait | 5 minutes | -| Authorization accepted age | 10 minutes | -| Future clock tolerance | 5 minutes | -| systemd readiness check | 60 seconds after restart returns | -| Probation | 60 seconds | -| Error text retained | 1,024 characters | -| Operator scan interval | 2 seconds | +| HTTPS connect / complete request | 15 seconds / 5 minutes | +| Artifact size | 100 MiB | +| NATS connection attempt / retry window | 15 seconds / 3 minutes | +| Candidate probe | 4 minutes | +| systemd stop / updater stop bound | 60 / 65 seconds | +| systemd start / updater start bound | 4 minutes / 4 minutes 15 seconds | +| Socket request-read / status response / upgrade response | 10 seconds / 15 seconds / 20 minutes | +| Socket request or response | 1 MiB | +| Concurrent socket handlers | 32 | +| Retained transitions | 16 | +| Detail or error | 1,024 characters | -These are compiled policy, not Score configuration. Change them only with a -clear operational requirement; each additional knob expands the compatibility -surface. +Only one mutating socket request runs at a time; `status` remains available +during preparation and recovery. Versioned binaries are not garbage-collected. +Devices currently have many GB available, so collection is deferred until disk +pressure makes retention an operational concern. -## Ownership priorities +Rollback changes only the executable. Configuration, local databases, Podman +labels, and any persistent format touched before commit must remain readable by +the previous release. -Before broad production rollout: +## Bootstrap -1. Make `rollback-failed` a hard quarantine that blocks new attempts until an - explicit repair clears it. -2. Run VM fault injection at every journal write, binary rename, symlink rename, - readiness transition, and rollback transition. -3. Separate artifact-signing and rollout-authorization keys. -4. Add explicit retry generation and immediate cancellation to the Device API. -5. Gate commit on attempt-bound behavioral health, not process stability alone. -6. Define bootstrap updater compatibility and update procedures. -7. Add attempt history and key-rotation support before scaling fleet rollout. - -The local transaction is intentionally conservative and has clear failure -boundaries. Its long-term risk lies outside the atomic symlink operation: key -lifecycle, semantic health, bootstrap maintenance, migration discipline, and -real systemd and power-loss testing. +`FleetDeviceSetupScore` installs the bootstrap binary, updater and agent units, +socket ownership, state directories, and active symlink. The root updater is not +self-updated. Repairing or replacing it requires another device setup operation. +Running device setup during an active transaction is unsupported. ## Related decisions - [ADR-016: Harmony agent and global mesh](../adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md) - [ADR-022: Fleet agent upgrade procedure](../adr/022-fleet-agent-upgrade.md) - [ADR-023: Deploy architecture](../adr/023-deploy-architecture.md) -- [Agent reconciliation and upgrade implementation plan](https://git.nationtech.io/Nationtech/harmony/src/branch/master/fleet/agent-reconciliation-and-upgrade-plan.md) diff --git a/docs/diagrams/fleet-agent-upgrade-architecture.svg b/docs/diagrams/fleet-agent-upgrade-architecture.svg index 380f6e77..9d413e22 100644 --- a/docs/diagrams/fleet-agent-upgrade-architecture.svg +++ b/docs/diagrams/fleet-agent-upgrade-architecture.svg @@ -1,62 +1,63 @@ - Harmony fleet agent upgrade architecture - A Device desired release flows through the fleet operator and NATS to an unprivileged device agent. A root updater controls binaries, the active symlink, and systemd. Podman workloads remain independent. + IoT fleet agent upgrade ownership and trust architecture + The fleet operator writes one authenticated NATS upgrade intent. The active agent forwards it while continuing Podman reconciliation. A root updater downloads by HTTPS, verifies SHA-256, probes as fleet-agent, and performs the systemd stop, symlink switch, and start transaction. - + - Upgrade control crosses two trust boundaries - Kubernetes declares intent. NATS carries durable state. Only the root helper mutates the active binary. + One intent, one active workload owner + Authenticated NATS carries upgrade intent. HTTPS and SHA-256 bind the candidate bytes. CONTROL PLANE - DESIRED STATE + DESIRED RELEASE Device.spec - release metadata + version and artifact metadata FLEET OPERATOR - attempt + authorization - CAS · status reflection + current attempt + CAS and status reflection - DURABLE MESH + AUTHENTICATED MESH NATS KV - upgrade-intent - upgrade-authorize - upgrade-status - per-device permissions + agent-upgrade-intent + agent-upgrade-status + device-scoped data subjects + identity from verified JWT - DEVICE + IOT PODMAN DEVICE UNPRIVILEGED Fleet agent - drain · probe · report + forward · reconcile · report ROOT Updater - verify · switch · rollback + prepare · switch · recover - Unix socket + upgrade Podman workloads - continue independently - - systemd + symlink - exclusive cutover + reconcile until SIGTERM + + systemd + symlink + stop · inactive · switch + start · strict readiness - + - The agent owns workload intent; the updater owns only the binary transaction. + NationTech administration is one trust domain; device_id comes from a JWKS-verified Zitadel JWT. diff --git a/docs/diagrams/fleet-agent-upgrade-recovery.svg b/docs/diagrams/fleet-agent-upgrade-recovery.svg index 73a77b3f..19790a1d 100644 --- a/docs/diagrams/fleet-agent-upgrade-recovery.svg +++ b/docs/diagrams/fleet-agent-upgrade-recovery.svg @@ -1,45 +1,45 @@ - Fleet agent upgrade durable transaction and recovery - Staged is safe before switch. Switching and probation are uncommitted post-switch states that attempt rollback. Committed keeps the candidate. Rollback failure leaves selection uncertain and requires manual repair. + Fleet agent durable upgrade states and recovery + Preparing keeps the previous agent active; interruption selects the previous binary and fails, while recovery failure quarantines the updater. Activating covers stop, switch, strict start, and readiness; interruption enters rollback. Readiness commits immediately. - + - The journal decides recovery; process memory does not - Any crash before commit attempts rollback; failure is recorded instead of hidden. + The durable journal controls recovery + Every interruption before commit attempts to restore the previous binary. Rollback failure blocks new attempts. - PRE-SWITCH · PREVIOUS AGENT IS ACTIVE - no journalnothing staged - stagedcandidate verified + PREPARATION: PREVIOUS AGENT RECONCILES + no journalexisting target stays active + preparingdownload, verify, probe - probe ok + upgrade request - POST-SWITCH · NOT COMMITTED - switchingpointer may change - probationcandidate is active - authorized - + ACTIVATION: STOP, SWITCH, STRICT START + activatingnot committed + committedtarget retained + probe passes + READY=1 + immediate commit, no wait period - committedcandidate retained - 60 s stable + rolling-backrestore previous target + + activation interrupted before commit - rolling-backrestore previous first - - - crash, timeout, restart, or boot before commit + failedprevious service selected + rollback-failedhard quarantine + restore and start succeed + preparation interrupted: select previous, fail + restore or start fails + restore or start fails - failedprevious restored - rollback-failedselection is uncertain - restart succeeds - restore or restart fails - - Durability boundary - journal fsync → atomic rename → directory fsync + Durability sequence + temporary write, file sync, atomic rename, directory sync + No automatic rollback after committed. diff --git a/docs/diagrams/fleet-agent-upgrade-sequence.svg b/docs/diagrams/fleet-agent-upgrade-sequence.svg index bf7ebc13..21b574ac 100644 --- a/docs/diagrams/fleet-agent-upgrade-sequence.svg +++ b/docs/diagrams/fleet-agent-upgrade-sequence.svg @@ -1,41 +1,38 @@ - Fleet agent upgrade sequence - The operator publishes immutable intent, the active agent drains, the updater verifies and probes a candidate, the operator signs authorization, and systemd starts exactly one new agent before probation commits. + IoT fleet agent stop, switch, start upgrade sequence + The operator CAS-writes the current NATS attempt. The active agent forwards one upgrade request and keeps reconciling while the updater downloads, verifies, installs, and probes. The updater stops the old service, waits for inactivity, switches the symlink, starts the candidate, and commits after strict readiness. The restarted agent publishes completion. - + - One attempt, two signatures, one active process - Artifact trust permits execution as a probe. Operator authorization permits the exclusive switch. + Prepare live, then stop, switch, start + One authenticated intent controls the complete transaction. Strict readiness commits immediately. Fleet operator NATS KV Active agent Root updater - systemd / new agent + systemd / candidate - 1CAS immutable intent - 2attempt delivery - drain mutationscontainers keep running - 3stage signed artifact - HTTPS + SHA-256Ed25519 + fsyncprobe as fleet-agent - 4staged - awaiting-authorization - probe status - 5signed exact authorization - - 6journal · symlink · restart - snapshot + READY=1one process lock - 7stable InvocationID - committed journal - new agent publishes complete + 1CAS current attempt + 2authenticated intent + one upgrade request + keep reconcilingno early drain + 3preparingHTTPS + SHA-256install + fsyncprobe as fleet-agent + 4activating: systemctl stop + SIGTERM + attempt current drainstatus + flush if done60 s systemd bound + inactive observed + 5atomic symlink switchstart + strict READY=13 min NATS bound + 6READY=1: commit locally + agent reflects complete status diff --git a/fleet/agent-reconciliation-and-upgrade-plan.md b/fleet/agent-reconciliation-and-upgrade-plan.md index 10d65b06..543c977e 100644 --- a/fleet/agent-reconciliation-and-upgrade-plan.md +++ b/fleet/agent-reconciliation-and-upgrade-plan.md @@ -19,7 +19,7 @@ - An existing container owned by another deployment or by a human is a conflict. The agent reports it and does not delete the container. - Agent upgrade never runs two workload reconcilers. The candidate runs only a - non-mutating probe. A root-owned updater then switches the single permanent + non-workload-mutating probe. A root-owned updater then switches the permanent systemd service and rolls back if the new agent does not become ready. ## Deployment reconciliation @@ -106,105 +106,18 @@ out of scope until the secret source exposes a revision or watch contract. ## Agent upgrade -This section replaces ADR-022's dual-active cutover and its guarantee that the -old agent remains active until the operator observes a full new agent. That -guarantee conflicts with the single-active-agent decision. The replacement -accepts a bounded interval with no reconciler while systemd starts the new -binary; workloads continue under Podman's restart policy, and the updater rolls -back on failed readiness. ADR-022 must be updated in the same change. +The signed authorization and probation design was replaced by stop-switch-start. +The maintained protocol is +[`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md); +the original implementation plan remains as a link-preserving pointer in +[`agent-upgrade-stop-switch-start-plan.md`](agent-upgrade-stop-switch-start-plan.md). -### Components +Deferred follow-up: -- The unprivileged fleet agent drains workload mutations, stages upgrade intent, - and reports attempt-scoped status. -- A candidate binary runs `--self-test`. It parses configuration, authenticates, - connects to NATS, reads its permitted desired-state snapshot, checks Podman, - reports probe success for the attempt, and exits. It never publishes a normal - heartbeat, subscribes as an active reconciler, or mutates workloads. -- A narrow root-owned helper owns versioned binaries, signature and digest - verification, the active symlink, systemd restart, rollback, and one durable - transaction file. `FleetDeviceSetupScore` installs it as a root systemd - service with a Unix socket owned by root and writable only by the - `fleet-agent` group. Its fixed request protocol accepts stage, signed switch, - cancellation, and read-only transaction status for paths derived under one - compiled artifact root. Finalization, recovery, and rollback are internal. It - accepts no shell command or caller-selected path. -- The operator owns desired version and attempt identity. Every intent, status, - probe, and switch authorization carries the same attempt ID. - -### Control-plane contract - -- `agent-upgrade-intent.` contains the latest immutable attempt. The - operator may write it; that device may read it. -- `agent-upgrade-authorize..` carries a signed authorization - for the exclusive switch after the operator observes probe success. The - operator may write it; that device may read it. -- `agent-upgrade-status.` contains the agent's latest attempt-scoped - phase and bounded error. The device may write it; the operator may read it. - -These are JetStream KV entries. Attempts use UUIDs and revisions are applied -monotonically. Replayed intent for an attempt whose durable status is terminal -is a no-op. `authorize-switch` is the operator transition; `finalize` is the -helper's local transaction completion. Callout permissions expose only the -device's own keys. - -### Upgrade transaction - -1. The operator publishes an immutable attempt containing device ID, source and - target versions, architecture, artifact URL, size limit, SHA-256 digest, - signature, signing key ID, and creation time. -2. The active agent rejects stale, duplicate, wrong-device, wrong-source, - wrong-architecture, or unsupported-version attempts. -3. The agent enters draining. Existing workloads continue running; the current - runtime mutation finishes, and newer desired changes remain queued. -4. The helper downloads to a same-filesystem temporary file, enforces HTTPS and - size limits, verifies architecture, SHA-256, and an Ed25519 signature against - `/etc/fleet-agent/trusted-upgrade-keys/.pub`, fsyncs, and atomically - installs the immutable versioned binary. The candidate always runs as - `fleet-agent`. -5. The candidate runs the non-mutating probe. Failure leaves the active agent - and symlink unchanged. -6. The operator observes probe success for the exact attempt and publishes - `authorize-switch`. If authorization does not arrive within five minutes, - the agent leaves draining, resumes queued workload reconciliation, and - retains the staged binary for inspection or a new attempt. -7. The updater records the previous target, switches the symlink atomically, - and restarts the one permanent `fleet-agent.service`. -8. The new agent acquires an OS advisory process lock, loads configuration, - authenticates, reaches Podman when enabled, initializes a complete desired - snapshot, starts the reconciliation worker, sends systemd readiness, and - reports ready for the attempt. Workload health is not an agent-readiness - predicate. -9. The helper allows 60 seconds for systemd readiness, then finalizes after a - further 60-second probation in which the service remains active and systemd - reports no restart. If startup, readiness, or probation fails, it restores - the previous symlink and restarts the previous version. - -The root-owned transaction file records `staged`, `switching`, `probation`, -`committed`, `rolling-back`, or `failed`, with attempt ID and previous and target -paths. File and parent-directory fsync happen before and after binary rename, -symlink rename, and state transitions. On helper startup, pre-switch states are -safe to resume; uncommitted post-switch states attempt rollback. Rollback failure -is durably reported rather than claimed as recovery. A failed attempt is -terminal until a new attempt ID arrives. - -### Upgrade edge cases - -The required test matrix covers successful upgrade, duplicate and stale attempts, downgrade, -download failure, oversized artifact, digest mismatch, bad signature, wrong -architecture, probe failure, commit before probe, cancellation, operator outage -before commit, queued workload recovery after timeout, stale commit, helper -unavailability, missing previous binary, readiness arriving after timeout, -power loss at every persisted transition, new-agent startup failure, readiness -timeout, helper restart, rollback failure reporting, repeated failed intent, and -reboot after commit. At every pre-switch failure the old agent remains active. -At every post-switch failure the helper automatically attempts rollback. - -The first updater-capable release is bootstrapped by one final -`FleetDeviceSetupScore` run, which installs the versioned binary layout, helper, -socket permissions, and revised systemd unit. The operator remains compatible -with old agents whose heartbeat has no version. Automatic upgrades begin only -after this bootstrap release is observed healthy. +- retry and classify activation readiness failures before rollback or quarantine; +- add an isolated VM agent-upgrade E2E covering success and failure recovery; +- replace the operator's two-second full-device scan before fleet scale; +- add binary garbage collection when disk pressure makes retention relevant. ## Delivery checkpoints diff --git a/fleet/agent-upgrade-stop-switch-start-plan.md b/fleet/agent-upgrade-stop-switch-start-plan.md new file mode 100644 index 00000000..214e01cf --- /dev/null +++ b/fleet/agent-upgrade-stop-switch-start-plan.md @@ -0,0 +1,18 @@ +# Fleet agent upgrade: stop, switch, start + +## Status + +Implemented. This plan replaced the signed two-step authorization protocol for +the IoT Podman fleet agent. + +The maintained documents are: + +- [`docs/adr/022-fleet-agent-upgrade.md`](../docs/adr/022-fleet-agent-upgrade.md) + for the decision and invariants; +- [`docs/design/fleet-agent-upgrades.md`](../docs/design/fleet-agent-upgrades.md) + for the current protocol, recovery behavior, limits, and operational caveats; +- `harmony-reconciler-contracts/src/upgrade.rs` for wire fields and phases; +- `fleet/harmony-fleet-agent/src/updater.rs` for the local transaction. + +This file is retained so links from the implementation work remain valid. It is +not an independent protocol specification. -- 2.39.5 From 3d9db8746b40f0122727ceed7025bca538512cce Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 22 Jul 2026 20:38:55 -0400 Subject: [PATCH 21/47] chore: add code-derived documentation review skill --- .../skills/code-derived-doc-review/SKILL.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .opencode/skills/code-derived-doc-review/SKILL.md diff --git a/.opencode/skills/code-derived-doc-review/SKILL.md b/.opencode/skills/code-derived-doc-review/SKILL.md new file mode 100644 index 00000000..535c3fcf --- /dev/null +++ b/.opencode/skills/code-derived-doc-review/SKILL.md @@ -0,0 +1,70 @@ +--- +name: code-derived-doc-review +description: Use when writing or reviewing architecture and design documentation to derive claims from code, map claims to tests, run an external review, and improve clarity and information density with the humanizer skill. +license: AGPL-3.0-only +compatibility: opencode +--- + +# Code-derived documentation review + +Keep design documentation factual, concise, and difficult to drift away from +the codebase. + +## Workflow + +1. Read the complete document set before editing. +2. Identify the authoritative document for decisions, operational behavior, + wire contracts, and implementation details. Remove duplicate specifications. +3. Extract every architecturally significant claim: ownership, trust boundary, + ordering, state transition, failure policy, durability guarantee, limit, and + security constraint. +4. For each claim, locate: + - the implementation owner with file and symbol; + - the test that observes the behavior; + - the coverage level: direct, partial, or missing. +5. Treat an unsupported claim as a finding. Correct the prose, add a behavioral + test when the guarantee is required, or label the gap explicitly. +6. Ask an independent read-only reviewer to compare the finished documentation + with code and tests. Address findings, then request confirmation. +7. Load the `humanizer` skill. Remove repetition, inflated certainty, generic + conclusions, and AI-style transitions without removing facts or caveats. +8. Run a density pass: each fact has one authoritative home; diagrams summarize + and do not introduce unique guarantees; planning files point to maintained + documentation instead of restating it. +9. Validate links, documentation builds, diagram syntax, formatting, and stale + terminology. + +## Claim table + +Use this shape during review: + +| Architectural claim | Documentation | Implementation owner | Validating test | Coverage | +|---|---|---|---|---| + +Coverage means: + +- **Direct:** the test observes the documented behavior at the owning boundary. +- **Partial:** the test covers a component or serialization detail but not the + full guarantee. +- **Missing:** no automated test proves the claim. + +Do not count a compile check, helper-only test, or repeated production logic as +behavioral proof. Do not imply integration coverage when only a unit test exists. + +## External review prompt + +Ask the reviewer to report findings first with file and line references. Require +checks for factual drift, hidden failure cases, security overclaims, duplicate +authority, unsupported certainty, unclear ownership, and missing tests. The +reviewer must not edit the files. + +## Completion standard + +Documentation is complete when: + +- each significant claim is traceable to code; +- each required guarantee has a direct test or an explicit coverage gap; +- one source owns each fact; +- external review findings are resolved; +- the prose passes the humanizer and density checks; +- documentation and diagram validation pass. -- 2.39.5 From 02584a7eec6bd6a729d7f64155073d7a64224504 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 08:29:11 -0400 Subject: [PATCH 22/47] feat(fleet): Self test checks expected version against bin packaged version, imprive socket handling on upgrades --- Cargo.lock | 2 +- docs/design/fleet-agent-upgrades.md | 3 +- fleet/harmony-fleet-agent/Cargo.toml | 2 +- fleet/harmony-fleet-agent/src/main.rs | 12 +++++++- fleet/harmony-fleet-agent/src/updater.rs | 37 ++++++++++++++++++++---- 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 302ed60b..4b74e3e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4030,7 +4030,7 @@ dependencies = [ [[package]] name = "harmony-fleet-agent" -version = "0.1.0" +version = "0.1.2" dependencies = [ "anyhow", "async-nats", diff --git a/docs/design/fleet-agent-upgrades.md b/docs/design/fleet-agent-upgrades.md index 5a529845..79529c2b 100644 --- a/docs/design/fleet-agent-upgrades.md +++ b/docs/design/fleet-agent-upgrades.md @@ -78,7 +78,8 @@ writes `preparing` to its journal before it: 2. enforces the attempt limit and 100 MiB compiled ceiling; 3. verifies SHA-256; 4. installs or re-verifies `/usr/lib/harmony-fleet/fleet-agent-v`; -5. runs ` --self-test` as `fleet-agent`. +5. runs ` --self-test` as `fleet-agent` and requires its compiled + version to match the target. The live agent continues reconciling during these steps. diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index f127d7db..08975da8 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "harmony-fleet-agent" -version = "0.1.0" +version = "0.1.2" edition = "2024" rust-version = "1.85" diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 6bafddd4..88eab67d 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -50,6 +50,8 @@ struct Cli { config: std::path::PathBuf, #[arg(long)] self_test: bool, + #[arg(long, requires = "self_test", hide = true)] + expected_version: Option, #[arg(long)] updater: bool, #[arg(long, default_value = updater::DEFAULT_SOCKET)] @@ -96,7 +98,7 @@ async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result tracing::error!(error = %e, "NATS server error"), Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"), - Event::Closed => tracing::error!("NATS connection closed"), + Event::Closed => tracing::debug!("NATS connection closed"), other => tracing::debug!(?other, "NATS event"), } }) @@ -348,6 +350,14 @@ async fn main() -> Result<()> { if cli.updater { return updater::run_server(&cli.updater_socket).await; } + if let Some(expected) = cli.expected_version.as_deref() + && expected != env!("CARGO_PKG_VERSION") + { + anyhow::bail!( + "candidate version mismatch: expected {expected}, binary reports {}", + env!("CARGO_PKG_VERSION") + ); + } let _process_lock = (!cli.self_test).then(acquire_process_lock).transpose()?; let cfg = config::load_config(&cli.config)?; tracing::info!( diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs index c6accc47..cd82cdb4 100644 --- a/fleet/harmony-fleet-agent/src/updater.rs +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -97,6 +97,12 @@ impl Transaction { }); } if self.phase != phase { + tracing::info!( + attempt_id = %self.attempt_id, + from = ?self.phase, + to = ?phase, + "upgrade transaction advancing" + ); let previous = self.transitions.last_mut().expect("transition exists"); previous.exited_at = Some(now); previous.duration_ms = Some( @@ -276,8 +282,14 @@ async fn handle(stream: UnixStream, transaction_lock: Arc transaction: None, }, }; - writer.write_all(&serde_json::to_vec(&response)?).await?; - writer.write_all(b"\n").await?; + let mut payload = serde_json::to_vec(&response)?; + payload.push(b'\n'); + if let Err(error) = writer.write_all(&payload).await { + if error.kind() != std::io::ErrorKind::BrokenPipe { + return Err(error.into()); + } + tracing::debug!("upgrade caller exited before the durable result was returned"); + } Ok(()) } @@ -341,6 +353,11 @@ async fn upgrade(attempt: &AgentUpgradeAttempt) -> Result { write_transaction(&transaction).await?; let target = &transaction.target; + tracing::info!( + attempt_id = %attempt.attempt_id, + target_version = %attempt.target_version, + "preparing agent upgrade" + ); let preparation = async { if target.exists() { verify_file(target, attempt).await?; @@ -374,13 +391,16 @@ async fn upgrade(attempt: &AgentUpgradeAttempt) -> Result { tokio::fs::rename(&temporary, target).await?; sync_directory(Path::new(ROOT))?; } - let mut command = tokio::process::Command::new("runuser"); + tracing::info!(path = %target.display(), "candidate artifact verified"); + let mut command = command_without_systemd_notify("runuser"); command.kill_on_drop(true).args([ "-u", "fleet-agent", "--", target.to_str().context("non-UTF-8 target path")?, "--self-test", + "--expected-version", + &attempt.target_version, ]); let status = tokio::time::timeout(SELF_TEST_TIMEOUT, command.status()) .await @@ -388,6 +408,7 @@ async fn upgrade(attempt: &AgentUpgradeAttempt) -> Result { if !status.success() { bail!("candidate self-test failed: {status}"); } + tracing::info!(target_version = %attempt.target_version, "candidate self-test passed"); Ok(()) } .await; @@ -643,7 +664,7 @@ async fn wait_ready() -> Result<()> { } async fn systemctl(arguments: &[&str]) -> Result<()> { - let status = tokio::process::Command::new("systemctl") + let status = command_without_systemd_notify("systemctl") .kill_on_drop(true) .args(arguments) .status() @@ -655,7 +676,7 @@ async fn systemctl(arguments: &[&str]) -> Result<()> { } async fn systemctl_property(property: &str) -> Result { - let output = tokio::process::Command::new("systemctl") + let output = command_without_systemd_notify("systemctl") .kill_on_drop(true) .args([ "show", @@ -675,6 +696,12 @@ async fn systemctl_property(property: &str) -> Result { Ok(String::from_utf8(output.stdout)?.trim().to_string()) } +fn command_without_systemd_notify(program: &str) -> tokio::process::Command { + let mut command = tokio::process::Command::new(program); + command.env_remove("NOTIFY_SOCKET"); + command +} + async fn read_transaction() -> Result { Ok(serde_json::from_slice(&tokio::fs::read(JOURNAL).await?)?) } -- 2.39.5 From 4b3029a1d7c53021cc971afb459ba29032323502 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 14:04:35 -0400 Subject: [PATCH 23/47] feat(fleet): add dashboard device commands --- docs/guides/operator-dashboard-sso.md | 17 +- .../harmony-fleet-agent/src/command_server.rs | 332 ++++++++++++++++-- fleet/harmony-fleet-e2e/Cargo.toml | 4 + fleet/harmony-fleet-e2e/tests/exec.rs | 39 ++ fleet/harmony-fleet-operator/src/commands.rs | 27 +- .../src/frontend/server.rs | 89 ++++- .../src/frontend/views/devices.rs | 47 ++- fleet/harmony-fleet-operator/src/main.rs | 39 +- .../src/service/mock.rs | 4 + .../harmony-fleet-operator/src/service/mod.rs | 1 + .../src/service/real.rs | 49 ++- harmony-reconciler-contracts/src/commands.rs | 40 ++- harmony-reconciler-contracts/src/lib.rs | 2 +- harmony_zitadel_auth/src/jwks.rs | 42 +++ harmony_zitadel_auth/src/login.rs | 24 +- harmony_zitadel_auth/src/session.rs | 31 ++ 16 files changed, 727 insertions(+), 60 deletions(-) create mode 100644 fleet/harmony-fleet-e2e/tests/exec.rs diff --git a/docs/guides/operator-dashboard-sso.md b/docs/guides/operator-dashboard-sso.md index b3c6b6ee..a83e45a0 100644 --- a/docs/guides/operator-dashboard-sso.md +++ b/docs/guides/operator-dashboard-sso.md @@ -14,6 +14,12 @@ ID. `FleetOperatorScore` builds `ZitadelAuthConfig` from that output and the dashboard Ingress, then generates and retains the session cookie key in the operator Secret. No client ID or cookie key is entered by hand. +The dashboard requires the exact Zitadel project role `fleet-admin`. Create the +role on the fleet project and grant it to each operator. Login automatically +requests `urn:zitadel:iam:org:project:roles`, so the project-level **Assert Roles +on Authentication** setting is not required. Users without the role receive a +403 response that retains the session cookie and includes a sign-out link. + ## Local dev (`serve-web`) `fleet/harmony-fleet-operator/dev.sh` sets the same config as two @@ -32,16 +38,15 @@ on the app's **Development Mode** (Zitadel rejects non-HTTPS redirects otherwise - **Cookie key** — `cookie_key_b64` must decode to ≥64 bytes, else the dashboard refuses to start (`cookie_key_b64 must decode to at least 64 bytes`; reconcile keeps running). +- **403 after login** — confirm the user has the exact `fleet-admin` project role + and that the aggregate roles claim is present in the ID token. ## Config reference The operator reads `ZitadelAuthConfig` and `OperatorCookieKey` through ConfigClient. The deploy derives `zitadel_base`, `base_url`, client ID, trusted -audience, logout URI, and `scope = openid profile email`. All endpoints derive -from `zitadel_base`: +audience, logout URI, and `scope = openid profile email`. The login flow adds the +aggregate roles scope once if it is absent. All endpoints derive from +`zitadel_base`: `/.well-known/openid-configuration`, `/oauth/v2/authorize`, `/oauth/v2/token`, `/oidc/v1/end_session`. - -> The dashboard only checks that the user authenticated — no role gate yet -> ([web-auth-security](./web-auth-security.md) §3, -> [ROADMAP/09](../../ROADMAP/09-sso-config-hardening.md)). diff --git a/fleet/harmony-fleet-agent/src/command_server.rs b/fleet/harmony-fleet-agent/src/command_server.rs index 0e31e372..90e5b128 100644 --- a/fleet/harmony-fleet-agent/src/command_server.rs +++ b/fleet/harmony-fleet-agent/src/command_server.rs @@ -1,10 +1,7 @@ //! Agent-side request/reply command server. //! -//! Subscribes to `device-commands..>` and dispatches one -//! handler per verb. Single-shot replies for v1; streaming verbs -//! (logs, exec follow-up) will reuse this loop and write multiple -//! frames to the inbox, terminating with the `X-Harmony-Final` -//! header. +//! Subscribes to `device-commands..>` and returns one reply +//! for each ping or bounded exec request. //! //! Runs alongside the KV reconciler in the agent's top-level //! `tokio::select!`. Independent of the podman runtime: when @@ -12,23 +9,42 @@ //! the command server still runs (ping is useful for "is this device //! online" health-checks regardless). +use std::io; +use std::os::unix::process::CommandExt; +use std::process::Stdio; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use async_nats::Client; use async_nats::Subject; use futures_util::StreamExt; use harmony_reconciler_contracts::{ - HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subscription, + CommandRequest, ExecReply, HDR_REQUEST_ID, Id, PingReply, Verb, device_command_subject, + device_command_subscription, }; use serde::Serialize; use thiserror::Error; +use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::sync::Semaphore; + +const EXEC_MAX_COMMAND: usize = 16 * 1024; +const EXEC_MAX_OUTPUT: usize = 128 * 1024; +const EXEC_DEADLINE: Duration = Duration::from_secs(25); +const EXEC_CONCURRENCY: usize = 4; +const COMMAND_CONCURRENCY: usize = 32; +const SIGKILL: i32 = 9; +const ESRCH: i32 = 3; + +unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; +} pub struct CommandServer { device_id: Id, client: Client, agent_version: &'static str, started_at: Instant, + exec_permits: Semaphore, } impl CommandServer { @@ -36,46 +52,39 @@ impl CommandServer { Self { device_id, client, - agent_version: env!("CARGO_PKG_VERSION"), + agent_version: crate::VERSION, started_at: Instant::now(), + exec_permits: Semaphore::new(EXEC_CONCURRENCY), } } pub async fn run(self: Arc) -> Result<(), CommandServerError> { let subject = device_command_subscription(&self.device_id.to_string()); tracing::info!(subject = %subject, "command server subscribing"); - let mut sub = self.client.subscribe(subject.clone()).await.map_err(|e| { + let sub = self.client.subscribe(subject.clone()).await.map_err(|e| { CommandServerError::Subscribe { subject: subject.clone(), source: e, } })?; - while let Some(msg) = sub.next().await { + sub.for_each_concurrent(COMMAND_CONCURRENCY, |msg| { let me = self.clone(); - tokio::spawn(async move { + async move { match me.dispatch(msg).await { Ok(()) => tracing::debug!("command handled"), Err(e) => { tracing::error!(command_error = %e, "failed to handle command") } }; - }); - } + } + }) + .await; tracing::warn!("command server subscription ended"); Ok(()) } async fn dispatch(&self, msg: async_nats::Message) -> Result<(), CommandError> { - // Subject token after the device id is the verb. Pattern is - // `device-commands..` — we own both ends so this - // unwrap shape is safe under normal routing. - // FIXME do not unwrap here, we cannot affoard to crash an entire fleet because a verb is - // added or removed or format changed. Log an error and move on maybe we could list supported verbs. - let verb_token = if let Some(verb) = msg.subject.rsplit('.').next() { - verb - } else { - return Err(CommandError::InvalidFormat(msg.subject.to_string())); - }; + let verb_token = msg.subject.rsplit('.').next().unwrap_or_default(); let request_id = msg .headers .as_ref() @@ -96,9 +105,19 @@ impl CommandServer { } }; - if verb_token == Verb::Ping.as_subject_token() { + let device_id = self.device_id.to_string(); + if msg.subject.as_str() == device_command_subject(&device_id, Verb::Ping) { self.reply_ping(reply_to).await?; Ok(()) + } else if msg.subject.as_str() == device_command_subject(&device_id, Verb::Exec) { + let reply = match serde_json::from_slice(&msg.payload) { + Ok(CommandRequest::Exec { command }) => { + run_shell(&command, EXEC_DEADLINE, &self.exec_permits).await + } + Ok(_) => exec_error("expected exec request body"), + Err(error) => exec_error(format!("invalid exec request: {error}")), + }; + self.publish_reply(reply_to, &reply).await } else { tracing::warn!(verb = %verb_token, "unknown command verb"); Err(CommandError::UnknownVerb(verb_token.to_string())) @@ -111,7 +130,15 @@ impl CommandServer { agent_version: self.agent_version.to_string(), uptime_s: self.started_at.elapsed().as_secs(), }; - let payload = serde_json::to_vec(&reply).map_err(CommandError::SerializeReply)?; + self.publish_reply(reply_to, &reply).await + } + + async fn publish_reply( + &self, + reply_to: Subject, + reply: &impl Serialize, + ) -> Result<(), CommandError> { + let payload = serde_json::to_vec(reply).map_err(CommandError::SerializeReply)?; self.client .publish(reply_to, payload.into()) .await @@ -119,13 +146,164 @@ impl CommandServer { } } +async fn run_shell(command: &str, deadline: Duration, permits: &Semaphore) -> ExecReply { + if command.trim().is_empty() { + return exec_error("command must not be empty"); + } + if command.len() > EXEC_MAX_COMMAND { + return exec_error(format!("command exceeds {EXEC_MAX_COMMAND} byte limit")); + } + + let _permit = match permits.try_acquire() { + Ok(permit) => permit, + Err(_) => return exec_error("too many commands are already running"), + }; + + let mut command_process = std::process::Command::new("sh"); + command_process + .arg("-c") + // The outer shell terminates same-group background jobs after the command. + .arg("sh -c \"$1\"; status=$?; trap '' TERM; kill -TERM 0; exit $status") + .arg("harmony-exec") + .arg(command) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // A dedicated process group lets timeout cleanup reach shell descendants. + .process_group(0); + let mut child = match tokio::process::Command::from(command_process) + .kill_on_drop(true) + .spawn() + { + Ok(child) => child, + Err(error) => return exec_error(format!("failed to spawn shell: {error}")), + }; + let Some(process_group) = child.id().and_then(|id| i32::try_from(id).ok()) else { + return exec_error("spawned shell has no process id"); + }; + + let mut stdout = child.stdout.take().expect("piped stdout"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let completed = tokio::time::timeout( + deadline, + collect_output(&mut child, &mut stdout, &mut stderr), + ) + .await; + let (status, stdout, stderr, truncated) = match completed { + Ok(Ok(result)) => result, + outcome => { + // SAFETY: process_group is the positive PID returned for the child whose + // process group was set to that PID before exec. + if unsafe { kill(-process_group, SIGKILL) } == -1 { + tracing::warn!(error = %io::Error::last_os_error(), "failed to kill exec process group"); + let _ = child.start_kill(); + } + let _ = tokio::time::timeout(Duration::from_secs(1), child.wait()).await; + return match outcome { + Ok(Err(error)) => exec_error(format!("failed waiting for command output: {error}")), + Err(_) => exec_error(format!( + "command exceeded {}s deadline and was killed", + deadline.as_secs_f64() + )), + Ok(Ok(_)) => unreachable!(), + }; + } + }; + // The wrapper sends SIGTERM before exiting. Kill any same-group process + // that ignored it; ESRCH means the group is already gone. + let cleanup = unsafe { kill(-process_group, SIGKILL) }; + if cleanup == -1 { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(ESRCH) { + tracing::warn!(%error, "failed to clean up exec process group"); + } + } + + let (stdout, stderr, utf8_truncated) = bounded_strings(stdout, stderr); + ExecReply { + exit_code: status.code().unwrap_or(-1), + stdout, + stderr, + truncated: truncated || utf8_truncated, + } +} + +async fn collect_output( + child: &mut tokio::process::Child, + stdout: &mut (impl AsyncRead + Unpin), + stderr: &mut (impl AsyncRead + Unpin), +) -> io::Result<(std::process::ExitStatus, Vec, Vec, bool)> { + let (mut stdout_open, mut stderr_open) = (true, true); + let (mut stdout_output, mut stderr_output) = (Vec::new(), Vec::new()); + let (mut stdout_chunk, mut stderr_chunk) = ([0; 8192], [0; 8192]); + let (mut remaining, mut truncated) = (EXEC_MAX_OUTPUT, false); + let mut status = None; + + while status.is_none() || stdout_open || stderr_open { + tokio::select! { + result = child.wait(), if status.is_none() => status = Some(result?), + result = stdout.read(&mut stdout_chunk), if stdout_open => { + let read = result?; + stdout_open = read != 0; + retain_output(&mut stdout_output, &stdout_chunk[..read], &mut remaining, &mut truncated); + } + result = stderr.read(&mut stderr_chunk), if stderr_open => { + let read = result?; + stderr_open = read != 0; + retain_output(&mut stderr_output, &stderr_chunk[..read], &mut remaining, &mut truncated); + } + } + } + Ok(( + status.expect("loop waits for status"), + stdout_output, + stderr_output, + truncated, + )) +} + +fn retain_output(output: &mut Vec, chunk: &[u8], remaining: &mut usize, truncated: &mut bool) { + let retained = chunk.len().min(*remaining); + output.extend_from_slice(&chunk[..retained]); + *remaining -= retained; + *truncated |= retained < chunk.len(); +} + +fn bounded_strings(stdout: Vec, stderr: Vec) -> (String, String, bool) { + let mut left = EXEC_MAX_OUTPUT; + let mut truncated = false; + let mut convert = |bytes: Vec| { + let text = String::from_utf8_lossy(&bytes); + let end = text + .char_indices() + .map(|(index, _)| index) + .take_while(|index| *index <= left) + .last() + .unwrap_or(0); + let end = if text.len() <= left { text.len() } else { end }; + truncated |= end < text.len(); + left -= end; + text[..end].to_string() + }; + let stdout = convert(stdout); + let stderr = convert(stderr); + (stdout, stderr, truncated) +} + +fn exec_error(message: impl Into) -> ExecReply { + ExecReply { + exit_code: -1, + stdout: String::new(), + stderr: message.into(), + truncated: false, + } +} + /// Failure modes the per-message dispatcher can report. Stays /// `pub(crate)` for now — the run loop logs and continues on each /// variant rather than surfacing them to a caller. #[derive(Debug, Error, Serialize)] pub(crate) enum CommandError { - #[error("invalid command subject: {0}")] - InvalidFormat(String), #[error("unknown verb: {0}")] UnknownVerb(String), #[error("command message had no reply inbox")] @@ -151,3 +329,103 @@ pub enum CommandServerError { source: async_nats::SubscribeError, }, } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn exec_captures_stdout_and_nonzero_stderr() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + let reply = run_shell( + "printf output; printf error >&2; exit 7", + EXEC_DEADLINE, + &permits, + ) + .await; + assert_eq!(reply.exit_code, 7); + assert_eq!(reply.stdout, "output"); + assert_eq!(reply.stderr, "error"); + assert!(!reply.truncated); + } + + #[tokio::test] + async fn exec_rejects_empty_and_oversized_commands() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + for command in ["", " \t\n"] { + let reply = run_shell(command, EXEC_DEADLINE, &permits).await; + assert_eq!(reply.exit_code, -1); + assert!(reply.stderr.contains("empty")); + } + let oversized = "x".repeat(EXEC_MAX_COMMAND + 1); + let reply = run_shell(&oversized, EXEC_DEADLINE, &permits).await; + assert_eq!(reply.exit_code, -1); + assert!(reply.stderr.contains("limit")); + } + + #[tokio::test] + async fn exec_caps_combined_output() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + let reply = run_shell( + "yes o | head -c 100000; yes e | head -c 100000 >&2", + EXEC_DEADLINE, + &permits, + ) + .await; + assert_eq!(reply.exit_code, 0); + assert!(reply.truncated); + assert!(reply.stdout.len() + reply.stderr.len() <= EXEC_MAX_OUTPUT); + } + + #[tokio::test] + async fn exec_timeout_kills_process_group() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + let marker = std::env::temp_dir().join(format!( + "harmony-exec-timeout-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let command = format!("(sleep 0.3; touch '{}') & wait", marker.to_string_lossy()); + let reply = run_shell(&command, Duration::from_millis(100), &permits).await; + assert_eq!(reply.exit_code, -1); + assert!(reply.stderr.contains("deadline")); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!marker.exists(), "background process survived timeout"); + } + + #[tokio::test] + async fn exec_success_terminates_detached_same_group_processes() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + let marker = std::env::temp_dir().join(format!( + "harmony-exec-success-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let command = format!( + "(trap '' TERM; sleep 0.2; touch '{}') >/dev/null 2>&1 &", + marker.to_string_lossy() + ); + let reply = run_shell(&command, EXEC_DEADLINE, &permits).await; + assert_eq!(reply.exit_code, 0); + tokio::time::sleep(Duration::from_millis(250)).await; + assert!( + !marker.exists(), + "background process survived successful exec" + ); + } + + #[tokio::test] + async fn exec_rejects_when_concurrency_is_exhausted() { + let permits = Semaphore::new(EXEC_CONCURRENCY); + let _held = permits.acquire_many(EXEC_CONCURRENCY as u32).await.unwrap(); + let reply = run_shell("printf started", Duration::from_secs(2), &permits).await; + assert_eq!(reply.exit_code, -1); + assert!(reply.stderr.contains("already running")); + } +} diff --git a/fleet/harmony-fleet-e2e/Cargo.toml b/fleet/harmony-fleet-e2e/Cargo.toml index 987b173b..6a9f96e4 100644 --- a/fleet/harmony-fleet-e2e/Cargo.toml +++ b/fleet/harmony-fleet-e2e/Cargo.toml @@ -14,6 +14,10 @@ path = "src/lib.rs" name = "ping" path = "tests/ping.rs" +[[test]] +name = "exec" +path = "tests/exec.rs" + [[test]] name = "operator" path = "tests/operator.rs" diff --git a/fleet/harmony-fleet-e2e/tests/exec.rs b/fleet/harmony-fleet-e2e/tests/exec.rs new file mode 100644 index 00000000..a606d4cb --- /dev/null +++ b/fleet/harmony-fleet-e2e/tests/exec.rs @@ -0,0 +1,39 @@ +//! Operator-to-agent exec over Core NATS request/reply. + +use harmony_fleet_e2e::{StackOptions, shared_stack}; +use harmony_fleet_operator::commands::{CommandError, FleetCommandsClient}; + +const E2E_ENV: &str = "HARMONY_FLEET_E2E"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn operator_can_execute_a_bounded_agent_command() -> anyhow::Result<()> { + if !matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true")) { + eprintln!("skipping {E2E_ENV}-gated exec e2e test"); + return Ok(()); + } + + let stack = shared_stack(StackOptions::default()).await?; + let client = FleetCommandsClient::new(stack.nats_client.clone()); + let reply = client + .exec( + &stack.device_ids[0], + "printf stdout; printf stderr >&2; exit 7", + ) + .await?; + + assert_eq!(reply.exit_code, 7); + assert_eq!(reply.stdout, "stdout"); + assert_eq!(reply.stderr, "stderr"); + assert!(!reply.truncated); + + let delayed = client + .exec(&stack.device_ids[0], "sleep 11; printf delayed") + .await?; + assert_eq!(delayed.stdout, "delayed"); + + assert!(matches!( + client.exec("missing-device", "true").await, + Err(CommandError::DeviceOffline) + )); + Ok(()) +} diff --git a/fleet/harmony-fleet-operator/src/commands.rs b/fleet/harmony-fleet-operator/src/commands.rs index a696bb12..eb611c54 100644 --- a/fleet/harmony-fleet-operator/src/commands.rs +++ b/fleet/harmony-fleet-operator/src/commands.rs @@ -15,12 +15,15 @@ use std::time::Duration; use async_nats::Client; use async_nats::error::Error as NatsError; -use harmony_reconciler_contracts::{PingReply, Verb, device_command_subject}; +use harmony_reconciler_contracts::{ + CommandRequest, ExecReply, HDR_REQUEST_ID, PingReply, Verb, device_command_subject, +}; /// Default reply timeout for a single-shot command. 5 s is plenty for /// a healthy device on a LAN; offline devices get short-circuited /// earlier by NATS's `no_responders` reply. pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); +pub const EXEC_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, thiserror::Error)] pub enum CommandError { @@ -76,6 +79,28 @@ impl FleetCommandsClient { let reply: PingReply = serde_json::from_slice(&resp.payload)?; Ok(reply) } + + pub async fn exec(&self, device_id: &str, command: &str) -> Result { + let subject = device_command_subject(device_id, Verb::Exec); + let payload = serde_json::to_vec(&CommandRequest::Exec { + command: command.to_string(), + }) + .expect("command request is serializable"); + let mut headers = async_nats::HeaderMap::new(); + headers.insert(HDR_REQUEST_ID, uuid::Uuid::new_v4().to_string()); + let response = self + .nc + .send_request( + subject, + async_nats::Request::new() + .headers(headers) + .payload(payload.into()) + .timeout(Some(EXEC_TIMEOUT)), + ) + .await + .map_err(|error| map_request_error(error, EXEC_TIMEOUT))?; + Ok(serde_json::from_slice(&response.payload)?) + } } /// Map an async-nats `RequestError` to our typed surface. The `kind` diff --git a/fleet/harmony-fleet-operator/src/frontend/server.rs b/fleet/harmony-fleet-operator/src/frontend/server.rs index 967e0197..1f6a3dce 100644 --- a/fleet/harmony-fleet-operator/src/frontend/server.rs +++ b/fleet/harmony-fleet-operator/src/frontend/server.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::Result; use axum::Router; use axum::body::Body; -use axum::extract::{Extension, FromRef, Path, Query, State}; +use axum::extract::{Extension, Form, FromRef, Path, Query, State}; use axum::http::Request; use axum::http::{HeaderValue, Method, StatusCode, header}; use axum::middleware::{self, Next}; @@ -30,6 +30,7 @@ use crate::service::FleetService; use harmony_zitadel_auth::ZitadelAuthConfig; pub const DEFAULT_PORT: u16 = 18080; +const REQUIRED_ROLE: &str = "fleet-admin"; #[derive(Clone)] pub struct AppState { @@ -89,6 +90,7 @@ pub fn router(state: AppState) -> Router { let public_routes = Router::new() .route("/login", get(auth::login_handler)) .route("/auth/callback", get(auth::callback_handler)) + .route("/logout", get(auth::logout_handler)) .route("/static/tailwind.css", get(tailwind_css)) .route("/static/htmx.min.js", get(htmx_js)) .route("/static/app.js", get(app_js)); @@ -99,6 +101,7 @@ pub fn router(state: AppState) -> Router { // Devices .route("/devices", get(devices_handler)) .route("/devices/{id}/blacklist", post(blacklist_handler)) + .route("/devices/{id}/exec", post(device_exec_handler)) // Device detail .route("/device/{id}", get(device_detail_handler)) // Deployments @@ -107,8 +110,6 @@ pub fn router(state: AppState) -> Router { // Alerts .route("/alerts", get(alerts_handler)) .route("/alerts/{id}/ack", post(ack_alert_handler)) - // Logout - .route("/logout", get(auth::logout_handler)) .route_layer(middleware::from_fn_with_state(state.clone(), csrf_protect)) .route_layer(middleware::from_fn_with_state(state.clone(), require_auth)); @@ -137,6 +138,13 @@ async fn require_auth( match state.jwks.verify(cookie.value(), &state.config).await { Ok(session) => { + if !session.has_role(REQUIRED_ROLE) { + tracing::warn!( + roles = ?session.roles, + "dashboard access denied: missing {REQUIRED_ROLE} role" + ); + return forbidden_response(); + } req.extensions_mut().insert(session); next.run(req).await } @@ -148,6 +156,27 @@ async fn require_auth( } } +fn forbidden_response() -> Response { + let body = maud::html! { + (maud::DOCTYPE) + html lang="en" { + head { + meta charset="utf-8"; + meta name="viewport" content="width=device-width, initial-scale=1"; + title { "Access denied - Harmony Fleet" } + } + body { + main { + h1 { "Access denied" } + p { "The " code { (REQUIRED_ROLE) } " role is required to use this dashboard." } + p { "Ask your administrator to grant the role, or " a href="/logout" { "sign out" } "." } + } + } + } + }; + (StatusCode::FORBIDDEN, body).into_response() +} + async fn csrf_protect(State(state): State, req: Request, next: Next) -> Response { if !is_mutating_method(req.method()) { return next.run(req).await; @@ -400,6 +429,60 @@ async fn device_detail_handler( )) } +#[derive(Deserialize)] +struct ExecForm { + command: String, +} + +async fn device_exec_handler( + State(s): State, + Path(id): Path, + Extension(session): Extension, + Form(form): Form, +) -> Result { + if form.command.trim().is_empty() { + return Ok(devices_view::command_output( + &form.command, + "Command failed: command must not be empty", + )); + } + if form.command.len() > 16 * 1024 { + return Ok(devices_view::command_output( + "", + "Command failed: command exceeds 16384 byte limit", + )); + } + tracing::info!( + operator = %session.subject, + device = %id, + command_bytes = form.command.len(), + "dashboard device command requested" + ); + let started = tokio::time::Instant::now(); + let output = match s.fleet.run_command(&id, &form.command).await { + Ok(output) => { + tracing::info!( + operator = %session.subject, + device = %id, + duration_ms = started.elapsed().as_millis(), + "dashboard device command completed" + ); + output + } + Err(error) => { + tracing::warn!( + operator = %session.subject, + device = %id, + duration_ms = started.elapsed().as_millis(), + %error, + "dashboard device command failed" + ); + format!("Command failed: {error}") + } + }; + Ok(devices_view::command_output(&form.command, &output)) +} + // ── Deployments ──────────────────────────────────────────────────────── async fn deployments_handler( diff --git a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs index e257df53..a9ba883e 100644 --- a/fleet/harmony-fleet-operator/src/frontend/views/devices.rs +++ b/fleet/harmony-fleet-operator/src/frontend/views/devices.rs @@ -199,6 +199,7 @@ pub fn detail(device: &DeviceDetail, deployment_version: Option<&str>) -> Markup } (overview_tab(device, deployment_version)) + (command_panel(device)) } } } @@ -253,6 +254,40 @@ fn overview_tab(device: &DeviceDetail, deployment_version: Option<&str>) -> Mark } } +fn command_panel(device: &DeviceDetail) -> Markup { + html! { + div class="card p-5" { + div class="section-title mb-3" { "Run command" } + form + class="flex gap-2" + hx-post={"/devices/" (device.id) "/exec"} + hx-target="#command-output" + hx-swap="innerHTML" { + input + class="flex-1 rounded border px-3 py-2 font-mono text-[12px] text-slate-100" + style="background:#050608; border-color:var(--border)" + type="text" + name="command" + maxlength="16384" + autocomplete="off" + placeholder="uname -a" + required; + button class="btn" type="submit" { "Run" } + } + pre id="command-output" class="mt-3 min-h-16 overflow-auto whitespace-pre-wrap rounded p-3 font-mono text-[11.5px] text-slate-300" style="background:#050608" { + "Command output appears here." + } + } + } +} + +pub fn command_output(command: &str, output: &str) -> Markup { + html! { + span class="text-slate-500" { "$ " (command) "\n" } + (output) + } +} + // ── Helpers ──────────────────────────────────────────────────────────── fn agent_version(d: &DeviceDetail) -> &str { @@ -310,18 +345,14 @@ mod tests { } #[test] - fn detail_only_shows_implemented_device_features() { + fn detail_shows_command_form_and_only_implemented_device_features() { let html = detail(&sample(), Some("v2.14.1")).into_string(); assert!(html.contains("Device info")); assert!(html.contains("v1.2.3"), "agent version from heartbeat"); assert!(html.contains("aarch64")); - for unsupported in [ - "Reconcile", - "Restart", - "Suspend", - "Recent logs", - "Run command", - ] { + assert!(html.contains("Run command")); + assert!(html.contains("/devices/hf-edge-001/exec")); + for unsupported in ["Reconcile", "Restart", "Suspend", "Recent logs"] { assert!(!html.contains(unsupported)); } } diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 723a202a..0493e1cc 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -159,7 +159,18 @@ async fn main() -> Result<()> { addr, css_from, live_reload, - } => serve_web(mock, addr, css_from, live_reload, &cli.tenant_namespace).await, + } => { + serve_web( + mock, + addr, + css_from, + live_reload, + &cli.tenant_namespace, + &cli.nats_url, + &credentials_toml, + ) + .await + } } } @@ -173,6 +184,8 @@ async fn serve_web( css_from: Option, live_reload: bool, tenant_namespace: &str, + nats_url: &str, + credentials_toml: &str, ) -> Result<()> { use std::sync::Arc; @@ -181,9 +194,13 @@ async fn serve_web( let fleet: Arc = if mock { Arc::new(MockFleetService::default()) } else { + let commands = harmony_fleet_operator::commands::FleetCommandsClient::new( + connect_with_retry(nats_url, credentials_toml).await?, + ); Arc::new(RealFleetService::new( Client::try_default().await?, tenant_namespace, + commands, )) }; serve_dashboard(fleet, addr, css_from, live_reload).await @@ -244,7 +261,11 @@ async fn serve_dashboard( /// (e.g. Zitadel not yet reachable for JWKS) is logged but never tears /// down reconcile — the read UI is best-effort, the controller is not. #[cfg(feature = "web-frontend")] -fn spawn_dashboard(client: Client, tenant_namespace: &str) { +fn spawn_dashboard( + client: Client, + tenant_namespace: &str, + commands: harmony_fleet_operator::commands::FleetCommandsClient, +) { use std::net::SocketAddr; use std::sync::Arc; @@ -253,7 +274,7 @@ fn spawn_dashboard(client: Client, tenant_namespace: &str) { let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT)); let tenant_namespace = tenant_namespace.to_string(); tokio::spawn(async move { - let fleet = Arc::new(RealFleetService::new(client, tenant_namespace)); + let fleet = Arc::new(RealFleetService::new(client, tenant_namespace, commands)); if let Err(e) = serve_dashboard(fleet, addr, None, false).await { tracing::error!(error = %e, "dashboard server exited; reconcile continues"); } @@ -270,7 +291,7 @@ async fn run( ) -> Result<()> { let nats = connect_with_retry(nats_url, credentials_toml).await?; tracing::info!(url = %nats_url, "connected to NATS"); - let js = jetstream::new(nats); + let js = jetstream::new(nats.clone()); let desired_state_kv = js .create_key_value(jetstream::kv::Config { bucket: bucket.to_string(), @@ -328,11 +349,13 @@ async fn run( } }; - // Serve the read-only dashboard in the same process (best-effort; - // it reads CRs only, no NATS). Built only with the web-frontend - // feature; absent from the lean reconcile-only image. + // Dashboard state comes from CRs; interactive commands use NATS request/reply. #[cfg(feature = "web-frontend")] - spawn_dashboard(client.clone(), tenant_namespace); + spawn_dashboard( + client.clone(), + tenant_namespace, + harmony_fleet_operator::commands::FleetCommandsClient::new(nats), + ); // Concurrent tasks: // controller — CR validation + finalizer-cleanup diff --git a/fleet/harmony-fleet-operator/src/service/mock.rs b/fleet/harmony-fleet-operator/src/service/mock.rs index eece7b85..c7aff917 100644 --- a/fleet/harmony-fleet-operator/src/service/mock.rs +++ b/fleet/harmony-fleet-operator/src/service/mock.rs @@ -430,6 +430,10 @@ impl FleetService for MockFleetService { Ok(dev.clone()) } + async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result { + Ok(format!("[{device_id}] {command}\n[exit 0]")) + } + async fn list_alerts(&self) -> anyhow::Result> { Ok(self.alerts.lock().unwrap().clone()) } diff --git a/fleet/harmony-fleet-operator/src/service/mod.rs b/fleet/harmony-fleet-operator/src/service/mod.rs index db6fef88..f43beee7 100644 --- a/fleet/harmony-fleet-operator/src/service/mod.rs +++ b/fleet/harmony-fleet-operator/src/service/mod.rs @@ -16,6 +16,7 @@ pub trait FleetService: Send + Sync + 'static { async fn get_deployment(&self, name: &str) -> anyhow::Result>; async fn get_deployment_devices(&self, name: &str) -> anyhow::Result>; async fn blacklist_device(&self, id: &str) -> anyhow::Result; + async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result; async fn list_alerts(&self) -> anyhow::Result>; async fn ack_alert(&self, id: &str) -> anyhow::Result; async fn filtered_devices( diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index cc2b998b..e8c3ccdd 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -1,4 +1,4 @@ -//! Live [`FleetService`] as a read-only projection of Kubernetes CRs. +//! Live [`FleetService`] backed by Kubernetes CRs and the NATS command channel. //! //! The operator is the write side: `device_reconciler` materializes //! `Device` CRs (labels + inventory), `device_status` reflects liveness @@ -17,11 +17,12 @@ use kube::api::{Api, ListParams, Patch, PatchParams}; use kube::{Client, ResourceExt}; use serde_json::json; +use harmony_fleet_operator::commands::FleetCommandsClient; use harmony_fleet_operator::crd::{ Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability, }; use harmony_fleet_operator::fleet_aggregator::selector_matches; -use harmony_reconciler_contracts::ReconcileScore; +use harmony_reconciler_contracts::{ExecReply, ReconcileScore}; use super::{ Alert, AlertSeverity, DashboardDetail, DeploymentDetail, DeploymentStatus, DeviceDetail, @@ -36,16 +37,18 @@ const REGION_LABEL: &str = "region"; pub struct RealFleetService { kube: Client, namespace: String, + commands: FleetCommandsClient, /// In-memory ack set. Alerts are derived from live CR state and /// have no store of their own, so acks don't survive a restart. acked_alerts: Mutex>, } impl RealFleetService { - pub fn new(kube: Client, namespace: impl Into) -> Self { + pub fn new(kube: Client, namespace: impl Into, commands: FleetCommandsClient) -> Self { Self { kube, namespace: namespace.into(), + commands, acked_alerts: Mutex::new(HashSet::new()), } } @@ -85,6 +88,24 @@ impl RealFleetService { } } +fn format_exec(reply: ExecReply) -> String { + let mut output = reply.stdout; + if !reply.stderr.is_empty() { + if !output.is_empty() && !output.ends_with('\n') { + output.push('\n'); + } + output.push_str(&reply.stderr); + } + if !output.is_empty() && !output.ends_with('\n') { + output.push('\n'); + } + output.push_str(&format!("[exit {}]", reply.exit_code)); + if reply.truncated { + output.push_str(" [output truncated]"); + } + output +} + fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime) -> DeviceDetail { let labels = cr.metadata.labels.clone().unwrap_or_default(); let blacklisted = labels.get(BLACKLIST_LABEL).map(String::as_str) == Some("true"); @@ -341,6 +362,15 @@ impl FleetService for RealFleetService { .ok_or_else(|| anyhow::anyhow!("device {id} not found after blacklist")) } + async fn run_command(&self, device_id: &str, command: &str) -> anyhow::Result { + Ok(format_exec( + self.commands + .exec(device_id, command) + .await + .with_context(|| format!("running command on device {device_id}"))?, + )) + } + async fn list_alerts(&self) -> anyhow::Result> { let devices = self.devices().await?; let deployments = self.deployments().await?; @@ -413,6 +443,19 @@ mod tests { assert_eq!(device_status(false, None), DeviceStatus::Unknown); } + #[test] + fn exec_output_includes_stderr_exit_and_truncation() { + assert_eq!( + format_exec(ExecReply { + exit_code: 7, + stdout: "out".into(), + stderr: "err".into(), + truncated: true, + }), + "out\nerr\n[exit 7] [output truncated]" + ); + } + #[test] fn version_from_image_tag() { let score = ReconcileScore::PodmanV0(PodmanV0Score { diff --git a/harmony-reconciler-contracts/src/commands.rs b/harmony-reconciler-contracts/src/commands.rs index 9d7d448b..ee027885 100644 --- a/harmony-reconciler-contracts/src/commands.rs +++ b/harmony-reconciler-contracts/src/commands.rs @@ -42,6 +42,7 @@ pub const SUBJECT_PREFIX: &str = "device-commands"; #[serde(rename_all = "lowercase")] pub enum Verb { Ping, + Exec, } impl Verb { @@ -51,6 +52,7 @@ impl Verb { pub fn as_subject_token(&self) -> &'static str { match self { Verb::Ping => "ping", + Verb::Exec => "exec", } } } @@ -67,12 +69,12 @@ pub fn device_command_subscription(device_id: &str) -> String { format!("{SUBJECT_PREFIX}.{device_id}.>") } -/// JSON body of an outbound command request. v1 carries only `Ping` -/// (which has no payload). Future verbs add their own struct + variant. +/// JSON body of an outbound command request. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "verb", rename_all = "lowercase")] pub enum CommandRequest { Ping, + Exec { command: String }, } /// JSON body of a `Verb::Ping` reply. @@ -83,6 +85,16 @@ pub struct PingReply { pub uptime_s: u64, } +/// JSON body of a `Verb::Exec` reply. `exit_code` is `-1` when no exit +/// status is available, including validation failures and timeouts. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ExecReply { + pub exit_code: i32, + pub stdout: String, + pub stderr: String, + pub truncated: bool, +} + /// Stable error categories the agent reports on the reply payload /// when a verb can't be handled. The operator-side client maps these /// to its own typed error enum; everything else (no_responders, @@ -155,4 +167,28 @@ mod tests { let json = serde_json::to_string(&CommandRequest::Ping).unwrap(); assert_eq!(json, r#"{"verb":"ping"}"#); } + + #[test] + fn exec_wire_format_is_stable() { + assert_eq!( + device_command_subject("vm-device-00", Verb::Exec), + "device-commands.vm-device-00.exec" + ); + let json = serde_json::to_string(&CommandRequest::Exec { + command: "printf ok".into(), + }) + .unwrap(); + assert_eq!(json, r#"{"verb":"exec","command":"printf ok"}"#); + + let reply = ExecReply { + exit_code: 3, + stdout: String::new(), + stderr: "failed\n".into(), + truncated: false, + }; + assert_eq!( + serde_json::to_string(&reply).unwrap(), + r#"{"exit_code":3,"stdout":"","stderr":"failed\n","truncated":false}"# + ); + } } diff --git a/harmony-reconciler-contracts/src/lib.rs b/harmony-reconciler-contracts/src/lib.rs index 4f1b3533..9fd457c5 100644 --- a/harmony-reconciler-contracts/src/lib.rs +++ b/harmony-reconciler-contracts/src/lib.rs @@ -27,7 +27,7 @@ pub mod upgrade; pub use access::{DeploymentSecretGrants, DeviceGroupSource, GroupSourceError, SecretAccessError}; pub use commands::{ - CommandRequest, ErrorKind, ErrorReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB, + CommandRequest, ErrorKind, ErrorReply, ExecReply, HDR_DEADLINE, HDR_FINAL, HDR_OPERATOR_SUB, HDR_REQUEST_ID, PingReply, SUBJECT_PREFIX, Verb, device_command_subject, device_command_subscription, }; diff --git a/harmony_zitadel_auth/src/jwks.rs b/harmony_zitadel_auth/src/jwks.rs index ed8947cf..1a88a1e8 100644 --- a/harmony_zitadel_auth/src/jwks.rs +++ b/harmony_zitadel_auth/src/jwks.rs @@ -7,6 +7,8 @@ use serde::Deserialize; use crate::config::ZitadelAuthConfig; use crate::session::VerifiedSession; +pub(crate) const ZITADEL_PROJECT_ROLES_CLAIM: &str = "urn:zitadel:iam:org:project:roles"; + struct JwksCacheInner { set: jsonwebtoken::jwk::JwkSet, last_forced_refresh: Option, @@ -165,6 +167,8 @@ fn verify_with_jwk( email: Option, name: Option, nonce: Option, + #[serde(flatten)] + other: serde_json::Map, } let claims = decode::(token, &decoding_key, &validation) @@ -177,9 +181,23 @@ fn verify_with_jwk( name: claims.name, expires_at: claims.exp, nonce: claims.nonce, + roles: extract_zitadel_roles(&claims.other), }) } +fn extract_zitadel_roles(claims: &serde_json::Map) -> Vec { + let mut roles: Vec<_> = claims + .get(ZITADEL_PROJECT_ROLES_CLAIM) + .and_then(serde_json::Value::as_object) + .into_iter() + .flat_map(|roles| roles.keys()) + .cloned() + .collect(); + roles.sort(); + roles.dedup(); + roles +} + async fn discover_jwks_uri(issuer_url: &str, http: &reqwest::Client) -> Result { let url = format!( "{}/.well-known/openid-configuration", @@ -208,3 +226,27 @@ async fn fetch_jwks(jwks_uri: &str, http: &reqwest::Client) -> Result() .await?) } + +#[cfg(test)] +mod tests { + use super::extract_zitadel_roles; + use serde_json::json; + + #[test] + fn extracts_only_roles_asserted_for_the_dashboard_project() { + let claims = json!({ + "urn:zitadel:iam:org:project:roles": { + "fleet-admin": { "org": "Example" }, + "viewer": {} + }, + "urn:zitadel:iam:org:project:123:roles": { + "foreign-admin": {} + } + }); + + assert_eq!( + extract_zitadel_roles(claims.as_object().unwrap()), + ["fleet-admin", "viewer"] + ); + } +} diff --git a/harmony_zitadel_auth/src/login.rs b/harmony_zitadel_auth/src/login.rs index a87c8578..9cd59289 100644 --- a/harmony_zitadel_auth/src/login.rs +++ b/harmony_zitadel_auth/src/login.rs @@ -144,6 +144,18 @@ pub fn build_logout_url(config: &ZitadelAuthConfig, id_token: &str) -> Result String { + let roles_scope = crate::jwks::ZITADEL_PROJECT_ROLES_CLAIM; + if scope + .split_whitespace() + .any(|candidate| candidate == roles_scope) + { + scope.to_string() + } else { + format!("{scope} {roles_scope}") + } +} + pub fn build_login_attempt(config: &ZitadelAuthConfig) -> Result { let state = random_url_token(32); let pkce_code_verifier = random_url_token(32); @@ -155,7 +167,7 @@ pub fn build_login_attempt(config: &ZitadelAuthConfig) -> Result { .append_pair("client_id", &config.client_id) .append_pair("redirect_uri", &config.redirect_uri()) .append_pair("response_type", "code") - .append_pair("scope", &config.scope) + .append_pair("scope", &ensure_roles_scope(&config.scope)) .append_pair("code_challenge", &code_challenge) .append_pair("code_challenge_method", "S256") .append_pair("state", &state) @@ -235,4 +247,14 @@ mod tests { let challenge = pkce_s256_challenge(code_verifier); assert_eq!(challenge, "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM"); } + + #[test] + fn roles_scope_is_added_once() { + assert_eq!( + ensure_roles_scope("openid profile email"), + "openid profile email urn:zitadel:iam:org:project:roles" + ); + let scope = "openid urn:zitadel:iam:org:project:roles email"; + assert_eq!(ensure_roles_scope(scope), scope); + } } diff --git a/harmony_zitadel_auth/src/session.rs b/harmony_zitadel_auth/src/session.rs index 68f9dbf9..21bdd5b0 100644 --- a/harmony_zitadel_auth/src/session.rs +++ b/harmony_zitadel_auth/src/session.rs @@ -9,6 +9,14 @@ pub struct VerifiedSession { pub expires_at: i64, /// OIDC nonce from the ID token, used to bind callback tokens to login attempts. pub nonce: Option, + /// Project role names asserted by Zitadel. + pub roles: Vec, +} + +impl VerifiedSession { + pub fn has_role(&self, role: &str) -> bool { + self.roles.iter().any(|candidate| candidate == role) + } } /// PKCE state persisted in the encrypted login-attempt cookie during the @@ -20,3 +28,26 @@ pub struct LoginAttemptCookie { pub nonce: String, pub next: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + fn session(roles: &[&str]) -> VerifiedSession { + VerifiedSession { + subject: "user".into(), + email: None, + name: None, + expires_at: 0, + nonce: None, + roles: roles.iter().map(|role| (*role).into()).collect(), + } + } + + #[test] + fn role_matching_is_exact() { + let other_roles = session(&["fleet-admin-extra", "viewer"]); + assert!(!other_roles.has_role("fleet-admin")); + assert!(session(&["fleet-admin"]).has_role("fleet-admin")); + } +} -- 2.39.5 From f4f0824c898e67b084ddcf01591c8d16e8ab504d Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 14:04:54 -0400 Subject: [PATCH 24/47] feat(fleet): add OCI agent releases --- Cargo.lock | 121 ++++++++++ docs/design/fleet-agent-upgrades.md | 18 +- fleet/README.md | 32 +++ fleet/harmony-fleet-agent/Cargo.toml | 1 + .../src/fleet_publisher.rs | 2 +- fleet/harmony-fleet-agent/src/main.rs | 18 +- fleet/harmony-fleet-agent/src/updater.rs | 175 ++++++++++++-- fleet/harmony-fleet-agent/src/upgrade.rs | 14 +- fleet/harmony-fleet-e2e/tests/ping.rs | 2 +- fleet/harmony-fleet-e2e/tests/vm_ping.rs | 2 +- fleet/scripts/release.sh | 222 ++++++++++++++++++ 11 files changed, 562 insertions(+), 45 deletions(-) create mode 100755 fleet/scripts/release.sh diff --git a/Cargo.lock b/Cargo.lock index 4b74e3e0..bdfe5f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1750,6 +1750,27 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const_format" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -4042,6 +4063,7 @@ dependencies = [ "harmony-fleet-auth", "harmony-reconciler-contracts", "harmony_secret", + "oci-client", "podman-api", "reqwest 0.12.28", "sd-notify", @@ -4809,6 +4831,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "0.4.6" @@ -5583,6 +5614,21 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64 0.13.1", + "crypto-common 0.1.7", + "digest 0.10.7", + "hmac 0.12.1", + "serde", + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "k3d-rs" version = "0.1.0" @@ -5615,6 +5661,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kube" version = "1.1.0" @@ -6277,6 +6338,49 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-client" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b74df13319e08bc386d333d3dc289c774c88cc543cae31f5347db07b5ec2172" +dependencies = [ + "bytes 1.11.1", + "chrono", + "futures-util", + "http 1.4.0", + "http-auth", + "jwt", + "lazy_static", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.18", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3da52b83ce3258fbf29f66ac784b279453c2ac3c22c5805371b921ede0d308" +dependencies = [ + "const_format", + "derive_builder 0.20.2", + "getset", + "regex", + "serde", + "serde_json", + "strum 0.27.2", + "strum_macros 0.27.2", + "thiserror 2.0.18", +] + [[package]] name = "octocrab" version = "0.44.1" @@ -6333,6 +6437,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -9498,6 +9613,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" diff --git a/docs/design/fleet-agent-upgrades.md b/docs/design/fleet-agent-upgrades.md index 79529c2b..08209e58 100644 --- a/docs/design/fleet-agent-upgrades.md +++ b/docs/design/fleet-agent-upgrades.md @@ -38,11 +38,14 @@ limit, SHA-256, and UUID. It has no source version or expiry: a device returning after days or months can upgrade directly to the current target. Any required data migration belongs in the binary and must handle the versions it supports. -HTTPS authenticates transport. SHA-256 binds the downloaded bytes to the NATS -attempt. There is no artifact signature, second cutover authorization, device -upgrade key, or OpenBao dependency. NationTech administrators, the operator, -and NATS share one administrative trust domain, so a separate release-signing -domain is not required today. +Artifacts are either direct HTTPS URLs or anonymous `oci://` references with an +explicit tag or manifest digest. OCI artifacts must declare the Harmony agent +artifact type and contain exactly one Harmony agent binary layer. HTTPS +authenticates transport; SHA-256 binds the downloaded bytes to the NATS attempt. +There is no artifact signature, second cutover authorization, device upgrade +key, or OpenBao dependency. NationTech administrators, the operator, and NATS +share one administrative trust domain, so a separate release-signing domain is +not required today. The updater trusts callers admitted by the `fleet-agent` socket group to forward NATS intent faithfully. Compromise of that account includes control of agent @@ -213,6 +216,11 @@ socket ownership, state directories, and active symlink. The root updater is not self-updated. Repairing or replacing it requires another device setup operation. Running device setup during an active transaction is unsupported. +Devices installed before OCI support require one device setup operation before +their first `oci://` upgrade. This replaces the bootstrap updater; an older +updater cannot fetch an OCI artifact even when the active agent can accept the +intent. + ## Related decisions - [ADR-016: Harmony agent and global mesh](../adr/016-Harmony-Agent-And-Global-Mesh-For-Decentralized-Workload-Management.md) diff --git a/fleet/README.md b/fleet/README.md index f435f444..7064e65e 100644 --- a/fleet/README.md +++ b/fleet/README.md @@ -142,6 +142,38 @@ Run this from the private deploy repository whose binary calls See [`deployment-process.md`](deployment-process.md) for the clickable CD workflow and the in-cluster runner bootstrap. +### Releases + +`scripts/release.sh` publishes immutable releases. Control-plane and agent +versions are independent; the operator and NATS callout share one control-plane +version. + +```bash +# Publish either release independently. +fleet/scripts/release.sh --control-version 0.4.0 --publish-only +fleet/scripts/release.sh --agent-version 0.2.0 --publish-only + +# Publish both, deploy through a private deploy crate, and upgrade one Device. +fleet/scripts/release.sh \ + --control-version 0.4.0 \ + --agent-version 0.2.0 \ + --deploy-manifest /path/to/Cargo.toml \ + --deploy-bin private-deploy \ + --context production \ + --namespace fleet \ + --device device-1 +``` + +Control-plane images publish to `hub.nationtech.io/harmony`. The agent is a +raw, single-layer OCI artifact and Devices receive a manifest-digest reference. +Registry pulls are anonymous, while publication uses `REGISTRY_USER` and +`REGISTRY_TOKEN`. Run the script with `--help` for its tool and deployment +inputs. + +Before the first OCI-based upgrade, rerun `FleetDeviceSetupScore` for each +device to install an updater that understands `oci://` references. The updater +does not update itself. Direct `https://` agent artifacts remain supported. + ### Connecting to the operator The operator runs as a single-replica Deployment in the context namespace. diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 08975da8..d5eb54be 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -25,5 +25,6 @@ thiserror = { workspace = true } podman-api = "0.9" sd-notify = "0.4" reqwest.workspace = true +oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } uuid.workspace = true fs2.workspace = true diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index 5ea04148..0cdeb387 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -94,7 +94,7 @@ impl FleetPublisher { let hb = HeartbeatPayload { device_id: self.device_id.clone(), at: chrono::Utc::now(), - agent_version: Some(env!("CARGO_PKG_VERSION").to_string()), + agent_version: Some(crate::VERSION.to_string()), }; let key = device_heartbeat_key(&self.device_id.to_string()); match serde_json::to_vec(&hb) { diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 88eab67d..a400f5da 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -36,9 +36,17 @@ use crate::reconciler::{Reconciler, SnapshotEntry}; const RECONCILE_INTERVAL: Duration = Duration::from_secs(30); const NATS_CONNECT_WINDOW: Duration = Duration::from_secs(3 * 60); const NATS_CONNECT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15); +pub(crate) const VERSION: &str = match option_env!("HARMONY_FLEET_AGENT_VERSION") { + Some(version) => version, + None => env!("CARGO_PKG_VERSION"), +}; #[derive(Parser)] -#[command(name = "fleet-agent-v0", about = "IoT agent for Raspberry Pi devices")] +#[command( + name = "fleet-agent-v0", + version = VERSION, + about = "IoT agent for Raspberry Pi devices" +)] struct Cli { #[arg( long, @@ -316,7 +324,7 @@ fn local_inventory() -> InventorySnapshot { .map(|n| n.get() as u32) .unwrap_or(0), memory_mb: sys_memory_total_mb().unwrap_or(0), - agent_version: env!("CARGO_PKG_VERSION").to_string(), + agent_version: VERSION.to_string(), } } @@ -351,11 +359,11 @@ async fn main() -> Result<()> { return updater::run_server(&cli.updater_socket).await; } if let Some(expected) = cli.expected_version.as_deref() - && expected != env!("CARGO_PKG_VERSION") + && expected != VERSION { anyhow::bail!( "candidate version mismatch: expected {expected}, binary reports {}", - env!("CARGO_PKG_VERSION") + VERSION ); } let _process_lock = (!cli.self_test).then(acquire_process_lock).transpose()?; @@ -435,7 +443,7 @@ async fn main() -> Result<()> { let generation = reconciler.generation().await; reconciler.replace_snapshot(snapshot, generation).await?; } - tracing::info!(version = env!("CARGO_PKG_VERSION"), "self-test ok"); + tracing::info!(version = VERSION, "self-test ok"); return Ok(()); } diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs index cd82cdb4..8324f958 100644 --- a/fleet/harmony-fleet-agent/src/updater.rs +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -1,15 +1,20 @@ use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context as TaskContext, Poll}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use futures_util::StreamExt; use harmony_reconciler_contracts::AgentUpgradeAttempt; +use oci_client::client::{ClientConfig, ClientProtocol}; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client as OciClient, Reference}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader}; use tokio::net::{UnixListener, UnixStream}; pub const DEFAULT_SOCKET: &str = "/run/harmony-fleet-updater/updater.sock"; @@ -26,6 +31,8 @@ const MAX_TRANSITIONS: usize = 16; const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10); const STATUS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(15); const UPGRADE_RESPONSE_TIMEOUT: Duration = Duration::from_secs(20 * 60); +const OCI_ARTIFACT_TYPE: &str = "application/vnd.nationtech.harmony.fleet-agent.v1"; +const OCI_LAYER_MEDIA_TYPE: &str = "application/vnd.nationtech.harmony.fleet-agent.binary.v1"; #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "operation", content = "data", rename_all = "kebab-case")] @@ -362,25 +369,7 @@ async fn upgrade(attempt: &AgentUpgradeAttempt) -> Result { if target.exists() { verify_file(target, attempt).await?; } else { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(15)) - .timeout(Duration::from_secs(300)) - .redirect(reqwest::redirect::Policy::none()) - .build()?; - let response = client - .get(&attempt.artifact_url) - .send() - .await? - .error_for_status()?; - let mut bytes = Vec::new(); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - if bytes.len() as u64 + chunk.len() as u64 > attempt.max_bytes { - bail!("artifact exceeds {} bytes", attempt.max_bytes); - } - bytes.extend_from_slice(&chunk); - } + let bytes = download_artifact(attempt).await?; verify_bytes(&bytes, attempt).await?; tokio::fs::create_dir_all(ROOT).await?; let temporary = target.with_extension("tmp"); @@ -528,11 +517,7 @@ fn validate_attempt(attempt: &AgentUpgradeAttempt) -> Result<()> { if attempt.architecture != std::env::consts::ARCH { bail!("artifact architecture does not match this device"); } - let artifact_url = - reqwest::Url::parse(&attempt.artifact_url).context("invalid artifact URL")?; - if artifact_url.scheme() != "https" || artifact_url.host_str().is_none() { - bail!("artifact URL must use HTTPS"); - } + validate_artifact_reference(&attempt.artifact_url)?; if attempt.max_bytes == 0 || attempt.max_bytes > MAX_ARTIFACT_BYTES { bail!("artifact size limit must be between 1 and {MAX_ARTIFACT_BYTES} bytes"); } @@ -542,6 +527,124 @@ fn validate_attempt(attempt: &AgentUpgradeAttempt) -> Result<()> { Ok(()) } +fn validate_artifact_reference(value: &str) -> Result<()> { + if value.starts_with("oci://") { + parse_oci_reference(value)?; + return Ok(()); + } + let url = reqwest::Url::parse(value).context("invalid artifact URL")?; + if url.scheme() != "https" || url.host_str().is_none() { + bail!("artifact URL must use HTTPS or an oci:// reference"); + } + Ok(()) +} + +fn parse_oci_reference(value: &str) -> Result { + let raw = value + .strip_prefix("oci://") + .context("OCI artifact reference must start with oci://")?; + let last_segment = raw.rsplit('/').next().unwrap_or_default(); + if !raw.contains('@') && !last_segment.contains(':') { + bail!("OCI artifact reference requires an explicit tag or digest"); + } + Reference::try_from(raw).context("invalid OCI artifact reference") +} + +async fn download_artifact(attempt: &AgentUpgradeAttempt) -> Result> { + if attempt.artifact_url.starts_with("oci://") { + return download_oci_artifact(&attempt.artifact_url, attempt.max_bytes).await; + } + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(300)) + .redirect(reqwest::redirect::Policy::none()) + .build()?; + let response = client + .get(&attempt.artifact_url) + .send() + .await? + .error_for_status()?; + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if bytes.len() as u64 + chunk.len() as u64 > attempt.max_bytes { + bail!("artifact exceeds {} bytes", attempt.max_bytes); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +async fn download_oci_artifact(value: &str, max_bytes: u64) -> Result> { + let reference = parse_oci_reference(value)?; + let client = OciClient::try_from(ClientConfig { + protocol: ClientProtocol::Https, + connect_timeout: Some(Duration::from_secs(15)), + read_timeout: Some(Duration::from_secs(300)), + max_concurrent_download: 1, + ..Default::default() + })?; + let (manifest, _) = client + .pull_image_manifest(&reference, &RegistryAuth::Anonymous) + .await + .context("pulling OCI artifact manifest")?; + if manifest.artifact_type.as_deref() != Some(OCI_ARTIFACT_TYPE) { + bail!("OCI artifact has an unexpected artifact type"); + } + let [layer] = manifest.layers.as_slice() else { + bail!("OCI artifact must contain exactly one binary layer"); + }; + if layer.media_type != OCI_LAYER_MEDIA_TYPE { + bail!("OCI artifact has an unexpected layer media type"); + } + if layer.size < 0 || layer.size as u64 > max_bytes { + bail!("artifact exceeds {max_bytes} bytes"); + } + let mut output = BoundedWriter::new(max_bytes); + client + .pull_blob(&reference, layer, &mut output) + .await + .context("pulling OCI artifact binary")?; + Ok(output.bytes) +} + +struct BoundedWriter { + bytes: Vec, + limit: u64, +} + +impl BoundedWriter { + fn new(limit: u64) -> Self { + Self { + bytes: Vec::new(), + limit, + } + } +} + +impl AsyncWrite for BoundedWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + buffer: &[u8], + ) -> Poll> { + if self.bytes.len() as u64 + buffer.len() as u64 > self.limit { + return Poll::Ready(Err(std::io::Error::other("artifact exceeds byte limit"))); + } + self.bytes.extend_from_slice(buffer); + Poll::Ready(Ok(buffer.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + fn target_path(version: &str) -> Result { if version.contains('/') || version.contains("..") { bail!("invalid target version path"); @@ -767,6 +870,15 @@ mod tests { #[test] fn upgrade_rejects_invalid_identity_artifact_and_path_inputs() { assert!(validate_attempt(&attempt()).is_ok()); + let mut oci = attempt(); + oci.artifact_url = + "oci://hub.nationtech.io/harmony/harmony-fleet-agent:0.2.0-x86_64".into(); + assert!(validate_attempt(&oci).is_ok()); + oci.artifact_url = format!( + "oci://hub.nationtech.io/harmony/harmony-fleet-agent@sha256:{}", + "b".repeat(64) + ); + assert!(validate_attempt(&oci).is_ok()); for invalid in [ { let mut attempt = attempt(); @@ -793,6 +905,11 @@ mod tests { attempt.artifact_url = "http://example.invalid/agent".into(); attempt }, + { + let mut attempt = attempt(); + attempt.artifact_url = "oci://hub.nationtech.io/harmony/harmony-fleet-agent".into(); + attempt + }, { let mut attempt = attempt(); attempt.max_bytes = 0; @@ -808,6 +925,14 @@ mod tests { } } + #[tokio::test] + async fn oci_download_writer_enforces_the_attempt_limit() { + let mut writer = BoundedWriter::new(3); + writer.write_all(b"abc").await.unwrap(); + assert!(writer.write_all(b"d").await.is_err()); + assert_eq!(writer.bytes, b"abc"); + } + #[test] fn updater_wire_protocol_only_accepts_upgrade_and_status() { let encoded = serde_json::to_value(Request::Upgrade(attempt())).unwrap(); diff --git a/fleet/harmony-fleet-agent/src/upgrade.rs b/fleet/harmony-fleet-agent/src/upgrade.rs index 40c0827f..bf3ad863 100644 --- a/fleet/harmony-fleet-agent/src/upgrade.rs +++ b/fleet/harmony-fleet-agent/src/upgrade.rs @@ -158,7 +158,7 @@ impl UpgradeController { return Ok(()); } if transaction.phase == TransactionPhase::Activating - && transaction.target_version != env!("CARGO_PKG_VERSION") + && transaction.target_version != crate::VERSION { return Ok(()); } @@ -204,11 +204,11 @@ impl UpgradeController { }; let phase = match transaction.phase { TransactionPhase::Activating => { - if transaction.target_version != env!("CARGO_PKG_VERSION") { + if transaction.target_version != crate::VERSION { bail!( "activating updater transaction expects version {}, but running agent is {}", transaction.target_version, - env!("CARGO_PKG_VERSION") + crate::VERSION ); } AgentUpgradePhase::Starting @@ -696,7 +696,7 @@ mod tests { fn intent_is_timeless_and_may_target_the_running_version() { let device_id = Id::from("device-1".to_string()); let mut attempt = attempt(); - attempt.target_version = env!("CARGO_PKG_VERSION").into(); + attempt.target_version = crate::VERSION.into(); let transaction = transaction(&attempt, TransactionPhase::Committed); validate_intent(&attempt, &device_id).unwrap(); @@ -776,7 +776,7 @@ mod tests { let publisher = Arc::new(Publisher::default()); let controller = controller(backend, publisher.clone()); let mut attempt = attempt(); - attempt.target_version = env!("CARGO_PKG_VERSION").into(); + attempt.target_version = crate::VERSION.into(); let prepared_at = Utc::now() - chrono::Duration::seconds(2); let activated_at = prepared_at + chrono::Duration::seconds(1); let mut activating = transaction(&attempt, TransactionPhase::Activating); @@ -931,7 +931,7 @@ mod tests { let publisher = Arc::new(Publisher::default()); let controller = controller(backend.clone(), publisher.clone()); let mut attempt = attempt(); - attempt.target_version = env!("CARGO_PKG_VERSION").into(); + attempt.target_version = crate::VERSION.into(); let cases = [ (TransactionPhase::Preparing, AgentUpgradePhase::Preparing), (TransactionPhase::Activating, AgentUpgradePhase::Starting), @@ -988,7 +988,7 @@ mod tests { assert!(controller.ensure_active_startup().await.is_err()); let mut matching = attempt; - matching.target_version = env!("CARGO_PKG_VERSION").into(); + matching.target_version = crate::VERSION.into(); *backend.transaction.lock().unwrap() = Some(transaction(&matching, TransactionPhase::Activating)); assert!(controller.ensure_active_startup().await.unwrap()); diff --git a/fleet/harmony-fleet-e2e/tests/ping.rs b/fleet/harmony-fleet-e2e/tests/ping.rs index 3fa5d1d7..de50ebac 100644 --- a/fleet/harmony-fleet-e2e/tests/ping.rs +++ b/fleet/harmony-fleet-e2e/tests/ping.rs @@ -64,7 +64,7 @@ async fn operator_can_ping_agent() -> anyhow::Result<()> { ); assert!( !reply.agent_version.is_empty(), - "agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)" + "agent_version must be non-empty" ); Ok(()) } diff --git a/fleet/harmony-fleet-e2e/tests/vm_ping.rs b/fleet/harmony-fleet-e2e/tests/vm_ping.rs index 5a204341..a39c2aac 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_ping.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_ping.rs @@ -61,7 +61,7 @@ async fn agent_on_vm_replies_to_ping() -> anyhow::Result<()> { ); assert!( !reply.agent_version.is_empty(), - "agent_version must be non-empty (env!(CARGO_PKG_VERSION) at compile time)", + "agent_version must be non-empty", ); Ok(()) } diff --git a/fleet/scripts/release.sh b/fleet/scripts/release.sh new file mode 100755 index 00000000..f511be01 --- /dev/null +++ b/fleet/scripts/release.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Build, publish, and deploy independent Harmony Fleet releases. + +Usage: + fleet/scripts/release.sh [options] + +At least one release version is required: + --control-version VERSION Operator and NATS callout version + --agent-version VERSION Standalone fleet-agent version + +Agent options: + --agent-arch ARCH x86_64 (default) or aarch64 + --namespace NAMESPACE Namespace containing Device CRs + --device NAME Device to upgrade; repeat for multiple devices + +Control-plane deployment options: + --deploy-manifest PATH Private deployment Cargo.toml + --deploy-bin NAME Deployment binary + --context NAME Harmony deployment context + +Registry options: + --registry HOST Default: hub.nationtech.io + --repository PATH Default: harmony + +Other: + --kube-context NAME kubectl context used for Device patches + --publish-only Build and push without deploying + -h, --help + +Environment equivalents: + CONTROL_VERSION, AGENT_VERSION, AGENT_ARCH, FLEET_DEPLOY_MANIFEST, + FLEET_DEPLOY_BIN, FLEET_CONTEXT, FLEET_NAMESPACE, KUBECTL_CONTEXT, + REGISTRY_USER, REGISTRY_TOKEN, OPENBAO_TOKEN, RUST_LOG. +EOF +} + +fail() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +validate_version() { + [[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] || + fail "version must be SemVer without a leading v or build metadata: $1" +} + +control_version="${CONTROL_VERSION:-}" +agent_version="${AGENT_VERSION:-}" +agent_arch="${AGENT_ARCH:-x86_64}" +deploy_manifest="${FLEET_DEPLOY_MANIFEST:-}" +deploy_bin="${FLEET_DEPLOY_BIN:-}" +deploy_context="${FLEET_CONTEXT:-}" +namespace="${FLEET_NAMESPACE:-}" +kube_context="${KUBECTL_CONTEXT:-}" +registry="${FLEET_REGISTRY:-hub.nationtech.io}" +repository="${FLEET_REGISTRY_REPOSITORY:-harmony}" +publish_only=0 +devices=() + +while (($#)); do + case "$1" in + --control-version) control_version="${2:?missing control version}"; shift 2 ;; + --agent-version) agent_version="${2:?missing agent version}"; shift 2 ;; + --agent-arch) agent_arch="${2:?missing agent architecture}"; shift 2 ;; + --deploy-manifest) deploy_manifest="${2:?missing deployment manifest}"; shift 2 ;; + --deploy-bin) deploy_bin="${2:?missing deployment binary}"; shift 2 ;; + --context) deploy_context="${2:?missing deployment context}"; shift 2 ;; + --namespace) namespace="${2:?missing namespace}"; shift 2 ;; + --device) devices+=("${2:?missing device name}"); shift 2 ;; + --kube-context) kube_context="${2:?missing kubectl context}"; shift 2 ;; + --registry) registry="${2:?missing registry}"; shift 2 ;; + --repository) repository="${2:?missing repository}"; shift 2 ;; + --publish-only) publish_only=1; shift ;; + -h|--help) usage; exit 0 ;; + *) fail "unknown argument: $1" ;; + esac +done + +[[ -n "$control_version" || -n "$agent_version" ]] || fail "set a control or agent version" +[[ -z "$control_version" ]] || validate_version "$control_version" +[[ -z "$agent_version" ]] || validate_version "$agent_version" +[[ "$agent_arch" == "x86_64" || "$agent_arch" == "aarch64" ]] || + fail "agent architecture must be x86_64 or aarch64" +if [[ -n "$control_version" && ("$registry" != "hub.nationtech.io" || "$repository" != "harmony") ]]; then + fail "control-plane releases publish to hub.nationtech.io/harmony" +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../.." && pwd)" +cd "$repo_root" + +require_command cargo +require_command sha256sum +require_command stat +[[ -z "$control_version" ]] || require_command docker +[[ -z "$agent_version" ]] || require_command oras + +if [[ -n "$control_version" || -n "$agent_version" ]]; then + : "${REGISTRY_USER:?REGISTRY_USER is required for publication}" + : "${REGISTRY_TOKEN:?REGISTRY_TOKEN is required for publication}" +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +agent_binary="" +agent_size="" +agent_sha256="" +agent_tag_ref="" +agent_oci_ref="" + +if [[ -n "$agent_version" ]]; then + printf '==> Building fleet agent %s for %s\n' "$agent_version" "$agent_arch" + if [[ "$agent_arch" == "x86_64" ]]; then + [[ "$(uname -m)" == "x86_64" ]] || fail "x86_64 agent build requires an x86_64 host" + HARMONY_FLEET_AGENT_VERSION="$agent_version" \ + cargo build --release --locked -p harmony-fleet-agent + agent_binary="$repo_root/target/release/harmony-fleet-agent" + version_output="$("$agent_binary" --version)" + [[ "$version_output" == *" $agent_version" ]] || + fail "built agent reports the wrong version: $version_output" + else + require_command rustup + rustup target add aarch64-unknown-linux-gnu + HARMONY_FLEET_AGENT_VERSION="$agent_version" \ + cargo build --release --locked --target aarch64-unknown-linux-gnu \ + -p harmony-fleet-agent + agent_binary="$repo_root/target/aarch64-unknown-linux-gnu/release/harmony-fleet-agent" + fi + [[ -x "$agent_binary" ]] || fail "agent binary not produced: $agent_binary" + agent_size="$(stat -c %s "$agent_binary")" + agent_sha256="$(sha256sum "$agent_binary" | cut -d' ' -f1)" + agent_tag_ref="$registry/$repository/harmony-fleet-agent:$agent_version-$agent_arch" + if oras manifest fetch "$agent_tag_ref" >/dev/null 2>&1; then + fail "agent artifact tag already exists: $agent_tag_ref" + fi +fi + +operator_ref="" +callout_ref="" +if [[ -n "$control_version" ]]; then + for image in harmony-fleet-operator harmony-nats-callout; do + ref="$registry/$repository/$image:$control_version" + if docker manifest inspect "$ref" >/dev/null 2>&1; then + fail "control-plane image tag already exists: $ref" + fi + done + printf '==> Building and publishing Fleet control plane %s\n' "$control_version" + release_log="$tmp/control-release.log" + REGISTRY_USER="$REGISTRY_USER" REGISTRY_TOKEN="$REGISTRY_TOKEN" \ + RUST_LOG="${RUST_LOG:-info}" \ + cargo run --release -p harmony-fleet-deploy --bin harmony-fleet-release -- \ + --tag "$control_version" --push | tee "$release_log" + operator_ref="$(awk -F= '$1 == "operator" { value=$2 } END { print value }' "$release_log")" + callout_ref="$(awk -F= '$1 == "callout" { value=$2 } END { print value }' "$release_log")" + [[ "$operator_ref" =~ @sha256:[0-9a-f]{64}$ ]] || fail "operator digest missing from release output" + [[ "$callout_ref" =~ @sha256:[0-9a-f]{64}$ ]] || fail "callout digest missing from release output" +fi + +if [[ -n "$agent_version" ]]; then + printf '==> Publishing %s\n' "$agent_tag_ref" + printf '%s' "$REGISTRY_TOKEN" | oras login "$registry" \ + --username "$REGISTRY_USER" --password-stdin >/dev/null + cp "$agent_binary" "$tmp/harmony-fleet-agent" + ( + cd "$tmp" + oras push \ + --artifact-type application/vnd.nationtech.harmony.fleet-agent.v1 \ + --annotation "org.opencontainers.image.version=$agent_version" \ + --annotation "org.opencontainers.image.os=linux" \ + --annotation "org.opencontainers.image.architecture=$agent_arch" \ + "$agent_tag_ref" \ + "harmony-fleet-agent:application/vnd.nationtech.harmony.fleet-agent.binary.v1" + ) + manifest_digest="$(oras resolve "$agent_tag_ref")" + [[ "$manifest_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "could not resolve agent manifest digest" + agent_oci_ref="oci://$registry/$repository/harmony-fleet-agent@$manifest_digest" +fi + +if ((publish_only == 0)) && [[ -n "$control_version" ]]; then + [[ -n "$deploy_manifest" ]] || fail "--deploy-manifest is required to deploy the control plane" + [[ -n "$deploy_bin" ]] || fail "--deploy-bin is required to deploy the control plane" + [[ -n "$deploy_context" ]] || fail "--context is required to deploy the control plane" + : "${OPENBAO_TOKEN:?OPENBAO_TOKEN is required for deployment}" + printf '==> Deploying Fleet control plane %s\n' "$control_version" + OPENBAO_TOKEN="$OPENBAO_TOKEN" RUST_LOG="${RUST_LOG:-info}" \ + cargo run --release --manifest-path "$deploy_manifest" --bin "$deploy_bin" -- \ + deploy --context "$deploy_context" --tag "$control_version" \ + --image "operator=$operator_ref" --image "callout=$callout_ref" +fi + +if ((publish_only == 0)) && [[ -n "$agent_version" && ${#devices[@]} -gt 0 ]]; then + [[ -n "$namespace" ]] || fail "--namespace is required when --device is used" + require_command kubectl + printf 'note: Device setup must have refreshed the bootstrap updater with OCI support.\n' + kubectl_args=() + [[ -z "$kube_context" ]] || kubectl_args+=(--context "$kube_context") + for device in "${devices[@]}"; do + printf '==> Targeting agent %s on %s\n' "$agent_version" "$device" + kubectl "${kubectl_args[@]}" -n "$namespace" patch \ + devices.fleet.nationtech.io "$device" --type merge -p \ + "{\"spec\":{\"agentUpgrade\":{\"version\":\"$agent_version\",\"architecture\":\"$agent_arch\",\"artifactUrl\":\"$agent_oci_ref\",\"maxBytes\":$agent_size,\"sha256\":\"$agent_sha256\"}}}" + done +fi + +printf '\nRelease outputs:\n' +[[ -z "$operator_ref" ]] || printf ' operator=%s\n callout=%s\n' "$operator_ref" "$callout_ref" +if [[ -n "$agent_oci_ref" ]]; then + printf ' agent.version=%s\n' "$agent_version" + printf ' agent.architecture=%s\n' "$agent_arch" + printf ' agent.artifactUrl=%s\n' "$agent_oci_ref" + printf ' agent.maxBytes=%s\n' "$agent_size" + printf ' agent.sha256=%s\n' "$agent_sha256" +fi -- 2.39.5 From 18d88d369d63726cbafb49fe96119b61dac8c50f Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 14:04:59 -0400 Subject: [PATCH 25/47] docs(fleet): update delivery checks --- .../agent-reconciliation-and-upgrade-plan.md | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/fleet/agent-reconciliation-and-upgrade-plan.md b/fleet/agent-reconciliation-and-upgrade-plan.md index 543c977e..399d4230 100644 --- a/fleet/agent-reconciliation-and-upgrade-plan.md +++ b/fleet/agent-reconciliation-and-upgrade-plan.md @@ -116,9 +116,25 @@ Deferred follow-up: - retry and classify activation readiness failures before rollback or quarantine; - add an isolated VM agent-upgrade E2E covering success and failure recovery; +- rerun `FleetDeviceSetupScore` to refresh the non-self-updating bootstrap updater, + then verify an anonymous pull from a digest-pinned OCI artifact and a + target/binary version mismatch. The mismatch must fail during candidate + self-test without stopping the active agent; updater child processes must not + emit stray systemd notifications or expected broken-pipe warnings; - replace the operator's two-second full-device scan before fleet scale; - add binary garbage collection when disk pressure makes retention relevant. +## Device commands + +The dashboard sends bounded one-shot exec requests over Core NATS request/reply. +Exec is an administrative capability, not a sandbox: it runs as the +`fleet-agent` Unix account and can access that account's credentials and rootless +Podman runtime. Only a verified dashboard session with the exact `fleet-admin` +project role may use it. +Manual QA must cover stdout, stderr and non-zero exit status, timeout cleanup, +output truncation, an offline device, and denial for a dashboard user without the +`fleet-admin` role. + ## Delivery checkpoints 1. Rework deployment reconciliation around a complete snapshot and a fakeable @@ -130,5 +146,6 @@ Deferred follow-up: operator coordination. 5. Run unit tests, compile environment-gated E2E, run focused local VM tests if available without production credentials, and complete independent reviews. -6. Production QA is manual and stepwise: deployment recovery first, then one - agent upgrade attempt with rollback rehearsed before any canary rollout. +6. Production QA is manual and stepwise: deployment recovery, bounded device + commands, bootstrap-updater diagnostics, then one agent upgrade attempt with + rollback rehearsed before any canary rollout. -- 2.39.5 From 3403b3458fe6b264d338a4a292f14d446cdb7010 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 15:23:08 -0400 Subject: [PATCH 26/47] fix(fleet): make releases idempotent --- Cargo.lock | 2 + Cargo.toml | 1 + fleet/harmony-fleet-agent/Cargo.toml | 2 +- fleet/harmony-fleet-agent/src/updater.rs | 10 +- fleet/harmony-fleet-deploy/Cargo.toml | 2 + .../src/bin/harmony-fleet-release.rs | 176 +++++++++++++++++- fleet/scripts/release.sh | 46 ++--- harmony-reconciler-contracts/src/upgrade.rs | 4 + 8 files changed, 196 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdfe5f4c..f0c4a283 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4108,6 +4108,7 @@ dependencies = [ "harmony", "harmony-fleet-auth", "harmony-fleet-operator", + "harmony-reconciler-contracts", "harmony_app", "harmony_cli", "harmony_config", @@ -4119,6 +4120,7 @@ dependencies = [ "kube", "log", "non-blank-string-rs", + "oci-client", "schemars 0.8.22", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 616522a3..bb16f0e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ reqwest = { version = "0.12", features = [ "http2", "json", ], default-features = false } +oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } assertor = "0.0.4" tokio-test = "0.4" anyhow = "1.0" diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index d5eb54be..c111c9d1 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -25,6 +25,6 @@ thiserror = { workspace = true } podman-api = "0.9" sd-notify = "0.4" reqwest.workspace = true -oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } +oci-client.workspace = true uuid.workspace = true fs2.workspace = true diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs index 8324f958..3725981c 100644 --- a/fleet/harmony-fleet-agent/src/updater.rs +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -8,7 +8,9 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use futures_util::StreamExt; -use harmony_reconciler_contracts::AgentUpgradeAttempt; +use harmony_reconciler_contracts::upgrade::{ + AGENT_OCI_ARTIFACT_TYPE, AGENT_OCI_LAYER_MEDIA_TYPE, AgentUpgradeAttempt, +}; use oci_client::client::{ClientConfig, ClientProtocol}; use oci_client::secrets::RegistryAuth; use oci_client::{Client as OciClient, Reference}; @@ -31,8 +33,6 @@ const MAX_TRANSITIONS: usize = 16; const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10); const STATUS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(15); const UPGRADE_RESPONSE_TIMEOUT: Duration = Duration::from_secs(20 * 60); -const OCI_ARTIFACT_TYPE: &str = "application/vnd.nationtech.harmony.fleet-agent.v1"; -const OCI_LAYER_MEDIA_TYPE: &str = "application/vnd.nationtech.harmony.fleet-agent.binary.v1"; #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "operation", content = "data", rename_all = "kebab-case")] @@ -589,13 +589,13 @@ async fn download_oci_artifact(value: &str, max_bytes: u64) -> Result> { .pull_image_manifest(&reference, &RegistryAuth::Anonymous) .await .context("pulling OCI artifact manifest")?; - if manifest.artifact_type.as_deref() != Some(OCI_ARTIFACT_TYPE) { + if manifest.artifact_type.as_deref() != Some(AGENT_OCI_ARTIFACT_TYPE) { bail!("OCI artifact has an unexpected artifact type"); } let [layer] = manifest.layers.as_slice() else { bail!("OCI artifact must contain exactly one binary layer"); }; - if layer.media_type != OCI_LAYER_MEDIA_TYPE { + if layer.media_type != AGENT_OCI_LAYER_MEDIA_TYPE { bail!("OCI artifact has an unexpected layer media type"); } if layer.size < 0 || layer.size as u64 > max_bytes { diff --git a/fleet/harmony-fleet-deploy/Cargo.toml b/fleet/harmony-fleet-deploy/Cargo.toml index 7833f995..e359a8c1 100644 --- a/fleet/harmony-fleet-deploy/Cargo.toml +++ b/fleet/harmony-fleet-deploy/Cargo.toml @@ -33,6 +33,7 @@ harmony_types = { path = "../../harmony_types" } harmony_macros = { path = "../../harmony_macros" } harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-operator = { path = "../harmony-fleet-operator" } +harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } anyhow = { workspace = true } @@ -44,6 +45,7 @@ kube = { workspace = true, features = ["runtime", "derive"] } log = { workspace = true } env_logger = { workspace = true } non-blank-string-rs = "1" +oci-client.workspace = true inquire.workspace = true schemars = "0.8" serde = { workspace = true } diff --git a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs index 0f9c421d..24552752 100644 --- a/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs +++ b/fleet/harmony-fleet-deploy/src/bin/harmony-fleet-release.rs @@ -1,24 +1,91 @@ -use clap::Parser; +use std::collections::BTreeMap; +use std::path::PathBuf; + +use anyhow::{Context, bail}; +use clap::{Parser, Subcommand}; use harmony_app::{PublicationTopology, publish::build_images}; use harmony_fleet_deploy::FleetApp; +use harmony_reconciler_contracts::upgrade::{AGENT_OCI_ARTIFACT_TYPE, AGENT_OCI_LAYER_MEDIA_TYPE}; +use oci_client::annotations::ORG_OPENCONTAINERS_IMAGE_VERSION; +use oci_client::client::{Config, ImageLayer}; +use oci_client::errors::{OciDistributionError, OciErrorCode}; +use oci_client::manifest::OciImageManifest; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client, Reference}; #[derive(Parser)] struct Args { - #[arg(long)] - tag: String, - #[arg(long)] - push: bool, + #[command(subcommand)] + release: Release, } -fn main() -> anyhow::Result<()> { +#[derive(Subcommand)] +enum Release { + Control { + #[arg(long)] + version: String, + #[arg(long)] + push: bool, + }, + Agent { + #[arg(long)] + binary: PathBuf, + #[arg(long)] + reference: String, + #[arg(long)] + version: String, + }, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { harmony_cli::cli_logger::init(); - let args = Args::parse(); - let images = FleetApp::official_images(&args.tag); + match Args::parse().release { + Release::Control { version, push } => publish_control(&version, push).await, + Release::Agent { + binary, + reference, + version, + } => publish_agent(binary, &reference, &version).await, + } +} + +async fn publish_control(version: &str, push: bool) -> anyhow::Result<()> { + let images = FleetApp::official_images(version); let registry = PublicationTopology::Registry { registry: "hub.nationtech.io".to_string(), }; + if push { + let client = Client::default(); + let auth = registry_auth()?; + let mut existing = Vec::new(); + for image in &images { + let reference = Reference::try_from(image.image.as_str())?; + match client.pull_manifest(&reference, &auth).await { + Ok((_, digest)) => existing.push(( + image.name.clone(), + format!( + "{}/{}@{digest}", + reference.registry(), + reference.repository() + ), + )), + Err(error) if manifest_is_missing(&error) => {} + Err(error) => return Err(error).context("checking control-plane release"), + } + } + if existing.len() == images.len() { + for (name, image) in existing { + println!("{name}={image}"); + } + return Ok(()); + } + if !existing.is_empty() { + bail!("control-plane release is incomplete; refusing to overwrite existing tags"); + } + } let refs = build_images(&images, ®istry)?; - let refs = if args.push { + let refs = if push { harmony_app::publish::publish_images(&images, &refs, ®istry)? } else { refs @@ -28,3 +95,94 @@ fn main() -> anyhow::Result<()> { } Ok(()) } + +async fn publish_agent(binary: PathBuf, reference: &str, version: &str) -> anyhow::Result<()> { + let reference = Reference::try_from(reference).context("invalid agent OCI reference")?; + if reference.tag().is_none() || reference.digest().is_some() { + bail!("agent publication requires a tag reference"); + } + let auth = registry_auth()?; + let client = Client::default(); + let layer = ImageLayer::new( + std::fs::read(&binary) + .with_context(|| format!("reading agent binary {}", binary.display()))?, + AGENT_OCI_LAYER_MEDIA_TYPE.to_string(), + None, + ); + match client.pull_manifest(&reference, &auth).await { + Ok((oci_client::manifest::OciManifest::Image(manifest), digest)) + if manifest.artifact_type.as_deref() == Some(AGENT_OCI_ARTIFACT_TYPE) + && manifest + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ORG_OPENCONTAINERS_IMAGE_VERSION)) + .map(String::as_str) + == Some(version) + && matches!(manifest.layers.as_slice(), [existing] + if existing.media_type == AGENT_OCI_LAYER_MEDIA_TYPE + && existing.digest == layer.sha256_digest()) => + { + println!( + "agent=oci://{}/{}@{digest}", + reference.registry(), + reference.repository() + ); + return Ok(()); + } + Ok(_) => bail!( + "agent artifact tag exists with different content: {}", + reference.whole() + ), + Err(error) if manifest_is_missing(&error) => {} + Err(error) => return Err(error).context("checking agent artifact tag"), + } + + let config = Config::new( + b"{}".to_vec(), + "application/vnd.oci.empty.v1+json".to_string(), + None, + ); + let mut annotations = BTreeMap::new(); + annotations.insert( + ORG_OPENCONTAINERS_IMAGE_VERSION.to_string(), + version.to_string(), + ); + let mut manifest = + OciImageManifest::build(std::slice::from_ref(&layer), &config, Some(annotations)); + manifest.artifact_type = Some(AGENT_OCI_ARTIFACT_TYPE.to_string()); + client + .push(&reference, &[layer], config, &auth, Some(manifest)) + .await + .context("publishing agent OCI artifact")?; + let (_, digest) = client + .pull_manifest(&reference, &auth) + .await + .context("resolving published agent manifest")?; + println!( + "agent=oci://{}/{}@{digest}", + reference.registry(), + reference.repository() + ); + Ok(()) +} + +fn registry_auth() -> anyhow::Result { + Ok(RegistryAuth::Basic( + std::env::var("REGISTRY_USER").context("REGISTRY_USER is required")?, + std::env::var("REGISTRY_TOKEN").context("REGISTRY_TOKEN is required")?, + )) +} + +fn manifest_is_missing(error: &OciDistributionError) -> bool { + matches!(error, OciDistributionError::ImageManifestNotFoundError(_)) + || matches!( + error, + OciDistributionError::RegistryError { envelope, .. } + if envelope.errors.iter().any(|error| matches!( + &error.code, + OciErrorCode::ManifestUnknown + | OciErrorCode::NameUnknown + | OciErrorCode::NotFound + )) + ) +} diff --git a/fleet/scripts/release.sh b/fleet/scripts/release.sh index f511be01..fe1b0497 100755 --- a/fleet/scripts/release.sh +++ b/fleet/scripts/release.sh @@ -101,7 +101,6 @@ require_command cargo require_command sha256sum require_command stat [[ -z "$control_version" ]] || require_command docker -[[ -z "$agent_version" ]] || require_command oras if [[ -n "$control_version" || -n "$agent_version" ]]; then : "${REGISTRY_USER:?REGISTRY_USER is required for publication}" @@ -139,52 +138,35 @@ if [[ -n "$agent_version" ]]; then agent_size="$(stat -c %s "$agent_binary")" agent_sha256="$(sha256sum "$agent_binary" | cut -d' ' -f1)" agent_tag_ref="$registry/$repository/harmony-fleet-agent:$agent_version-$agent_arch" - if oras manifest fetch "$agent_tag_ref" >/dev/null 2>&1; then - fail "agent artifact tag already exists: $agent_tag_ref" - fi fi operator_ref="" callout_ref="" +if [[ -n "$agent_version" ]]; then + printf '==> Publishing %s\n' "$agent_tag_ref" + agent_release_log="$tmp/agent-release.log" + REGISTRY_USER="$REGISTRY_USER" REGISTRY_TOKEN="$REGISTRY_TOKEN" \ + RUST_LOG="${RUST_LOG:-info}" \ + cargo run --release -p harmony-fleet-deploy --bin harmony-fleet-release -- \ + agent --binary "$agent_binary" --reference "$agent_tag_ref" \ + --version "$agent_version" | tee "$agent_release_log" + agent_oci_ref="$(awk -F= '$1 == "agent" { value=$2 } END { print value }' "$agent_release_log")" + [[ "$agent_oci_ref" =~ @sha256:[0-9a-f]{64}$ ]] || fail "agent digest missing from release output" +fi + if [[ -n "$control_version" ]]; then - for image in harmony-fleet-operator harmony-nats-callout; do - ref="$registry/$repository/$image:$control_version" - if docker manifest inspect "$ref" >/dev/null 2>&1; then - fail "control-plane image tag already exists: $ref" - fi - done - printf '==> Building and publishing Fleet control plane %s\n' "$control_version" + printf '==> Resolving or publishing Fleet control plane %s\n' "$control_version" release_log="$tmp/control-release.log" REGISTRY_USER="$REGISTRY_USER" REGISTRY_TOKEN="$REGISTRY_TOKEN" \ RUST_LOG="${RUST_LOG:-info}" \ cargo run --release -p harmony-fleet-deploy --bin harmony-fleet-release -- \ - --tag "$control_version" --push | tee "$release_log" + control --version "$control_version" --push | tee "$release_log" operator_ref="$(awk -F= '$1 == "operator" { value=$2 } END { print value }' "$release_log")" callout_ref="$(awk -F= '$1 == "callout" { value=$2 } END { print value }' "$release_log")" [[ "$operator_ref" =~ @sha256:[0-9a-f]{64}$ ]] || fail "operator digest missing from release output" [[ "$callout_ref" =~ @sha256:[0-9a-f]{64}$ ]] || fail "callout digest missing from release output" fi -if [[ -n "$agent_version" ]]; then - printf '==> Publishing %s\n' "$agent_tag_ref" - printf '%s' "$REGISTRY_TOKEN" | oras login "$registry" \ - --username "$REGISTRY_USER" --password-stdin >/dev/null - cp "$agent_binary" "$tmp/harmony-fleet-agent" - ( - cd "$tmp" - oras push \ - --artifact-type application/vnd.nationtech.harmony.fleet-agent.v1 \ - --annotation "org.opencontainers.image.version=$agent_version" \ - --annotation "org.opencontainers.image.os=linux" \ - --annotation "org.opencontainers.image.architecture=$agent_arch" \ - "$agent_tag_ref" \ - "harmony-fleet-agent:application/vnd.nationtech.harmony.fleet-agent.binary.v1" - ) - manifest_digest="$(oras resolve "$agent_tag_ref")" - [[ "$manifest_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "could not resolve agent manifest digest" - agent_oci_ref="oci://$registry/$repository/harmony-fleet-agent@$manifest_digest" -fi - if ((publish_only == 0)) && [[ -n "$control_version" ]]; then [[ -n "$deploy_manifest" ]] || fail "--deploy-manifest is required to deploy the control plane" [[ -n "$deploy_bin" ]] || fail "--deploy-bin is required to deploy the control plane" diff --git a/harmony-reconciler-contracts/src/upgrade.rs b/harmony-reconciler-contracts/src/upgrade.rs index bc46eaef..24581d3a 100644 --- a/harmony-reconciler-contracts/src/upgrade.rs +++ b/harmony-reconciler-contracts/src/upgrade.rs @@ -4,6 +4,10 @@ use sha2::{Digest, Sha256}; use crate::Id; +pub const AGENT_OCI_ARTIFACT_TYPE: &str = "application/vnd.nationtech.harmony.fleet-agent.v1"; +pub const AGENT_OCI_LAYER_MEDIA_TYPE: &str = + "application/vnd.nationtech.harmony.fleet-agent.binary.v1"; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AgentUpgradeAttempt { -- 2.39.5 From 6358121a270e1c550cbbb9d9c6b7425670f3f67f Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 15:23:18 -0400 Subject: [PATCH 27/47] fix(fleet): include dashboard roles in tokens --- fleet/harmony-fleet-deploy/src/app.rs | 1 + harmony/src/modules/zitadel/setup.rs | 28 ++++++++++++++++++++++++--- harmony_app/src/capabilities.rs | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/fleet/harmony-fleet-deploy/src/app.rs b/fleet/harmony-fleet-deploy/src/app.rs index e70eade2..f5cd483b 100644 --- a/fleet/harmony-fleet-deploy/src/app.rs +++ b/fleet/harmony-fleet-deploy/src/app.rs @@ -87,6 +87,7 @@ impl HarmonyApp for FleetApp { ZitadelAppType::WebPkce { redirect_uris: vec![format!("https://{host}/auth/callback")], post_logout_redirect_uris: vec![format!("https://{host}/")], + id_token_role_assertion: true, }, ) } else { diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index a458b0ba..d27b0c97 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -88,6 +88,7 @@ pub enum ZitadelAppType { WebPkce { redirect_uris: Vec, post_logout_redirect_uris: Vec, + id_token_role_assertion: bool, }, } @@ -1264,6 +1265,7 @@ impl ZitadelSetupInterpret { &self, redirect_uris: &[String], post_logout_redirect_uris: &[String], + id_token_role_assertion: bool, ) -> serde_json::Value { serde_json::json!({ "redirectUris": redirect_uris, @@ -1272,7 +1274,8 @@ impl ZitadelSetupInterpret { "grantTypes": ["OIDC_GRANT_TYPE_AUTHORIZATION_CODE", "OIDC_GRANT_TYPE_REFRESH_TOKEN"], "appType": "OIDC_APP_TYPE_USER_AGENT", "authMethodType": "OIDC_AUTH_METHOD_TYPE_NONE", - "idTokenUserinfoAssertion": true + "idTokenUserinfoAssertion": true, + "idTokenRoleAssertion": id_token_role_assertion, }) } @@ -1284,8 +1287,13 @@ impl ZitadelSetupInterpret { app_name: &str, redirect_uris: &[String], post_logout_redirect_uris: &[String], + id_token_role_assertion: bool, ) -> Result { - let mut body = self.web_pkce_oidc_config_body(redirect_uris, post_logout_redirect_uris); + let mut body = self.web_pkce_oidc_config_body( + redirect_uris, + post_logout_redirect_uris, + id_token_role_assertion, + ); body["name"] = serde_json::json!(app_name); self.create_oidc_app(client, pat, project_id, body).await } @@ -1383,13 +1391,18 @@ impl ZitadelSetupInterpret { ZitadelAppType::WebPkce { redirect_uris, post_logout_redirect_uris, + id_token_role_assertion, } => self .update_oidc_config( client, pat, &project_id, &found.id, - self.web_pkce_oidc_config_body(redirect_uris, post_logout_redirect_uris), + self.web_pkce_oidc_config_body( + redirect_uris, + post_logout_redirect_uris, + *id_token_role_assertion, + ), ) .await .map_err(InterpretError::new)?, @@ -1408,6 +1421,7 @@ impl ZitadelSetupInterpret { ZitadelAppType::WebPkce { redirect_uris, post_logout_redirect_uris, + id_token_role_assertion, } => self .create_web_pkce_app( client, @@ -1416,6 +1430,7 @@ impl ZitadelSetupInterpret { &app.app_name, redirect_uris, post_logout_redirect_uris, + *id_token_role_assertion, ) .await .map_err(InterpretError::new)?, @@ -2985,6 +3000,7 @@ mod tests { app_type: ZitadelAppType::WebPkce { redirect_uris: vec!["https://timesheet.nationtech.io/callback".to_string()], post_logout_redirect_uris: vec![], + id_token_role_assertion: false, }, }], api_apps: vec![], @@ -3112,6 +3128,12 @@ mod tests { ZitadelSetupInterpret { score } } + #[test] + fn web_pkce_can_assert_roles_in_id_token() { + let body = interp(score("zitadel.example.com")).web_pkce_oidc_config_body(&[], &[], true); + assert_eq!(body["idTokenRoleAssertion"], serde_json::json!(true)); + } + #[test] fn api_url_https_default_port_omits_port() { let i = interp(score("zitadel.example.com")); diff --git a/harmony_app/src/capabilities.rs b/harmony_app/src/capabilities.rs index 58d0530e..15c118ff 100644 --- a/harmony_app/src/capabilities.rs +++ b/harmony_app/src/capabilities.rs @@ -237,6 +237,8 @@ impl Capability for ZitadelAuth { app_type: ZitadelAppType::WebPkce { redirect_uris: self.redirect_uris.clone(), post_logout_redirect_uris: self.post_logout_redirect_uris.clone(), + // TODO this should be configurable + id_token_role_assertion: false, }, }], api_apps: vec![], -- 2.39.5 From 1236e1a9ac7f3db59de3cfaf907fbedee5a8685c Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 18:07:35 -0400 Subject: [PATCH 28/47] fix(fleet): enforce group-gated placement --- docs/guides/fleet-device-secrets.md | 3 + examples/fleet_load_test/src/main.rs | 4 +- examples/fleet_server_install/src/main.rs | 8 +- examples/harmony_apply_deployment/src/main.rs | 5 +- .../src/operator/chart.rs | 38 ++++- .../src/operator/score.rs | 9 ++ fleet/harmony-fleet-e2e/src/stack.rs | 5 + fleet/harmony-fleet-e2e/tests/operator.rs | 16 +- fleet/harmony-fleet-operator/src/access.rs | 7 +- fleet/harmony-fleet-operator/src/crd.rs | 30 ++-- .../src/fleet_aggregator.rs | 149 ++++++++++++------ fleet/harmony-fleet-operator/src/main.rs | 61 +++---- .../src/service/real.rs | 106 ++++++++----- fleet/scripts/load-test.sh | 2 + fleet/scripts/smoke-a1.sh | 12 +- fleet/scripts/smoke-a4.sh | 9 +- harmony_zitadel_auth/src/device_groups.rs | 51 +++++- 17 files changed, 359 insertions(+), 156 deletions(-) diff --git a/docs/guides/fleet-device-secrets.md b/docs/guides/fleet-device-secrets.md index 1df7e2cc..6cac8541 100644 --- a/docs/guides/fleet-device-secrets.md +++ b/docs/guides/fleet-device-secrets.md @@ -70,6 +70,9 @@ policy to each allowed group. Existing device sessions pick the grant up immediately — rolling out a new deployment never touches logins, tokens, or per-device configuration. +`allowedGroups` is required. An empty list authorizes and targets no +devices; a selector never bypasses the group boundary. + ## Managing device group membership Until Zitadel's first-class groups are GA, a fleet group is a Zitadel diff --git a/examples/fleet_load_test/src/main.rs b/examples/fleet_load_test/src/main.rs index cd2845ee..01576104 100644 --- a/examples/fleet_load_test/src/main.rs +++ b/examples/fleet_load_test/src/main.rs @@ -46,6 +46,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio::task::JoinSet; +const LOAD_TEST_GROUP: &str = "fleet-load-test"; + #[derive(Parser, Debug, Clone)] #[command( name = "fleet_load_test", @@ -406,7 +408,7 @@ async fn apply_one_cr( let cr = Deployment::new( &group.cr_name, DeploymentSpec { - allowed_groups: None, + allowed_groups: vec![LOAD_TEST_GROUP.to_string()], target_selector: LabelSelector { match_labels: Some(match_labels), match_expressions: None, diff --git a/examples/fleet_server_install/src/main.rs b/examples/fleet_server_install/src/main.rs index f21d9104..29b3fec8 100644 --- a/examples/fleet_server_install/src/main.rs +++ b/examples/fleet_server_install/src/main.rs @@ -95,6 +95,9 @@ struct Cli { /// `RUST_LOG` value injected into the operator pod's env. #[arg(long, default_value = "info,kube_runtime=warn")] log_level: String, + /// Static `device=group|group;...` membership for local testing. + #[arg(long)] + device_groups: Option, /// Hostname Zitadel should answer on. When set, Zitadel + its /// PostgreSQL cluster are installed alongside the operator. @@ -138,12 +141,15 @@ async fn main() -> Result<()> { // NatsScore install creates. ClusterIP and LoadBalancer both // expose the same `.:4222` for in-cluster // callers. - let operator = FleetOperatorScore::new(&cli.operator_image) + let mut operator = FleetOperatorScore::new(&cli.operator_image) .namespace(&cli.operator_namespace) .release_name(&cli.operator_release) .image_pull_policy(&cli.operator_image_pull_policy) .messaging(&nats.client_ref()) .log_level(&cli.log_level); + if let Some(groups) = cli.device_groups { + operator = operator.device_groups(groups); + } // FleetServerScore now takes NatsK8sScore (auth-callout-aware, // OKD-Route-aware) — see `fleet_staging_install` for the diff --git a/examples/harmony_apply_deployment/src/main.rs b/examples/harmony_apply_deployment/src/main.rs index f5c9b713..7de76f62 100644 --- a/examples/harmony_apply_deployment/src/main.rs +++ b/examples/harmony_apply_deployment/src/main.rs @@ -70,6 +70,9 @@ struct Cli { /// `--target-device` when provided. All pairs AND together. #[arg(long = "selector", value_name = "KEY=VALUE")] selectors: Vec, + /// Zitadel fleet group authorized to run this deployment. Repeatable. + #[arg(long = "allowed-group", required = true)] + allowed_groups: Vec, /// Container image to run. #[arg(long, default_value = "docker.io/library/nginx:latest")] image: String, @@ -226,7 +229,7 @@ fn build_cr(cli: &Cli) -> Deployment { Deployment::new( &cli.name, DeploymentSpec { - allowed_groups: None, + allowed_groups: cli.allowed_groups.clone(), target_selector: LabelSelector { match_labels: Some(match_labels), match_expressions: None, diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 272a9686..eeaaf136 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -75,6 +75,7 @@ pub struct ChartOptions { pub identity: Option, pub identity_version: Option, pub image_pull_secret: Option, + pub device_groups: Option, } #[derive(Debug, Clone, Serialize)] @@ -109,6 +110,7 @@ impl Default for ChartOptions { identity: None, identity_version: None, image_pull_secret: None, + device_groups: None, } } } @@ -462,8 +464,7 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { env.push(secret_env(ENV_WEB_AUTH_CONFIG)); env.push(secret_env(ENV_WEB_COOKIE_KEY)); // Secret-grant sync (OpenBao) + the device-group scheduling gate - // (Zitadel role grants) — ADR-025. All optional: absent, the - // operator logs and runs ungated/without grant sync. + // (Zitadel role grants) — ADR-025. Missing group data fails closed. for name in [ "OPENBAO_URL", "OPENBAO_TOKEN", @@ -473,6 +474,13 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { ] { env.push(secret_env(name)); } + if let Some(groups) = &opts.device_groups { + env.push(EnvVar { + name: "FLEET_DEVICE_GROUPS".to_string(), + value: Some(groups.clone()), + ..Default::default() + }); + } // Namespace deliberately omitted — same rationale as the // ServiceAccount: helm fills in the release namespace at install @@ -705,6 +713,32 @@ mod tests { ); } + #[test] + fn deployment_injects_static_device_groups() { + let deployment = operator_deployment(&ChartOptions { + device_groups: Some("pi-01=edge-a".to_string()), + ..Default::default() + }); + let env = deployment + .spec + .unwrap() + .template + .spec + .unwrap() + .containers + .into_iter() + .next() + .unwrap() + .env + .unwrap(); + assert_eq!( + env.iter() + .find(|env| env.name == "FLEET_DEVICE_GROUPS") + .and_then(|env| env.value.as_deref()), + Some("pi-01=edge-a") + ); + } + // The chart bakes these env names at publish time; the operator's // ConfigClient derives them from the struct names at runtime. Lock // them together so a rename can't silently desync the two. diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index 57833e2e..10f2945e 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -167,6 +167,8 @@ pub struct FleetOperatorScore { pub web_auth: Option, pub identity: Option, pub image_pull_secret: Option, + /// Static `device=group|group;...` membership for dev and tests. + pub device_groups: Option, } impl FleetOperatorScore { @@ -189,6 +191,7 @@ impl FleetOperatorScore { web_auth: None, identity: None, image_pull_secret: None, + device_groups: None, } } @@ -246,6 +249,11 @@ impl FleetOperatorScore { self } + pub fn device_groups(mut self, groups: impl Into) -> Self { + self.device_groups = Some(groups.into()); + self + } + /// Set the operator's NATS auth-callout credentials (zitadel-jwt /// `[credentials]` TOML). Applied as the operator Secret before the /// helm install — including on the published-chart (CD) path. @@ -520,6 +528,7 @@ impl Interpret for FleetOperatorInterp identity: self.score.identity.clone(), identity_version, image_pull_secret: self.score.image_pull_secret.clone(), + device_groups: self.score.device_groups.clone(), }; let expected_config_hash = chart::config_hash(&chart_options); if let Some(secret) = operator_secret(&chart_options) { diff --git a/fleet/harmony-fleet-e2e/src/stack.rs b/fleet/harmony-fleet-e2e/src/stack.rs index c3f4d94f..9de4a039 100644 --- a/fleet/harmony-fleet-e2e/src/stack.rs +++ b/fleet/harmony-fleet-e2e/src/stack.rs @@ -85,6 +85,7 @@ pub struct StackOptions { pub log_level: String, pub deploy_agent: bool, pub deploy_operator: bool, + pub device_groups: Option, } impl Default for StackOptions { @@ -97,6 +98,7 @@ impl Default for StackOptions { log_level: "info".to_string(), deploy_agent: true, deploy_operator: false, + device_groups: None, } } } @@ -313,6 +315,9 @@ impl Stack { .namespace(namespace.clone()) .messaging(&nats_ref) .log_level(opts.log_level.clone()); + if let Some(groups) = &opts.device_groups { + operator = operator.device_groups(groups); + } if let Some(handles) = callout_handles.as_ref() { operator = operator.identity( diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index 76ac5505..cf04d759 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -51,9 +51,13 @@ async fn operator_writes_desired_state_for_matching_device() -> anyhow::Result<( let deployments: Api = Api::namespaced(client, &stack.namespace); create_device(&devices, "desired-state-device").await?; + create_device(&devices, "unauthorized-device").await?; create_fleet_deployment(&deployments, "desired-state-test").await?; wait_for_desired_state_entry(&stack, "desired-state-device", "desired-state-test", true) .await?; + tokio::time::sleep(Duration::from_secs(2)).await; + wait_for_desired_state_entry(&stack, "unauthorized-device", "desired-state-test", false) + .await?; Ok(()) } @@ -230,6 +234,16 @@ async fn operator_stack() -> anyhow::Result> { let stack = shared_stack(StackOptions { deploy_agent: false, deploy_operator: true, + device_groups: Some( + [ + "desired-state-device", + "cleanup-device", + "retarget-device", + "other-tenant-device", + ] + .map(|device| format!("{device}=e2e")) + .join(";"), + ), ..StackOptions::default() }) .await?; @@ -271,7 +285,7 @@ async fn create_fleet_deployment(deployments: &Api, name: &str) -> a ..Default::default() }, spec: DeploymentSpec { - allowed_groups: None, + allowed_groups: vec!["e2e".to_string()], target_selector: LabelSelector::default(), score: ReconcileScore::PodmanV0(PodmanV0Score { services: vec![PodmanService { diff --git a/fleet/harmony-fleet-operator/src/access.rs b/fleet/harmony-fleet-operator/src/access.rs index 7a81bb7f..ad70e63f 100644 --- a/fleet/harmony-fleet-operator/src/access.rs +++ b/fleet/harmony-fleet-operator/src/access.rs @@ -12,6 +12,8 @@ pub struct StaticDeviceGroups { } impl StaticDeviceGroups { + /// Parse `device=group|group;...`. The reserved `*` device supplies + /// membership to every device in local test and load-test setups. pub fn parse(spec: &str) -> Self { let map = spec .split(';') @@ -86,10 +88,11 @@ mod tests { #[tokio::test] async fn static_groups_parse_membership() { - let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk"); + let source = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b; *=load ;;junk"); let groups = source.device_groups().await.unwrap(); - assert_eq!(groups.len(), 2); + assert_eq!(groups.len(), 3); assert_eq!(groups["pi-01"].len(), 2); + assert!(groups["*"].contains("load")); } #[tokio::test] diff --git a/fleet/harmony-fleet-operator/src/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs index 7796d9e5..6a1d395c 100644 --- a/fleet/harmony-fleet-operator/src/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -22,14 +22,10 @@ pub use harmony_reconciler_contracts::{PodmanService, PodmanV0Score, ReconcileSc )] #[serde(rename_all = "camelCase")] pub struct DeploymentSpec { - /// Device groups allowed to *view* this deployment — run it and - /// read its secrets (ADR-025). Membership is admin-managed identity - /// (Zitadel role grants), never device-reported labels: labels can - /// narrow placement below, but only groups grant. Absent = no group - /// gating and no secret access — the selector alone places a - /// secretless workload. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub allowed_groups: Option>, + /// Device groups allowed to run this deployment and read its secrets + /// (ADR-025). Membership is admin-managed identity, never + /// device-reported labels. Empty means no device is authorized. + pub allowed_groups: Vec, /// Which devices this deployment targets, *within* the allowed /// groups. Matches against `Device.metadata.labels`. pub target_selector: LabelSelector, @@ -60,9 +56,9 @@ pub struct DeploymentStatus { #[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeploymentAggregate { - /// How many Device CRs currently match `spec.targetSelector`. - /// The three phase counters below sum to this; targeted-but- - /// unreported devices are folded into `pending`. + /// How many authorized, selected devices have acknowledged desired + /// state. The three phase counters below sum to this; devices that + /// have not reported state are folded into `pending`. pub matched_device_count: u32, pub succeeded: u32, pub failed: u32, @@ -190,10 +186,20 @@ pub enum Reachability { mod tests { use kube::CustomResourceExt; - use super::Device; + use super::{Deployment, Device}; #[test] fn device_is_namespaced() { assert_eq!(Device::crd().spec.scope, "Namespaced"); } + + #[test] + fn deployment_requires_allowed_groups() { + let crd = serde_json::to_value(Deployment::crd()).unwrap(); + let required = crd + .pointer("/spec/versions/0/schema/openAPIV3Schema/properties/spec/required") + .and_then(serde_json::Value::as_array) + .unwrap(); + assert!(required.iter().any(|field| field == "allowedGroups")); + } } diff --git a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs index 6cdbbd76..5c689593 100644 --- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs +++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs @@ -78,9 +78,8 @@ pub struct DevicePair { pub struct CachedDeployment { key: DeploymentKey, deployment_name: DeploymentName, - /// `spec.allowedGroups` — the security gate. `None` = ungated (and - /// secretless); `Some` = only group members are schedulable. - allowed_groups: Option>, + /// `spec.allowedGroups`, the authorization boundary for placement. + allowed_groups: Vec, selector: LabelSelector, /// JSON-serialized score payload ready to `put` into /// desired-state. Cached because the same bytes are written to @@ -151,20 +150,9 @@ pub fn selector_matches(selector: &LabelSelector, labels: &BTreeMap>, - device_groups: Option<&HashSet>, -) -> bool { - match allowed { - None => true, - Some(allowed) => { - device_groups.is_some_and(|groups| allowed.iter().any(|g| groups.contains(g))) - } - } +/// Is the device in one of the deployment's allowed groups? +pub fn group_allows(allowed: &[String], device_groups: Option<&HashSet>) -> bool { + device_groups.is_some_and(|groups| allowed.iter().any(|group| groups.contains(group))) } /// Is `deployment` allowed (groups) and placed (labels) on this device? @@ -172,8 +160,10 @@ fn device_eligible( deployment: &CachedDeployment, labels: &BTreeMap, device_groups: Option<&HashSet>, + default_groups: Option<&HashSet>, ) -> bool { - group_allows(&deployment.allowed_groups, device_groups) + (group_allows(&deployment.allowed_groups, device_groups) + || group_allows(&deployment.allowed_groups, default_groups)) && selector_matches(&deployment.selector, labels) } @@ -183,12 +173,40 @@ fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet .devices .iter() .filter(|(name, labels)| { - device_eligible(deployment, labels, state.device_groups.get(*name)) + device_eligible( + deployment, + labels, + state.device_groups.get(*name), + state.device_groups.get("*"), + ) }) .map(|(name, _)| name.clone()) .collect() } +impl FleetState { + /// Current group-authorized and selector-matched deployments per device. + pub fn eligible_deployments_by_device(&self) -> HashMap> { + if !reconciliation_ready(self) { + return HashMap::new(); + } + let mut eligible: HashMap> = HashMap::new(); + for deployment in self.deployments.values() { + for device in matched_devices(deployment, self) { + eligible + .entry(device) + .or_default() + .push(deployment.deployment_name.to_string()); + } + } + for deployments in eligible.values_mut() { + deployments.sort(); + deployments.dedup(); + } + eligible + } +} + fn upsert_deployment_state( state: &mut FleetState, cached: CachedDeployment, @@ -281,6 +299,7 @@ pub async fn run( js: async_nats::jetstream::Context, secret_grants: Option>, group_source: Option>, + state: SharedFleetState, ) -> anyhow::Result<()> { let state_bucket = js .create_key_value(async_nats::jetstream::kv::Config { @@ -298,10 +317,7 @@ pub async fn run( // Cold-start: initialize owned_targets from the current contents // of the desired-state bucket so we don't orphan entries written // by a previous operator run. - let state: SharedFleetState = Arc::new(Mutex::new(FleetState { - group_source_ready: group_source.is_none(), - ..Default::default() - })); + state.lock().await.group_source_ready = group_source.is_none(); seed_owned_targets(&desired_bucket, &state).await?; let deployments_api: Api = Api::namespaced(client.clone(), namespace); @@ -366,7 +382,10 @@ pub async fn run( ticker.tick().await; match source.device_groups().await { Ok(snapshot) => apply_device_groups(&state, snapshot).await, - Err(e) => warn!(error = %e, "aggregator: group source fetch failed"), + Err(e) => { + warn!(error = %e, "aggregator: group source fetch failed; revoking cached membership"); + apply_device_groups(&state, HashMap::new()).await; + } } } }) @@ -594,10 +613,7 @@ async fn on_deployment_upsert( if let Some(grants) = grants { sync_grant_batch( grants, - &[( - deployment_name.clone(), - cached.allowed_groups.clone().unwrap_or_default(), - )], + &[(deployment_name.clone(), cached.allowed_groups.clone())], ) .await; } @@ -965,7 +981,7 @@ mod tests { name: name.to_string(), }, deployment_name: dn(name), - allowed_groups: None, + allowed_groups: vec!["edge-a".to_string()], selector: LabelSelector { match_labels: Some(ml), match_expressions: None, @@ -975,7 +991,7 @@ mod tests { } fn with_groups(mut cached: CachedDeployment, groups: &[&str]) -> CachedDeployment { - cached.allowed_groups = Some(groups.iter().map(|s| s.to_string()).collect()); + cached.allowed_groups = groups.iter().map(|s| s.to_string()).collect(); cached } @@ -1014,21 +1030,14 @@ mod tests { fn group_gate_semantics() { let edge_a = HashSet::from(["edge-a".to_string()]); - // Undeclared = ungated. - assert!(group_allows(&None, Some(&edge_a))); - assert!(group_allows(&None, None)); - - // Declared = member devices only; unknown membership fails closed. - let allowed = Some(vec!["edge-a".to_string(), "edge-b".to_string()]); + let allowed = vec!["edge-a".to_string(), "edge-b".to_string()]; assert!(group_allows(&allowed, Some(&edge_a))); assert!(!group_allows( &allowed, Some(&HashSet::from(["other".to_string()])) )); assert!(!group_allows(&allowed, None)); - - // Declared-but-empty is a revocation, not a wildcard. - assert!(!group_allows(&Some(vec![]), Some(&edge_a))); + assert!(!group_allows(&[], Some(&edge_a))); } #[test] @@ -1046,16 +1055,13 @@ mod tests { s.device_groups .insert("pi-02".to_string(), HashSet::from(["edge-b".to_string()])); - let gated = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]); - assert_eq!(matched_devices(&gated, &s), devs(&["pi-01"])); + let deployment = cached("ns", "web", "zone", "lab"); + assert_eq!(matched_devices(&deployment, &s), devs(&["pi-01"])); // A device-chosen label can never widen access: the label // matches, the group doesn't, the device is not targeted. - let ungated = cached("ns", "web", "zone", "lab"); - assert_eq!( - matched_devices(&ungated, &s), - devs(&["pi-01", "pi-02", "pi-03"]) - ); + let revoked = with_groups(deployment, &[]); + assert!(matched_devices(&revoked, &s).is_empty()); } #[test] @@ -1074,6 +1080,9 @@ mod tests { "pi-01".to_string(), BTreeMap::from([("device-id".to_string(), "pi-01".to_string())]), ); + state + .device_groups + .insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()])); let deployment = cached("fleet", "web", "device-id", "pi-01"); assert_eq!(upsert_deployment_state(&mut state, deployment), Some("new")); @@ -1110,6 +1119,9 @@ mod tests { device.to_string(), BTreeMap::from([("zone".to_string(), "lab".to_string())]), ); + state + .device_groups + .insert(device.to_string(), HashSet::from(["edge-a".to_string()])); } let deployment = cached("fleet", "web", "zone", "lab"); state @@ -1157,6 +1169,51 @@ mod tests { assert_eq!(state.desired_dirty, HashSet::from([dn("web")])); } + #[test] + fn eligibility_projection_uses_groups_and_selectors() { + let mut state = FleetState { + deployment_watch_ready: true, + device_watch_ready: true, + group_source_ready: true, + ..Default::default() + }; + state.devices.insert( + "pi-01".into(), + BTreeMap::from([("zone".into(), "lab".into())]), + ); + state.devices.insert( + "pi-02".into(), + BTreeMap::from([("zone".into(), "lab".into())]), + ); + state + .device_groups + .insert("pi-01".into(), HashSet::from(["edge-a".into()])); + state + .device_groups + .insert("pi-02".into(), HashSet::from(["edge-b".into()])); + for name in ["web-b", "web-a"] { + let deployment = cached("fleet", name, "zone", "lab"); + state.deployments.insert(deployment.key.clone(), deployment); + } + + assert_eq!( + state.eligible_deployments_by_device()["pi-01"], + ["web-a", "web-b"] + ); + assert!(!state.eligible_deployments_by_device().contains_key("pi-02")); + + state + .device_groups + .insert("*".into(), HashSet::from(["edge-a".into()])); + assert_eq!( + state.eligible_deployments_by_device()["pi-02"], + ["web-a", "web-b"] + ); + + state.group_source_ready = false; + assert!(state.eligible_deployments_by_device().is_empty()); + } + #[test] fn compute_aggregate_counts_matched_devices() { let cached = cached("fleet-demo", "hello", "zone", "lab"); diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 0493e1cc..dff7eae1 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -159,50 +159,27 @@ async fn main() -> Result<()> { addr, css_from, live_reload, - } => { - serve_web( - mock, - addr, - css_from, - live_reload, - &cli.tenant_namespace, - &cli.nats_url, - &credentials_toml, - ) - .await - } + } => serve_web(mock, addr, css_from, live_reload).await, } } -/// `serve-web` subcommand: dashboard on its own (mock data, or the live -/// CR-reading service). The deployed operator instead serves the -/// dashboard alongside the reconcile loop — see [`spawn_dashboard`]. +/// `serve-web` subcommand for frontend iteration with mock data. The +/// live dashboard runs alongside the reconcile loop; see [`spawn_dashboard`]. #[cfg(feature = "web-frontend")] async fn serve_web( mock: bool, addr: std::net::SocketAddr, css_from: Option, live_reload: bool, - tenant_namespace: &str, - nats_url: &str, - credentials_toml: &str, ) -> Result<()> { use std::sync::Arc; - use service::{FleetService, mock::MockFleetService, real::RealFleetService}; + use service::{FleetService, mock::MockFleetService}; - let fleet: Arc = if mock { - Arc::new(MockFleetService::default()) - } else { - let commands = harmony_fleet_operator::commands::FleetCommandsClient::new( - connect_with_retry(nats_url, credentials_toml).await?, - ); - Arc::new(RealFleetService::new( - Client::try_default().await?, - tenant_namespace, - commands, - )) - }; + if !mock { + anyhow::bail!("live dashboard runs in-process with the operator; use serve-web --mock"); + } + let fleet: Arc = Arc::new(MockFleetService::default()); serve_dashboard(fleet, addr, css_from, live_reload).await } @@ -265,6 +242,7 @@ fn spawn_dashboard( client: Client, tenant_namespace: &str, commands: harmony_fleet_operator::commands::FleetCommandsClient, + fleet_state: fleet_aggregator::SharedFleetState, ) { use std::net::SocketAddr; use std::sync::Arc; @@ -274,7 +252,12 @@ fn spawn_dashboard( let addr = SocketAddr::from(([0, 0, 0, 0], frontend::server::DEFAULT_PORT)); let tenant_namespace = tenant_namespace.to_string(); tokio::spawn(async move { - let fleet = Arc::new(RealFleetService::new(client, tenant_namespace, commands)); + let fleet = Arc::new(RealFleetService::new( + client, + tenant_namespace, + commands, + fleet_state, + )); if let Err(e) = serve_dashboard(fleet, addr, None, false).await { tracing::error!(error = %e, "dashboard server exited; reconcile continues"); } @@ -325,7 +308,7 @@ async fn run( }; // Group membership (the scheduling gate): Zitadel role grants in - // prod, a static map for dev/e2e, or ungated when neither is set. + // prod or a static map for dev/e2e. Missing membership fails closed. let group_source: Option> = match ( std::env::var("ZITADEL_URL"), std::env::var("ZITADEL_PAT"), @@ -343,18 +326,24 @@ async fn run( _ => { tracing::warn!( "no device-group source (ZITADEL_URL/PAT/PROJECT_ID or FLEET_DEVICE_GROUPS); \ - deployments with allowedGroups will match no devices" + deployments will match no devices" ); None } }; - // Dashboard state comes from CRs; interactive commands use NATS request/reply. + let fleet_state = Arc::new(tokio::sync::Mutex::new( + fleet_aggregator::FleetState::default(), + )); + + // Dashboard associations use the aggregator's authorization decision; + // interactive commands use NATS request/reply. #[cfg(feature = "web-frontend")] spawn_dashboard( client.clone(), tenant_namespace, harmony_fleet_operator::commands::FleetCommandsClient::new(nats), + fleet_state.clone(), ); // Concurrent tasks: @@ -376,7 +365,7 @@ async fn run( r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r, r = device_status::run(ds_client, tenant_namespace, ds_js) => r, r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js) => r, - r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source) => r, + r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source, fleet_state) => r, } } diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index e8c3ccdd..2ed54d54 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -2,10 +2,9 @@ //! //! The operator is the write side: `device_reconciler` materializes //! `Device` CRs (labels + inventory), `device_status` reflects liveness -//! onto `Device.status`, and `fleet_aggregator` writes -//! `Deployment.status.aggregate`. This dashboard is the read side — it -//! only reads those CRs and projects them to view DTOs. No NATS: the -//! CR is the single contract between the two sides. +//! onto `Device.status`, and `fleet_aggregator` owns target eligibility +//! and writes `Deployment.status.aggregate`. This dashboard projects +//! those sources to view DTOs. use std::collections::{BTreeMap, HashSet}; use std::sync::Mutex; @@ -21,7 +20,7 @@ use harmony_fleet_operator::commands::FleetCommandsClient; use harmony_fleet_operator::crd::{ Deployment as DeploymentCr, Device as DeviceCr, DeviceStatus as DeviceLiveness, Reachability, }; -use harmony_fleet_operator::fleet_aggregator::selector_matches; +use harmony_fleet_operator::fleet_aggregator::SharedFleetState; use harmony_reconciler_contracts::{ExecReply, ReconcileScore}; use super::{ @@ -38,17 +37,24 @@ pub struct RealFleetService { kube: Client, namespace: String, commands: FleetCommandsClient, + fleet_state: SharedFleetState, /// In-memory ack set. Alerts are derived from live CR state and /// have no store of their own, so acks don't survive a restart. acked_alerts: Mutex>, } impl RealFleetService { - pub fn new(kube: Client, namespace: impl Into, commands: FleetCommandsClient) -> Self { + pub fn new( + kube: Client, + namespace: impl Into, + commands: FleetCommandsClient, + fleet_state: SharedFleetState, + ) -> Self { Self { kube, namespace: namespace.into(), commands, + fleet_state, acked_alerts: Mutex::new(HashSet::new()), } } @@ -64,13 +70,23 @@ impl RealFleetService { } async fn devices(&self) -> anyhow::Result> { - let deployments = self.deployment_crs().await?; + let eligible = self + .fleet_state + .lock() + .await + .eligible_deployments_by_device(); let now = Utc::now(); let mut devices: Vec = self .device_crs() .await? .iter() - .map(|cr| map_device(cr, &deployments, now)) + .map(|cr| { + let deployment = eligible + .get(&cr.name_any()) + .and_then(|deployments| deployments.first()) + .cloned(); + map_device(cr, deployment, now) + }) .collect(); devices.sort_by(|a, b| a.id.cmp(&b.id)); Ok(devices) @@ -106,7 +122,7 @@ fn format_exec(reply: ExecReply) -> String { output } -fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime) -> DeviceDetail { +fn map_device(cr: &DeviceCr, deployment: Option, now: DateTime) -> DeviceDetail { let labels = cr.metadata.labels.clone().unwrap_or_default(); let blacklisted = labels.get(BLACKLIST_LABEL).map(String::as_str) == Some("true"); let last_seen = cr @@ -123,7 +139,7 @@ fn map_device(cr: &DeviceCr, deployments: &[DeploymentCr], now: DateTime) - status: device_status(blacklisted, cr.status.as_ref()), last_seen, minutes_ago: (now - last_seen).num_minutes().max(0), - deployment: primary_deployment(&labels, deployments), + deployment, region: labels .get(REGION_LABEL) .cloned() @@ -148,22 +164,6 @@ fn device_status(blacklisted: bool, liveness: Option<&DeviceLiveness>) -> Device } } -/// First deployment (by name) whose selector matches the device — the -/// canonical [`selector_matches`] over CR labels, the same matcher the -/// aggregator uses. No reconstruction. -fn primary_deployment( - labels: &BTreeMap, - deployments: &[DeploymentCr], -) -> Option { - let mut matched: Vec = deployments - .iter() - .filter(|d| selector_matches(&d.spec.target_selector, labels)) - .map(ResourceExt::name_any) - .collect(); - matched.sort(); - matched.into_iter().next() -} - /// Routing labels rendered as `k=v` chips, minus internal keys. fn tags_from_labels(labels: &BTreeMap) -> Vec { labels @@ -334,21 +334,26 @@ impl FleetService for RealFleetService { } async fn get_deployment_devices(&self, name: &str) -> anyhow::Result> { - let deployments = self.deployment_crs().await?; - let Some(cr) = deployments.iter().find(|c| c.name_any() == name) else { - return Ok(Vec::new()); - }; - let selector = cr.spec.target_selector.clone(); + let eligible = self + .fleet_state + .lock() + .await + .eligible_deployments_by_device(); let now = Utc::now(); - Ok(self + let mut devices: Vec<_> = self .device_crs() .await? .iter() - .filter(|dev| { - selector_matches(&selector, &dev.metadata.labels.clone().unwrap_or_default()) + .filter_map(|device| { + let deployments = eligible.get(&device.name_any())?; + deployments + .iter() + .any(|deployment| deployment == name) + .then(|| map_device(device, deployments.first().cloned(), now)) }) - .map(|dev| map_device(dev, &deployments, now)) - .collect()) + .collect(); + devices.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(devices) } async fn blacklist_device(&self, id: &str) -> anyhow::Result { @@ -390,15 +395,15 @@ impl FleetService for RealFleetService { search: Option, ) -> anyhow::Result> { let search = search.map(|s| s.to_lowercase()); - Ok(self - .devices() - .await? + let devices = if let Some(deployment) = deployment.as_deref() { + self.get_deployment_devices(deployment).await? + } else { + self.devices().await? + }; + Ok(devices .into_iter() .filter(|d| { status.is_none_or(|s| d.status == s) - && deployment - .as_deref() - .is_none_or(|dep| d.deployment.as_deref() == Some(dep)) && region.as_deref().is_none_or(|r| d.region == r) && search.as_deref().is_none_or(|q| { d.id.to_lowercase().contains(q) @@ -415,6 +420,7 @@ impl FleetService for RealFleetService { #[cfg(test)] mod tests { use super::*; + use harmony_fleet_operator::crd::{Device, DeviceSpec}; use harmony_reconciler_contracts::{PodmanService, PodmanV0Score}; fn liveness(r: Reachability) -> DeviceLiveness { @@ -443,6 +449,22 @@ mod tests { assert_eq!(device_status(false, None), DeviceStatus::Unknown); } + #[test] + fn device_mapping_uses_authoritative_deployment() { + let device = Device::new( + "pi-01", + DeviceSpec { + inventory: None, + agent_upgrade: None, + }, + ); + assert_eq!( + map_device(&device, Some("web".into()), Utc::now()).deployment, + Some("web".into()) + ); + assert_eq!(map_device(&device, None, Utc::now()).deployment, None); + } + #[test] fn exec_output_includes_stderr_exit_and_truncation() { assert_eq!( diff --git a/fleet/scripts/load-test.sh b/fleet/scripts/load-test.sh index 2ccfa9eb..b139b9c6 100755 --- a/fleet/scripts/load-test.sh +++ b/fleet/scripts/load-test.sh @@ -42,6 +42,7 @@ GROUP_SIZES="${GROUP_SIZES:-55,5,5,5,5,5,5,5,5,5}" TICK_MS="${TICK_MS:-1000}" DURATION="${DURATION:-60}" NAMESPACE="${NAMESPACE:-fleet-load}" +LOAD_GROUP="fleet-load-test" # Keep the stack alive after the test completes so the user can poke # at CRs + NATS interactively. Ctrl-C to tear everything down. @@ -180,6 +181,7 @@ fi --operator-release "$OPERATOR_RELEASE" \ --operator-image "$OPERATOR_IMAGE" \ --operator-image-pull-policy IfNotPresent \ + --device-groups "*=$LOAD_GROUP" \ --log-level "$OPERATOR_RUST_LOG" ) diff --git a/fleet/scripts/smoke-a1.sh b/fleet/scripts/smoke-a1.sh index 2b13befa..1424ec64 100755 --- a/fleet/scripts/smoke-a1.sh +++ b/fleet/scripts/smoke-a1.sh @@ -33,6 +33,7 @@ NATS_IMAGE="${NATS_IMAGE:-docker.io/library/nats:2.10-alpine}" NATSBOX_IMAGE="${NATSBOX_IMAGE:-docker.io/natsio/nats-box:latest}" NATS_PORT="${NATS_PORT:-4222}" TARGET_DEVICE="${TARGET_DEVICE:-pi-demo-01}" +GROUP="${GROUP:-smoke}" DEPLOY_NAME="${DEPLOY_NAME:-hello-world}" DEPLOY_NS="${DEPLOY_NS:-fleet-demo}" HELLO_CONTAINER="${HELLO_CONTAINER:-hello}" @@ -148,7 +149,10 @@ metadata: name: bad-discriminator namespace: $DEPLOY_NS spec: - targetDevices: [$TARGET_DEVICE] + allowedGroups: [$GROUP] + targetSelector: + matchLabels: + device-id: $TARGET_DEVICE score: type: "has spaces" data: {} @@ -178,6 +182,7 @@ log "phase 3: start operator" ) NATS_URL="nats://127.0.0.1:$NATS_PORT" \ KV_BUCKET="desired-state" \ +FLEET_DEVICE_GROUPS="$TARGET_DEVICE=$GROUP" \ RUST_LOG="info,kube_runtime=warn" \ "$REPO_ROOT/target/debug/harmony-fleet-operator" \ >"$OPERATOR_LOG" 2>&1 & @@ -247,7 +252,10 @@ metadata: name: $DEPLOY_NAME namespace: $DEPLOY_NS spec: - targetDevices: [$TARGET_DEVICE] + allowedGroups: [$GROUP] + targetSelector: + matchLabels: + device-id: $TARGET_DEVICE score: type: PodmanV0 data: diff --git a/fleet/scripts/smoke-a4.sh b/fleet/scripts/smoke-a4.sh index b57d590b..37bee7e6 100755 --- a/fleet/scripts/smoke-a4.sh +++ b/fleet/scripts/smoke-a4.sh @@ -208,6 +208,7 @@ log "phase 4: start operator (host-side) connected to nats://localhost:$NATS_NOD ) NATS_URL="nats://localhost:$NATS_NODE_PORT" \ KV_BUCKET="desired-state" \ +FLEET_DEVICE_GROUPS="$DEVICE_ID=$GROUP" \ RUST_LOG="info,kube_runtime=warn" \ "$REPO_ROOT/target/release/harmony-fleet-operator" \ >"$OPERATOR_LOG" 2>&1 & @@ -377,6 +378,7 @@ if [[ "$AUTO" == "1" ]]; then --namespace "$DEPLOY_NS" \ --name "$DEPLOY_NAME" \ --target-device "$DEVICE_ID" \ + --allowed-group "$GROUP" \ --image "$V1_IMAGE" \ --port "$DEPLOY_PORT" ) @@ -424,6 +426,7 @@ if [[ "$AUTO" == "1" ]]; then --namespace "$DEPLOY_NS" \ --name "$DEPLOY_NAME" \ --target-device "$DEVICE_ID" \ + --allowed-group "$GROUP" \ --image "$V2_IMAGE" \ --port "$DEPLOY_PORT" ) @@ -450,6 +453,7 @@ if [[ "$AUTO" == "1" ]]; then --namespace "$DEPLOY_NS" \ --name "$DEPLOY_NAME" \ --target-device "$DEVICE_ID" \ + --allowed-group "$GROUP" \ --delete ) for _ in $(seq 1 60); do @@ -490,16 +494,17 @@ $(printf '\033[1mApply an nginx deployment (typed Rust):\033[0m\n') --namespace $DEPLOY_NS \\ --name $DEPLOY_NAME \\ --target-device $DEVICE_ID \\ + --allowed-group $GROUP \\ --image docker.io/library/nginx:latest $(printf '\033[1mUpgrade it:\033[0m\n') cargo run -q -p example_harmony_apply_deployment -- \\ - --namespace $DEPLOY_NS --name $DEPLOY_NAME --target-device $DEVICE_ID \\ + --namespace $DEPLOY_NS --name $DEPLOY_NAME --target-device $DEVICE_ID --allowed-group $GROUP \\ --image docker.io/library/nginx:1.26 $(printf '\033[1mPreview the CR as JSON (and apply via kubectl):\033[0m\n') cargo run -q -p example_harmony_apply_deployment -- \\ - --name $DEPLOY_NAME --target-device $DEVICE_ID \\ + --name $DEPLOY_NAME --target-device $DEVICE_ID --allowed-group $GROUP \\ --image docker.io/library/nginx:latest --print | kubectl apply -f - $(printf '\033[1mConnect to the device:\033[0m\n') diff --git a/harmony_zitadel_auth/src/device_groups.rs b/harmony_zitadel_auth/src/device_groups.rs index fa677dd6..dd9af2ab 100644 --- a/harmony_zitadel_auth/src/device_groups.rs +++ b/harmony_zitadel_auth/src/device_groups.rs @@ -47,6 +47,22 @@ struct GrantEntry { } const PAGE: u64 = 1000; +const DEVICE_USERNAME_PREFIX: &str = "device-"; + +fn insert_grant(map: &mut HashMap>, grant: GrantEntry) { + let roles: HashSet = grant.role_keys.into_iter().collect(); + if !grant.user_name.is_empty() && grant.user_name != grant.user_id { + map.entry(grant.user_name.clone()) + .or_default() + .extend(roles.iter().cloned()); + if let Some(device_id) = grant.user_name.strip_prefix(DEVICE_USERNAME_PREFIX) { + map.entry(device_id.to_string()) + .or_default() + .extend(roles.iter().cloned()); + } + } + map.entry(grant.user_id).or_default().extend(roles); +} #[async_trait] impl DeviceGroupSource for ZitadelDeviceGroups { @@ -84,16 +100,10 @@ impl DeviceGroupSource for ZitadelDeviceGroups { let entries = page.result.unwrap_or_default(); let n = entries.len() as u64; for grant in entries { - let roles: HashSet = grant.role_keys.into_iter().collect(); // Fleets key devices by machine-user id or by user name // depending on provisioning vintage; expose the grant under - // both so either convention resolves. - if !grant.user_name.is_empty() && grant.user_name != grant.user_id { - map.entry(grant.user_name) - .or_default() - .extend(roles.iter().cloned()); - } - map.entry(grant.user_id).or_default().extend(roles); + // both and under the device id used by NATS and Device CRs. + insert_grant(&mut map, grant); } if n < PAGE { return Ok(map); @@ -102,3 +112,28 @@ impl DeviceGroupSource for ZitadelDeviceGroups { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefixed_username_maps_to_device_id() { + let mut map = HashMap::new(); + let response: GrantSearchResponse = serde_json::from_value(serde_json::json!({ + "result": [{ + "userId": "123", + "userName": "device-pi-42", + "roleKeys": ["edge-a"] + }] + })) + .unwrap(); + for grant in response.result.unwrap() { + insert_grant(&mut map, grant); + } + + for alias in ["123", "device-pi-42", "pi-42"] { + assert!(map[alias].contains("edge-a")); + } + } +} -- 2.39.5 From dbecc562d23f7f4bbe8619e99e6720b604c0e124 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 23 Jul 2026 18:07:35 -0400 Subject: [PATCH 29/47] docs(fleet): design scheduled tasks --- docs/SUMMARY.md | 2 + docs/design/fleet-system-upgrades.md | 173 ++++++++++++++++ docs/design/fleet-tasks.md | 289 +++++++++++++++++++++++++++ 3 files changed, 464 insertions(+) create mode 100644 docs/design/fleet-system-upgrades.md create mode 100644 docs/design/fleet-tasks.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 33c87f77..415dbc98 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -39,6 +39,8 @@ - [Fleet Score References](./reference/fleet-score-references.md) - [Fleet Agent Upgrades](./design/fleet-agent-upgrades.md) +- [Fleet Tasks](./design/fleet-tasks.md) + - [System-upgrade Executor](./design/fleet-system-upgrades.md) ## Architecture Decision Records diff --git a/docs/design/fleet-system-upgrades.md b/docs/design/fleet-system-upgrades.md new file mode 100644 index 00000000..e531e1b8 --- /dev/null +++ b/docs/design/fleet-system-upgrades.md @@ -0,0 +1,173 @@ +# Fleet system-upgrade executor + +## Summary + +System upgrade is a privileged built-in Fleet task. `TaskSchedule` and +`TaskRun` own scheduling, placement, sealed target plans, canary rollout, +deadlines, status retention, and dashboard history as described in +[Fleet tasks](./fleet-tasks.md). This document covers only the device executor +and the additional trust boundary required for root package mutation. + +The agent relays expiring, operator-signed intent to the root updater. The +updater verifies the signature and runs one compiled policy. It accepts no shell +command, package name, repository, path, unit, or reboot argument. + +System upgrades share the updater process and privileged mutation lock with +agent upgrades, but use a separate protocol operation, journal, and state +machine. Package upgrades are not atomically reversible. Canary containment and +explicit repair status replace agent-binary rollback. + +The first executor supports only apt full upgrades on Debian-family devices +from a preconfigured immutable repository snapshot, followed by a mandatory +reboot. It does not accept dnf, rpm-ostree, pacman, apk, mutable repositories, +package lists, shell commands, package rollback, cancellation, or automatic +retry after terminal failure. + +## Authorization and protocol + +The existing updater socket admits the `fleet-agent` Unix group. That is enough +for installing a candidate that still runs unprivileged, but not for root +package maintainer scripts and host reboot. A system-upgrade attempt therefore +contains an operator signature over: + +```text +attemptId +runUid +deviceId +policy: AptFullUpgradeV1 +repositorySnapshot +notBefore +expiresAt +``` + +Device setup pins the system-upgrade authorization public key in the updater's +root-owned configuration. The operator private key stays in a Kubernetes +Secret. The updater verifies the signature, exact device identity, time window, +policy, and repository snapshot before mutation. The agent and updater both +refuse to start an expired attempt. Run termination removes pending intents; +an operation already applying packages is allowed to finish and is reported as +late without changing the run result. + +The updater publishes its protocol and policy capabilities through ordinary +device observation. Task planning fails if any selected device lacks the +required updater protocol or policy; selected devices are never silently +omitted. Enabling this workload requires a completed out-of-band updater rollout +through `FleetDeviceSetupScore`; the privileged updater does not self-update. + +The updater does not have NATS credentials. Its completion is relayed by the +agent, so canary gating assumes trusted canary devices and detects honest +upgrade or boot failures. Protecting the gate from a compromised agent would +require a root-held device key and signed completion receipt; that is outside +the first release. + +## Privileged state machine + +Agent and system upgrades share one durable mutation lock. System upgrades use +one journal per attempt under +`/var/lib/harmony-fleet-updater/system-upgrades/`. + +Device-reported phases are: + +```text +pending +blocked +preflight +applying +rebooting +verifying +complete +failed +repair-required +``` + +`blocked` is non-terminal and covers an active agent upgrade or a bounded wait +for apt/dpkg locks. `repair-required` forbids automatic retry. + +`AptFullUpgradeV1` uses a fixed command and conffile policy compiled into the +updater. The configured apt source must identify the signed immutable snapshot +from the attempt. Each device verifies that identity before applying packages. + +Durable recovery is: + +| Journal state | Recovery | +|---|---| +| `preflight` | Repeat non-mutating checks. | +| `applying` | Wait for any surviving package process, run bounded dpkg repair and audit, then rerun the fixed idempotent transaction or enter `repair-required`. | +| `rebooting`, old boot ID | Retry the fixed reboot request up to a bound, then report `failed`. | +| `rebooting`, changed boot ID | Persist `verifying`. | +| `verifying` | Audit package state and snapshot identity, then complete or require repair. | +| `complete`, `failed`, `repair-required` | Return the exact terminal result for duplicate intent. | + +The updater persists `applying` before spawning apt and `rebooting` before +requesting reboot. Startup quarantines privileged mutation if more than one +journal appears active or a journal is corrupt. Journal writes use temporary +files, fsync, atomic rename, bounded transition history, and content-bound UUID +handling as the agent-upgrade transaction does. Terminal journals are removed +after `expiresAt` plus a 30-day replay window. `repair-required` journals remain +until explicit operator repair. + +## Task result + +Disconnect after `rebooting` is expected. Request/reply timeout is not a failure +signal. A device succeeds only after: + +- the updater observes a boot ID different from the pre-upgrade boot ID; +- the immutable repository snapshot still matches; +- post-boot dpkg state is healthy; +- the agent reconnects and relays the updater's `complete` state; +- the Device heartbeat is fresh. + +The operator owns the run and cohort deadlines. A canary that never returns +fails the run. Already upgraded devices are not rolled back. + +## Invariants + +- A device executes at most one privileged mutation. +- Attempts cannot start outside their signed time window. +- Restart does not repeat a completed device attempt. +- Corrupt or incomplete package state fails closed. + +## Implementation plan + +### 1. Trust and contracts + +- Define operator signing-key provisioning and root-owned public-key delivery. +- Add updater protocol/capability observation. +- Add signed attempt, status, phases, time bounds, and policy contracts to + `harmony-reconciler-contracts`. +- Test canonical signing bytes, signature rejection, expiry, identity mismatch, + duplicate UUIDs, and protocol negotiation. + +### 2. Updater engine + +- Add the signed socket operation and one lock shared with agent upgrades. +- Implement immutable-snapshot verification, apt preflight, fixed full upgrade, + dpkg repair/audit, durable reboot, boot-ID verification, and the complete + recovery table. +- Version the updater protocol and harden its systemd unit before production. +- Test power loss at every journal boundary, package-lock timeout, apt failure, + reboot command failure, unchanged boot ID, corrupt journals, and concurrent + agent upgrades. + +### 3. Agent bridge + +- Add the device-scoped watch-plus-snapshot bridge and status publication. +- Test malformed, expired, wrong-device, duplicate, blocked, rebooting, and late + attempts plus status publication retry. + +### 4. End-to-end proof + +- Use Debian VMs and a controlled immutable apt snapshot. +- Prove package failure, reboot timeout, expired intent, and verification + failure produce the correct terminal task result. +- Prove operator/agent/updater restart recovery and exclusion with agent upgrade. + +## Open decisions + +This executor should not be implemented until these are answered: + +1. Which immutable apt snapshot service and identifier are available to every + target network? +2. Where is the operator signing key generated, backed up, and rotated, and how + are key IDs and overlapping trusted public keys rolled out to existing + devices? diff --git a/docs/design/fleet-tasks.md b/docs/design/fleet-tasks.md new file mode 100644 index 00000000..e0eb49a0 --- /dev/null +++ b/docs/design/fleet-tasks.md @@ -0,0 +1,289 @@ +# Fleet tasks + +## Summary + +Fleet tasks are finite jobs selected with the same group and label rules as +Fleet deployments. Deployments continuously reconcile desired state. Tasks +instead freeze their target set, execute once per selected device, and retain a +terminal result. + +Two namespaced resources separate recurring policy from execution: + +- `TaskSchedule` creates runs at typed calendar times. +- `TaskRun` is one immutable execution. Users also create it directly for + one-time work. + +The task envelope is generic. A workload is either a one-shot container or a +versioned built-in operation. The first built-ins are agent and system upgrades. +Each built-in keeps its own validation, device protocol, and recovery behavior. + +## First release + +The first release includes: + +- direct one-time `TaskRun` creation; +- daily, weekly, yearly, and typed five-field cron schedules; +- explicit IANA time zones for recurring schedules; +- `allowedGroups` and `targetSelector` placement; +- immediate and canary-then-all rollout; +- one active run per schedule; +- one-shot container, agent-upgrade, and system-upgrade workloads; +- deadlines, aggregate status, bounded run history, and dashboard alerts; +- recovery across operator and agent restarts. + +It does not include overlapping runs from one schedule, mutable run specs, +shell-string workloads, percentage batches, cancellation, automatic retry, +schedule editing, or unbounded per-device status in Kubernetes resources. + +## Resources + +### `TaskSchedule` + +`TaskSchedule` owns a recurring calendar policy and a run template: + +```yaml +apiVersion: fleet.nationtech.io/v1alpha1 +kind: TaskSchedule +metadata: + name: weekly-system-upgrade +spec: + schedule: + weekly: + day: friday + at: "23:00" + timeZone: America/New_York + suspend: false + runTemplate: + allowedGroups: [production] + targetSelector: + matchLabels: + fleet.nationtech.io/os-family: debian + rollout: + canaryThenAll: + selector: + matchLabels: + fleet.nationtech.io/operator-canary: "true" + canaryDeadlineSeconds: 3600 + deadlineSeconds: 21600 + workload: + builtin: + systemUpgradeV1: + repositorySnapshot: debian-12-2026-07-26 +``` + +Schedule variants are: + +```text +Daily { at, timeZone } +Weekly { day, at, timeZone } +Yearly { month, day, at, timeZone } +Cron { minute, hour, dayOfMonth, month, dayOfWeek, timeZone } +``` + +The cron fields are typed expressions, not one opaque string. Each field accepts +`Any`, a value, a range, a list, or a stepped expression, with unit-specific +validated values. Five-field cron order is minute, hour, day of month, month, +and day of week. It has no seconds or year field. Friendly daily, weekly, and +yearly forms compile to the same canonical field representation used by the +scheduler. + +An explicit time zone makes daylight-saving behavior part of the resource. For +example, Friday 23:00 in `America/New_York` occurs at Saturday 03:00 UTC during +EDT and Saturday 04:00 UTC during EST. A fixed Saturday 03:00 UTC schedule is a +different policy. One-time runs use an absolute timestamp and need no time zone. + +Schedule status contains `lastScheduleTime`, `activeRun`, +`lastSuccessfulRun`, and conditions. The operator chooses the latest eligible +missed tick and creates at most one catch-up run. Ticks while a run is active +are skipped permanently. Suspension prevents new runs but does not stop an +active run. + +The run name is derived from the schedule UID and scheduled instant. The +scheduler claims `status.activeRun` with a resource-version patch before +creating that deterministic run. Recovery creates a claimed but missing run. +This prevents overlapping operator pods from creating different runs around a +calendar boundary. Schedule deletion waits for an active run to finish. + +### `TaskRun` + +A scheduled run contains an immutable copy of its scheduled instant and run +template. A directly created run has the same fields without a schedule owner: + +```yaml +apiVersion: fleet.nationtech.io/v1alpha1 +kind: TaskRun +metadata: + name: inventory-check-2026-07-24 +spec: + scheduledFor: "2026-07-24T03:00:00Z" + allowedGroups: [production] + targetSelector: + matchLabels: + fleet.nationtech.io/site: montreal + rollout: + immediate: {} + deadlineSeconds: 1800 + workload: + containerV1: + image: registry.example.test/inventory-check@sha256:... + args: ["--report"] +``` + +`scheduledFor` is an absolute instant. If omitted, the run is eligible to start +immediately. Admission rejects spec mutation after creation. A run has one +phase: + +```text +Planning | Running | Complete | Failed +``` + +Status also carries aggregate counts, start and completion times, the latest +bounded error, and a reason. Operator-derived device outcomes such as +`TimedOut`, `Expired`, and `SkippedRevoked` remain distinct from workload +status reported by a device. + +Per-device maps do not live in the CR. Detailed NATS status is retained for the +active and most recent terminal run. The operator keeps the three most recent +successful and three failed runs per schedule. + +## Workloads + +The workload is a closed, versioned union: + +```text +ContainerV1 +Builtin::AgentUpgradeV1 +Builtin::SystemUpgradeV1 +``` + +`ContainerV1` is a one-shot container with a digest-pinned image, argument +array, and bounded resources. It does not accept a shell string. Completion is +the container exit result, not continued service health. Long-running services +remain Deployments even when they share image, argument, environment, or +resource component types with container tasks. + +Built-ins exist when Harmony owns more behavior than a container exit code can +express. Agent upgrade retains candidate activation, health verification, and +rollback. System upgrade retains signed root authorization, package recovery, +reboot verification, and repair-required state. These operations do not become +Deployment workload variants because they have terminal, edge-triggered +semantics. + +## Placement and sealed plans + +Placement reuses the deployment aggregator's `device_eligible` rule: + +```text +allowed group membership AND targetSelector +``` + +`matchLabels` remains a conjunction, an empty selector matches every device, +and `matchExpressions` fails closed until implemented. Groups remain the +authorization boundary. `allowedGroups` is required; an empty list authorizes +no target. + +Canary designation uses an operator-owned label namespace. The Device +reconciler rejects that namespace from agent-reported labels and preserves +administrator-set values. A device therefore cannot elect itself as a canary. + +Planning writes one assignment per selected device under a planning generation. +The operator then CAS-creates a seal containing the target count, canary count, +and digest of sorted assignments. No intent is written until the seal validates. +An interrupted, unsealed plan is resumed or discarded, never executed. + +The assignment freezes membership, but group authorization is checked again +before each cohort is released. A revoked device becomes `SkippedRevoked`; no +newly eligible replacement enters the run. A revoked canary fails the run. A +revoked non-canary contributes a failed outcome after its cohort starts. + +A run with no targets completes. A canary rollout with targets but no canaries +fails with `NoCanaries`; it never falls back to immediate release. + +Suggested JetStream KV keys are: + +```text +task-assignments: . +task-plan: +task-intent: . +task-status: . +``` + +The buckets are file-backed, use the deployment NATS replica policy, and have +explicit byte and history limits. Assignment and intent keys are removed after +terminal aggregation. Detailed status has short bounded retention. + +## Rollout + +`Immediate` releases every assignment after planning. `CanaryThenAll` advances +as follows: + +1. Seal the complete assignment plan. +2. Revalidate canary group authorization and write all canary intents. +3. Wait for every canary to succeed. +4. On canary failure or deadline, fail the run and write no remaining intent. +5. Revalidate group authorization and release all remaining devices in one + reconciliation pass. +6. Complete when every released device is terminal. A failure or run deadline + fails the run but does not stop work already released. + +Releasing the second cohort together may disrupt all selected service replicas +and load the registry, NATS, and network concurrently. The selector must cover +devices whose simultaneous disruption is acceptable. Bounded concurrency is a +separate future rollout strategy. + +Run phase changes use Kubernetes resource versions, intent creation is +idempotent, and duplicate controllers may reconcile the same run without +creating a second attempt. No global leader election is required. A terminal +run never advances. Late status remains diagnostic and cannot reopen a run or +release a blocked cohort. + +## Dashboard + +Tasks have Schedules and Runs views. Schedules show their readable calendar, +time zone, suspension state, selectors, workload, next run, and last result. +Fleet admins can create and suspend schedules. Runs show phase, cohort progress, +start, duration, result, and recent per-device status from NATS. + +Editing is deferred because schedule changes need generation-specific +missed-tick semantics. Deletion is available only when no run is active. All +mutations use the existing `fleet-admin` session and CSRF middleware. + +Failed runs become durable conditions on the run CR. The dashboard derives +critical alerts with IDs containing the run UID and links them to run detail. +This reuses the current alert list, badge, acknowledgement, and dashboard strip. +Acknowledgement remains process-local. Email, webhook delivery, notification +persistence, and sinks remain separate work. + +## Invariants + +- A schedule has at most one active run. +- A deterministic schedule tick creates at most one run. +- A run UID identifies one immutable target plan and workload. +- Group authorization is current when each intent is released. +- A device executes a run assignment at most once. +- Non-canary intents never exist before all canaries succeed. +- A terminal failed run never advances. +- Restart does not repeat a completed device attempt. +- No Kubernetes status map grows with fleet size. + +## Implementation order + +1. Add typed schedule, workload, rollout, `TaskSchedule`, and `TaskRun` + contracts, CRDs, status subresources, RBAC, and admission rules. +2. Implement calendar evaluation with `croner`, deterministic catch-up, + active-run claims, suspension, and bounded history. +3. Reuse deployment placement, protect operator-owned labels, and implement + sealed plans, rollout, deadlines, CAS transitions, and finalizer cleanup. +4. Add the device-scoped watch and snapshot bridge, container execution, status + publication, and bounded NATS retention. +5. Connect the existing agent-upgrade flow and the system-upgrade executor as + built-in workloads without duplicating their state machines. +6. Add schedule and run dashboard views, alerts, and end-to-end restart and + authorization tests. + +## Open decisions + +1. Should the initial weekly system-upgrade policy be Friday 23:00 + `America/New_York` or Saturday 03:00 UTC? +2. What maximum target count and shared infrastructure capacity are supported + by an immediate or all-at-once cohort? -- 2.39.5 From 2f70410efd24ae8b55cfeae705f1453178a9d1de Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 24 Jul 2026 09:48:34 -0400 Subject: [PATCH 30/47] feat(fleet): add system upgrade tasks --- docs/design/fleet-system-upgrades.md | 168 ++--- docs/design/fleet-tasks.md | 315 +++------ examples/fleet_load_test/src/main.rs | 1 + .../src/fleet_publisher.rs | 37 +- fleet/harmony-fleet-agent/src/main.rs | 81 ++- .../harmony-fleet-agent/src/system_upgrade.rs | 640 ++++++++++++++++++ .../src/system_upgrade_service.rs | 267 ++++++++ fleet/harmony-fleet-agent/src/updater.rs | 171 ++++- fleet/harmony-fleet-deploy/src/app.rs | 22 + .../src/operator/chart.rs | 41 +- .../src/operator/score.rs | 7 +- fleet/harmony-fleet-e2e/tests/operator.rs | 1 + fleet/harmony-fleet-operator/src/crd.rs | 4 +- .../src/device_reconciler.rs | 50 +- .../src/fleet_aggregator.rs | 15 +- fleet/harmony-fleet-operator/src/lib.rs | 3 + fleet/harmony-fleet-operator/src/main.rs | 8 +- .../src/service/real.rs | 1 + fleet/harmony-fleet-operator/src/task.rs | 111 +++ .../src/task_run_controller.rs | 630 +++++++++++++++++ harmony-reconciler-contracts/src/fleet.rs | 27 +- harmony-reconciler-contracts/src/kv.rs | 19 + harmony-reconciler-contracts/src/lib.rs | 12 +- .../src/system_upgrade.rs | 127 ++++ nats/callout/src/main.rs | 16 +- nats/callout/src/permissions.rs | 188 ++++- 26 files changed, 2532 insertions(+), 430 deletions(-) create mode 100644 fleet/harmony-fleet-agent/src/system_upgrade.rs create mode 100644 fleet/harmony-fleet-agent/src/system_upgrade_service.rs create mode 100644 fleet/harmony-fleet-operator/src/task.rs create mode 100644 fleet/harmony-fleet-operator/src/task_run_controller.rs create mode 100644 harmony-reconciler-contracts/src/system_upgrade.rs diff --git a/docs/design/fleet-system-upgrades.md b/docs/design/fleet-system-upgrades.md index e531e1b8..2a722664 100644 --- a/docs/design/fleet-system-upgrades.md +++ b/docs/design/fleet-system-upgrades.md @@ -2,63 +2,52 @@ ## Summary -System upgrade is a privileged built-in Fleet task. `TaskSchedule` and -`TaskRun` own scheduling, placement, sealed target plans, canary rollout, -deadlines, status retention, and dashboard history as described in -[Fleet tasks](./fleet-tasks.md). This document covers only the device executor -and the additional trust boundary required for root package mutation. +System upgrade is a privileged built-in Fleet task. `TaskRun` owns placement, +the frozen target plan, deadlines, and aggregate status as described in +[Fleet tasks](./fleet-tasks.md). This document covers the device executor. -The agent relays expiring, operator-signed intent to the root updater. The -updater verifies the signature and runs one compiled policy. It accepts no shell -command, package name, repository, path, unit, or reboot argument. +The unprivileged agent requests one fixed operation from the existing root +updater over its Unix socket. The updater accepts no shell command, package +name, repository, path, unit, or reboot argument. -System upgrades share the updater process and privileged mutation lock with -agent upgrades, but use a separate protocol operation, journal, and state -machine. Package upgrades are not atomically reversible. Canary containment and -explicit repair status replace agent-binary rollback. +System and agent upgrades share the updater process and mutation lock. System +upgrades have a separate protocol operation, journal, and state machine because +package upgrades are not atomically reversible. -The first executor supports only apt full upgrades on Debian-family devices -from a preconfigured immutable repository snapshot, followed by a mandatory -reboot. It does not accept dnf, rpm-ostree, pacman, apk, mutable repositories, -package lists, shell commands, package rollback, cancellation, or automatic -retry after terminal failure. +The first executor runs an apt full upgrade using the device's configured, +signed repositories and then requires a reboot. It supports Debian and +Raspberry Pi OS. It does not configure repositories, select packages, roll back +packages, cancel an active apt/dpkg process, or retry a terminal failure. -## Authorization and protocol +## Trust boundary -The existing updater socket admits the `fleet-agent` Unix group. That is enough -for installing a candidate that still runs unprivileged, but not for root -package maintainer scripts and host reboot. A system-upgrade attempt therefore -contains an operator signature over: +The updater socket admits only the `fleet-agent` Unix group. The agent can ask +for `AptFullUpgradeV1`, but cannot alter what that operation does. A compromised +agent can invoke the upgrade and is treated as a compromised device; adding a +second key held by the same agent would not improve that boundary. + +A system-upgrade request contains only: ```text attemptId runUid deviceId -policy: AptFullUpgradeV1 -repositorySnapshot -notBefore expiresAt ``` -Device setup pins the system-upgrade authorization public key in the updater's -root-owned configuration. The operator private key stays in a Kubernetes -Secret. The updater verifies the signature, exact device identity, time window, -policy, and repository snapshot before mutation. The agent and updater both -refuse to start an expired attempt. Run termination removes pending intents; -an operation already applying packages is allowed to finish and is reported as -late without changing the run result. +The updater validates the identifiers and expiry before starting. Duplicate +requests with the same content return the durable result. Reusing an attempt ID +with different content fails. Expiry prevents an intent delayed in NATS from +starting later; it does not interrupt apt/dpkg after mutation begins. -The updater publishes its protocol and policy capabilities through ordinary -device observation. Task planning fails if any selected device lacks the -required updater protocol or policy; selected devices are never silently -omitted. Enabling this workload requires a completed out-of-band updater rollout -through `FleetDeviceSetupScore`; the privileged updater does not self-update. +The updater publishes protocol and `AptFullUpgradeV1` capability through device +observation. Planning fails if a selected device lacks that capability. The +privileged updater is installed out of band by `FleetDeviceSetupScore` and does +not self-update. -The updater does not have NATS credentials. Its completion is relayed by the -agent, so canary gating assumes trusted canary devices and detects honest -upgrade or boot failures. Protecting the gate from a compromised agent would -require a root-held device key and signed completion receipt; that is outside -the first release. +The updater has no NATS credentials. The agent relays updater status. A +compromised agent can therefore falsify status, which is consistent with the +device-compromise trust model. ## Privileged state machine @@ -69,7 +58,6 @@ one journal per attempt under Device-reported phases are: ```text -pending blocked preflight applying @@ -83,91 +71,59 @@ repair-required `blocked` is non-terminal and covers an active agent upgrade or a bounded wait for apt/dpkg locks. `repair-required` forbids automatic retry. -`AptFullUpgradeV1` uses a fixed command and conffile policy compiled into the -updater. The configured apt source must identify the signed immutable snapshot -from the attempt. Each device verifies that identity before applying packages. +`AptFullUpgradeV1` has one compiled transaction: + +1. Confirm package state is healthy and no configured apt source disables + repository authentication. +2. Run `apt-get update`. +3. Run noninteractive `apt-get full-upgrade`, preserving locally modified + configuration files. +4. Audit dpkg state. +5. Record `rebooting` and the current boot ID, then request reboot. +6. After startup, require a changed boot ID and healthy dpkg state. Durable recovery is: | Journal state | Recovery | |---|---| | `preflight` | Repeat non-mutating checks. | -| `applying` | Wait for any surviving package process, run bounded dpkg repair and audit, then rerun the fixed idempotent transaction or enter `repair-required`. | +| `applying` | Audit dpkg and enter `repair-required`; never rerun the interrupted full upgrade. | | `rebooting`, old boot ID | Retry the fixed reboot request up to a bound, then report `failed`. | | `rebooting`, changed boot ID | Persist `verifying`. | -| `verifying` | Audit package state and snapshot identity, then complete or require repair. | -| `complete`, `failed`, `repair-required` | Return the exact terminal result for duplicate intent. | +| `verifying` | Audit package state, then complete or require repair. | +| `complete`, `failed`, `repair-required` | Return the same terminal result for duplicate intent. | The updater persists `applying` before spawning apt and `rebooting` before -requesting reboot. Startup quarantines privileged mutation if more than one -journal appears active or a journal is corrupt. Journal writes use temporary -files, fsync, atomic rename, bounded transition history, and content-bound UUID -handling as the agent-upgrade transaction does. Terminal journals are removed -after `expiresAt` plus a 30-day replay window. `repair-required` journals remain -until explicit operator repair. +requesting reboot. Startup fails if more than one journal appears active or a +journal is corrupt. Journal writes use a temporary +file, fsync, and atomic rename. `repair-required` remains until an operator +repairs the device. ## Task result -Disconnect after `rebooting` is expected. Request/reply timeout is not a failure +Disconnect after `rebooting` is expected; request timeout is not a failure signal. A device succeeds only after: - the updater observes a boot ID different from the pre-upgrade boot ID; -- the immutable repository snapshot still matches; - post-boot dpkg state is healthy; -- the agent reconnects and relays the updater's `complete` state; +- the agent reconnects and relays `complete`; - the Device heartbeat is fresh. -The operator owns the run and cohort deadlines. A canary that never returns -fails the run. Already upgraded devices are not rolled back. +The operator owns the run deadline. A device that never returns fails the run. +Already upgraded devices are not rolled back. ## Invariants -- A device executes at most one privileged mutation. -- Attempts cannot start outside their signed time window. -- Restart does not repeat a completed device attempt. +- A device runs at most one privileged mutation at a time. +- An expired attempt cannot start. +- Restart does not repeat a completed attempt. - Corrupt or incomplete package state fails closed. +- All apt, dpkg, and reboot arguments are compiled into the updater. -## Implementation plan +## Remaining work -### 1. Trust and contracts - -- Define operator signing-key provisioning and root-owned public-key delivery. -- Add updater protocol/capability observation. -- Add signed attempt, status, phases, time bounds, and policy contracts to - `harmony-reconciler-contracts`. -- Test canonical signing bytes, signature rejection, expiry, identity mismatch, - duplicate UUIDs, and protocol negotiation. - -### 2. Updater engine - -- Add the signed socket operation and one lock shared with agent upgrades. -- Implement immutable-snapshot verification, apt preflight, fixed full upgrade, - dpkg repair/audit, durable reboot, boot-ID verification, and the complete - recovery table. -- Version the updater protocol and harden its systemd unit before production. -- Test power loss at every journal boundary, package-lock timeout, apt failure, - reboot command failure, unchanged boot ID, corrupt journals, and concurrent - agent upgrades. - -### 3. Agent bridge - -- Add the device-scoped watch-plus-snapshot bridge and status publication. -- Test malformed, expired, wrong-device, duplicate, blocked, rebooting, and late - attempts plus status publication retry. - -### 4. End-to-end proof - -- Use Debian VMs and a controlled immutable apt snapshot. -- Prove package failure, reboot timeout, expired intent, and verification - failure produce the correct terminal task result. -- Prove operator/agent/updater restart recovery and exclusion with agent upgrade. - -## Open decisions - -This executor should not be implemented until these are answered: - -1. Which immutable apt snapshot service and identifier are available to every - target network? -2. Where is the operator signing key generated, backed up, and rotated, and how - are key IDs and overlapping trusted public keys rolled out to existing - devices? +1. Prove the path on a disposable Debian VM, then Raspberry Pi OS. +2. Test updater and agent restart, package failure, reboot failure, expiry, and + duplicate delivery against real systemd, apt, and dpkg. +3. Add all-at-once multi-device rollout and UTC recurring schedules after the + direct run passes those tests. diff --git a/docs/design/fleet-tasks.md b/docs/design/fleet-tasks.md index e0eb49a0..7cb8cd73 100644 --- a/docs/design/fleet-tasks.md +++ b/docs/design/fleet-tasks.md @@ -3,287 +3,136 @@ ## Summary Fleet tasks are finite jobs selected with the same group and label rules as -Fleet deployments. Deployments continuously reconcile desired state. Tasks -instead freeze their target set, execute once per selected device, and retain a -terminal result. +Fleet deployments. A task freezes its targets, runs once on each device, and +retains a terminal result. -Two namespaced resources separate recurring policy from execution: +The first release supports one directly created `TaskRun`, one selected device, +and the built-in system upgrade. Multi-device runs and recurring schedules come +after the direct path is proven on Debian and Raspberry Pi OS. -- `TaskSchedule` creates runs at typed calendar times. -- `TaskRun` is one immutable execution. Users also create it directly for - one-time work. - -The task envelope is generic. A workload is either a one-shot container or a -versioned built-in operation. The first built-ins are agent and system upgrades. -Each built-in keeps its own validation, device protocol, and recovery behavior. - -## First release - -The first release includes: - -- direct one-time `TaskRun` creation; -- daily, weekly, yearly, and typed five-field cron schedules; -- explicit IANA time zones for recurring schedules; -- `allowedGroups` and `targetSelector` placement; -- immediate and canary-then-all rollout; -- one active run per schedule; -- one-shot container, agent-upgrade, and system-upgrade workloads; -- deadlines, aggregate status, bounded run history, and dashboard alerts; -- recovery across operator and agent restarts. - -It does not include overlapping runs from one schedule, mutable run specs, -shell-string workloads, percentage batches, cancellation, automatic retry, -schedule editing, or unbounded per-device status in Kubernetes resources. - -## Resources - -### `TaskSchedule` - -`TaskSchedule` owns a recurring calendar policy and a run template: - -```yaml -apiVersion: fleet.nationtech.io/v1alpha1 -kind: TaskSchedule -metadata: - name: weekly-system-upgrade -spec: - schedule: - weekly: - day: friday - at: "23:00" - timeZone: America/New_York - suspend: false - runTemplate: - allowedGroups: [production] - targetSelector: - matchLabels: - fleet.nationtech.io/os-family: debian - rollout: - canaryThenAll: - selector: - matchLabels: - fleet.nationtech.io/operator-canary: "true" - canaryDeadlineSeconds: 3600 - deadlineSeconds: 21600 - workload: - builtin: - systemUpgradeV1: - repositorySnapshot: debian-12-2026-07-26 -``` - -Schedule variants are: - -```text -Daily { at, timeZone } -Weekly { day, at, timeZone } -Yearly { month, day, at, timeZone } -Cron { minute, hour, dayOfMonth, month, dayOfWeek, timeZone } -``` - -The cron fields are typed expressions, not one opaque string. Each field accepts -`Any`, a value, a range, a list, or a stepped expression, with unit-specific -validated values. Five-field cron order is minute, hour, day of month, month, -and day of week. It has no seconds or year field. Friendly daily, weekly, and -yearly forms compile to the same canonical field representation used by the -scheduler. - -An explicit time zone makes daylight-saving behavior part of the resource. For -example, Friday 23:00 in `America/New_York` occurs at Saturday 03:00 UTC during -EDT and Saturday 04:00 UTC during EST. A fixed Saturday 03:00 UTC schedule is a -different policy. One-time runs use an absolute timestamp and need no time zone. - -Schedule status contains `lastScheduleTime`, `activeRun`, -`lastSuccessfulRun`, and conditions. The operator chooses the latest eligible -missed tick and creates at most one catch-up run. Ticks while a run is active -are skipped permanently. Suspension prevents new runs but does not stop an -active run. - -The run name is derived from the schedule UID and scheduled instant. The -scheduler claims `status.activeRun` with a resource-version patch before -creating that deterministic run. Recovery creates a claimed but missing run. -This prevents overlapping operator pods from creating different runs around a -calendar boundary. Schedule deletion waits for an active run to finish. - -### `TaskRun` - -A scheduled run contains an immutable copy of its scheduled instant and run -template. A directly created run has the same fields without a schedule owner: +## `TaskRun` ```yaml apiVersion: fleet.nationtech.io/v1alpha1 kind: TaskRun metadata: - name: inventory-check-2026-07-24 + name: upgrade-device-1 spec: - scheduledFor: "2026-07-24T03:00:00Z" allowedGroups: [production] targetSelector: matchLabels: - fleet.nationtech.io/site: montreal - rollout: - immediate: {} - deadlineSeconds: 1800 - workload: - containerV1: - image: registry.example.test/inventory-check@sha256:... - args: ["--report"] + device-id: device-1 + deadlineSeconds: 21600 + systemUpgradeV1: {} ``` -`scheduledFor` is an absolute instant. If omitted, the run is eligible to start -immediately. Admission rejects spec mutation after creation. A run has one +`systemUpgradeV1` is empty because apt, dpkg, repository, and reboot policy is +compiled into the updater. The API cannot pass package names, repository paths, +commands, or reboot arguments. + +Admission rejects changes to the spec. The run starts immediately and has one phase: ```text Planning | Running | Complete | Failed ``` -Status also carries aggregate counts, start and completion times, the latest -bounded error, and a reason. Operator-derived device outcomes such as -`TimedOut`, `Expired`, and `SkippedRevoked` remain distinct from workload -status reported by a device. +During `Planning`, the operator requires exactly one device that supports +`AptFullUpgradeV1`. No match fails with `NoTargets`; more than one match fails +with `TooManyTargets`. -Per-device maps do not live in the CR. Detailed NATS status is retained for the -active and most recent terminal run. The operator keeps the three most recent -successful and three failed runs per schedule. +`status.selectedDeviceId` freezes the selected device. Status also records the +target, success, and failure counts; start and completion times; a reason; and +the latest bounded error. The CR never contains an unbounded per-device map. -## Workloads +## Placement -The workload is a closed, versioned union: - -```text -ContainerV1 -Builtin::AgentUpgradeV1 -Builtin::SystemUpgradeV1 -``` - -`ContainerV1` is a one-shot container with a digest-pinned image, argument -array, and bounded resources. It does not accept a shell string. Completion is -the container exit result, not continued service health. Long-running services -remain Deployments even when they share image, argument, environment, or -resource component types with container tasks. - -Built-ins exist when Harmony owns more behavior than a container exit code can -express. Agent upgrade retains candidate activation, health verification, and -rollback. System upgrade retains signed root authorization, package recovery, -reboot verification, and repair-required state. These operations do not become -Deployment workload variants because they have terminal, edge-triggered -semantics. - -## Placement and sealed plans - -Placement reuses the deployment aggregator's `device_eligible` rule: +Task placement uses the deployment aggregator's rule: ```text allowed group membership AND targetSelector ``` -`matchLabels` remains a conjunction, an empty selector matches every device, -and `matchExpressions` fails closed until implemented. Groups remain the -authorization boundary. `allowedGroups` is required; an empty list authorizes -no target. +`allowedGroups` requires at least one group. `matchLabels` is a conjunction, and +an empty selector matches every authorized device. +`matchExpressions` fails closed until implemented. -Canary designation uses an operator-owned label namespace. The Device -reconciler rejects that namespace from agent-reported labels and preserves -administrator-set values. A device therefore cannot elect itself as a canary. +The operator checks group authorization again before creating the intent. A +device revoked before release fails the run with `TargetRevoked`; another device +is never substituted into the frozen plan. A group-source error retries +planning instead of becoming `NoTargets`. -Planning writes one assignment per selected device under a planning generation. -The operator then CAS-creates a seal containing the target count, canary count, -and digest of sorted assignments. No intent is written until the seal validates. -An interrupted, unsealed plan is resumed or discarded, never executed. +The `Planning` to `Running` status transition uses the Kubernetes resource +version as its claim. Two operator instances cannot freeze different targets +for the same run. The run UID is also the attempt ID, so intent creation and +updater execution are idempotent across restarts. -The assignment freezes membership, but group authorization is checked again -before each cohort is released. A revoked device becomes `SkippedRevoked`; no -newly eligible replacement enters the run. A revoked canary fails the run. A -revoked non-canary contributes a failed outcome after its cohort starts. +## Execution -A run with no targets completes. A canary rollout with targets but no canaries -fails with `NoCanaries`; it never falls back to immediate release. - -Suggested JetStream KV keys are: +The operator writes the fixed system-upgrade attempt to: ```text -task-assignments: . -task-plan: -task-intent: . -task-status: . +system-upgrade-intent: . ``` -The buckets are file-backed, use the deployment NATS replica policy, and have -explicit byte and history limits. Assignment and intent keys are removed after -terminal aggregation. Detailed status has short bounded retention. +The agent publishes updater-owned status to: -## Rollout +```text +system-upgrade-status: . +``` -`Immediate` releases every assignment after planning. `CanaryThenAll` advances -as follows: +The agent watches current and new intents, so reboot does not lose the active +run. The operator accepts terminal status only when its attempt and run IDs +match. Success also requires a fresh heartbeat published after the terminal +status. A late result cannot reopen a failed run. -1. Seal the complete assignment plan. -2. Revalidate canary group authorization and write all canary intents. -3. Wait for every canary to succeed. -4. On canary failure or deadline, fail the run and write no remaining intent. -5. Revalidate group authorization and release all remaining devices in one - reconciliation pass. -6. Complete when every released device is terminal. A failure or run deadline - fails the run but does not stop work already released. +Intent and status buckets are file-backed, keep one value per key, and have byte +and age limits. The operator removes a terminal intent; terminal status remains +for bounded diagnostics. -Releasing the second cohort together may disrupt all selected service replicas -and load the registry, NATS, and network concurrently. The selector must cover -devices whose simultaneous disruption is acceptable. Bounded concurrency is a -separate future rollout strategy. +The deadline is copied into the attempt as `expiresAt`. Expiry prevents a +delayed intent from starting. It does not interrupt apt or dpkg after the +updater has accepted the attempt. -Run phase changes use Kubernetes resource versions, intent creation is -idempotent, and duplicate controllers may reconcile the same run without -creating a second attempt. No global leader election is required. A terminal -run never advances. Late status remains diagnostic and cannot reopen a run or -release a blocked cohort. +## Multi-device runs -## Dashboard +A later multi-device run freezes every authorized matching device, then releases +all intents together. There is no concurrency setting or canary phase. A +failure does not stop work already released, and completed devices are not +rolled back. -Tasks have Schedules and Runs views. Schedules show their readable calendar, -time zone, suspension state, selectors, workload, next run, and last result. -Fleet admins can create and suspend schedules. Runs show phase, cohort progress, -start, duration, result, and recent per-device status from NATS. +The frozen target set must remain bounded outside the CR, while the CR retains +aggregate counts and the latest error. -Editing is deferred because schedule changes need generation-specific -missed-tick semantics. Deletion is available only when no run is active. All -mutations use the existing `fleet-admin` session and CSRF middleware. +## Recurring schedules -Failed runs become durable conditions on the run CR. The dashboard derives -critical alerts with IDs containing the run UID and links them to run detail. -This reuses the current alert list, badge, acknowledgement, and dashboard strip. -Acknowledgement remains process-local. Email, webhook delivery, notification -persistence, and sinks remain separate work. +A later namespaced `TaskSchedule` creates immutable `TaskRun` resources. Daily, +weekly, yearly, and five-field cron forms use UTC. Time-zone configuration is +not supported. + +One schedule has at most one active run. Ticks during an active run are skipped, +and restart creates at most one catch-up run for the latest eligible tick. +Suspension prevents new runs without stopping an active run. + +## Other workloads + +Container and agent-upgrade tasks remain deferred. Add a workload discriminator +only when a second workload is implemented. Agent upgrades continue to use +their existing activation, health-check, and rollback state machine. ## Invariants -- A schedule has at most one active run. -- A deterministic schedule tick creates at most one run. -- A run UID identifies one immutable target plan and workload. -- Group authorization is current when each intent is released. -- A device executes a run assignment at most once. -- Non-canary intents never exist before all canaries succeed. -- A terminal failed run never advances. +- A run UID identifies one immutable target plan and operation. +- Group authorization is current when an intent is released. +- A device executes one run attempt at most once. +- A terminal run never advances. - Restart does not repeat a completed device attempt. - No Kubernetes status map grows with fleet size. -## Implementation order +## Remaining work -1. Add typed schedule, workload, rollout, `TaskSchedule`, and `TaskRun` - contracts, CRDs, status subresources, RBAC, and admission rules. -2. Implement calendar evaluation with `croner`, deterministic catch-up, - active-run claims, suspension, and bounded history. -3. Reuse deployment placement, protect operator-owned labels, and implement - sealed plans, rollout, deadlines, CAS transitions, and finalizer cleanup. -4. Add the device-scoped watch and snapshot bridge, container execution, status - publication, and bounded NATS retention. -5. Connect the existing agent-upgrade flow and the system-upgrade executor as - built-in workloads without duplicating their state machines. -6. Add schedule and run dashboard views, alerts, and end-to-end restart and - authorization tests. - -## Open decisions - -1. Should the initial weekly system-upgrade policy be Friday 23:00 - `America/New_York` or Saturday 03:00 UTC? -2. What maximum target count and shared infrastructure capacity are supported - by an immediate or all-at-once cohort? +1. Prove package upgrade, reboot, restart recovery, expiry, and duplicate intent + handling on a disposable Debian VM and Raspberry Pi OS device. +2. Add all-at-once multi-device target storage and aggregation. +3. Add UTC schedules. +4. Add dashboard run history and alerts. diff --git a/examples/fleet_load_test/src/main.rs b/examples/fleet_load_test/src/main.rs index 01576104..04ad28cf 100644 --- a/examples/fleet_load_test/src/main.rs +++ b/examples/fleet_load_test/src/main.rs @@ -470,6 +470,7 @@ async fn publish_one_info(bucket: kv::Store, device: DevicePlan) -> Result<()> { device_id: Id::from(device.device_id.clone()), labels: BTreeMap::from([("group".to_string(), device.cr_name.clone())]), inventory: None, + updater: None, updated_at: Utc::now(), }; let key = device_info_key(&device.device_id); diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index 0cdeb387..3b3bfeec 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -6,8 +6,8 @@ use async_nats::jetstream::{self, kv}; use harmony_reconciler_contracts::{ BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, DeploymentName, - DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, device_heartbeat_key, - device_info_key, device_state_key, + DeploymentState, DeviceInfo, HeartbeatPayload, Id, InventorySnapshot, UpdaterCapabilities, + device_heartbeat_key, device_info_key, device_state_key, }; use std::collections::BTreeMap; @@ -30,36 +30,13 @@ pub trait DeploymentStatePublisher: Send + Sync { } impl FleetPublisher { - /// Open every bucket the agent needs, creating those that don't - /// exist yet. Idempotent with operator-side creation. + /// Open the operator-owned buckets used by the agent. pub async fn connect(client: async_nats::Client, device_id: Id) -> anyhow::Result { let jetstream = jetstream::new(client.clone()); - let info_bucket = jetstream - .create_key_value(kv::Config { - bucket: BUCKET_DEVICE_INFO.to_string(), - // If this is as I think, it would be useful to keep a history of the last 10 device - // info, with a timestamp - history: 1, - ..Default::default() - }) - .await?; - let state_bucket = jetstream - .create_key_value(kv::Config { - bucket: BUCKET_DEVICE_STATE.to_string(), - // If this is as I think, it would be useful to keep a history of the last 10 states - // a device had, with a timestamp - history: 1, - ..Default::default() - }) - .await?; - let heartbeat_bucket = jetstream - .create_key_value(kv::Config { - bucket: BUCKET_DEVICE_HEARTBEAT.to_string(), - history: 1, - ..Default::default() - }) - .await?; + let info_bucket = jetstream.get_key_value(BUCKET_DEVICE_INFO).await?; + let state_bucket = jetstream.get_key_value(BUCKET_DEVICE_STATE).await?; + let heartbeat_bucket = jetstream.get_key_value(BUCKET_DEVICE_HEARTBEAT).await?; Ok(Self { device_id, @@ -76,11 +53,13 @@ impl FleetPublisher { &self, labels: BTreeMap, inventory: Option, + updater: Option, ) -> anyhow::Result<()> { let info = DeviceInfo { device_id: self.device_id.clone(), labels, inventory, + updater, updated_at: chrono::Utc::now(), }; let key = device_info_key(&self.device_id.to_string()); diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index a400f5da..da36da10 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -3,6 +3,8 @@ mod config; mod fleet_publisher; mod podman; mod reconciler; +mod system_upgrade; +mod system_upgrade_service; mod updater; mod upgrade; @@ -21,9 +23,9 @@ type Creds = Arc; use futures_util::StreamExt; use harmony_reconciler_contracts::{ BUCKET_AGENT_UPGRADE_INTENT, BUCKET_AGENT_UPGRADE_STATUS, BUCKET_DESIRED_STATE, - BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, Id, InventorySnapshot, - agent_upgrade_intent_key, agent_upgrade_status_key, desired_state_watch_filter, - device_heartbeat_key, device_info_key, + BUCKET_DEVICE_HEARTBEAT, BUCKET_DEVICE_INFO, BUCKET_DEVICE_STATE, BUCKET_SYSTEM_UPGRADE_INTENT, + BUCKET_SYSTEM_UPGRADE_STATUS, Id, InventorySnapshot, agent_upgrade_intent_key, + agent_upgrade_status_key, desired_state_watch_filter, device_heartbeat_key, device_info_key, }; use crate::command_server::CommandServer; @@ -160,12 +162,19 @@ async fn probe_services( .await? .status() .await?; - tokio::time::timeout( - Duration::from_secs(15), - updater::UpdaterClient::new(updater_socket).status(), - ) - .await - .context("updater status probe timed out")??; + for bucket in [BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS] { + jetstream.get_key_value(bucket).await?.status().await?; + } + let updater = updater::UpdaterClient::new(updater_socket); + tokio::time::timeout(Duration::from_secs(15), updater.status()) + .await + .context("updater status probe timed out")??; + let capabilities = tokio::time::timeout(Duration::from_secs(15), updater.capabilities()) + .await + .context("updater capabilities probe timed out")??; + if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 { + anyhow::bail!("updater does not support AptFullUpgradeV1"); + } load_desired_snapshot(&desired, device_id).await } @@ -179,7 +188,7 @@ async fn load_desired_snapshot( bucket.prefix, desired_state_watch_filter(&device_id.to_string()) ); - let mut consumer = bucket + let consumer = bucket .stream .create_consumer(async_nats::jetstream::consumer::pull::OrderedConfig { filter_subject: filter, @@ -187,7 +196,7 @@ async fn load_desired_snapshot( ..Default::default() }) .await?; - let pending = consumer.info().await?.num_pending; + let pending = consumer.cached_info().num_pending; if pending == 0 { return Ok(Vec::new()); } @@ -466,8 +475,27 @@ async fn main() -> Result<()> { startup_labels .entry("device-id".to_string()) .or_insert_with(|| device_id.to_string()); + let updater_capabilities = if cli.updater_socket.exists() { + match updater::UpdaterClient::new(&cli.updater_socket) + .capabilities() + .await + { + Ok(capabilities) => Some(capabilities), + Err(error) if !cfg.agent.runtime_enabled => { + tracing::warn!(%error, "updater unavailable; capabilities omitted"); + None + } + Err(error) => return Err(error.context("reading required updater capabilities")), + } + } else { + None + }; fleet - .publish_device_info(startup_labels, Some(inventory_snapshot.clone())) + .publish_device_info( + startup_labels, + Some(inventory_snapshot.clone()), + updater_capabilities, + ) .await .context("publishing device registration")?; @@ -511,6 +539,29 @@ async fn main() -> Result<()> { ); None }; + let system_upgrade_service = if cli.updater_socket.exists() { + match system_upgrade_service::SystemUpgradeService::connect( + client.clone(), + device_id.clone(), + &cli.updater_socket, + ) + .await + { + Ok(service) => Some(service), + Err(error) if !cfg.agent.runtime_enabled => { + tracing::warn!(%error, "updater unavailable; system upgrades disabled"); + None + } + Err(error) => return Err(error.context("connecting required system upgrade relay")), + } + } else if cfg.agent.runtime_enabled { + anyhow::bail!( + "required fleet updater socket '{}' is unavailable", + cli.updater_socket.display() + ); + } else { + None + }; let desired_bucket = desired_state_store(client.clone()).await?; let snapshot = load_desired_snapshot(&desired_bucket, &device_id).await?; @@ -581,6 +632,11 @@ async fn main() -> Result<()> { Ok(()) }), }; + let system_upgrades: std::pin::Pin> + Send>> = + match system_upgrade_service { + Some(service) => Box::pin(service.run()), + None => Box::pin(std::future::pending()), + }; enum Shutdown { Interrupt, @@ -593,6 +649,7 @@ async fn main() -> Result<()> { _ = snapshots => anyhow::bail!("desired-state snapshot loop exited unexpectedly"), r = worker_finished => { r?; anyhow::bail!("reconciler worker exited unexpectedly") } r = upgrades => { r?; anyhow::bail!("agent upgrade loop exited unexpectedly") } + r = system_upgrades => { r?; anyhow::bail!("system upgrade relay exited unexpectedly") } _ = heartbeat => anyhow::bail!("heartbeat loop exited unexpectedly"), r = commands => { r?; anyhow::bail!("command server exited unexpectedly") } }; diff --git a/fleet/harmony-fleet-agent/src/system_upgrade.rs b/fleet/harmony-fleet-agent/src/system_upgrade.rs new file mode 100644 index 00000000..30307ac7 --- /dev/null +++ b/fleet/harmony-fleet-agent/src/system_upgrade.rs @@ -0,0 +1,640 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use chrono::{DateTime, Utc}; +use harmony_reconciler_contracts::{SystemUpgradeAttempt, SystemUpgradePhase, SystemUpgradeStatus}; +use serde::{Deserialize, Serialize}; + +use crate::updater::{bounded_error, safe_token, sync_directory}; + +const JOURNAL_DIR: &str = "/var/lib/harmony-fleet-updater/system-upgrades"; +const BOOT_ID: &str = "/proc/sys/kernel/random/boot_id"; +const MAX_REBOOT_REQUESTS: u8 = 3; +const TERMINAL_REPLAY_DAYS: i64 = 2; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CommandSpec { + pub program: &'static str, + pub args: &'static [&'static str], + pub env: &'static [(&'static str, &'static str)], +} + +const AUDIT: CommandSpec = CommandSpec { + program: "/usr/bin/dpkg", + args: &["--audit"], + env: &[], +}; +const CHECK: CommandSpec = CommandSpec { + program: "/usr/bin/apt-get", + args: &["-o", "DPkg::Lock::Timeout=300", "check"], + env: &[], +}; +const UPDATE: CommandSpec = CommandSpec { + program: "/usr/bin/apt-get", + args: &[ + "-o", + "DPkg::Lock::Timeout=300", + "-o", + "APT::Update::Error-Mode=any", + "-o", + "Acquire::AllowInsecureRepositories=false", + "-o", + "Acquire::AllowDowngradeToInsecureRepositories=false", + "-o", + "Acquire::AllowWeakRepositories=false", + "update", + ], + env: &[], +}; +const FULL_UPGRADE: CommandSpec = CommandSpec { + program: "/usr/bin/apt-get", + args: &[ + "-o", + "DPkg::Lock::Timeout=300", + "-o", + "Dpkg::Use-Pty=0", + "-o", + "Dpkg::Options::=--force-confold", + "-o", + "APT::Get::AllowUnauthenticated=false", + "-y", + "full-upgrade", + ], + env: &[ + ("DEBIAN_FRONTEND", "noninteractive"), + ("APT_LISTCHANGES_FRONTEND", "none"), + ("NEEDRESTART_MODE", "a"), + ], +}; +const REBOOT: CommandSpec = CommandSpec { + program: "/bin/systemctl", + args: &["reboot", "--no-wall"], + env: &[], +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct Journal { + attempt_id: String, + run_uid: String, + device_id: harmony_reconciler_contracts::Id, + attempt_digest: String, + phase: SystemUpgradePhase, + started_at: DateTime, + updated_at: DateTime, + error: Option, + boot_id: Option, + reboot_requests: u8, +} + +impl Journal { + pub(super) fn status(&self) -> SystemUpgradeStatus { + SystemUpgradeStatus { + attempt_id: self.attempt_id.clone(), + run_uid: self.run_uid.clone(), + phase: self.phase, + started_at: self.started_at, + updated_at: self.updated_at, + error: self.error.clone(), + } + } + + fn transition(&mut self, phase: SystemUpgradePhase, error: Option) { + self.phase = phase; + self.error = error; + self.updated_at = Utc::now(); + } + + fn validate(&self, expected_attempt_id: &str) -> Result<()> { + if self.attempt_id != expected_attempt_id + || uuid::Uuid::parse_str(&self.attempt_id).is_err() + || !safe_token(&self.run_uid) + || !safe_token(&self.device_id.to_string()) + || self.attempt_digest.len() != 64 + || !self.attempt_digest.chars().all(|c| c.is_ascii_hexdigit()) + || self.reboot_requests > MAX_REBOOT_REQUESTS + || self.phase == SystemUpgradePhase::Blocked + || matches!( + self.phase, + SystemUpgradePhase::Rebooting | SystemUpgradePhase::Verifying + ) && self.boot_id.as_deref().is_none_or(str::is_empty) + { + bail!("invalid system upgrade journal"); + } + Ok(()) + } +} + +pub(super) enum Acceptance { + Existing(SystemUpgradeStatus), + New(Journal), +} + +pub(super) fn accept( + attempt: &SystemUpgradeAttempt, + existing: Option<&Journal>, + now: DateTime, +) -> Result { + validate(attempt)?; + if let Some(journal) = existing { + if journal.attempt_digest != attempt.digest() { + bail!("system upgrade attempt id was reused with different content"); + } + return Ok(Acceptance::Existing(journal.status())); + } + if attempt.expires_at <= now { + bail!("system upgrade attempt has expired"); + } + Ok(Acceptance::New(Journal { + attempt_id: attempt.attempt_id.clone(), + run_uid: attempt.run_uid.clone(), + device_id: attempt.device_id.clone(), + attempt_digest: attempt.digest(), + phase: SystemUpgradePhase::Preflight, + started_at: now, + updated_at: now, + error: None, + boot_id: None, + reboot_requests: 0, + })) +} + +fn validate(attempt: &SystemUpgradeAttempt) -> Result<()> { + uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid attempt id")?; + if !safe_token(&attempt.run_uid) { + bail!("invalid run uid"); + } + if !safe_token(&attempt.device_id.to_string()) { + bail!("invalid device id"); + } + Ok(()) +} + +pub(super) async fn read(attempt_id: &str) -> Result> { + uuid::Uuid::parse_str(attempt_id).context("invalid attempt id")?; + let path = journal_path(attempt_id); + match tokio::fs::read(&path).await { + Ok(bytes) => { + let journal: Journal = serde_json::from_slice(&bytes) + .with_context(|| format!("reading system upgrade journal {}", path.display()))?; + journal.validate(attempt_id)?; + Ok(Some(journal)) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } +} + +pub(super) async fn write(journal: &Journal) -> Result<()> { + let path = journal_path(&journal.attempt_id); + let parent = path + .parent() + .context("system upgrade journal has no parent")?; + tokio::fs::create_dir_all(parent).await?; + let temporary = path.with_extension("tmp"); + let mut file = tokio::fs::File::create(&temporary).await?; + use tokio::io::AsyncWriteExt; + file.write_all(&serde_json::to_vec(journal)?).await?; + file.sync_all().await?; + tokio::fs::rename(&temporary, &path).await?; + sync_directory(parent) +} + +pub(super) async fn recover_active() -> Result> { + let mut entries = match tokio::fs::read_dir(JOURNAL_DIR).await { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let mut active = None; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let journal: Journal = serde_json::from_slice(&tokio::fs::read(&path).await?) + .with_context(|| format!("corrupt system upgrade journal {}", path.display()))?; + let attempt_id = path + .file_stem() + .and_then(|value| value.to_str()) + .context("invalid system upgrade journal filename")?; + journal.validate(attempt_id)?; + if !retain_journal(&journal, Utc::now()) { + tokio::fs::remove_file(path).await?; + continue; + } + if !journal.phase.is_terminal() && active.replace(journal).is_some() { + bail!("multiple active system upgrade journals"); + } + } + Ok(active) +} + +fn retain_journal(journal: &Journal, now: DateTime) -> bool { + journal.phase == SystemUpgradePhase::RepairRequired + || !journal.phase.is_terminal() + || journal.updated_at + chrono::Duration::days(TERMINAL_REPLAY_DAYS) >= now +} + +pub(super) async fn run(mut journal: Journal) -> Result { + let result = match recovery_action(&journal, ¤t_boot_id()?) { + RecoveryAction::Apply => apply(&mut journal).await, + RecoveryAction::Repair => { + let audit = run_command(&AUDIT, true).await; + let message = match audit { + Ok(()) => "system upgrade was interrupted while applying packages".into(), + Err(error) => format!("system upgrade was interrupted; {error}"), + }; + journal.transition(SystemUpgradePhase::RepairRequired, Some(message)); + write(&journal).await + } + RecoveryAction::Reboot => request_reboot(&mut journal).await, + RecoveryAction::Verify => verify(&mut journal).await, + RecoveryAction::None => Ok(()), + }; + if let Err(error) = result { + if journal.phase == SystemUpgradePhase::Rebooting { + journal.error = Some(bounded_error(&error.to_string())); + journal.updated_at = Utc::now(); + write(&journal).await?; + } else if !journal.phase.is_terminal() { + let phase = if journal.phase == SystemUpgradePhase::Applying { + SystemUpgradePhase::RepairRequired + } else { + SystemUpgradePhase::Failed + }; + journal.transition(phase, Some(bounded_error(&error.to_string()))); + write(&journal).await?; + } + return Err(error); + } + Ok(journal.status()) +} + +async fn apply(journal: &mut Journal) -> Result<()> { + run_command(&AUDIT, true).await?; + run_command(&CHECK, false).await?; + validate_sources().await?; + run_command(&UPDATE, false).await?; + journal.transition(SystemUpgradePhase::Applying, None); + write(journal).await?; + let upgrade = run_command(&FULL_UPGRADE, false).await; + let health = match run_command(&AUDIT, true).await { + Ok(()) => run_command(&CHECK, false).await, + error => error, + }; + if let Err(error) = health { + journal.transition( + SystemUpgradePhase::RepairRequired, + Some(bounded_error(&error.to_string())), + ); + write(journal).await?; + return Ok(()); + } + if let Err(error) = upgrade { + journal.transition( + SystemUpgradePhase::Failed, + Some(bounded_error(&error.to_string())), + ); + write(journal).await?; + return Ok(()); + } + journal.boot_id = Some(current_boot_id()?); + journal.transition(SystemUpgradePhase::Rebooting, None); + request_reboot(journal).await +} + +async fn validate_sources() -> Result<()> { + let mut paths = vec![PathBuf::from("/etc/apt/sources.list")]; + match tokio::fs::read_dir("/etc/apt/sources.list.d").await { + Ok(mut entries) => { + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if matches!( + path.extension().and_then(|extension| extension.to_str()), + Some("list" | "sources") + ) { + paths.push(path); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + for path in paths { + let contents = match tokio::fs::read_to_string(&path).await { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.into()), + }; + if source_disables_authentication(&contents) { + bail!( + "apt source {} disables repository authentication", + path.display() + ); + } + } + Ok(()) +} + +fn source_disables_authentication(contents: &str) -> bool { + contents.lines().any(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return false; + } + let lower = line.to_ascii_lowercase(); + let options = lower + .strip_prefix("deb ") + .or_else(|| lower.strip_prefix("deb-src ")) + .and_then(|line| line.strip_prefix('[')) + .and_then(|line| line.split_once(']').map(|(options, _)| options)); + if let Some(options) = options { + let options = options.replace(" =", "=").replace("= ", "="); + return options.split_ascii_whitespace().any(|option| { + matches!( + option.split_once('='), + Some(( + "trusted" | "allow-insecure" | "allow-weak", + "yes" | "true" | "1" + )) + ) + }); + } + matches!( + lower + .split_once(':') + .map(|(key, value)| (key.trim(), value.trim())), + Some(( + "trusted" | "allow-insecure" | "allow-weak", + "yes" | "true" | "1" + )) + ) + }) +} + +async fn request_reboot(journal: &mut Journal) -> Result<()> { + while journal.reboot_requests < MAX_REBOOT_REQUESTS { + journal.reboot_requests += 1; + write(journal).await?; + match run_command(&REBOOT, false).await { + Ok(()) => tokio::time::sleep(std::time::Duration::from_secs(30)).await, + Err(error) => { + journal.error = Some(bounded_error(&error.to_string())); + journal.updated_at = Utc::now(); + write(journal).await?; + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + } + } + journal.transition( + SystemUpgradePhase::Failed, + Some("reboot did not change the boot id".into()), + ); + write(journal).await?; + Ok(()) +} + +async fn verify(journal: &mut Journal) -> Result<()> { + journal.transition(SystemUpgradePhase::Verifying, None); + write(journal).await?; + let result = run_command(&AUDIT, true).await; + let result = match result { + Ok(()) => run_command(&CHECK, false).await, + error => error, + }; + if let Err(error) = result { + journal.transition( + SystemUpgradePhase::RepairRequired, + Some(bounded_error(&error.to_string())), + ); + return write(journal).await; + } + journal.transition(SystemUpgradePhase::Complete, None); + write(journal).await +} + +async fn run_command(spec: &CommandSpec, require_empty_stdout: bool) -> Result<()> { + let mut command = tokio::process::Command::new(spec.program); + command + .env_remove("NOTIFY_SOCKET") + .envs(spec.env.iter().copied()) + .args(spec.args) + .kill_on_drop(true); + let output = command.output().await?; + if !output.status.success() { + bail!("{} failed: {}", spec.program, output.status); + } + if require_empty_stdout && !output.stdout.iter().all(u8::is_ascii_whitespace) { + bail!("dpkg audit reported incomplete package state"); + } + Ok(()) +} + +fn current_boot_id() -> Result { + Ok(std::fs::read_to_string(BOOT_ID)?.trim().to_string()) +} + +#[derive(Debug, PartialEq, Eq)] +enum RecoveryAction { + Apply, + Repair, + Reboot, + Verify, + None, +} + +fn recovery_action(journal: &Journal, boot_id: &str) -> RecoveryAction { + match journal.phase { + SystemUpgradePhase::Preflight => RecoveryAction::Apply, + SystemUpgradePhase::Applying => RecoveryAction::Repair, + SystemUpgradePhase::Rebooting if journal.boot_id.as_deref() == Some(boot_id) => { + RecoveryAction::Reboot + } + SystemUpgradePhase::Rebooting => RecoveryAction::Verify, + SystemUpgradePhase::Verifying => RecoveryAction::Verify, + _ => RecoveryAction::None, + } +} + +fn journal_path(attempt_id: &str) -> PathBuf { + Path::new(JOURNAL_DIR).join(format!("{attempt_id}.json")) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeDelta; + use harmony_reconciler_contracts::Id; + + fn attempt() -> SystemUpgradeAttempt { + SystemUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + run_uid: "run-1".into(), + device_id: Id::from("device-1"), + expires_at: Utc::now() + TimeDelta::minutes(5), + } + } + + fn journal(phase: SystemUpgradePhase) -> Journal { + let attempt = attempt(); + let Acceptance::New(mut journal) = accept(&attempt, None, Utc::now()).unwrap() else { + unreachable!() + }; + journal.phase = phase; + journal.boot_id = Some("old-boot".into()); + journal + } + + #[test] + fn validates_expiry_and_duplicate_digest() { + let now = Utc::now(); + let original = attempt(); + let Acceptance::New(journal) = accept(&original, None, now).unwrap() else { + unreachable!() + }; + assert!(matches!( + accept(&original, Some(&journal), now + TimeDelta::hours(1)).unwrap(), + Acceptance::Existing(_) + )); + let mut changed = original.clone(); + changed.run_uid = "changed".into(); + assert!(accept(&changed, Some(&journal), now).is_err()); + + let mut expired = attempt(); + expired.expires_at = now; + assert!(accept(&expired, None, now).is_err()); + expired.expires_at = now + TimeDelta::minutes(1); + expired.attempt_id = "invalid".into(); + assert!(accept(&expired, None, now).is_err()); + expired.attempt_id = uuid::Uuid::new_v4().to_string(); + expired.run_uid = "unsafe/run".into(); + assert!(accept(&expired, None, now).is_err()); + } + + #[test] + fn command_policy_is_exact() { + assert_eq!( + AUDIT, + CommandSpec { + program: "/usr/bin/dpkg", + args: &["--audit"], + env: &[] + } + ); + assert_eq!(CHECK.args, &["-o", "DPkg::Lock::Timeout=300", "check"]); + assert_eq!( + UPDATE.args, + &[ + "-o", + "DPkg::Lock::Timeout=300", + "-o", + "APT::Update::Error-Mode=any", + "-o", + "Acquire::AllowInsecureRepositories=false", + "-o", + "Acquire::AllowDowngradeToInsecureRepositories=false", + "-o", + "Acquire::AllowWeakRepositories=false", + "update" + ] + ); + assert_eq!( + FULL_UPGRADE.args, + &[ + "-o", + "DPkg::Lock::Timeout=300", + "-o", + "Dpkg::Use-Pty=0", + "-o", + "Dpkg::Options::=--force-confold", + "-o", + "APT::Get::AllowUnauthenticated=false", + "-y", + "full-upgrade" + ] + ); + assert_eq!( + FULL_UPGRADE.env, + &[ + ("DEBIAN_FRONTEND", "noninteractive"), + ("APT_LISTCHANGES_FRONTEND", "none"), + ("NEEDRESTART_MODE", "a") + ] + ); + assert_eq!( + REBOOT, + CommandSpec { + program: "/bin/systemctl", + args: &["reboot", "--no-wall"], + env: &[] + } + ); + } + + #[test] + fn apt_sources_cannot_disable_authentication() { + for source in [ + "deb [trusted=yes] https://example.invalid stable main", + "deb [allow-insecure = yes] https://example.invalid stable main", + "deb [allow-weak=yes] https://example.invalid stable main", + "Types: deb\nTrusted: yes\nURIs: https://example.invalid", + "Types: deb\nAllow-Insecure: true\nURIs: https://example.invalid", + "Types: deb\nAllow-Weak: 1\nURIs: https://example.invalid", + ] { + assert!( + source_disables_authentication(source), + "accepted insecure source: {source}" + ); + } + assert!(!source_disables_authentication( + "deb [signed-by=/etc/apt/keyrings/vendor.gpg] https://example.invalid stable main" + )); + assert!(!source_disables_authentication( + "deb https://trusted=yes.example.invalid stable main" + )); + assert!(!source_disables_authentication( + "# deb [trusted=yes] disabled" + )); + } + + #[test] + fn recovery_never_reapplies_an_interrupted_upgrade() { + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Preflight), "old-boot"), + RecoveryAction::Apply + ); + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Applying), "old-boot"), + RecoveryAction::Repair + ); + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Rebooting), "old-boot"), + RecoveryAction::Reboot + ); + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Rebooting), "new-boot"), + RecoveryAction::Verify + ); + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Verifying), "new-boot"), + RecoveryAction::Verify + ); + assert_eq!( + recovery_action(&journal(SystemUpgradePhase::Complete), "new-boot"), + RecoveryAction::None + ); + } + + #[test] + fn terminal_journals_expire_but_repair_state_remains() { + let now = Utc::now(); + let mut complete = journal(SystemUpgradePhase::Complete); + complete.updated_at = now - chrono::Duration::days(TERMINAL_REPLAY_DAYS + 1); + assert!(!retain_journal(&complete, now)); + + let mut repair = complete; + repair.phase = SystemUpgradePhase::RepairRequired; + assert!(retain_journal(&repair, now)); + } +} diff --git a/fleet/harmony-fleet-agent/src/system_upgrade_service.rs b/fleet/harmony-fleet-agent/src/system_upgrade_service.rs new file mode 100644 index 00000000..54a86dc4 --- /dev/null +++ b/fleet/harmony-fleet-agent/src/system_upgrade_service.rs @@ -0,0 +1,267 @@ +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use async_nats::jetstream::kv::{Operation, Store}; +use chrono::Utc; +use futures_util::StreamExt; +use harmony_reconciler_contracts::{ + BUCKET_SYSTEM_UPGRADE_INTENT, BUCKET_SYSTEM_UPGRADE_STATUS, Id, SystemUpgradeAttempt, + SystemUpgradePhase, SystemUpgradeStatus, system_upgrade_intent_key, + system_upgrade_intent_watch_filter, system_upgrade_status_key, +}; + +use crate::updater::{UpdaterClient, safe_token}; + +const RETRY_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Clone)] +pub struct SystemUpgradeService { + device_id: Id, + intent: Store, + status: Store, + updater: UpdaterClient, +} + +impl SystemUpgradeService { + pub async fn connect( + client: async_nats::Client, + device_id: Id, + updater_socket: &std::path::Path, + ) -> Result { + let jetstream = async_nats::jetstream::new(client); + let intent = jetstream + .get_key_value(BUCKET_SYSTEM_UPGRADE_INTENT) + .await?; + let status = jetstream + .get_key_value(BUCKET_SYSTEM_UPGRADE_STATUS) + .await?; + let updater = UpdaterClient::new(updater_socket); + let capabilities = updater.capabilities().await?; + if capabilities.protocol != 1 || !capabilities.apt_full_upgrade_v1 { + bail!("updater does not support AptFullUpgradeV1"); + } + Ok(Self { + device_id, + intent, + status, + updater, + }) + } + + pub async fn run(self) -> Result<()> { + let filter = system_upgrade_intent_watch_filter(&self.device_id.to_string()); + loop { + let mut intents = match self.intent.watch_with_history(&filter).await { + Ok(intents) => intents, + Err(error) => { + tracing::warn!(%error, "system upgrade intent watch start failed"); + tokio::time::sleep(RETRY_INTERVAL).await; + continue; + } + }; + while let Some(entry) = intents.next().await { + match entry { + Ok(entry) if entry.operation == Operation::Put => { + let result = serde_json::from_slice(&entry.value) + .context("decoding system upgrade attempt") + .and_then(|attempt| { + validate_intent(&self.device_id, &entry.key, &attempt) + .map(|()| attempt) + }); + match result { + Ok(attempt) => self.relay(attempt).await, + Err(error) => { + tracing::warn!(key = %entry.key, %error, "system upgrade attempt rejected") + } + } + } + Ok(_) => {} + Err(error) => { + tracing::warn!(%error, "system upgrade intent watch failed; restarting"); + break; + } + } + } + tokio::time::sleep(RETRY_INTERVAL).await; + } + } + + async fn relay(&self, attempt: SystemUpgradeAttempt) { + let mut blocked_since = None; + let recovered = match self + .updater + .system_upgrade_status(&attempt.attempt_id) + .await + { + Ok(Some(status)) + if status.attempt_id == attempt.attempt_id && status.run_uid == attempt.run_uid => + { + Some(status) + } + Ok(Some(_)) => { + tracing::warn!(attempt_id = %attempt.attempt_id, "updater status does not match system upgrade intent"); + return; + } + Ok(None) | Err(_) => None, + }; + let accepted = if let Some(status) = recovered { + status + } else { + loop { + if blocked_since.is_some() && attempt.expires_at <= Utc::now() { + return; + } + match self.updater.start_system_upgrade(&attempt).await { + Ok(status) => break status, + Err(error) + if error + .to_string() + .contains("another upgrade is already in progress") => + { + let now = Utc::now(); + let started_at = *blocked_since.get_or_insert(now); + let status = SystemUpgradeStatus { + attempt_id: attempt.attempt_id.clone(), + run_uid: attempt.run_uid.clone(), + phase: SystemUpgradePhase::Blocked, + started_at, + updated_at: now, + error: Some(error.to_string()), + }; + if let Err(error) = self.publish(&attempt, status).await { + tracing::warn!(%error, "publishing blocked system upgrade status failed"); + } + if attempt.expires_at <= Utc::now() { + return; + } + tokio::time::sleep(RETRY_INTERVAL).await; + } + Err(error) if temporary_updater_error(&error) => { + tracing::warn!(%error, "updater unavailable during system upgrade submit; retrying"); + if attempt.expires_at <= Utc::now() { + return; + } + tokio::time::sleep(RETRY_INTERVAL).await; + } + Err(error) => { + tracing::warn!(%error, "system upgrade submit failed"); + return; + } + } + } + }; + let accepted_published = match self.publish(&attempt, accepted.clone()).await { + Ok(()) => true, + Err(error) => { + tracing::warn!(%error, "publishing accepted system upgrade status failed"); + false + } + }; + if accepted.phase.is_terminal() && accepted_published { + return; + } + let mut last = accepted_published.then_some(accepted); + loop { + match self + .updater + .system_upgrade_status(&attempt.attempt_id) + .await + { + Ok(Some(status)) if last.as_ref() != Some(&status) => { + let terminal = status.phase.is_terminal(); + let published = match self.publish(&attempt, status.clone()).await { + Ok(()) => { + last = Some(status); + true + } + Err(error) => { + tracing::warn!(%error, "publishing system upgrade status failed"); + false + } + }; + if terminal && published { + return; + } + } + Ok(Some(status)) if status.phase.is_terminal() => return, + Ok(Some(_)) => {} + Ok(_) => {} + Err(error) => { + tracing::warn!(%error, "updater system upgrade status unavailable; retrying") + } + } + tokio::time::sleep(RETRY_INTERVAL).await; + } + } + + async fn publish( + &self, + attempt: &SystemUpgradeAttempt, + status: SystemUpgradeStatus, + ) -> Result<()> { + if status.attempt_id != attempt.attempt_id || status.run_uid != attempt.run_uid { + bail!("updater system upgrade status does not match current attempt"); + } + let key = system_upgrade_status_key(&self.device_id.to_string(), &status.run_uid); + self.status + .put(&key, serde_json::to_vec(&status)?.into()) + .await?; + Ok(()) + } +} + +fn temporary_updater_error(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() + || error.to_string().starts_with("updater response timed out") +} + +fn validate_intent(device_id: &Id, key: &str, attempt: &SystemUpgradeAttempt) -> Result<()> { + uuid::Uuid::parse_str(&attempt.attempt_id).context("invalid system upgrade attempt id")?; + if &attempt.device_id != device_id { + bail!("system upgrade attempt targets another device"); + } + if !safe_token(&attempt.run_uid) { + bail!("invalid system upgrade run uid"); + } + if key != system_upgrade_intent_key(&device_id.to_string(), &attempt.run_uid) { + bail!("system upgrade intent key does not match its device and run"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeDelta; + + fn attempt(device_id: &str) -> SystemUpgradeAttempt { + SystemUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + run_uid: "run-1".into(), + device_id: Id::from(device_id), + expires_at: Utc::now() + TimeDelta::minutes(5), + } + } + + #[test] + fn rejects_wrong_device_and_bad_keys() { + let device_id = Id::from("device-1"); + let wrong = attempt("device-2"); + assert!(validate_intent(&device_id, "device-1.run-1", &wrong).is_err()); + + let invalid = attempt("device-1"); + assert!(validate_intent(&device_id, "device-1.wrong", &invalid).is_err()); + } + + #[test] + fn intent_and_status_keys_map_to_the_attempt_run() { + let device_id = Id::from("device-1"); + let attempt = attempt("device-1"); + assert!(validate_intent(&device_id, "device-1.run-1", &attempt).is_ok()); + assert!(validate_intent(&device_id, "device-1.other", &attempt).is_err()); + assert_eq!( + system_upgrade_status_key(&device_id.to_string(), &attempt.run_uid), + "device-1.run-1" + ); + } +} diff --git a/fleet/harmony-fleet-agent/src/updater.rs b/fleet/harmony-fleet-agent/src/updater.rs index 3725981c..7f0952db 100644 --- a/fleet/harmony-fleet-agent/src/updater.rs +++ b/fleet/harmony-fleet-agent/src/updater.rs @@ -11,6 +11,9 @@ use futures_util::StreamExt; use harmony_reconciler_contracts::upgrade::{ AGENT_OCI_ARTIFACT_TYPE, AGENT_OCI_LAYER_MEDIA_TYPE, AgentUpgradeAttempt, }; +use harmony_reconciler_contracts::{ + SystemUpgradeAttempt, SystemUpgradeStatus, UpdaterCapabilities, +}; use oci_client::client::{ClientConfig, ClientProtocol}; use oci_client::secrets::RegistryAuth; use oci_client::{Client as OciClient, Reference}; @@ -19,6 +22,8 @@ use sha2::{Digest, Sha256}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader}; use tokio::net::{UnixListener, UnixStream}; +use crate::system_upgrade; + pub const DEFAULT_SOCKET: &str = "/run/harmony-fleet-updater/updater.sock"; const ROOT: &str = "/usr/lib/harmony-fleet"; const BOOTSTRAP_BINARY: &str = "/usr/lib/harmony-fleet/fleet-agent-bootstrap"; @@ -39,6 +44,12 @@ const UPGRADE_RESPONSE_TIMEOUT: Duration = Duration::from_secs(20 * 60); enum Request { Upgrade(AgentUpgradeAttempt), Status, + Capabilities, + AptFullUpgradeV1(SystemUpgradeAttempt), + SystemUpgradeStatus { + #[serde(rename = "attemptId")] + attempt_id: String, + }, } #[derive(Debug, Serialize, Deserialize)] @@ -46,6 +57,10 @@ struct Response { ok: bool, error: Option, transaction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + capabilities: Option, + #[serde(rename = "systemUpgrade", skip_serializing_if = "Option::is_none")] + system_upgrade: Option, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] @@ -155,7 +170,46 @@ impl UpdaterClient { self.request(Request::Status, STATUS_RESPONSE_TIMEOUT).await } + pub async fn capabilities(&self) -> Result { + self.request_response(Request::Capabilities, STATUS_RESPONSE_TIMEOUT) + .await? + .capabilities + .context("updater omitted capabilities") + } + + pub async fn start_system_upgrade( + &self, + attempt: &SystemUpgradeAttempt, + ) -> Result { + self.request_response( + Request::AptFullUpgradeV1(attempt.clone()), + STATUS_RESPONSE_TIMEOUT, + ) + .await? + .system_upgrade + .context("updater omitted system upgrade status") + } + + pub async fn system_upgrade_status( + &self, + attempt_id: &str, + ) -> Result> { + Ok(self + .request_response( + Request::SystemUpgradeStatus { + attempt_id: attempt_id.to_string(), + }, + STATUS_RESPONSE_TIMEOUT, + ) + .await? + .system_upgrade) + } + async fn request(&self, request: Request, timeout: Duration) -> Result> { + Ok(self.request_response(request, timeout).await?.transaction) + } + + async fn request_response(&self, request: Request, timeout: Duration) -> Result { tokio::time::timeout(timeout, async { let mut stream = UnixStream::connect(&self.socket).await?; let mut payload = serde_json::to_vec(&request)?; @@ -173,7 +227,7 @@ impl UpdaterClient { if !response.ok { bail!(response.error.unwrap_or_else(|| "updater failed".into())); } - Ok(response.transaction) + Ok(response) }) .await .map_err(|_| anyhow!("updater response timed out after {timeout:?}"))? @@ -188,6 +242,17 @@ pub async fn run_server(socket: &Path) -> Result<()> { Err(error) if is_not_found(&error) => None, Err(error) => return Err(error.context("reading updater transaction during recovery")), }; + let system_recovery = system_upgrade::recover_active() + .await + .context("reading system upgrade journals during recovery")?; + if recovery.is_some() && system_recovery.is_some() { + bail!("agent and system upgrade journals are both active"); + } + if let Some(journal) = system_recovery { + system_upgrade::run(journal) + .await + .context("recovering active system upgrade before readiness")?; + } if socket.exists() { tokio::fs::remove_file(socket).await?; } @@ -266,27 +331,73 @@ async fn handle(stream: UnixStream, transaction_lock: Arc bail!("updater request exceeds {MAX_REQUEST_BYTES} bytes"); } let request: Request = serde_json::from_str(&line)?; - let result = match request { + let result: Result<( + Option, + Option, + Option, + )> = match request { Request::Upgrade(attempt) => match transaction_lock.try_lock() { - Ok(_guard) => upgrade(&attempt).await.map(Some), + Ok(_guard) => upgrade(&attempt) + .await + .map(|transaction| (Some(transaction), None, None)), Err(_) => Err(anyhow!("another upgrade is already in progress")), }, Request::Status => match read_transaction().await { - Ok(transaction) => Ok(Some(transaction)), - Err(error) if is_not_found(&error) => Ok(None), + Ok(transaction) => Ok((Some(transaction), None, None)), + Err(error) if is_not_found(&error) => Ok((None, None, None)), Err(error) => Err(error), }, + Request::Capabilities => Ok(( + None, + Some(UpdaterCapabilities { + protocol: 1, + apt_full_upgrade_v1: true, + }), + None, + )), + Request::SystemUpgradeStatus { attempt_id } => Ok(( + None, + None, + system_upgrade::read(&attempt_id) + .await? + .map(|journal| journal.status()), + )), + Request::AptFullUpgradeV1(attempt) => { + let existing = system_upgrade::read(&attempt.attempt_id).await?; + match system_upgrade::accept(&attempt, existing.as_ref(), Utc::now())? { + system_upgrade::Acceptance::Existing(status) => Ok((None, None, Some(status))), + system_upgrade::Acceptance::New(journal) => { + let guard = transaction_lock + .clone() + .try_lock_owned() + .map_err(|_| anyhow!("another upgrade is already in progress"))?; + system_upgrade::write(&journal).await?; + let status = journal.status(); + tokio::spawn(async move { + let _guard = guard; + if let Err(error) = system_upgrade::run(journal).await { + tracing::error!(%error, "system upgrade failed"); + } + }); + Ok((None, None, Some(status))) + } + } + } }; let response = match result { - Ok(transaction) => Response { + Ok((transaction, capabilities, system_upgrade)) => Response { ok: true, error: None, transaction, + capabilities, + system_upgrade, }, Err(error) => Response { ok: false, error: Some(error.to_string()), transaction: None, + capabilities: None, + system_upgrade: None, }, }; let mut payload = serde_json::to_vec(&response)?; @@ -670,7 +781,7 @@ async fn verify_bytes(bytes: &[u8], attempt: &AgentUpgradeAttempt) -> Result<()> Ok(()) } -fn safe_token(value: &str) -> bool { +pub(super) fn safe_token(value: &str) -> bool { !value.is_empty() && value .chars() @@ -821,7 +932,7 @@ async fn write_transaction(transaction: &Transaction) -> Result<()> { sync_directory(parent) } -fn sync_directory(path: &Path) -> Result<()> { +pub(super) fn sync_directory(path: &Path) -> Result<()> { std::fs::File::open(path)?.sync_all()?; Ok(()) } @@ -845,7 +956,7 @@ fn is_not_found(error: &anyhow::Error) -> bool { .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) } -fn bounded_error(error: &str) -> String { +pub(super) fn bounded_error(error: &str) -> String { error.chars().take(1024).collect() } @@ -946,6 +1057,48 @@ mod tests { .unwrap(), Request::Status )); + assert_eq!( + serde_json::to_value(Request::Capabilities).unwrap(), + serde_json::json!({ "operation": "capabilities" }) + ); + assert_eq!( + serde_json::to_value(Response { + ok: true, + error: None, + transaction: None, + capabilities: Some(UpdaterCapabilities { + protocol: 1, + apt_full_upgrade_v1: true, + }), + system_upgrade: None, + }) + .unwrap(), + serde_json::json!({ + "ok": true, + "error": null, + "transaction": null, + "capabilities": { "protocol": 1, "aptFullUpgradeV1": true } + }) + ); + let system_attempt = harmony_reconciler_contracts::SystemUpgradeAttempt { + attempt_id: uuid::Uuid::new_v4().to_string(), + run_uid: "run-1".into(), + device_id: Id::from("device-1"), + expires_at: Utc::now(), + }; + let encoded = serde_json::to_value(Request::AptFullUpgradeV1(system_attempt)).unwrap(); + assert_eq!(encoded["operation"], "apt-full-upgrade-v1"); + assert!(encoded["data"].get("attemptId").is_some()); + assert_eq!( + serde_json::to_value(Request::SystemUpgradeStatus { + attempt_id: "attempt-1".into() + }) + .unwrap(), + serde_json::json!({ + "operation": "system-upgrade-status", + "data": { "attemptId": "attempt-1" } + }) + ); for removed in ["stage", "switch", "cancel"] { assert!( serde_json::from_value::(serde_json::json!({ "operation": removed })) diff --git a/fleet/harmony-fleet-deploy/src/app.rs b/fleet/harmony-fleet-deploy/src/app.rs index f5cd483b..736baf4b 100644 --- a/fleet/harmony-fleet-deploy/src/app.rs +++ b/fleet/harmony-fleet-deploy/src/app.rs @@ -418,6 +418,18 @@ fn fleet_deployer_rules() -> Vec { verbs: verbs(), ..Default::default() }, + PolicyRule { + api_groups: Some(vec!["fleet.nationtech.io".to_string()]), + resources: Some(vec!["taskruns".to_string()]), + verbs: ["get", "list", "watch"].map(String::from).to_vec(), + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["fleet.nationtech.io".to_string()]), + resources: Some(vec!["taskruns/status".to_string()]), + verbs: ["get", "update", "patch"].map(String::from).to_vec(), + ..Default::default() + }, ] } @@ -551,6 +563,16 @@ mod tests { .any(|resource| resource == "customresourcedefinitions") }) })); + assert!(rules.iter().any(|rule| { + rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()]) + && rule.resources.as_deref() == Some(&["taskruns".to_string()]) + && rule.verbs == ["get", "list", "watch"].map(String::from) + })); + assert!(rules.iter().any(|rule| { + rule.api_groups.as_deref() == Some(&["fleet.nationtech.io".to_string()]) + && rule.resources.as_deref() == Some(&["taskruns/status".to_string()]) + && rule.verbs == ["get", "update", "patch"].map(String::from) + })); } #[tokio::test] diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index eeaaf136..c93c34a0 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -290,7 +290,7 @@ fn role() -> Role { // Device liveness: the device-status reconciler patches the // status subresource — a distinct RBAC resource from `devices`. PolicyRule { - api_groups: Some(vec![group]), + api_groups: Some(vec![group.clone()]), resources: Some(vec!["devices/status".to_string()]), verbs: vec!["get", "update", "patch"] .into_iter() @@ -298,6 +298,24 @@ fn role() -> Role { .collect(), ..Default::default() }, + PolicyRule { + api_groups: Some(vec![group.clone()]), + resources: Some(vec!["taskruns".to_string()]), + verbs: vec!["get", "list", "watch"] + .into_iter() + .map(String::from) + .collect(), + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec![group]), + resources: Some(vec!["taskruns/status".to_string()]), + verbs: vec!["get", "update", "patch"] + .into_iter() + .map(String::from) + .collect(), + ..Default::default() + }, ]), } } @@ -655,6 +673,27 @@ mod tests { }; assert!(grants_patch("deployments/status")); assert!(grants_patch("devices/status")); + assert!(grants_patch("taskruns/status")); + } + + #[test] + fn role_grants_only_required_taskrun_access() { + let rules = role().rules.unwrap(); + assert!(rules.iter().any(|rule| { + rule.resources.as_deref() == Some(&["taskruns".to_string()]) + && rule.verbs == ["get", "list", "watch"].map(String::from) + })); + assert!(rules.iter().any(|rule| { + rule.resources.as_deref() == Some(&["taskruns/status".to_string()]) + && rule.verbs == ["get", "update", "patch"].map(String::from) + })); + assert!(rules.iter().all(|rule| { + !rule.resources.as_ref().is_some_and(|resources| { + resources.iter().any(|resource| { + resource.contains("schedule") || resource == "taskruns/finalizers" + }) + }) + })); } #[test] diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index 10f2945e..c534bd7b 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -41,7 +41,7 @@ use harmony::modules::nats::NatsClientRef; use harmony::modules::zitadel::{OidcApplicationRef, OidcMachineIdentityRef, OidcProviderRef}; use harmony::score::Score; use harmony::topology::{HelmCommand, K8sclient, Topology}; -use harmony_fleet_operator::{Deployment, Device}; +use harmony_fleet_operator::{Deployment, Device, TaskRun}; use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret}; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::CustomResourceExt; @@ -56,7 +56,7 @@ use crate::operator::chart::{ pub struct FleetCrdsScore; fn fleet_crds() -> Vec { - vec![Deployment::crd(), Device::crd()] + vec![Deployment::crd(), Device::crd(), TaskRun::crd()] } impl Score for FleetCrdsScore { @@ -666,7 +666,8 @@ mod tests { #[test] fn fleet_crds_are_namespaced_resources() { let crds = fleet_crds(); - assert_eq!(crds.len(), 2); + assert_eq!(crds.len(), 3); + assert!(crds.iter().any(|crd| crd.spec.names.kind == "TaskRun")); for crd in crds { assert_eq!( crd.spec.scope, diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index cf04d759..560acd44 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -271,6 +271,7 @@ async fn create_device(devices: &Api, name: &str) -> anyhow::Result<()> name, DeviceSpec { inventory: None, + updater: None, agent_upgrade: None, }, ); diff --git a/fleet/harmony-fleet-operator/src/crd.rs b/fleet/harmony-fleet-operator/src/crd.rs index 6a1d395c..71f78434 100644 --- a/fleet/harmony-fleet-operator/src/crd.rs +++ b/fleet/harmony-fleet-operator/src/crd.rs @@ -1,4 +1,4 @@ -use harmony_reconciler_contracts::InventorySnapshot; +use harmony_reconciler_contracts::{InventorySnapshot, UpdaterCapabilities}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; use kube::CustomResource; use schemars::JsonSchema; @@ -105,6 +105,8 @@ pub struct DeviceSpec { #[serde(skip_serializing_if = "Option::is_none")] pub inventory: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub updater: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub agent_upgrade: Option, } diff --git a/fleet/harmony-fleet-operator/src/device_reconciler.rs b/fleet/harmony-fleet-operator/src/device_reconciler.rs index 37c0e8d4..5cd2b4f9 100644 --- a/fleet/harmony-fleet-operator/src/device_reconciler.rs +++ b/fleet/harmony-fleet-operator/src/device_reconciler.rs @@ -13,7 +13,7 @@ use anyhow::Result; use async_nats::jetstream::kv::{Operation, Store}; use futures_util::StreamExt; -use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo}; +use harmony_reconciler_contracts::{BUCKET_DEVICE_INFO, DeviceInfo, device_info_key}; use kube::Client; use kube::api::{Api, DeleteParams, Patch, PatchParams}; use std::collections::BTreeMap; @@ -62,6 +62,10 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()> continue; } }; + if !info_key_matches(&entry.key, &info) { + tracing::warn!(key = %entry.key, device = %info.device_id, "device-reconciler: key does not match payload device_id"); + continue; + } if let Err(e) = upsert_device(&devices, namespace, &info).await { tracing::warn!( device = %info.device_id, @@ -85,13 +89,7 @@ async fn run_loop(client: Client, namespace: &str, bucket: Store) -> Result<()> async fn upsert_device(api: &Api, namespace: &str, info: &DeviceInfo) -> Result<()> { let name = info.device_id.to_string(); - let mut device = Device::new( - &name, - DeviceSpec { - inventory: info.inventory.clone(), - agent_upgrade: None, - }, - ); + let mut device = device_from_info(info); device.metadata.namespace = Some(namespace.to_string()); device.metadata.labels = Some(clean_labels(&info.labels)); @@ -105,6 +103,21 @@ async fn upsert_device(api: &Api, namespace: &str, info: &DeviceInfo) -> Ok(()) } +fn device_from_info(info: &DeviceInfo) -> Device { + Device::new( + &info.device_id.to_string(), + DeviceSpec { + inventory: info.inventory.clone(), + updater: info.updater.clone(), + agent_upgrade: None, + }, + ) +} + +fn info_key_matches(key: &str, info: &DeviceInfo) -> bool { + key == device_info_key(&info.device_id.to_string()) +} + async fn delete_device(api: &Api, name: &str) -> Result<()> { match api.delete(name, &DeleteParams::default()).await { Ok(_) => { @@ -150,6 +163,9 @@ fn is_label_value(s: &str) -> bool { #[cfg(test)] mod tests { + use chrono::Utc; + use harmony_reconciler_contracts::{Id, UpdaterCapabilities}; + use super::*; #[test] @@ -168,4 +184,22 @@ mod tests { assert!(!is_label_value("has space")); assert!(!is_label_value(&"x".repeat(64))); } + + #[test] + fn device_info_identity_and_capability_are_reflected() { + let info = DeviceInfo { + device_id: Id::from("device-1"), + labels: BTreeMap::new(), + inventory: None, + updater: Some(UpdaterCapabilities { + protocol: 1, + apt_full_upgrade_v1: true, + }), + updated_at: Utc::now(), + }; + + assert!(info_key_matches("info.device-1", &info)); + assert!(!info_key_matches("info.device-2", &info)); + assert_eq!(device_from_info(&info).spec.updater, info.updater); + } } diff --git a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs index 5c689593..d63722c9 100644 --- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs +++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs @@ -155,16 +155,16 @@ pub fn group_allows(allowed: &[String], device_groups: Option<&HashSet>) device_groups.is_some_and(|groups| allowed.iter().any(|group| groups.contains(group))) } -/// Is `deployment` allowed (groups) and placed (labels) on this device? -fn device_eligible( - deployment: &CachedDeployment, +/// Is this device allowed by groups and placed by labels? +pub fn device_eligible( + allowed_groups: &[String], + selector: &LabelSelector, labels: &BTreeMap, device_groups: Option<&HashSet>, default_groups: Option<&HashSet>, ) -> bool { - (group_allows(&deployment.allowed_groups, device_groups) - || group_allows(&deployment.allowed_groups, default_groups)) - && selector_matches(&deployment.selector, labels) + (group_allows(allowed_groups, device_groups) || group_allows(allowed_groups, default_groups)) + && selector_matches(selector, labels) } /// Set of Device names the deployment is currently allowed and placed on. @@ -174,7 +174,8 @@ fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet .iter() .filter(|(name, labels)| { device_eligible( - deployment, + &deployment.allowed_groups, + &deployment.selector, labels, state.device_groups.get(*name), state.device_groups.get("*"), diff --git a/fleet/harmony-fleet-operator/src/lib.rs b/fleet/harmony-fleet-operator/src/lib.rs index 095cfdad..c360932e 100644 --- a/fleet/harmony-fleet-operator/src/lib.rs +++ b/fleet/harmony-fleet-operator/src/lib.rs @@ -16,9 +16,12 @@ pub mod crd; pub mod device_reconciler; pub mod device_status; pub mod fleet_aggregator; +pub mod task; +pub mod task_run_controller; pub use crd::{ AgentUpgradeTarget, AggregateLastError, Deployment, DeploymentAggregate, DeploymentSpec, DeploymentStatus, Device, DeviceSpec, DeviceStatus, DeviceUpgradeStatus, Reachability, Rollout, RolloutStrategy, }; +pub use task::{SystemUpgradeTaskV1, TaskRun, TaskRunPhase, TaskRunSpec, TaskRunStatus}; diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index dff7eae1..189352bd 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -6,7 +6,9 @@ mod frontend; mod service; use harmony_fleet_operator::access::StaticDeviceGroups; -use harmony_fleet_operator::{agent_upgrade, device_reconciler, device_status, fleet_aggregator}; +use harmony_fleet_operator::{ + agent_upgrade, device_reconciler, device_status, fleet_aggregator, task_run_controller, +}; use harmony_reconciler_contracts::{DeploymentSecretGrants, DeviceGroupSource}; use harmony_secret::OpenBaoDeploymentSecretGrants; use harmony_zitadel_auth::ZitadelDeviceGroups; @@ -360,11 +362,15 @@ async fn run( let ds_js = js.clone(); let upgrade_client = client.clone(); let upgrade_js = js.clone(); + let task_client = client.clone(); + let task_js = js.clone(); + let task_group_source = group_source.clone(); tokio::select! { r = controller::run(ctl_client, tenant_namespace, desired_state_kv) => r, r = device_reconciler::run(dr_client, tenant_namespace, dr_js) => r, r = device_status::run(ds_client, tenant_namespace, ds_js) => r, r = agent_upgrade::run(upgrade_client, tenant_namespace, upgrade_js) => r, + r = task_run_controller::run(task_client, tenant_namespace, task_js, task_group_source) => r, r = fleet_aggregator::run(client, tenant_namespace, js, secret_grants, group_source, fleet_state) => r, } } diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index 2ed54d54..266ec090 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -455,6 +455,7 @@ mod tests { "pi-01", DeviceSpec { inventory: None, + updater: None, agent_upgrade: None, }, ); diff --git a/fleet/harmony-fleet-operator/src/task.rs b/fleet/harmony-fleet-operator/src/task.rs new file mode 100644 index 00000000..33d39224 --- /dev/null +++ b/fleet/harmony-fleet-operator/src/task.rs @@ -0,0 +1,111 @@ +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, Time}; +use kube::{CustomResource, KubeSchema}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::AggregateLastError; + +#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, KubeSchema)] +#[kube( + group = "fleet.nationtech.io", + version = "v1alpha1", + kind = "TaskRun", + plural = "taskruns", + shortname = "fleettask", + namespaced, + status = "TaskRunStatus", + validation = Rule::new("self == oldSelf").message("TaskRun spec is immutable") +)] +#[serde(rename_all = "camelCase")] +pub struct TaskRunSpec { + #[schemars(length(min = 1))] + pub allowed_groups: Vec, + pub target_selector: LabelSelector, + #[schemars(range(min = 60, max = 86400))] + pub deadline_seconds: u32, + pub system_upgrade_v1: SystemUpgradeTaskV1, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq, JsonSchema)] +pub struct SystemUpgradeTaskV1 {} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, JsonSchema)] +pub enum TaskRunPhase { + Planning, + Running, + Complete, + Failed, +} + +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct TaskRunStatus { + pub phase: TaskRunPhase, + #[serde(skip_serializing_if = "Option::is_none")] + #[schemars(length(max = 253))] + pub selected_device_id: Option, + pub target_count: u32, + pub succeeded: u32, + pub failed: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub start_time: Option