diff --git a/Cargo.lock b/Cargo.lock index c96f3db5..964e836d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4277,6 +4277,23 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "harmony-tenant-backup" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "harmony", + "harmony_config", + "k8s-openapi", + "kube", + "schemars 0.8.22", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "harmony_agent" version = "0.1.0" diff --git a/examples/harmony_tenant_backup/Cargo.toml b/examples/harmony_tenant_backup/Cargo.toml new file mode 100644 index 00000000..eb7e9ba5 --- /dev/null +++ b/examples/harmony_tenant_backup/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "harmony-tenant-backup" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[[bin]] +name = "harmony-tenant-backup" +path = "src/main.rs" + +[[bin]] +name = "harmony-tenant-backup-setup" +path = "src/setup.rs" + +[dependencies] +harmony = { path = "../../harmony" } +harmony_config = { path = "../../harmony_config" } +tokio = { workspace = true, features = ["process", "time"] } +kube = { workspace = true } +k8s-openapi = { workspace = true } +clap = { workspace = true } +anyhow = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +schemars = "0.8" +chrono = { workspace = true } diff --git a/examples/harmony_tenant_backup/DESIGN.md b/examples/harmony_tenant_backup/DESIGN.md new file mode 100644 index 00000000..dfe474eb --- /dev/null +++ b/examples/harmony_tenant_backup/DESIGN.md @@ -0,0 +1,301 @@ +# Tenant backup design + +Status: this is the current plan. The code in this folder is an earlier +prototype (v0), kept for reference only. It does not implement this design; +see "What changes from v0". + +A tenant on a shared cluster opts in to offsite backups of their own data. The +backup job runs in the tenant's namespace with an identity that can write +backups and can never delete them through the S3 API. + +## Backup instance + +The unit is a backup instance. It is created when a tenant enables backup and +owns one config, one bucket, one restic password and one writer identity. A +second instance, or a replacement, gets all four new. Nothing is shared +between instances or between tenants. + +| Who | Does | +|---|---| +| Platform | `BackupInstanceScore`: bucket on the offsite RGW with versioning and Object Lock, a writer scoped to it, writer credentials into OpenBao at the tenant's path. Run by the platform team on request today, triggered by tenant self-service later. | +| Tenant | `TenantBackupScore` in their deploy code: schedule, CNPG clusters, buckets, PVCs. Data they can regenerate stays out. Generates the restic password in their own password manager. | +| Job | CronJob in the tenant namespace. Reads its instance config from OpenBao (see "Job identity"). | + +Tenants are admins of their namespace. They can read the job's secrets, edit +or delete the CronJob. That only affects their own backups, which is +acceptable because backup is their choice. + +## Storage and security + +- The bucket is owned by the platform's RGW account. The writer is an IAM + user in that account with an allow-only policy: `GetObject`, `PutObject`, + `ListBucket`, `GetBucketLocation`, and `DeleteObject` on `locks/*`. + Everything else is denied by default. The bucket owner is never the writer: + an owner keeps full control through ACLs whatever the policy says. +- `PutObject` can overwrite, so the bucket has versioning and Object Lock in + compliance mode, 90 days by default. An overwrite or delete only adds a + version or a delete marker; the locked version stays. restic never rewrites + a key except `locks/*`. The 2025 median intrusion dwell time is 14 days, 26 + when an outside party reports it (Mandiant M-Trends 2026); 90 days leaves + room for the slow tail. Cost: every locked version is stored for 90 days, + and a bucket can only be deleted 90 days after its last write. Compliance + retention can be extended, never shortened. +- One lifecycle rule, on `locks/` only, expires noncurrent lock versions and + stale delete markers after the retention period. No lifecycle rule ever + touches other prefixes; their old versions are what a rollback reads. +- A bucket quota caps growth. A full bucket fails the job loudly. +- Object Lock binds the S3 API only. The Admin Ops bucket purge, + `radosgw-admin bucket rm --purge-objects`, `radosgw-admin object rm` and raw + `rados rm` all delete without checking retention (only `rgw_op.cc`'s S3 + delete paths call `verify_object_lock`). On the offsite cluster, Ceph + keyrings that reach the RGW pools and RGW users with `buckets=write` caps, + including the one Rook creates, are held outside routine operator access. + Object Lock protects against tenants and stolen S3 keys, not against the + storage operator. +- RGW accounts and IAM users need Ceph Squid (19.2) or later. On Reef the + writer is a plain RGW user with an inline policy on a bucket owned by + another user. +- Writer credentials reach one bucket. A tenant who reads them learns nothing + about another tenant. +- An offsite breach alone yields ciphertext without the key. +- The job never runs `restic forget` or `prune`. +- TLS is never skipped: `sslmode=verify-full` to Postgres, the RGW CA mounted + if it is private. + +`BackupInstanceScore` proves its own policy before returning: the writer can +put an object, and cannot delete an object version or change the bucket +policy. + +Not covered: whoever can read the instance's OpenBao path (OpenBao root, or +cluster-admin acting as the job) gets the password and the writer +credentials, and with them all history. They could already read the live +data. restic is symmetric (no write-only key exists in restic, rustic, kopia +or borg), so the job necessarily holds a key that reads all history. + +## Restic password + +The tenant generates the password in their own password manager, so the +offline copy exists before the backup does. Nothing on the platform side +generates it. + +The password is a `#[config(secret)]` field of the instance's config, stored +by `harmony_config` like any other secret. On first apply, `TenantBackupScore` +calls `get_or_prompt` and the tenant pastes it. In CI a missing password +fails with "apply once interactively to set the restic password". On every +apply the Score checks that the password opens the existing repo, so a wrong +paste never creates a second repo or orphans history. + +| Event | Outcome | +|---|---| +| Password lost from the tenant's vault, OpenBao alive | Read it back with `harmony_config`. Same instance. | +| Lost from both | Backups are unreadable, permanently. | +| Password leaked | Read exposure only. New instance; the old bucket is deleted once the new one has a verified snapshot. | +| Writer credentials reissued by the platform | Update the tenant's vault copy; the Score picks them up on next apply. | + +The tenant's vault entry holds endpoint, bucket, region, writer credentials +and password: everything the restore below needs. The password is never +rotated in place and never escrowed by the platform. + +## Job identity (open) + +The job reads its config from OpenBao at run time, using its own short-lived +token rather than a stored secret. All auth goes through Zitadel today, and +Zitadel cannot exchange a Kubernetes ServiceAccount token for its own: token +exchange only accepts Zitadel-issued subject tokens, and the JWT IdP is a +browser flow. So one of these: + +| | Holds | OpenBao trusts | Revocation | Cost | +|---|---|---|---|---| +| A. Second OpenBao JWT mount on the cluster's service-account issuer, static `jwt_validation_pubkeys` | Projected token, audience `openbao`, about 1 h | Zitadel, plus each cluster issuer limited to backup roles | Token TTL | Mount per cluster; one role per instance bound to `system:serviceaccount::` and one path; update keys when the cluster rotates its signing key. `harmony_secret` reads the token file instead of signing with a Zitadel key. | +| B. Zitadel machine key per instance (v0) | Long-lived key in a namespace Secret | Zitadel only | Manual | None, but the stored secret returns | +| C. OpenBao `kubernetes` auth (TokenReview) | Projected token | Zitadel, plus each cluster API server | Immediate | OpenBao reaches each API server; new login method | + +Leaning A. Revisit if Zitadel ships workload identity federation. Rejected: +copying the config into a namespace Secret at apply time, or External Secrets +Operator; both leave the secret in etcd, and ESO adds a cluster-wide +operator. + +## The job + +Two CronJobs, so a PVC that cannot mount never blocks the databases: + +- `pg+s3`: Postgres dumps and tenant buckets. +- `pvc`: declared RWX volumes. Only created when the Score lists any. + +Settings: `concurrencyPolicy: Forbid`, `startingDeadlineSeconds: 3600`, +`backoffLimit: 0` (a retry duplicates snapshots), `restartPolicy: Never`, +`activeDeadlineSeconds`, `failedJobsHistoryLimit: 3`. Scratch is an `emptyDir` +with `sizeLimit` and a matching `ephemeral-storage` request, staged one item +at a time. Memory leaves room for the restic index, which grows forever +without prune. + +The Score emits one NetworkPolicy: egress from the job to the CNPG pods on +5432, the RGW on 443 and DNS; matching ingress on the CNPG pods. Shared OKD +projects often default-deny. + +Every run ends with `restic check --read-data-subset=/7`, so the +whole repo is read once a week. It is the only thing that notices a pack +overwritten with the writer's credentials. A killed run leaves a stale +non-exclusive lock that does not block the next backup, and orphan packs that +stay forever (bounded per run). + +Exit is zero or non-zero; the log line says fatal or partial and why. + +## Surfaces + +- CNPG: the Score names the credential Secret per cluster, `-app` by + default (CNPG disables superuser access by default). `pg_dumpall + --globals-only --no-role-passwords`, then `pg_dump --format=custom + --compress=0 --lock-wait-timeout=60s` per database over `-rw`, then + `pg_restore --list` to reject an unreadable archive. No compression because + restic compresses and compressed dumps defeat deduplication. Superuser is + required only for a cluster with several databases owned by different + roles; the Score checks this at apply time. The image's `pg_dump` is the + highest Postgres major on the platform. RPO is the last successful run; + there is no PITR. +- Tenant S3 buckets: `rclone copy` to scratch with read-only source + credentials, then restic. Old versions on a versioned source bucket are not + copied. +- PVCs: RWX only, decided at apply time. A declared RWO PVC is a Score error; + on Ceph most PVCs are RBD RWO, so this surface mostly covers CephFS. The + volume is `readOnly` at the `persistentVolumeClaim` source, not just the + mount, with `fsGroupChangePolicy: OnRootMismatch`, so kubelet never chowns + the tenant's live data. Files the job's UID cannot read make the run + partial, not fatal. CNPG data PVCs are never listed; the dump covers them. + +Each instance is one restic repo at the root of its bucket, with `--host` +pinned to `-`. Tags on every snapshot: `run:`, `ns:`, +`surface:pg|s3|pvc`, and `cluster:`, `bucket:` or `pvc:`. + +## Deploy and failure visibility + +`TenantBackupScore` returns only after one Job created from the CronJob +template exits zero and its snapshot is listed. NetworkPolicy, region, +path-style and TLS mistakes fail the deploy with an actionable message instead +of a 3 AM surprise. + +The Score emits a PrometheusRule for user-workload monitoring: alert when +`time() - kube_cronjob_status_last_successful_time` exceeds twice the +schedule interval. An optional dead-man's-switch URL is pinged on success +with the snapshot IDs, which also catches a deleted CronJob or a burned +cluster. + +## Schedule, growth and retention + +The schedule is a typed Score field: `Daily { at }` (default), `Hourly`, or +`Cron(String)` for anything else. There is no pruning, so the bucket only +grows; restic deduplication keeps each run to the chunks that changed, which +is why dumps are uncompressed. Each run logs the bytes it added, so the +tenant sees the real growth rate instead of an estimate. + +To drop old backups the tenant opens a ticket. The platform disables the +writer key; the grace period, no shorter than the Object Lock retention, +starts then. The tenant provisions a new instance, and the platform deletes +the old bucket after the grace period. + +## Restore + +Restore needs only the tenant's vault entry, restic and rclone, never the +burned cluster. Pick +the snapshot by the S3 `LastModified` of `snapshots/`, which the server +sets and Object Lock preserves, never by `latest` or restic's own timestamp: +someone holding the writer credentials and the password can write a fake +newer or backdated snapshot. restic 0.18 or later: + +```bash +export RESTIC_REPOSITORY="s3:https:///" +export RESTIC_PASSWORD='' +export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_DEFAULT_REGION=default +R=(restic -o s3.bucket-lookup=path) + +"${R[@]}" snapshots --tag "ns:tenant-x" + +# one database, into a NEW CNPG cluster, never over the broken one +"${R[@]}" restore --target /restore +psql -h new-cluster-rw -U postgres -f /restore/pg/app/globals.sql # first; "role exists" is fine +pg_restore -h new-cluster-rw -U postgres -C -d postgres /restore/pg/app/app.dump +# then reset app role passwords from the new cluster's -app Secret + +"${R[@]}" restore --target /restore # bucket or PVC snapshot +``` + +The restore target needs superuser enabled on the new cluster. Dumps are +staged at `pg//.dump` and `pg//globals.sql`; the job +keeps that layout fixed. + +If the repo was overwritten, read it as it was before the incident without +changing anything in the bucket: + +```bash +RCLONE_S3_VERSION_AT='