From a5e624f5427df497550fc9eb0c78d04b23260137 Mon Sep 17 00:00:00 2001 From: Nikholas Pcenicni <82239765+nikpcenicni@users.noreply.github.com> Date: Sat, 28 Mar 2026 01:17:22 -0400 Subject: [PATCH] Update CLUSTER-BUILD.md to reflect the current state of the Talos cluster, detailing progress through Phase D (observability) and advancements in Phase E (secrets). Include updates on Sealed Secrets, External Secrets Operator, and Vault configurations, along with deployment instructions and next steps for Kubernetes auth and ClusterSecretStore integration. Mark relevant tasks as completed and outline remaining objectives for future phases. --- .../noble/apps/external-secrets/README.md | 60 +++++++ .../examples/vault-cluster-secret-store.yaml | 31 ++++ .../apps/external-secrets/namespace.yaml | 5 + .../noble/apps/external-secrets/values.yaml | 10 ++ clusters/noble/apps/sealed-secrets/README.md | 48 ++++++ .../noble/apps/sealed-secrets/namespace.yaml | 5 + .../noble/apps/sealed-secrets/values.yaml | 11 ++ clusters/noble/apps/vault/README.md | 150 ++++++++++++++++++ clusters/noble/apps/vault/namespace.yaml | 5 + clusters/noble/apps/vault/unseal-cronjob.yaml | 63 ++++++++ clusters/noble/apps/vault/values.yaml | 48 ++++++ talos/CLUSTER-BUILD.md | 23 ++- 12 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 clusters/noble/apps/external-secrets/README.md create mode 100644 clusters/noble/apps/external-secrets/examples/vault-cluster-secret-store.yaml create mode 100644 clusters/noble/apps/external-secrets/namespace.yaml create mode 100644 clusters/noble/apps/external-secrets/values.yaml create mode 100644 clusters/noble/apps/sealed-secrets/README.md create mode 100644 clusters/noble/apps/sealed-secrets/namespace.yaml create mode 100644 clusters/noble/apps/sealed-secrets/values.yaml create mode 100644 clusters/noble/apps/vault/README.md create mode 100644 clusters/noble/apps/vault/namespace.yaml create mode 100644 clusters/noble/apps/vault/unseal-cronjob.yaml create mode 100644 clusters/noble/apps/vault/values.yaml diff --git a/clusters/noble/apps/external-secrets/README.md b/clusters/noble/apps/external-secrets/README.md new file mode 100644 index 0000000..04374c4 --- /dev/null +++ b/clusters/noble/apps/external-secrets/README.md @@ -0,0 +1,60 @@ +# External Secrets Operator (noble) + +Syncs secrets from external systems into Kubernetes **Secret** objects via **ExternalSecret** / **ClusterExternalSecret** CRDs. + +- **Chart:** `external-secrets/external-secrets` **2.2.0** (app **v2.2.0**) +- **Namespace:** `external-secrets` +- **Helm release name:** `external-secrets` (matches the operator **ServiceAccount** name `external-secrets`) + +## Install + +```bash +helm repo add external-secrets https://charts.external-secrets.io +helm repo update +kubectl apply -f clusters/noble/apps/external-secrets/namespace.yaml +helm upgrade --install external-secrets external-secrets/external-secrets -n external-secrets \ + --version 2.2.0 -f clusters/noble/apps/external-secrets/values.yaml --wait +``` + +Verify: + +```bash +kubectl -n external-secrets get deploy,pods +kubectl get crd | grep external-secrets +``` + +## Vault `ClusterSecretStore` (after Vault is deployed) + +The checklist expects a **Vault**-backed store. Install Vault first (`talos/CLUSTER-BUILD.md` Phase E — Vault on Longhorn + auto-unseal), then: + +1. Enable **KV v2** secrets engine and **Kubernetes** auth in Vault; create a **role** (e.g. `external-secrets`) that maps the cluster’s **`external-secrets` / `external-secrets`** service account to a policy that can read the paths you need. +2. Copy **`examples/vault-cluster-secret-store.yaml`**, set **`spec.provider.vault.server`** to your Vault URL. This repo’s Vault Helm values use **HTTP** on port **8200** (`global.tlsDisable: true`): **`http://vault.vault.svc.cluster.local:8200`**. Use **`https://`** if you enable TLS on the Vault listener. +3. If Vault uses a **private TLS CA**, configure **`caProvider`** or **`caBundle`** on the Vault provider — see [HashiCorp Vault provider](https://external-secrets.io/latest/provider/hashicorp-vault/). Do not commit private CA material to public git unless intended. +4. Apply: **`kubectl apply -f …/vault-cluster-secret-store.yaml`** +5. Confirm the store is ready: **`kubectl describe clustersecretstore vault`** + +Example **ExternalSecret** (after the store is healthy): + +```yaml +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: demo + namespace: default +spec: + refreshInterval: 1h + secretStoreRef: + name: vault + kind: ClusterSecretStore + target: + name: demo-synced + data: + - secretKey: password + remoteRef: + key: secret/data/myapp + property: password +``` + +## Upgrades + +Pin the chart version in `values.yaml` header comments; run the same **`helm upgrade --install`** with the new **`--version`** after reviewing [release notes](https://github.com/external-secrets/external-secrets/releases). diff --git a/clusters/noble/apps/external-secrets/examples/vault-cluster-secret-store.yaml b/clusters/noble/apps/external-secrets/examples/vault-cluster-secret-store.yaml new file mode 100644 index 0000000..bdbb1fc --- /dev/null +++ b/clusters/noble/apps/external-secrets/examples/vault-cluster-secret-store.yaml @@ -0,0 +1,31 @@ +# ClusterSecretStore for HashiCorp Vault (KV v2) using Kubernetes auth. +# +# Do not apply until Vault is running, reachable from the cluster, and configured with: +# - Kubernetes auth at mountPath (default: kubernetes) +# - A role (below: external-secrets) bound to this service account: +# name: external-secrets +# namespace: external-secrets +# - A policy allowing read on the KV path used below (e.g. secret/data/* for path "secret") +# +# Adjust server, mountPath, role, and path to match your Vault deployment. If Vault uses TLS +# with a private CA, set provider.vault.caProvider or caBundle (see README). +# +# kubectl apply -f clusters/noble/apps/external-secrets/examples/vault-cluster-secret-store.yaml +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: vault +spec: + provider: + vault: + server: "http://vault.vault.svc.cluster.local:8200" + path: secret + version: v2 + auth: + kubernetes: + mountPath: kubernetes + role: external-secrets + serviceAccountRef: + name: external-secrets + namespace: external-secrets diff --git a/clusters/noble/apps/external-secrets/namespace.yaml b/clusters/noble/apps/external-secrets/namespace.yaml new file mode 100644 index 0000000..eab4215 --- /dev/null +++ b/clusters/noble/apps/external-secrets/namespace.yaml @@ -0,0 +1,5 @@ +# External Secrets Operator — apply before Helm. +apiVersion: v1 +kind: Namespace +metadata: + name: external-secrets diff --git a/clusters/noble/apps/external-secrets/values.yaml b/clusters/noble/apps/external-secrets/values.yaml new file mode 100644 index 0000000..871f674 --- /dev/null +++ b/clusters/noble/apps/external-secrets/values.yaml @@ -0,0 +1,10 @@ +# External Secrets Operator — noble +# +# helm repo add external-secrets https://charts.external-secrets.io +# helm repo update +# kubectl apply -f clusters/noble/apps/external-secrets/namespace.yaml +# helm upgrade --install external-secrets external-secrets/external-secrets -n external-secrets \ +# --version 2.2.0 -f clusters/noble/apps/external-secrets/values.yaml --wait +# +# CRDs are installed by the chart (installCRDs: true). Vault ClusterSecretStore: see README + examples/. +commonLabels: {} diff --git a/clusters/noble/apps/sealed-secrets/README.md b/clusters/noble/apps/sealed-secrets/README.md new file mode 100644 index 0000000..2cd1aa1 --- /dev/null +++ b/clusters/noble/apps/sealed-secrets/README.md @@ -0,0 +1,48 @@ +# Sealed Secrets (noble) + +Encrypts `Secret` manifests so they can live in git; the controller decrypts **SealedSecret** resources into **Secret**s in-cluster. + +- **Chart:** `sealed-secrets/sealed-secrets` **2.18.4** (app **0.36.1**) +- **Namespace:** `sealed-secrets` + +## Install + +```bash +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm repo update +kubectl apply -f clusters/noble/apps/sealed-secrets/namespace.yaml +helm upgrade --install sealed-secrets sealed-secrets/sealed-secrets -n sealed-secrets \ + --version 2.18.4 -f clusters/noble/apps/sealed-secrets/values.yaml --wait +``` + +## Workstation: `kubeseal` + +Install a **kubeseal** build compatible with the controller (match **app** minor, e.g. **0.36.x** for **0.36.1**). Examples: + +- **Homebrew:** `brew install kubeseal` (check `kubeseal --version` against the chart’s `image.tag` in `helm show values`). +- **GitHub releases:** [bitnami-labs/sealed-secrets](https://github.com/bitnami-labs/sealed-secrets/releases) + +Fetch the cluster’s public seal cert (once per kube context): + +```bash +kubeseal --fetch-cert > /tmp/noble-sealed-secrets.pem +``` + +Create a sealed secret from a normal secret manifest: + +```bash +kubectl create secret generic example --from-literal=foo=bar --dry-run=client -o yaml \ + | kubeseal --cert /tmp/noble-sealed-secrets.pem -o yaml > example-sealedsecret.yaml +``` + +Commit `example-sealedsecret.yaml`; apply it with `kubectl apply -f`. The controller creates the **Secret** in the same namespace as the **SealedSecret**. + +## Backup the sealing key + +If the controller’s private key is lost, existing sealed files cannot be decrypted on a new cluster. Back up the key secret after install: + +```bash +kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key=active -o yaml > sealed-secrets-key-backup.yaml +``` + +Store `sealed-secrets-key-backup.yaml` in a safe offline location (not in public git). diff --git a/clusters/noble/apps/sealed-secrets/namespace.yaml b/clusters/noble/apps/sealed-secrets/namespace.yaml new file mode 100644 index 0000000..d2e9d85 --- /dev/null +++ b/clusters/noble/apps/sealed-secrets/namespace.yaml @@ -0,0 +1,5 @@ +# Sealed Secrets controller — apply before Helm. +apiVersion: v1 +kind: Namespace +metadata: + name: sealed-secrets diff --git a/clusters/noble/apps/sealed-secrets/values.yaml b/clusters/noble/apps/sealed-secrets/values.yaml new file mode 100644 index 0000000..cf9abfe --- /dev/null +++ b/clusters/noble/apps/sealed-secrets/values.yaml @@ -0,0 +1,11 @@ +# Sealed Secrets — noble (Git-encrypted Secret workflow) +# +# helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +# helm repo update +# kubectl apply -f clusters/noble/apps/sealed-secrets/namespace.yaml +# helm upgrade --install sealed-secrets sealed-secrets/sealed-secrets -n sealed-secrets \ +# --version 2.18.4 -f clusters/noble/apps/sealed-secrets/values.yaml --wait +# +# Client: install kubeseal (same minor as controller — see README). +# Defaults are sufficient for the lab; override here if you need key renewal, resources, etc. +commonLabels: {} diff --git a/clusters/noble/apps/vault/README.md b/clusters/noble/apps/vault/README.md new file mode 100644 index 0000000..13048c3 --- /dev/null +++ b/clusters/noble/apps/vault/README.md @@ -0,0 +1,150 @@ +# HashiCorp Vault (noble) + +Standalone Vault with **file** storage on a **Longhorn** PVC (`server.dataStorage`). The listener uses **HTTP** (`global.tlsDisable: true`) for in-cluster use; add TLS at the listener when exposing outside the cluster. + +- **Chart:** `hashicorp/vault` **0.32.0** (Vault **1.21.2**) +- **Namespace:** `vault` + +## Install + +```bash +helm repo add hashicorp https://helm.releases.hashicorp.com +helm repo update +kubectl apply -f clusters/noble/apps/vault/namespace.yaml +helm upgrade --install vault hashicorp/vault -n vault \ + --version 0.32.0 -f clusters/noble/apps/vault/values.yaml --wait --timeout 15m +``` + +Verify: + +```bash +kubectl -n vault get pods,pvc,svc +kubectl -n vault exec -i sts/vault -- vault status +``` + +## Initialize and unseal (first time) + +From a workstation with `kubectl` (or `kubectl exec` into any pod with `vault` CLI): + +```bash +kubectl -n vault exec -i sts/vault -- vault operator init -key-shares=1 -key-threshold=1 +``` + +**Lab-only:** `-key-shares=1 -key-threshold=1` keeps a single unseal key. For stronger Shamir splits, use more shares and store them safely. + +Save the **Unseal Key** and **Root Token** offline. Then unseal once: + +```bash +kubectl -n vault exec -i sts/vault -- vault operator unseal +# paste unseal key +``` + +Or create the Secret used by the optional CronJob and apply it: + +```bash +kubectl -n vault create secret generic vault-unseal-key --from-literal=key='YOUR_UNSEAL_KEY' +kubectl apply -f clusters/noble/apps/vault/unseal-cronjob.yaml +``` + +The CronJob runs every minute and unseals if Vault is sealed and the Secret is present. + +## Auto-unseal note + +Vault **OSS** auto-unseal uses cloud KMS (AWS, GCP, Azure, OCI), **Transit** (another Vault), etc. There is no first-class “Kubernetes Secret” seal. This repo uses an optional **CronJob** as a **lab** substitute. Production clusters should use a supported seal backend. + +## Kubernetes auth (External Secrets / ClusterSecretStore) + +Run these **from your workstation** (needs `kubectl`; no local `vault` binary required). Use a **short-lived admin token** or the root token **only in your shell** — do not paste tokens into logs or chat. + +**1. Enable the auth method** (skip if already done): + +```bash +kubectl -n vault exec -it sts/vault -- sh -c ' + export VAULT_ADDR=http://127.0.0.1:8200 + export VAULT_TOKEN="YOUR_ROOT_OR_ADMIN_TOKEN" + vault auth enable kubernetes +' +``` + +**2. Configure `auth/kubernetes`** — the API **issuer** must match the `iss` claim on service account JWTs. With **kube-vip** / a custom API URL, discover it from the cluster (do not assume `kubernetes.default`): + +```bash +ISSUER=$(kubectl get --raw /.well-known/openid-configuration | jq -r .issuer) +REVIEWER=$(kubectl -n vault create token vault --duration=8760h) +CA_B64=$(kubectl config view --raw --minify -o jsonpath='{.clusters[0].cluster.certificate-authority-data}') +``` + +Then apply config **inside** the Vault pod (environment variables are passed in with `env` so quoting stays correct): + +```bash +export VAULT_TOKEN="YOUR_ROOT_OR_ADMIN_TOKEN" +export ISSUER REVIEWER CA_B64 +kubectl -n vault exec -i sts/vault -- env \ + VAULT_ADDR=http://127.0.0.1:8200 \ + VAULT_TOKEN="$VAULT_TOKEN" \ + CA_B64="$CA_B64" \ + REVIEWER="$REVIEWER" \ + ISSUER="$ISSUER" \ + sh -ec ' + echo "$CA_B64" | base64 -d > /tmp/k8s-ca.crt + vault write auth/kubernetes/config \ + kubernetes_host="https://kubernetes.default.svc:443" \ + kubernetes_ca_cert=@/tmp/k8s-ca.crt \ + token_reviewer_jwt="$REVIEWER" \ + issuer="$ISSUER" +' +``` + +**3. KV v2** at path `secret` (skip if already enabled): + +```bash +kubectl -n vault exec -it sts/vault -- sh -c ' + export VAULT_ADDR=http://127.0.0.1:8200 + export VAULT_TOKEN="YOUR_ROOT_OR_ADMIN_TOKEN" + vault secrets enable -path=secret kv-v2 +' +``` + +**4. Policy + role** for the External Secrets operator SA (`external-secrets` / `external-secrets`): + +```bash +kubectl -n vault exec -it sts/vault -- sh -c ' + export VAULT_ADDR=http://127.0.0.1:8200 + export VAULT_TOKEN="YOUR_ROOT_OR_ADMIN_TOKEN" + vault policy write external-secrets - <