From 2fb86f59304f014b20b0b7108d956c98db766e12 Mon Sep 17 00:00:00 2001 From: Nikholas Pcenicni <82239765+nikpcenicni@users.noreply.github.com> Date: Fri, 15 May 2026 00:04:34 -0400 Subject: [PATCH] Enhance Authentik and Newt configurations to support Open WebUI integration. Add necessary environment variables and secrets management for Open WebUI in .env.sample and Ansible tasks. Update README to clarify setup steps for automating HTTP resources with Pangolin, ensuring consistency with new branding and deployment practices. --- .env.sample | 15 +- ansible/inventory/group_vars/all.yml | 2 + ansible/playbooks/noble.yml | 2 +- ansible/roles/noble_authentik/README.md | 6 +- .../roles/noble_authentik/defaults/main.yml | 8 + .../roles/noble_authentik/tasks/from_env.yml | 63 ++++ ansible/roles/noble_authentik/tasks/main.yml | 33 ++ .../templates/authentik-clients.json.j2 | 6 + .../authentik-worker-oidc-spec.json.j2 | 6 + ansible/roles/noble_newt/defaults/main.yml | 8 + ansible/roles/noble_newt/tasks/main.yml | 72 ++-- .../roles/noble_newt/tasks/pangolin_sync.yml | 95 +++++ .../noble/apps/open-webui/kustomization.yaml | 1 + clusters/noble/apps/open-webui/namespace.yaml | 6 + clusters/noble/apps/open-webui/values.yaml | 45 ++- clusters/noble/bootstrap/newt/README.md | 20 ++ ...nc_pangolin_http_resources.cpython-314.pyc | Bin 0 -> 19496 bytes .../scripts/sync_pangolin_http_resources.py | 331 ++++++++++++++++++ 18 files changed, 674 insertions(+), 45 deletions(-) create mode 100644 ansible/roles/noble_newt/tasks/pangolin_sync.yml create mode 100644 clusters/noble/apps/open-webui/namespace.yaml create mode 100644 clusters/noble/bootstrap/newt/scripts/__pycache__/sync_pangolin_http_resources.cpython-314.pyc create mode 100755 clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py diff --git a/.env.sample b/.env.sample index e875db4..c08c58a 100644 --- a/.env.sample +++ b/.env.sample @@ -12,13 +12,21 @@ PANGOLIN_ENDPOINT= NEWT_ID= NEWT_SECRET= +# Optional: Pangolin Integration API — automate public HTTP resources + Traefik targets (**noble_pangolin_sync_http_resources=true** in **group_vars**; see **clusters/noble/bootstrap/newt/README.md** §4). +# NOBLE_PANGOLIN_API_BASE=https://api.your-pangolin.example/v1 +# NOBLE_PANGOLIN_ORG_ID= +# NOBLE_PANGOLIN_API_TOKEN= +# NOBLE_PANGOLIN_SITE_ID= +# NOBLE_PANGOLIN_TRAEFIK_IP=192.168.50.211 +# NOBLE_PANGOLIN_TRAEFIK_PORT=443 + # Velero — when **noble_velero_install=true**, set bucket + S3 API URL and credentials (see clusters/noble/bootstrap/velero/README.md). NOBLE_VELERO_S3_BUCKET= NOBLE_VELERO_S3_URL= NOBLE_VELERO_AWS_ACCESS_KEY_ID= NOBLE_VELERO_AWS_SECRET_ACCESS_KEY= -# Authentik + OIDC — when **noble_authentik_install=true**, Ansible installs Authentik and reconfigures Argo CD, Grafana, Headlamp (native OIDC) and Prometheus/Alertmanager/Longhorn via oauth2-proxy (OIDC to Authentik + Traefik ForwardAuth). See **ansible/roles/noble_authentik/README.md**. +# Authentik + OIDC — when **noble_authentik_install=true**, Ansible installs Authentik and reconfigures Argo CD, Grafana, Headlamp, **Open WebUI** (native OIDC) and Prometheus/Alertmanager/Longhorn via oauth2-proxy (OIDC to Authentik + Traefik ForwardAuth). See **ansible/roles/noble_authentik/README.md**. NOBLE_AUTHENTIK_SECRET_KEY= NOBLE_AUTHENTIK_POSTGRES_PASSWORD= NOBLE_AUTHENTIK_BOOTSTRAP_TOKEN= @@ -28,6 +36,11 @@ NOBLE_AUTHENTIK_CLIENT_SECRET_ARGOCD= NOBLE_AUTHENTIK_CLIENT_SECRET_GRAFANA= NOBLE_AUTHENTIK_CLIENT_SECRET_HEADLAMP= NOBLE_AUTHENTIK_CLIENT_SECRET_OAUTH2_PROXY= +# Open WebUI (Argo app **clusters/noble/apps/open-webui**) — OIDC client secret + app secrets (see **clusters/noble/apps/open-webui/values.yaml**). +NOBLE_AUTHENTIK_CLIENT_SECRET_OPEN_WEBUI= +NOBLE_OPEN_WEBUI_OPENAI_API_KEY= +# e.g. openssl rand -hex 32 +NOBLE_OPEN_WEBUI_WEBUI_SECRET_KEY= # Random secret for oauth2-proxy session cookie (see oauth2-proxy Helm chart docs; e.g. openssl rand -base64 32 | head -c 32 | base64) NOBLE_AUTHENTIK_OAUTH2_PROXY_COOKIE_SECRET= # S3 media — **separate** bucket from Velero backups (**NOBLE_VELERO_S3_BUCKET**). Endpoint and keys default to the Velero vars above unless you set the Authentik-specific overrides. diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 3ae1adb..f7a7780 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -30,4 +30,6 @@ noble_authentik_install: true # Optional: public (or extra) Authentik hostnames on the same IdP — list of FQDNs. Pangolin: CNAME + resource → Newt → Traefik (see noble_authentik README). noble_authentik_ingress_extra_hosts: - auth.nikflix.ca +# Open WebUI — public Ingress host (Pangolin → Newt → Traefik). Must match **clusters/noble/apps/open-webui/values.yaml** **ingress.host** and Authentik redirect URI (Ansible uses this var). +noble_open_webui_public_host: webui.nikflix.ca noble_authentik_blueprints_enabled: true \ No newline at end of file diff --git a/ansible/playbooks/noble.yml b/ansible/playbooks/noble.yml index bd32053..0da8c89 100644 --- a/ansible/playbooks/noble.yml +++ b/ansible/playbooks/noble.yml @@ -225,7 +225,7 @@ - role: noble_cert_manager tags: [cert_manager, certs] - role: noble_newt - tags: [newt] + tags: [newt, pangolin] - role: noble_argocd tags: [argocd, gitops] - role: noble_platform diff --git a/ansible/roles/noble_authentik/README.md b/ansible/roles/noble_authentik/README.md index 1341600..93db7af 100644 --- a/ansible/roles/noble_authentik/README.md +++ b/ansible/roles/noble_authentik/README.md @@ -28,6 +28,8 @@ To expose the **same** Authentik instance on an **internet-facing** FQDN (while Then in **Pangolin**: link the domain, create an **HTTP** resource for that hostname, and set the **target** to your **Newt** site with **`ip:port`** pointing at the cluster **Traefik** HTTPS entry (same pattern as **`clusters/noble/bootstrap/newt/README.md`** — typically the MetalLB / LAN VIP and **443**). One Newt tunnel can front many hostnames. +**Open WebUI (public hostname only):** Set **`noble_open_webui_public_host`** in **`group_vars`** to the same FQDN as **`clusters/noble/apps/open-webui/values.yaml`** **`ingress.host`** (for example **`webui.example.com`**). Add a **Pangolin** HTTP resource for that hostname to the same Newt target. Point **`OPENID_PROVIDER_URL`** in **`values.yaml`** at your **public** Authentik host (usually the first **`noble_authentik_ingress_extra_hosts`** entry, e.g. **`https://auth.example.com/application/o/open-webui/.well-known/openid-configuration`**), and set **`OPENID_REDIRECT_URI`** / **`WEBUI_URL`** to **`https:///…`**. After changing **`noble_open_webui_public_host`**, re-run **`ansible-playbook playbooks/noble.yml --tags authentik`** so Authentik’s OAuth2 redirect URI matches. + ### Split routing, two Brands, and optional blueprints This role supports a **single Authentik deployment** with **two hostnames** (lab + public) and **different Brands** per **`Host`**, without Authentik’s separate-database **Tenancy** feature (see [Tenancy](https://docs.goauthentik.io/sys-mgmt/tenancy) — alpha / licensing). [Brands](https://docs.goauthentik.io/brands/) choose default **authentication** (and related) **flows** and branding for each FQDN. @@ -110,7 +112,7 @@ Authentik **tenancy** (multiple isolated tenants in one deployment, **`AUTHENTIK ## IdP configuration -When **`noble_authentik_configure_idp`** is true, Ansible creates/updates OAuth2 providers and applications for **argocd**, **grafana**, **headlamp**, and **oauth2-proxy** using either the **worker ORM path** (default **`noble_authentik_oidc_provision_via: worker`**: **`kubectl exec`** + **`ak shell`** + **`files/worker_upsert_oauth_oidc.py`**, which avoids **2026+** REST **403** on **`GET …/providers/oauth2/**`) or the **REST-only path** (**`noble_authentik_oidc_provision_via: rest`**: **`files/configure_authentik.py`** needs a token that can list/patch OAuth2 providers). With the worker path and a bootstrap email, it also runs **`files/worker_add_bootstrap_user_groups.py`** so **`User.groups.add`** does not depend on **`GET …/core/users/**`. It then runs **`configure_authentik.py`** with **`AUTHENTIK_SKIP_OIDC_REST`** / **`AUTHENTIK_SKIP_USER_GROUP_REST`** when those worker steps ran, so the script only calls **`ensure_group`** over the API (skipped when **`AUTHENTIK_NOBLE_*_GROUP_PK`** are set). +When **`noble_authentik_configure_idp`** is true, Ansible creates/updates OAuth2 providers and applications for **argocd**, **grafana**, **headlamp**, **open-webui**, and **oauth2-proxy** using either the **worker ORM path** (default **`noble_authentik_oidc_provision_via: worker`**: **`kubectl exec`** + **`ak shell`** + **`files/worker_upsert_oauth_oidc.py`**, which avoids **2026+** REST **403** on **`GET …/providers/oauth2/**`) or the **REST-only path** (**`noble_authentik_oidc_provision_via: rest`**: **`files/configure_authentik.py`** needs a token that can list/patch OAuth2 providers). With the worker path and a bootstrap email, it also runs **`files/worker_add_bootstrap_user_groups.py`** so **`User.groups.add`** does not depend on **`GET …/core/users/**`. It then runs **`configure_authentik.py`** with **`AUTHENTIK_SKIP_OIDC_REST`** / **`AUTHENTIK_SKIP_USER_GROUP_REST`** when those worker steps ran, so the script only calls **`ensure_group`** over the API (skipped when **`AUTHENTIK_NOBLE_*_GROUP_PK`** are set). ## RBAC notes @@ -136,7 +138,7 @@ When **`noble_authentik_configure_idp`** is true, Ansible creates/updates OAuth2 - **Headlamp OIDC authorize fails / `invalid_scope`**: Authentik often has no separate **`groups`** ScopeMapping (groups live under **`profile`**). Default **`noble_authentik_headlamp_oidc_scopes`** omits **`groups`**; add a **`groups`** mapping to the provider in Authentik and set **`noble_authentik_headlamp_oidc_scopes`** to include **`groups`** if you need that scope by name. - **Headlamp OIDC: Authentik flashes then back at login / page refresh**: Headlamp **does** support normal browser OAuth (redirect to Authentik and return to **`/oidc-callback`**). If the callback fails, the UI looks like it “drops” auth. Common causes: **`X-Forwarded-Proto`** not reaching Headlamp (callback built as **`http`** — see [Headlamp OIDC docs](https://headlamp.dev/docs/latest/installation/in-cluster/oidc)); **Traefik ForwardAuth** on the same Ingress (do not combine with native OIDC); **PKCE** state issues — this role defaults **`noble_authentik_headlamp_oidc_use_pkce: false`** for Authentik confidential clients (set **`true`** in **`group_vars`** if you need PKCE). - **Headlamp UI: `/me` works but `/clusters/main/version` (and other K8s calls) return 401**: Headlamp forwards your **OIDC id_token** to **kube-apiserver**. The API server must be configured with **OIDC** flags for the same **issuer** and **`oidc-client-id`** as Headlamp (see **`talos/talconfig.yaml`** patch and [Kubernetes OIDC authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens)). Apply regenerated Talos configs to control plane nodes, then **`kubectl apply -k clusters/noble/bootstrap/headlamp`** (or **`--tags authentik`**) for **`oidc-noble-admins-clusterrolebinding.yaml`**. Ensure your user is in Authentik group **`noble-admins`** and the id_token includes a **`groups`** claim if you rely on that binding. -- **Headlamp + Traefik ForwardAuth (`oauth2-proxy-forward-auth`)**: Do **not** put ForwardAuth on the **Headlamp** Ingress while using **native Headlamp OIDC**. Auth runs on **`/oidc-callback`** before Headlamp can finish the code exchange; ForwardAuth returns **401** and breaks login. Use **either** native OIDC (this repo’s **`values-authentik-oidc.yaml`**) **or** terminate auth at oauth2-proxy only (no **`config.oidc`**), not both. +- **Headlamp + Traefik ForwardAuth (`oauth2-proxy-forward-auth`)**: Do **not** put ForwardAuth on the **Headlamp** Ingress while using **native Headlamp OIDC**. Auth runs on **`/oidc-callback`** before Headlamp can finish the code exchange; ForwardAuth returns **401** and breaks login. Use **either** native OIDC (this repo’s **`values-authentik-oidc.yaml`**) **or** terminate auth at oauth2-proxy only (no **`config.oidc`**), not both. The same applies to **Open WebUI** native OIDC (**`/oauth/oidc/callback`** in **`clusters/noble/apps/open-webui/values.yaml`**). ### Fix admin access manually (worker shell, no Ansible) diff --git a/ansible/roles/noble_authentik/defaults/main.yml b/ansible/roles/noble_authentik/defaults/main.yml index d83a979..d7f5b8b 100644 --- a/ansible/roles/noble_authentik/defaults/main.yml +++ b/ansible/roles/noble_authentik/defaults/main.yml @@ -122,6 +122,7 @@ noble_authentik_client_id_argocd: argocd noble_authentik_client_id_grafana: grafana noble_authentik_client_id_headlamp: headlamp noble_authentik_client_id_oauth2_proxy: oauth2-proxy +noble_authentik_client_id_open_webui: open-webui # Headlamp **OIDC_SCOPES** for Secret **headlamp-oidc**. Omit **groups** unless the Authentik OAuth2 provider # includes a separate **groups** ScopeMapping (2026.x defaults often embed groups in **profile** only; requesting @@ -143,8 +144,15 @@ noble_authentik_client_secret_argocd: "" noble_authentik_client_secret_grafana: "" noble_authentik_client_secret_headlamp: "" noble_authentik_client_secret_oauth2_proxy: "" +noble_authentik_client_secret_open_webui: "" noble_authentik_oauth2_proxy_cookie_secret: "" +# **open-webui** namespace — Secret **open-webui-secrets** (Ansible **--tags authentik**). See **clusters/noble/apps/open-webui/values.yaml**. +noble_open_webui_openai_api_key: "" +noble_open_webui_webui_secret_key: "" +# Public FQDN for Open WebUI (Ingress + OIDC **redirect_uri**). Set in **group_vars** (e.g. **webui.example.com**); must match GitOps **values.yaml** **ingress.host** and **OPENID_REDIRECT_URI** / **WEBUI_URL**. +noble_open_webui_public_host: "" + # Optional: OAuth2 provider flow PKs (UUID strings). When **both** are set, **configure_authentik.py** # skips **GET /flows/instances/** (avoids 403 if the API token cannot view flows). If unset, the role # tries **kubectl exec** into **authentik-worker** + **ak shell** to read the same slugs from the DB. diff --git a/ansible/roles/noble_authentik/tasks/from_env.yml b/ansible/roles/noble_authentik/tasks/from_env.yml index f107981..5984286 100644 --- a/ansible/roles/noble_authentik/tasks/from_env.yml +++ b/ansible/roles/noble_authentik/tasks/from_env.yml @@ -195,6 +195,69 @@ - (noble_authentik_cs_o2_from_env.stdout | default('') | trim | length) > 0 no_log: true +- name: Load NOBLE_AUTHENTIK_CLIENT_SECRET_OPEN_WEBUI from .env when unset + ansible.builtin.shell: | + set -a + . "{{ noble_repo_root }}/.env" + set +a + printf '%s' "${NOBLE_AUTHENTIK_CLIENT_SECRET_OPEN_WEBUI:-}" + register: noble_authentik_cs_ow_from_env + when: + - noble_authentik_dotenv_stat.stat.exists | default(false) + - noble_authentik_client_secret_open_webui | default('') | length == 0 + changed_when: false + no_log: true + +- name: Apply NOBLE_AUTHENTIK_CLIENT_SECRET_OPEN_WEBUI from .env + ansible.builtin.set_fact: + noble_authentik_client_secret_open_webui: "{{ noble_authentik_cs_ow_from_env.stdout | trim }}" + when: + - noble_authentik_cs_ow_from_env is defined + - (noble_authentik_cs_ow_from_env.stdout | default('') | trim | length) > 0 + no_log: true + +- name: Load NOBLE_OPEN_WEBUI_OPENAI_API_KEY from .env when unset + ansible.builtin.shell: | + set -a + . "{{ noble_repo_root }}/.env" + set +a + printf '%s' "${NOBLE_OPEN_WEBUI_OPENAI_API_KEY:-}" + register: noble_open_webui_openai_from_env + when: + - noble_authentik_dotenv_stat.stat.exists | default(false) + - noble_open_webui_openai_api_key | default('') | length == 0 + changed_when: false + no_log: true + +- name: Apply NOBLE_OPEN_WEBUI_OPENAI_API_KEY from .env + ansible.builtin.set_fact: + noble_open_webui_openai_api_key: "{{ noble_open_webui_openai_from_env.stdout | trim }}" + when: + - noble_open_webui_openai_from_env is defined + - (noble_open_webui_openai_from_env.stdout | default('') | trim | length) > 0 + no_log: true + +- name: Load NOBLE_OPEN_WEBUI_WEBUI_SECRET_KEY from .env when unset + ansible.builtin.shell: | + set -a + . "{{ noble_repo_root }}/.env" + set +a + printf '%s' "${NOBLE_OPEN_WEBUI_WEBUI_SECRET_KEY:-}" + register: noble_open_webui_webui_secret_from_env + when: + - noble_authentik_dotenv_stat.stat.exists | default(false) + - noble_open_webui_webui_secret_key | default('') | length == 0 + changed_when: false + no_log: true + +- name: Apply NOBLE_OPEN_WEBUI_WEBUI_SECRET_KEY from .env + ansible.builtin.set_fact: + noble_open_webui_webui_secret_key: "{{ noble_open_webui_webui_secret_from_env.stdout | trim }}" + when: + - noble_open_webui_webui_secret_from_env is defined + - (noble_open_webui_webui_secret_from_env.stdout | default('') | trim | length) > 0 + no_log: true + - name: Load NOBLE_AUTHENTIK_OAUTH2_PROXY_COOKIE_SECRET from .env when unset ansible.builtin.shell: | set -a diff --git a/ansible/roles/noble_authentik/tasks/main.yml b/ansible/roles/noble_authentik/tasks/main.yml index 1e60e77..245f3b9 100644 --- a/ansible/roles/noble_authentik/tasks/main.yml +++ b/ansible/roles/noble_authentik/tasks/main.yml @@ -22,9 +22,16 @@ - noble_authentik_client_secret_grafana | default('') | length > 0 - noble_authentik_client_secret_headlamp | default('') | length > 0 - noble_authentik_client_secret_oauth2_proxy | default('') | length > 0 + - noble_authentik_client_secret_open_webui | default('') | length > 0 - noble_authentik_oauth2_proxy_cookie_secret | default('') | length > 0 + - noble_open_webui_openai_api_key | default('') | length > 0 + - noble_open_webui_webui_secret_key | default('') | length > 0 + - noble_open_webui_public_host | default('') | trim | length > 0 fail_msg: >- Authentik requires secrets in .env (see ansible/roles/noble_authentik/README.md) or matching -e extra-vars. + Includes Open WebUI: NOBLE_AUTHENTIK_CLIENT_SECRET_OPEN_WEBUI, NOBLE_OPEN_WEBUI_OPENAI_API_KEY, + NOBLE_OPEN_WEBUI_WEBUI_SECRET_KEY (see .env.sample). Set **noble_open_webui_public_host** (must match + **clusters/noble/apps/open-webui/values.yaml** ingress host; see README Pangolin section). - name: Require Authentik S3 media settings (same endpoint/keys as Velero; dedicated bucket) ansible.builtin.assert: @@ -566,6 +573,32 @@ no_log: true changed_when: true + - name: Ensure open-webui namespace exists (Secret before Argo first sync) + ansible.builtin.shell: | + set -euo pipefail + kubectl create namespace open-webui --dry-run=client -o yaml | kubectl apply -f - + environment: + KUBECONFIG: "{{ noble_kubeconfig }}" + when: noble_authentik_configure_idp | default(true) | bool + changed_when: true + + - name: Create Open WebUI secrets (OpenAI + WEBUI + OIDC client secret) + ansible.builtin.shell: | + set -euo pipefail + kubectl -n open-webui create secret generic open-webui-secrets \ + --from-literal=OPENAI_API_KEY="${OPENAI_API_KEY}" \ + --from-literal=WEBUI_SECRET_KEY="${WEBUI_SECRET_KEY}" \ + --from-literal=OAUTH_CLIENT_SECRET="${OAUTH_CLIENT_SECRET}" \ + --dry-run=client -o yaml | kubectl apply -f - + environment: + KUBECONFIG: "{{ noble_kubeconfig }}" + OPENAI_API_KEY: "{{ noble_open_webui_openai_api_key }}" + WEBUI_SECRET_KEY: "{{ noble_open_webui_webui_secret_key }}" + OAUTH_CLIENT_SECRET: "{{ noble_authentik_client_secret_open_webui }}" + no_log: true + when: noble_authentik_configure_idp | default(true) | bool + changed_when: true + - name: Create oauth2-proxy credentials Secret (OIDC to Authentik; not BasicAuth) ansible.builtin.shell: | set -euo pipefail diff --git a/ansible/roles/noble_authentik/templates/authentik-clients.json.j2 b/ansible/roles/noble_authentik/templates/authentik-clients.json.j2 index 84594a2..7b3a951 100644 --- a/ansible/roles/noble_authentik/templates/authentik-clients.json.j2 +++ b/ansible/roles/noble_authentik/templates/authentik-clients.json.j2 @@ -22,5 +22,11 @@ "client_id": {{ noble_authentik_client_id_oauth2_proxy | to_json }}, "client_secret": {{ noble_authentik_client_secret_oauth2_proxy | to_json }}, "redirect_uri": "https://{{ noble_authentik_oauth2_proxy_host }}/oauth2/callback" + }, + "open-webui": { + "name": "Open WebUI", + "client_id": {{ noble_authentik_client_id_open_webui | to_json }}, + "client_secret": {{ noble_authentik_client_secret_open_webui | to_json }}, + "redirect_uri": "https://{{ noble_open_webui_public_host | trim }}/oauth/oidc/callback" } } diff --git a/ansible/roles/noble_authentik/templates/authentik-worker-oidc-spec.json.j2 b/ansible/roles/noble_authentik/templates/authentik-worker-oidc-spec.json.j2 index 7f295b6..f350d84 100644 --- a/ansible/roles/noble_authentik/templates/authentik-worker-oidc-spec.json.j2 +++ b/ansible/roles/noble_authentik/templates/authentik-worker-oidc-spec.json.j2 @@ -27,6 +27,12 @@ "client_id": {{ noble_authentik_client_id_oauth2_proxy | to_json }}, "client_secret": {{ noble_authentik_client_secret_oauth2_proxy | to_json }}, "redirect_uri": "https://{{ noble_authentik_oauth2_proxy_host }}/oauth2/callback" + }, + "open-webui": { + "name": "Open WebUI", + "client_id": {{ noble_authentik_client_id_open_webui | to_json }}, + "client_secret": {{ noble_authentik_client_secret_open_webui | to_json }}, + "redirect_uri": "https://{{ noble_open_webui_public_host | trim }}/oauth/oidc/callback" } } } diff --git a/ansible/roles/noble_newt/defaults/main.yml b/ansible/roles/noble_newt/defaults/main.yml index b95cae1..d29881e 100644 --- a/ansible/roles/noble_newt/defaults/main.yml +++ b/ansible/roles/noble_newt/defaults/main.yml @@ -1,3 +1,11 @@ --- # Set true after creating the newt-pangolin-auth Secret (see role / cluster docs). noble_newt_install: true + +# Pangolin Integration API — idempotent HTTP resources + Traefik targets (see clusters/noble/bootstrap/newt/README.md §4). +noble_pangolin_sync_http_resources: false +# Extra FQDNs to sync (in addition to **noble_authentik_ingress_extra_hosts** + **noble_open_webui_public_host** when set). +noble_pangolin_http_fqdns_extra: [] +# Traefik HTTPS backend for Pangolin targets (MetalLB / LAN VIP). Empty → **kubectl** discovers the Traefik Service. +noble_pangolin_traefik_target_ip: "" +noble_pangolin_traefik_target_port: 443 diff --git a/ansible/roles/noble_newt/tasks/main.yml b/ansible/roles/noble_newt/tasks/main.yml index 8f5d529..13bb42b 100644 --- a/ansible/roles/noble_newt/tasks/main.yml +++ b/ansible/roles/noble_newt/tasks/main.yml @@ -3,40 +3,46 @@ ansible.builtin.debug: msg: "noble_newt_install is false — set PANGOLIN_ENDPOINT, NEWT_ID, NEWT_SECRET in repo .env (or create the Secret manually) and set noble_newt_install=true to deploy Newt." when: not (noble_newt_install | bool) + tags: [newt] -- name: Create Newt namespace - ansible.builtin.command: - argv: - - kubectl - - apply - - -f - - "{{ noble_repo_root }}/clusters/noble/bootstrap/newt/namespace.yaml" - environment: - KUBECONFIG: "{{ noble_kubeconfig }}" +- name: Deploy Newt (Pangolin tunnel) and optional Pangolin HTTP resource sync when: noble_newt_install | bool - changed_when: true + tags: [newt, pangolin] + block: + - name: Create Newt namespace + ansible.builtin.command: + argv: + - kubectl + - apply + - -f + - "{{ noble_repo_root }}/clusters/noble/bootstrap/newt/namespace.yaml" + environment: + KUBECONFIG: "{{ noble_kubeconfig }}" + changed_when: true -- name: Apply Newt Pangolin auth Secret from repository .env (optional) - ansible.builtin.include_tasks: from_env.yml - when: noble_newt_install | bool + - name: Apply Newt Pangolin auth Secret from repository .env (optional) + ansible.builtin.include_tasks: from_env.yml -- name: Install Newt chart - ansible.builtin.command: - argv: - - helm - - upgrade - - --install - - newt - - fossorial/newt - - --namespace - - newt - - --version - - "1.5.0" - - -f - - "{{ noble_repo_root }}/clusters/noble/bootstrap/newt/values.yaml" - - --force-conflicts - - --wait - environment: - KUBECONFIG: "{{ noble_kubeconfig }}" - when: noble_newt_install | bool - changed_when: true + - name: Install Newt chart + ansible.builtin.command: + argv: + - helm + - upgrade + - --install + - newt + - fossorial/newt + - --namespace + - newt + - --version + - "1.5.0" + - -f + - "{{ noble_repo_root }}/clusters/noble/bootstrap/newt/values.yaml" + - --force-conflicts + - --wait + environment: + KUBECONFIG: "{{ noble_kubeconfig }}" + changed_when: true + + - name: Optional Pangolin Integration API (HTTP resources + Traefik targets) + ansible.builtin.include_tasks: pangolin_sync.yml + when: noble_pangolin_sync_http_resources | default(false) | bool diff --git a/ansible/roles/noble_newt/tasks/pangolin_sync.yml b/ansible/roles/noble_newt/tasks/pangolin_sync.yml new file mode 100644 index 0000000..5e72ee7 --- /dev/null +++ b/ansible/roles/noble_newt/tasks/pangolin_sync.yml @@ -0,0 +1,95 @@ +--- +# Pangolin Integration API — public HTTP resources → Newt site → Traefik (see clusters/noble/bootstrap/newt/README.md §4). +# Included only when **noble_pangolin_sync_http_resources** is true. +- name: Build Pangolin HTTP FQDN list + ansible.builtin.set_fact: + noble_pangolin_http_fqdns_effective: >- + {{ + ( + noble_pangolin_http_fqdns_extra | default([]) + + (noble_authentik_ingress_extra_hosts | default([])) + + ([noble_open_webui_public_host | trim] if (noble_open_webui_public_host | default('') | trim | length) > 0 else []) + ) | unique | list + }} + +- name: Discover Traefik LoadBalancer IP for Pangolin targets (when not set explicitly) + ansible.builtin.command: + argv: + - kubectl + - get + - svc + - -n + - traefik + - -l + - app.kubernetes.io/name=traefik + - -o + - jsonpath={.items[0].status.loadBalancer.ingress[0].ip} + environment: + KUBECONFIG: "{{ noble_kubeconfig }}" + register: noble_pangolin_traefik_lb_ip + changed_when: false + failed_when: false + when: + - noble_pangolin_http_fqdns_effective | length > 0 + - noble_pangolin_traefik_target_ip | default('') | trim | length == 0 + +- name: Resolve Traefik IP for Pangolin sync + ansible.builtin.set_fact: + noble_pangolin_traefik_ip_resolved: >- + {{ + (noble_pangolin_traefik_target_ip | default('') | trim) + if (noble_pangolin_traefik_target_ip | default('') | trim | length > 0) + else (noble_pangolin_traefik_lb_ip.stdout | default('') | trim) + }} + when: noble_pangolin_http_fqdns_effective | length > 0 + +- name: Require Traefik IP for Pangolin sync + ansible.builtin.assert: + that: + - noble_pangolin_traefik_ip_resolved | length > 0 + fail_msg: >- + Set **noble_pangolin_traefik_target_ip** in inventory (Traefik Service LoadBalancer / MetalLB IP), or ensure + **kubectl** can read **traefik** Services (see **clusters/noble/bootstrap/traefik/**). + when: noble_pangolin_http_fqdns_effective | length > 0 + +- name: Stat repository .env for Pangolin API credentials + ansible.builtin.stat: + path: "{{ noble_repo_root }}/.env" + register: noble_pangolin_env_file + changed_when: false + when: noble_pangolin_http_fqdns_effective | length > 0 + +- name: Require .env for Pangolin Integration API secrets + ansible.builtin.assert: + that: + - noble_pangolin_env_file.stat.exists | default(false) + fail_msg: >- + Pangolin sync needs **.env** at the repo root with **NOBLE_PANGOLIN_*** variables (see **.env.sample**). + when: noble_pangolin_http_fqdns_effective | length > 0 + +- name: Sync Pangolin public HTTP resources (Integration API) + ansible.builtin.command: + argv: + - python3 + - "{{ noble_repo_root }}/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py" + - "--env-file" + - "{{ noble_repo_root }}/.env" + - "--fqdns" + - "{{ noble_pangolin_http_fqdns_effective | join(',') }}" + - "--traefik-ip" + - "{{ noble_pangolin_traefik_ip_resolved }}" + - "--traefik-port" + - "{{ noble_pangolin_traefik_target_port | int | string }}" + register: noble_pangolin_sync_cmd + changed_when: >- + '[create]' in (noble_pangolin_sync_cmd.stdout | default('')) + or '[target]' in (noble_pangolin_sync_cmd.stdout | default('')) + or 'target created' in (noble_pangolin_sync_cmd.stdout | default('')) + when: noble_pangolin_http_fqdns_effective | length > 0 + +- name: Skip Pangolin sync (no public FQDNs configured) + ansible.builtin.debug: + msg: >- + noble_pangolin_sync_http_resources is true but the FQDN list is empty + (set **noble_authentik_ingress_extra_hosts**, **noble_open_webui_public_host**, and/or **noble_pangolin_http_fqdns_extra**). + when: noble_pangolin_http_fqdns_effective | length == 0 diff --git a/clusters/noble/apps/open-webui/kustomization.yaml b/clusters/noble/apps/open-webui/kustomization.yaml index 6f311ba..68604c1 100644 --- a/clusters/noble/apps/open-webui/kustomization.yaml +++ b/clusters/noble/apps/open-webui/kustomization.yaml @@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - namespace.yaml - application.yaml diff --git a/clusters/noble/apps/open-webui/namespace.yaml b/clusters/noble/apps/open-webui/namespace.yaml new file mode 100644 index 0000000..a3dfd26 --- /dev/null +++ b/clusters/noble/apps/open-webui/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: open-webui + labels: + app.kubernetes.io/name: open-webui diff --git a/clusters/noble/apps/open-webui/values.yaml b/clusters/noble/apps/open-webui/values.yaml index 6798375..5a27571 100644 --- a/clusters/noble/apps/open-webui/values.yaml +++ b/clusters/noble/apps/open-webui/values.yaml @@ -1,10 +1,10 @@ # Open WebUI — https://github.com/open-webui/helm-charts (chart **open-webui**). # Ingress: Traefik + cert-manager (same pattern as **`clusters/noble/apps/homepage/values.yaml`**). # -# After sync: set an OpenAI-compatible API key (**`openaiApiKey`** below or **`openaiApiKeyExistingSecret`**), -# or enable in-cluster **Ollama** / **Pipelines** in this file. Chart defaults ship a placeholder key — override before use. -# -# Optional: protect with ForwardAuth like **`clusters/noble/bootstrap/longhorn/values-authentik-forwardauth.yaml`**. +# **Secrets** (**`OPENAI_API_KEY`**, **`WEBUI_SECRET_KEY`**, **`OAUTH_CLIENT_SECRET`**) are created in-cluster by +# **`ansible-playbook … --tags authentik`** as **`open-webui/open-webui-secrets`** (see **noble_authentik** role and **`.env.sample`**). +# **OIDC** uses Authentik provider slug **`open-webui`** (issuer **`…/application/o/open-webui/`**). Do **not** put ForwardAuth on this Ingress while using native OIDC (same pattern as Headlamp). +# **Public host only** (Pangolin → Newt → Traefik): keep **ingress.host**, **OPENID_PROVIDER_URL**, **OPENID_REDIRECT_URI**, and **WEBUI_URL** in sync with **`noble_open_webui_public_host`** in Ansible **group_vars** (see **noble_authentik** README). # ollama: enabled: false @@ -12,7 +12,6 @@ ollama: pipelines: enabled: false -# External Ollama (when not using the subchart), e.g. `http://ollama.ollama.svc.cluster.local:11434` ollamaUrls: [] ingress: @@ -20,7 +19,7 @@ ingress: class: traefik annotations: cert-manager.io/cluster-issuer: letsencrypt-prod - host: open-webui.apps.noble.lab.pcenicni.dev + host: webui.nikflix.ca additionalHosts: [] tls: true existingSecret: "" @@ -28,8 +27,38 @@ ingress: enableOpenaiApi: true openaiBaseApiUrl: "https://api.openai.com/v1" openaiApiKey: "" -# openaiApiKeyExistingSecret: open-webui-openai -# openaiApiKeyExistingSecretKey: api-key +openaiApiKeyExistingSecret: open-webui-secrets +openaiApiKeyExistingSecretKey: OPENAI_API_KEY + +extraEnvVars: + - name: ENABLE_OAUTH_SIGNUP + value: "true" + - name: OAUTH_MERGE_ACCOUNTS_BY_EMAIL + value: "true" + - name: OAUTH_PROVIDER_NAME + value: "Authentik" + - name: OAUTH_CLIENT_ID + value: "open-webui" + - name: OPENID_PROVIDER_URL + value: "https://auth.nikflix.ca/application/o/open-webui/.well-known/openid-configuration" + - name: OAUTH_SCOPES + value: "openid email profile offline_access" + - name: OPENID_REDIRECT_URI + value: "https://webui.nikflix.ca/oauth/oidc/callback" + - name: WEBUI_URL + value: "https://webui.nikflix.ca" + - name: ENABLE_OAUTH_PERSISTENT_CONFIG + value: "false" + - name: WEBUI_SECRET_KEY + valueFrom: + secretKeyRef: + name: open-webui-secrets + key: WEBUI_SECRET_KEY + - name: OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: open-webui-secrets + key: OAUTH_CLIENT_SECRET persistence: enabled: true diff --git a/clusters/noble/bootstrap/newt/README.md b/clusters/noble/bootstrap/newt/README.md index db5e5c3..f6cbdcb 100644 --- a/clusters/noble/bootstrap/newt/README.md +++ b/clusters/noble/bootstrap/newt/README.md @@ -88,6 +88,26 @@ curl -sS -X PUT -H "Authorization: Bearer ${TOKEN}" -H 'Content-Type: applicatio Exact JSON fields and IDs differ by domain type (**ns** vs **cname** vs **wildcard**); see [Common API routes](https://docs.pangolin.net/manage/common-api-routes) and Swagger. +## 4. Automate HTTP resources (Integration API + Ansible) + +You still **link domains** in Pangolin and create **CNAME** records at your DNS host manually (Pangolin does not replace your registrar). After that, this repository can **ensure** public **HTTP** resources and **Traefik** targets exist for the same FQDNs you use in GitOps / Ansible: + +- **`noble_authentik_ingress_extra_hosts`** (e.g. **`auth.example.com`**) +- **`noble_open_webui_public_host`** when set (e.g. **`webui.example.com`**) +- Optional extra list **`noble_pangolin_http_fqdns_extra`** in **`ansible/inventory/group_vars/all.yml`** + +Steps: + +1. In Pangolin, create an **organization API key** with permission to manage domains, resources, and targets ([Integration API](https://docs.pangolin.net/manage/integration-api)). +2. Add to repository **`.env`** (never commit secrets): **`NOBLE_PANGOLIN_API_BASE`**, **`NOBLE_PANGOLIN_ORG_ID`**, **`NOBLE_PANGOLIN_API_TOKEN`**, **`NOBLE_PANGOLIN_SITE_ID`** (numeric site that owns your **Newt** pair). Optionally **`NOBLE_PANGOLIN_TRAEFIK_IP`** / **`NOBLE_PANGOLIN_TRAEFIK_PORT`** — if unset, Ansible uses **`kubectl`** to read the Traefik Service **LoadBalancer** IP. +3. Set **`noble_pangolin_sync_http_resources: true`** in **`ansible/inventory/group_vars/all.yml`** (or pass **`-e noble_pangolin_sync_http_resources=true`**). +4. Run **`ansible-playbook playbooks/noble.yml --tags newt`** (or a full **`noble.yml`**) with **`KUBECONFIG`** pointed at the cluster. + +Implementation: **`clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py`** (stdlib **Python 3**). Dry run: +`python3 clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py --env-file .env --fqdns auth.example.com,webui.example.com --traefik-ip 192.168.50.211 --dry-run` + +The script matches each FQDN to the **longest** linked **`baseDomain`** in Pangolin, creates the HTTP resource if missing, then adds a **target** (**`siteId`** + Traefik **`ip`:`port`**, **`method`:** **`http`**) if none matches. Pangolin’s API is still evolving — if a call fails, compare with [Swagger](https://api.pangolin.net/v1/docs) for your deployment version. + ### Authentik on a public name Use **`noble_authentik_ingress_extra_hosts`** (see **`ansible/roles/noble_authentik/README.md`**) so the Authentik Ingress (and **cert-manager** SANs) include your public FQDN, then create the Pangolin **HTTP** resource + **target** to the same Traefik **:443** endpoint as other apps. One Newt site can carry many hostnames. diff --git a/clusters/noble/bootstrap/newt/scripts/__pycache__/sync_pangolin_http_resources.cpython-314.pyc b/clusters/noble/bootstrap/newt/scripts/__pycache__/sync_pangolin_http_resources.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ef5a9c0f75e5f260a3f09cb2015fd89b9fbf80f GIT binary patch literal 19496 zcmd6PZEzDwmSB}s`u>(>`4e3J!~$#qHeifRV{8M)U`*M_-HkylEZL|Lk|@c1=$Ky5 z%-xPn_l@o8-oU-vy)E|o4inoKc;fbMy?YxM^nAp!aS?lEifd`Az0urlM_kPPIeUR% zr#CL*US_FEvMr!{Vj}huoRwAiGV|rv%lBTsr)_4Fo`CTFhw8V#+D;IEk6xrei3##a z8%Yoo1VhA$a|A>7kVh49MFmPLdX#Y`eo}D?KUHxReyZbY{M5v?_^HEBWsm--A#SK3 z7^;Y1R7FI{8aFa!j2hlF=q*IXO^g=abU4q<=pk*uX$xb7v!?O>dcF2d8EnSWWng5KIMeR^7i#8lvp8k~sH+W*8?FDu_+ET#(qW+(ZOPSGn&AWa)eSflJXy zJP?kDuG1rIz!wkD{l4&6BoqwO(ed-4;0S$maIl|d1F^_BI}(V|+vq{o7Z?p*q~ktz zED(>;SAsq|ejz~jgyVrR))x;(!gNP}PorMn6&Z=O(ih_KXsp%k_QP9aG*f$HI1qPV z@`Zh40e3K4u)!A%>W6%xi}4F=WPI$xw$}5Of@~yw3Hn9X zM_9U{ArQXOFd7U6T=a-9OveK8Ry|Gkoji2B+uPsKcjV;po<1+&>^;;m(7kUbb0GL2Ueenx|G8ShAH5-VJv*Co<9~_CFfiK%>h|b<6pfUAm zguUd8$52~U!l?UnEWpOx;owDgSKvw@6p02{_l3x%Km&Y-4!cJ}1tnKxG@bvw3QE>@RiJ=i1_bh=KwgQVQLdwLV1bGgf*};~`MrKb zXRojo@C7fve-DWXB3)WO(YdTMP5S4CQo7pzuB)B5eOcg`>`WIpZ|$4i_x}FLuCEG; z=9)iw?ZemRn^I-&hh^?xS&P#a`&`X?zlTxjsDj7Onza!BTy5wq(ybvN=mdxPA)mkj4t;zf0Z}j% z5E2#4r-sqGRX|L1EL!J+`cfc%A>wDzPzuU%HYBLxk&7@Z)cJ`2`ZtAH!UDAGEPaFS zi-ZHL7NEb$Av!3J)xk#>$s&|QqJ!rI(*PW8MH#ISAwp0>R;;2ZZM3W?;rUo+Tv5Su zMGYgeqCqJwVW3xZD5WRVj#UGsSQ8WB#{RT=S$mQeTC1&`|7HAedqVyFf(Vp&%RUL_LPZ`+v1Wc#iG zQVajGMBsIey#4V=yNPNdXQUCKUl%HgL5(z)uOcAFcZlSJ&K(K+x(ea7-?8xfVco)Wg=`Wp~B3i zK+*!KcbNqWRV%3chsQ z6koeLRok{y+qO8w*LF{IE$gk*jZ=*&J-wu-(^^B?WJ_BL9~;Z&t|rHmW89|3w9&yi z>sM5iO%L>_rS0L~BQs`>-nC?Ddcdq`&}SWCsC+``4C;x_6^BaIOD>xVrq4~COPQ*d zOx4NGg#yl0oigoyWZJ#xNIOe!1!sfrUragcmz?$UMZ9z8(WErR4>iRw`COTMfx3H*D{0})`#9~sG%Tr>sg{|;yrGPv%Ch4k^N8eIH^ zgAc3&qjoC+Ta7pY6mA`JY#M*$k}nvJed7oPWAQUI1+@57kbyU&& zwJ0`v6r>4*_xTmE9Sp^ z{=6?1=)x@sgrJq$Vi6_#MzNhvZ0U>8AY@+*__G>H5sC(V(#QqS ziy@U5o#<|)j{_W5Nk)N%1sz}p4s@FGVK*8Cg4#&;h9g(Qbhd5!iZ29AW-Fag*M~u- zbb&;Q2r7F7MTin-WYQ1_qo6_Xi<$xI?MN^zsACZ}9`K6?rj9*@pc#ljYDIi2=@%5| z{qW-Vf*=ceNI$5{#?K2X-xb7BqMa-8leBomnvoJKbR__~14K_*KY(ER7XoTMK|IwG z#?rYO-cT{om3B8xbbV=ZOpOpN%Zk{ArLYXqoB)E_C|r)Y-YFq~qh( zl&S8asV;4Iaz(C>n&#qvv@d!2<94pZ#o4z`c79dh1Z~8cnv|anEI9u34X$ACRM-D7 zI-seWZF9{@^5eb9!THL6JM=}_cF{_iL%!Mm7p*8;CM%wDQrJL1#RJU|`n6HdO z%=Mp2y%kt`E5MDiskdT}s(L?dT(JFl;ZF+}Zty$1Qag`vJC8k3d{Na4z)O{JzI2?o zUYS$@>yb9^Uhx0)bvV}#Fl3q;N--lG{690oo${|JNz*9#M5(msflbrtL9#bmXUKoO zccyF3_6J8e%$)q@9}Gm*I2l92-Y?9>hmFQxY%7QOms$hFYj{m0gX4p<9uhC%HD#rT zo5~)ZS|mE4WhgE$b%Y)B2K}N;f+}VQQ1mLcCIUb)7j7QuGC-*bQL`S0Ynq&+LNkndj)SD3l!&dr5Fbm*d`Fr`7V9hE{=@?u%9=V6 z1CY1QF$&cwmhfouiRz$r%y3CLWVRfY0xe|z5;871C17+>2^d*z4rbaH_jV0t`TH!3 z_%FmnhS-P+=&^MkUEUt+q#j50Ik@VFK?KP^ihc*`-Ee`FpH*jh^euk{`skcd&ViD# z^Qc;2ujN8{C}1;kj%>s}F?xK^?wD` z>Xt7CUW>xzQ+q;Rhn@q#Ker3YBi)07PKFRvK~NjS1U*0m!&viy6p?Y@wx2@1iRym& zJ00_l2IAM@q9YQe>ksVSv4cj{##}A~3+NG8Ou|Y5K_83(Zyfi9M*>K13@3v+a1Hb* za5NxJ86P!*5-!EqH&8(=ei_(NlpaG-HHwf58Qr;z1q2<6y)o#3sK~(T3)OjaC<;o* zW1|2YL(HG7ub?vrj2$)a^@T#&>k>8rfS8aB;zK4)S50(+5CKPwwP2zrtv9D_C0N2Q z;BB>>wJsT$KRn;UHFe(ay1(Z^DK~hQ8#u?Ay_3piOW_9>leO~&ytA3N?4G2S4TUrQ zkGeo<@X$~RI@al9Q^#iFkBsGMqYcPEQaV4aAgpkveM+S)wGS<|D5D%_tdtS9!du!| z?OYjetxZ`Ea@K?Mj(f#-ix;-=u2%8G;&~wZmXf*JPaP>s-9t+qD(@7_7ZN(-w0=rI zv-^>@ByF}&UzoZubAvb6q|946^VWIAq!N0F6z6jq-bzD7fNkKT0TJ6zN9L>l&4q>9 zM|FEqokkH^13V_j-))`KJwhF_p)&+Jv7#Y#COArT_;{VEo^D;XmvfsA@%B#6*tt>W zf^{s?TpAKDIXZIH6i7i8%c?07a8bcZittyR6YoMZviNHvt*Zn?bKw>ASSZDBStJ1# z2Sm8>!Qj9L!wZfAS&W6NF@P3>ha5&-#9;#|uR;XFL*~1jk|ZXX>;v%z;)1$|gGWAi zs$Y3DfkgKTok(87sLbhwau{j-aM;ibAX?B1WnQg|%)Or=84C3F*>Ry>GA^fajd?Ha zHjE3%DQm_>=J%s2keuKc1>E4G1*ff)1kF15&3Uq&Ag-yt&CnBR`WE9BpRhoBhs zv*%Gl3H7jE1o#$2F{G*x6|f?^?2rhzS2SG^^&vk5D7@ewKmzn3Kn04Xie}1r-R6ly z%NpBE)dwRfP3c2T=^XpXwGXd-lK8`f#2$0nN>D9KZ<*RMvyInp211y5tLOD)DSaiU zubgAy{;q2K{J@{pq^kBltlB$SAlCm#w|Q~!l`S9&VZOP4t~zP|$OX*sWG^B+BOf)T z3>6Oz70U&c$$@0|yluXDzAowJ3wCkFUB5C|vfDGmkr2PF-iEyd!y>Ymr%(#d17=V&nOmk(AiNXp`ZM194> z2D=tIy=*F*>*7t76NkSvIYE50Sf|IQ#%Da!-@ExeB&nIJ;9}QY3#XI8{Nwg?f#c@U zRRduzT7@&nx^W$#sl!8Bk86La{!W2zUa=UW6;46`%;BjkrxnfdHR7cky!dr?4qJM zggd(k80FBe4KfFn!`V_V=E6lw3J(yvHSjM!5_V%kOXX8HWi29d4jovoSdz$JLRRPH zlz_1jC5dW~CScjB*|%ZDzR}A(t~;T1XUw$N5M;q+oM^PghT#n~w`*Bb_A-2BSrj4q zi>WiLc)Tj!Lm#S91P8WwN0wReK!U{uZ^YcB)WQsa;YIK-AOUpOR5Q;RwW(`! zI^I&1vb1xS_N4#Lh1(bAukv+!#Se?SIBk2{V4CRJxJA&E;bpZQWA@T5f=m-AM$721 zd4fK3*AT~ym!OG(e1PP1v_S+#Fe*y#R21|;Sj&y<6(XTm643=H9s@cA!}Od`k?K_x z%>X|JkNFsWEGqHiK5QWP_@Ehsh+U(gpcG^#&4_nQu6jGK8dKgo8&}miInP7397PsQ z5_$CxiR&)AuV81wfUXbO3;S$Yk?0~zMoCKZDAv^ugbS_eSK=^qS&7xi1$&1VG&pijWW9M6_>hMT;<-3>X8LI zB?8YP+B_4)*IL75$8`5p_e}k}CsRh6GtzUHd2MBKXi2;MIib*PUqR8b(K18cQq8Jn z{O=o5)=JI_>;w;+VRLC*w=TZE*3cQg2-2=#<54 z9HXU#Cr0K6 zR)X*q`$~MJaM;V)`3+®a|m&2TwU#O9+(S{FovM9EMBbRhYma;SU1Ot-UD<)zec z%Ox@UU=)*KUS8Njjuz)zx1jq@E$Fi+)VFTI@SR#P{uV8mzEcb4-=YOe{uT!F&$M;e zEX`~7YUj6T)yCMR+22TIm;#B0F^;_%`uD<>QOG!%BBuD2a3xGB*q@iLfwM_{$v&Gi z$tZ9<{UCg<4-&2JCjCD^^fap<%MUO0q+d2M%GM~g1#CFjkzR#8z*c({{YVA1E-;&< zdZhL}6Rv(Mj2i#e14S_En=257NLN@t(HlNNI* z1GFfwWN5J_+me*}owS%kDWFApZ@^V|-nG-DV04g^XO1Dq06DaTUG@n%CdkP%W{_in zoIHJNQZ%XHYMFw|Wo09}T&~aTrOsh#v*a3oy0WcFDQO={;2GQQIuKx~K1J_Vafd`z zQILMJmehXe8SMXUQZ3A8rlP+=ddogNvV;Mu9hp0W3uCgsF$*Q7`gcmH?31CT-4Fd# zk1KF5L5pyCwEtW5CPOK}tYH{g65-&HZ(N{%&}~EkqHv#-V`Lzz1hc)XlyJB}=8S9J znU^5@cT4SMpG=jM3%4}VfFtUYXPropFY6^3Z|2rQXIqj|->MdU&S{ogU!IltZR(R? zBE@U4%d>hV$d~moQj*0EMhc8t)(5GUNU28)(t5QAj1!m|kk$>u`y|-m{Zfjl9s0`+ z(z+4u%}m2^4vE2I96lhWCF(5=jChMQXpEXp>|9gg;0sG&8Qf#c%L&#=*ym6wIQ27k zdAT^5UX0Ti(A+he-gs}gQ^Id(%A=P?AZqq#tVFls@V^38m}R(IhWWozztPDY zwMK2jN94T!BjrWy9{cG3LTxiHW~;}7_BgW*c#`c*1JmfSx+#w>V+QW_*uV^2$+($% zH`z%>3x<27`j{Oa3&NJ!>9In*%VWd1%exe~AF}_L1f6}#GlIvf*AII<-+VXatcvDc zlt=F=Xr{zkq_K}WvTH17%{ubV*zW9!E|q`T`rSCMSvfLV=qZ$Xv0)DPcpMw%Fgs>@ zJ%zax6~@TienqH6TTU_mlI46RW8YoUntl$P3F$V<*)i`jO@A3_j(CcoZ0=0H zoC3gpE(a>s3rfWtN-ysGqqxQ%yz`5soe!%&yO)kh{i;B9Oo|*qPJVATpQ+CnvMHm= zC>4~9+SFjwb{s5h4d6+My$8WvHKEUXh$eRR``B23?(Ke~9UFrot1~bztB*vHXR9xi zekp>SOh(eX5) zX=o6=pNiMzcSs(n30*@2(r%$joa=xxuBZtk)FOKBHh^DS$vP;>`?RM&Va@^~MW#^p z0#M|rDk#B0HTa)FJ%jE{;@}6uCSj=~l#w}bY%p=O6Z~_1;LZuQc;H^_m)wf6|0-R7 z3EVqH1HKnn_r<_saG;|P3@H?X7CBcSM^^TaQ6tc4KYP7_9S6GgFGU}=~c0IZQ`FCuic-~o-zqOmF-TLvaJ zoel@CMFS&fU^2PL*E$M_2U_VqfkL*D33V7bztZgQqlUCmHWH7FL_&fxJ`!a?7fZ0e zhw8VV5k0ui%HG;^LZ7L%$KReXWX&?c(xw0OU_vKf6bO3hzNg2ZXy_C@!KF_!M)W0l z6hwFM9zR%0T4}oBfZQgu#J+>j*~Dag#`Bl5p9x$8J1$|~Od0{UZr8L?u;-v^ZF)<-(igj&c0W`<# z0ig)Zeq&oG0;@1~pi$1)0ox~50r>vouK`)keEM&O9+2<7Gv`Yd&2L=@r7WGCrSrcI z{RZoxe^|DeJe+XgF&CB4^vPZtZT@x^T_Ff$D?10ztm+#3I3B`0Nc4 z&HfmQB{Z1}qeMY&6XMG9CsbWv6x-->30k;Ef*lQX&fLL{jU!ikOgC9U;{|io5wBM; z`uu(`fZ1(hJM7_z6m;zeHi*hv;&QL#uJ1-m;C&rr+t zc2HPL{~u9*(aehm1Z^mF1{G9cBE%M>59n|f@4NmJN*zT}7m6;U2-^h|;LR4G!oAJA*QnzVLNdnV|4R1qyY6-GmAw5m6w+f+n+F1o9gD zbCdyRUXBeH)ET+tMb52;%+~jUD>0@EIhV{jF%R7`plq4~pxQ!WPCnfMWIj|5n3AB3w@EY$+Z6P)C0@^hmdR z;!v7W|LElRPtF|Yww#$b`G^{pKYUhoZ|mKy_Zsgua{Cx=(8F&XLIAHz0LMO4+|%FH z-!tDeb9?)^eul3fKmdahK+mV_oy6_Lop)}(!|mzkm_dHaX$0Vr0D3>O-z&XadT;aH z&0GrxaIFRC`|QxYV|S0;>$}^>?e}o6zsYZ3gJfD;ylk>ATMIrobnDpcv0Ht!eO&dy z`?b8|Xv%StbDVt8$y@tZG=$3Wlz>~66+NNSpzfx%wzRDwe)PTXe{b1Qk)-aJZi9_=W6IIIQmO*C2MWc_l0%as){Jw3qF4(n?5mpXi61(xMEN4V4UIxPV+9$B=w#N zIrUwf1(&{l&e{JdbLY+5Z{9g~`yAKY`+(tV`vJs|E-bnA=IooP!c$z~sZV#^X}#Tg z=fLd)+^*vf&hyo$07RcID!X-K_C~5`fGZlvofkZDTLu8em@eJ?iTgu$s&tqu9sZ2D z_vYO<@147Qj%yp@-Z;ZI3M$o)vp1NncYvPQ1(xw`?r@eUlBG|B`O=~-+68$ zboxnhRRP&6DneykB>?)llh9hQGk_X{N5gJ&oQo~mp32hw8tOUZ%2zU zRo}8y-?CWnsJ@*y_Ab@8CwKmJ(dR{fUGm7Z`Mq;<9bcF>!^eqk&e;2v!HQ}uY2?jr zPV4^KVxMWg)jrz}HuNb={gR~~tQ;*36Gvfz%{XqA&6e?|O`_K&Z>pU*jBffUG^^-3 zuyV;*3D^1{ zCN;i=D>=%WdpK>+*KiTpvT&NW?w^2kAsBd8G|c_pc_C0iW&GaTq zmI}5#*w4K_%pE(sVt~BKE|__fopaPB<8aUa_F~DR;r{*y*Z<}cXCHoKJPS4g(>JDX zOx{2)2+s0lYgyV+k}j&4c{`~}jwWB9e}i-EMV}nSPjo7q{fSajWS%_o)JhaMKNy+Y z^2xRjwuu?a0UE!J}OgO7|IkCAB^Ha$Je+mCR@BjSP^xivOBHW!*d#5>(! zagnm`S+eip?JX&LJ7;fS?BLA@CY4`V?5Jc#+v31q9N{)~@pjk--Cx@`&t01@m=Eyw z=1I+02J^d3Um$*kH=mwvnr^x&zoiVP9~w?$w7|1?UI!k(tqa>1efN!v?{N0MN5+$a z(ec>mm>Elh*TvSf(LQq=9GquH(E#ZuRZoqC&dgc2%qx@Q^G6p>FC5{_ZI6&ihh@5L zx^1!ziZ5#mz~(k>DVltHPM>T_)+VoVMmPFov^~|Sbb7H8b9G8vv)Zj7Y|f`g6@*GZ zx#jo!e)D8N38dmT|4>etj+6MOE&f(f{nj4!-)>by{NHvLz12o>HVedp)(iLi!7%)C z%zuF6(N)U+6+8sJ*E>26zjK3rQ3e!m1pAjLLTYjLzo2|&?0QTviq7ke*sqpF3Vgww z`IK=K2P?AcQ4l0VKV!rNid+fiPcYBoN8d14@@a8!2TUn01PM*`w2wg6ho4$l!2t4 z6%%Cj-xHQE3B#9!?n^@dB~kDtQTTU+^O+icBjJG^!MN+01I$&PS&@~%$~lUljFX8+%90g@Qsw-W*7##J9BowTKce5) t*z}YjRZj4HpVr>g&J;|8M<=|0gI#YvZ_;!eAU;1(a;QxCvr-b`{|Bs(opk^J literal 0 HcmV?d00001 diff --git a/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py b/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py new file mode 100755 index 0000000..9669a52 --- /dev/null +++ b/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +Idempotently create Pangolin public HTTP resources + Traefik targets via the Integration API. + +Docs: https://docs.pangolin.net/manage/integration-api +Walkthrough: https://docs.pangolin.net/manage/common-api-routes + +Environment (or --env-file) can set: + NOBLE_PANGOLIN_API_BASE, NOBLE_PANGOLIN_ORG_ID, NOBLE_PANGOLIN_API_TOKEN, + NOBLE_PANGOLIN_SITE_ID, NOBLE_PANGOLIN_TRAEFIK_IP, NOBLE_PANGOLIN_TRAEFIK_PORT (default 443) + +CLI overrides env. FQDNs: --fqdns a.example.com,b.example.com (required). +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from typing import Any + + +def load_dotenv(path: str) -> dict[str, str]: + out: dict[str, str] = {} + with open(path, encoding="utf-8") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k: + out[k] = v + return out + + +def api_request( + method: str, + url: str, + token: str, + body: dict[str, Any] | None = None, +) -> dict[str, Any]: + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request( + url, + data=data, + method=method, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + **({"Content-Type": "application/json"} if body is not None else {}), + }, + ) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + payload = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", errors="replace") + raise SystemExit(f"HTTP {e.code} {method} {url}\n{detail}") from e + if not payload: + return {} + return json.loads(payload) + + +def unwrap(resp: dict[str, Any]) -> Any: + if resp.get("error") or resp.get("success") is False: + raise SystemExit(f"Pangolin API error: {resp!r}") + return resp.get("data") + + +def normalize_fqdn(s: str) -> str: + return s.lower().strip().rstrip(".") + + +def resolve_domain( + fqdn: str, domains: list[dict[str, Any]] +) -> tuple[str, str | None]: + """Return (domainId, subdomain) for an HTTP resource. subdomain is None for apex / cname-only domains.""" + fqdn = normalize_fqdn(fqdn) + best: tuple[str, str | None, int] | None = None + for d in domains: + bd = (d.get("baseDomain") or "").strip() + if not bd: + continue + bd_n = normalize_fqdn(bd) + if fqdn == bd_n: + cand = (str(d["domainId"]), None, len(bd_n)) + elif fqdn.endswith("." + bd_n): + sub = fqdn[: -(len(bd_n) + 1)] + if not sub: + continue + cand = (str(d["domainId"]), sub, len(bd_n)) + else: + continue + if best is None or cand[2] > best[2]: + best = cand + if best is None: + avail = ", ".join(sorted({normalize_fqdn(d.get("baseDomain") or "") for d in domains if d.get("baseDomain")})) + raise SystemExit( + f"No linked Pangolin domain matches FQDN {fqdn!r}. " + f"Link the parent domain in Pangolin first. Known baseDomain values: {avail or '(none)'}" + ) + return best[0], best[1] + + +def list_all_resources(api_base: str, org_id: str, token: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + page = 1 + page_size = 100 + while True: + url = f"{api_base.rstrip('/')}/org/{org_id}/resources?page={page}&pageSize={page_size}" + data = unwrap(api_request("GET", url, token)) + if isinstance(data, list): + out.extend(data) + break + if not isinstance(data, dict): + break + batch = data.get("resources") or data.get("items") or [] + if not isinstance(batch, list): + break + out.extend(batch) + pag = data.get("pagination") or {} + total = pag.get("total") + if isinstance(total, int) and len(out) >= total: + break + if len(batch) < page_size: + break + page += 1 + if page > 500: + raise SystemExit("Pagination safety stop (>500 pages)") + return out + + +def resource_public_fqdn(res: dict[str, Any]) -> str | None: + fd = res.get("fullDomain") + if isinstance(fd, str) and fd.strip(): + return normalize_fqdn(fd) + sub = res.get("subdomain") + dom = res.get("domain") or {} + bd = dom.get("baseDomain") if isinstance(dom, dict) else None + if isinstance(bd, str) and bd.strip(): + bd_n = normalize_fqdn(bd) + if sub is None or sub == "": + return bd_n + if isinstance(sub, str): + return normalize_fqdn(f"{sub}.{bd_n}") + return None + + +def find_resource_for_fqdn(resources: list[dict[str, Any]], fqdn: str) -> dict[str, Any] | None: + target = normalize_fqdn(fqdn) + for r in resources: + if not r.get("http"): + continue + got = resource_public_fqdn(r) + if got == target: + return r + return None + + +def list_targets(api_base: str, resource_id: int, token: str) -> list[dict[str, Any]]: + url = f"{api_base.rstrip('/')}/resource/{resource_id}/targets" + data = unwrap(api_request("GET", url, token)) + if isinstance(data, list): + return data + if isinstance(data, dict): + return data.get("targets") or data.get("items") or [] + return [] + + +def target_matches(t: dict[str, Any], site_id: int, ip: str, port: int) -> bool: + return ( + int(t.get("siteId") or 0) == site_id + and str(t.get("ip") or "") == ip + and int(t.get("port") or 0) == port + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--env-file", help="Parse KEY=value lines (optional overrides for env)") + ap.add_argument("--api-base", default=os.environ.get("NOBLE_PANGOLIN_API_BASE", "")) + ap.add_argument("--org-id", default=os.environ.get("NOBLE_PANGOLIN_ORG_ID", "")) + ap.add_argument("--token", default=os.environ.get("NOBLE_PANGOLIN_API_TOKEN", "")) + ap.add_argument("--site-id", default=os.environ.get("NOBLE_PANGOLIN_SITE_ID", "")) + ap.add_argument("--traefik-ip", default=os.environ.get("NOBLE_PANGOLIN_TRAEFIK_IP", "")) + ap.add_argument( + "--traefik-port", + type=int, + default=int(os.environ.get("NOBLE_PANGOLIN_TRAEFIK_PORT", "443") or 443), + ) + ap.add_argument( + "--fqdns", + required=True, + help="Comma-separated public FQDNs (must match linked Pangolin domains / SANs)", + ) + ap.add_argument( + "--dry-run", + action="store_true", + help="Print actions only; do not call mutating endpoints", + ) + args = ap.parse_args() + + if args.env_file: + for k, v in load_dotenv(args.env_file).items(): + if k.startswith("NOBLE_PANGOLIN_"): + os.environ.setdefault(k, v) + args.api_base = args.api_base or os.environ.get("NOBLE_PANGOLIN_API_BASE", "") + args.org_id = args.org_id or os.environ.get("NOBLE_PANGOLIN_ORG_ID", "") + args.token = args.token or os.environ.get("NOBLE_PANGOLIN_API_TOKEN", "") + args.site_id = args.site_id or os.environ.get("NOBLE_PANGOLIN_SITE_ID", "") + args.traefik_ip = args.traefik_ip or os.environ.get("NOBLE_PANGOLIN_TRAEFIK_IP", "") + + missing = [ + n + for n, v in [ + ("NOBLE_PANGOLIN_API_BASE", args.api_base), + ("NOBLE_PANGOLIN_ORG_ID", args.org_id), + ("NOBLE_PANGOLIN_API_TOKEN", args.token), + ("NOBLE_PANGOLIN_SITE_ID", args.site_id), + ("NOBLE_PANGOLIN_TRAEFIK_IP", args.traefik_ip), + ] + if not str(v).strip() + ] + if missing: + raise SystemExit(f"Missing required settings: {', '.join(missing)}") + + fqdns = [normalize_fqdn(x) for x in args.fqdns.split(",") if x.strip()] + if not fqdns: + raise SystemExit("No FQDNs in --fqdns") + + api_base = str(args.api_base).rstrip("/") + org_id = str(args.org_id).strip() + token = str(args.token).strip() + site_id = int(str(args.site_id).strip()) + traefik_ip = str(args.traefik_ip).strip() + traefik_port = int(args.traefik_port) + + dom_url = f"{api_base}/org/{org_id}/domains" + domains_raw = unwrap(api_request("GET", dom_url, token)) + domains: list[dict[str, Any]] = [] + if isinstance(domains_raw, list): + domains = domains_raw + elif isinstance(domains_raw, dict): + domains = domains_raw.get("domains") or [] + if not isinstance(domains, list): + raise SystemExit(f"Unexpected domains response: {domains_raw!r}") + + resources = list_all_resources(api_base, org_id, token) + + for fqdn in fqdns: + domain_id, subdomain = resolve_domain(fqdn, domains) + res = find_resource_for_fqdn(resources, fqdn) + if res is None: + body = { + "name": f"noble {fqdn}", + "http": True, + "protocol": "tcp", + "domainId": domain_id, + **({"subdomain": subdomain} if subdomain is not None else {"subdomain": None}), + } + print(f"[create] resource {fqdn!r} domainId={domain_id} subdomain={subdomain!r}") + if args.dry_run: + continue + created = unwrap( + api_request( + "PUT", + f"{api_base}/org/{org_id}/resource", + token, + body, + ) + ) + rid = int(str(created.get("resourceId", "")).strip() or 0) + if not rid: + raise SystemExit(f"Create resource response missing resourceId: {created!r}") + print(f" -> resourceId={rid}") + resources.append( + { + "resourceId": rid, + "http": True, + "fullDomain": fqdn, + "subdomain": subdomain, + } + ) + res = find_resource_for_fqdn(resources, fqdn) + else: + rid = int(str(res.get("resourceId", "")).strip() or 0) + if not rid: + raise SystemExit(f"Resource missing resourceId: {res!r}") + print(f"[exists] resource {fqdn!r} resourceId={rid}") + + if res is None: + raise SystemExit("internal: resource missing after create") + rid = int(str(res.get("resourceId", "")).strip() or 0) + if not rid: + raise SystemExit(f"Resource missing resourceId: {res!r}") + + targets = list_targets(api_base, rid, token) + if any(target_matches(t, site_id, traefik_ip, traefik_port) for t in targets): + print(f" -> target OK site={site_id} {traefik_ip}:{traefik_port}") + continue + tbody = { + "siteId": site_id, + "ip": traefik_ip, + "port": traefik_port, + "method": "http", + } + print(f"[target] PUT /resource/{rid}/target {tbody}") + if args.dry_run: + continue + api_request( + "PUT", + f"{api_base}/resource/{rid}/target", + token, + tbody, + ) + print(" -> target created") + + print("Done.") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130)