From 6e76a400b62cd03e03f969f8bb529bd7c14ab4ab Mon Sep 17 00:00:00 2001 From: Nikholas Pcenicni <82239765+nikpcenicni@users.noreply.github.com> Date: Fri, 15 May 2026 01:10:51 -0400 Subject: [PATCH] Update .env.sample and Ansible configurations to enhance Pangolin Integration API setup. Add detailed comments for environment variables and clarify usage in README. Implement HTTP-01 challenge support in cert-manager configurations for Let's Encrypt, ensuring proper resource management for domain validation. --- .env.sample | 10 +- ansible/inventory/group_vars/all.yml | 1 + .../clusterissuer-letsencrypt-prod.yaml | 8 +- .../clusterissuer-letsencrypt-staging.yaml | 3 + clusters/noble/bootstrap/newt/README.md | 39 ++- ...nc_pangolin_http_resources.cpython-314.pyc | Bin 19496 -> 33839 bytes .../scripts/sync_pangolin_http_resources.py | 290 ++++++++++++++++-- 7 files changed, 318 insertions(+), 33 deletions(-) diff --git a/.env.sample b/.env.sample index c08c58a..c561f38 100644 --- a/.env.sample +++ b/.env.sample @@ -13,12 +13,16 @@ NEWT_ID= NEWT_SECRET= # Optional: Pangolin Integration API — automate public HTTP resources + Traefik targets (**noble_pangolin_sync_http_resources=true** in **group_vars**; see **clusters/noble/bootstrap/newt/README.md** §4). -# NOBLE_PANGOLIN_API_BASE=https://api.your-pangolin.example/v1 +# NOBLE_PANGOLIN_API_BASE=https://api.example.com/v1 # Integration API — separate host from the main Pangolin UI; see clusters/noble/bootstrap/newt/README.md §4 # NOBLE_PANGOLIN_ORG_ID= -# NOBLE_PANGOLIN_API_TOKEN= -# NOBLE_PANGOLIN_SITE_ID= +# NOBLE_PANGOLIN_API_TOKEN= # **apiKeyId.apiKeySecret** (one value, dot in the middle) from Organization → API keys — **not** login password; browser cookies do not apply. Alternatively: secret only here + **NOBLE_PANGOLIN_API_KEY_ID** below. +# NOBLE_PANGOLIN_API_KEY_ID= # optional; if set, **NOBLE_PANGOLIN_API_TOKEN** may be the secret half only +# NOBLE_PANGOLIN_SITE_ID= # numeric siteId, or Pangolin **niceId** (Sites UI slug, e.g. unruly-asian-badger) # NOBLE_PANGOLIN_TRAEFIK_IP=192.168.50.211 # NOBLE_PANGOLIN_TRAEFIK_PORT=443 +# Self-signed Integration API TLS: either trust your CA (preferred) or homelab-only skip verify: +# NOBLE_PANGOLIN_CA_BUNDLE=/path/to/ca.pem +# NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY=true # Velero — when **noble_velero_install=true**, set bucket + S3 API URL and credentials (see clusters/noble/bootstrap/velero/README.md). NOBLE_VELERO_S3_BUCKET= diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index f7a7780..7153e15 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -15,6 +15,7 @@ noble_skip_k8s_health_check: false # Pangolin / Newt — set true only after newt-pangolin-auth Secret exists (SOPS: clusters/noble/secrets/ or imperative — see clusters/noble/bootstrap/newt/README.md) noble_newt_install: true +noble_pangolin_sync_http_resources: true # cert-manager needs Secret cloudflare-dns-api-token in cert-manager namespace before ClusterIssuers work noble_cert_manager_require_cloudflare_secret: true diff --git a/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-prod.yaml b/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-prod.yaml index 65fcb9e..fd22692 100644 --- a/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-prod.yaml +++ b/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-prod.yaml @@ -11,8 +11,7 @@ spec: privateKeySecretRef: name: letsencrypt-prod-account-key solvers: - # DNS-01 — works when public HTTP to Traefik is wrong (e.g. hostname proxied through Cloudflare - # returns 404 for /.well-known/acme-challenge). Requires Secret cloudflare-dns-api-token in cert-manager. + # DNS-01 — Cloudflare token covers pcenicni.dev only. Requires Secret cloudflare-dns-api-token in cert-manager. - dns01: cloudflare: apiTokenSecretRef: @@ -21,3 +20,8 @@ spec: selector: dnsZones: - pcenicni.dev + # HTTP-01 fallback — used for all other zones (e.g. nikflix.ca via Pangolin → Newt → Traefik). + # Requires a Pangolin HTTP resource + target for each hostname before LE can reach /.well-known/acme-challenge/. + - http01: + ingress: + ingressClassName: traefik diff --git a/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-staging.yaml b/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-staging.yaml index 5c0c53f..317b4a7 100644 --- a/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-staging.yaml +++ b/clusters/noble/bootstrap/cert-manager/clusterissuer-letsencrypt-staging.yaml @@ -19,3 +19,6 @@ spec: selector: dnsZones: - pcenicni.dev + - http01: + ingress: + ingressClassName: traefik diff --git a/clusters/noble/bootstrap/newt/README.md b/clusters/noble/bootstrap/newt/README.md index f6cbdcb..525e710 100644 --- a/clusters/noble/bootstrap/newt/README.md +++ b/clusters/noble/bootstrap/newt/README.md @@ -98,8 +98,41 @@ You still **link domains** in Pangolin and create **CNAME** records at your DNS Steps: -1. In Pangolin, create an **organization API key** with permission to manage domains, resources, and targets ([Integration API](https://docs.pangolin.net/manage/integration-api)). -2. Add to repository **`.env`** (never commit secrets): **`NOBLE_PANGOLIN_API_BASE`**, **`NOBLE_PANGOLIN_ORG_ID`**, **`NOBLE_PANGOLIN_API_TOKEN`**, **`NOBLE_PANGOLIN_SITE_ID`** (numeric site that owns your **Newt** pair). Optionally **`NOBLE_PANGOLIN_TRAEFIK_IP`** / **`NOBLE_PANGOLIN_TRAEFIK_PORT`** — if unset, Ansible uses **`kubectl`** to read the Traefik Service **LoadBalancer** IP. +0. **Enable the Integration API** in Pangolin’s `config.yml` on the Pangolin host — it is **off by default**. Add to `config.yml`: + + ```yaml + flags: + enable_integration_api: true + server: + integration_port: 3003 # default; omit to keep 3003 + ``` + + Then expose it with a **Traefik route** in `config/traefik/dynamic_config.yml`. The Integration API is a *separate* process from the main Pangolin server and needs its own hostname (e.g. `api.pcenicni.dev`): + + ```yaml + routers: + int-api-router: + rule: "Host(`api.pcenicni.dev`)" + service: int-api-service + entryPoints: [websecure] + tls: { certResolver: letsencrypt } + int-api-router-redirect: + rule: "Host(`api.pcenicni.dev`)" + service: int-api-service + entryPoints: [web] + middlewares: [redirect-to-https] + services: + int-api-service: + loadBalancer: + servers: [{ url: "http://pangolin:3003" }] + ``` + + After restarting Pangolin, verify: `curl https://api.pcenicni.dev/v1/` should return `{"message":"Pangolin Integration API"}`. Also add a **CNAME** for `api.pcenicni.dev` pointing at the same upstream as `pangolin.pcenicni.dev`. + + > **Common mistake:** `https://pangolin.pcenicni.dev/api/v1` is the session-based **external API** — it will always return **401** to Bearer tokens. The Integration API must have its own Traefik-exposed hostname. + +1. In Pangolin, create an **organization API key** ([Integration API docs](https://docs.pangolin.net/self-host/advanced/integration-api)) with permission to manage domains, resources, and targets. The API expects **`Authorization: Bearer {apiKeyId}.{apiKeySecret}`** — paste **`id.secret`** as a single string into **`NOBLE_PANGOLIN_API_TOKEN`**, or set **`NOBLE_PANGOLIN_API_KEY_ID`** + only the **secret** in **`NOBLE_PANGOLIN_API_TOKEN`**. For **`NOBLE_PANGOLIN_SITE_ID`** as a **niceId** slug, enable **Site → List Sites** (the script falls back to listing sites if **Get Site** returns **404**). +2. Add to repository **`.env`** (never commit secrets): **`NOBLE_PANGOLIN_API_BASE`** is the Integration API hostname with `/v1` suffix — e.g. **`https://api.pcenicni.dev/v1`** (not `https://pangolin.pcenicni.dev/api/v1`). Also set **`NOBLE_PANGOLIN_ORG_ID`**, **`NOBLE_PANGOLIN_API_TOKEN`** (optionally **`NOBLE_PANGOLIN_API_KEY_ID`**), **`NOBLE_PANGOLIN_SITE_ID`** (numeric **siteId** *or* **niceId**). Optionally **`NOBLE_PANGOLIN_TRAEFIK_IP`** / **`NOBLE_PANGOLIN_TRAEFIK_PORT`** — if unset, Ansible uses **`kubectl`** to read the Traefik Service **LoadBalancer** IP. TLS: **`NOBLE_PANGOLIN_CA_BUNDLE`** or **`NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY=true`** for self-signed APIs 3. Set **`noble_pangolin_sync_http_resources: true`** in **`ansible/inventory/group_vars/all.yml`** (or pass **`-e noble_pangolin_sync_http_resources=true`**). 4. Run **`ansible-playbook playbooks/noble.yml --tags newt`** (or a full **`noble.yml`**) with **`KUBECONFIG`** pointed at the cluster. @@ -108,6 +141,8 @@ Implementation: **`clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_reso The script matches each FQDN to the **longest** linked **`baseDomain`** in Pangolin, creates the HTTP resource if missing, then adds a **target** (**`siteId`** + Traefik **`ip`:`port`**, **`method`:** **`http`**) if none matches. Pangolin’s API is still evolving — if a call fails, compare with [Swagger](https://api.pangolin.net/v1/docs) for your deployment version. +**`.env` vs shell:** If **`NOBLE_PANGOLIN_API_TOKEN`** (or other **`NOBLE_PANGOLIN_*`**) is set in your shell to an empty or old value, older script versions could ignore **`.env`**. Current script overwrites **`os.environ`** from **`.env`** when **`--env-file`** is passed — unset stray exports if you still see **401**. + ### Authentik on a public name Use **`noble_authentik_ingress_extra_hosts`** (see **`ansible/roles/noble_authentik/README.md`**) so the Authentik Ingress (and **cert-manager** SANs) include your public FQDN, then create the Pangolin **HTTP** resource + **target** to the same Traefik **:443** endpoint as other apps. One Newt site can carry many hostnames. diff --git a/clusters/noble/bootstrap/newt/scripts/__pycache__/sync_pangolin_http_resources.cpython-314.pyc b/clusters/noble/bootstrap/newt/scripts/__pycache__/sync_pangolin_http_resources.cpython-314.pyc index 4ef5a9c0f75e5f260a3f09cb2015fd89b9fbf80f..44cd6f784181bfcdc7d9a85626ac08f0fe4fa8b7 100644 GIT binary patch literal 33839 zcmeIbdvFw2nlG63em_)tp?B(uBv1(u<|!~j5+DRZPKn|r5QVBF)Cj4ZRWcwyM%~jf zE!mzc(e@4Gx$dz%)3X-SzKeQyBAVWr8;xf?V`lr_*c7F9sjIbNcCKS1dSm_&1B7Qh z5gYsaPF7|Wq6BW=yRmm8ZXz;IWuEu>&Ue1o`OXHX!^+{x{mA&8bH_REKhqE8(#rwO z!yjunZk!Xi2zQPXG;Nwg+K9G=MifTUgjFSP{0daE4$<*ulacp+<1xmy3Q4 zX(E~U?Z)qn#P2M@gI}2}Znltxa5f9)2ssGnvT&}j4&gi&UMJ)uT)@J4LLtIMESxVC zBV5A51p<$7DGL`0WeAtEaFI}fa3u>D3snfOXW^2wyimR4B*z7899PH{aA!+OxvhzQ zM(efZ>_5+)Ej4n1T#8e|oi%l~C_|BamU8F5w-w_-R|!|d(hYLEbw%8urlxjuTE}tL zwm|T5Xe1IGjtpJn`^2C>668Dm!viBjgTs7i^uo|!AAhL3yOS4#;gM0XFBs-G@ZF+6 z*gtrQkNCxbU?j}HGU(?c7lVA;a3nY&`XhrQ!+cX`o7ZY>8R-k};x9%bq42Ky`T%}; zLy6qI!@)@XW&f~$AXq<`Ojzd+4O&n5hb~1fiX)=~7r#O3zLCq9M}{e39g;+XVXL)u z_?1C%WcV`L#aEArd|h2|_?5c;!J%Lc-{&9Z!@(599Chmq)`9{z8zitwmkhgV)*uUOCVeMAt@Y zYf+csARms1gTn(G`M^koM=w*)T^i7%~uD4 z{r=IR2)}h}Lk+s)Sco9XKg4$*>Dsk=XquaP_aEaPnA4h#nat3g`s zZW`;sVRZMX7_0%5Hy>%^M_vhv;$Q&t1CZwB54_aU0m!eb>wh^g9OnJr;8p+S&`{6| zVBL5j^%q|)245Z>6oUa|Rila9{KLZ|5jKg!9r&$l8ot&?vp6x6Il{G=x|)YP0^^*I zJEz_8ec%hcE973%T+|2}K`ZD4eLZkRjp4UiM2qXGbK*LHUfc*!92JMhoPoi<$QiuZ zNDz7UCWpd|{UhRKeQhiJCmEh1wC@9ubH`U=i8eHEu1dwM@ zMlK8m>o1IqM39$1R6iWN5~&aOiG!g?xITPsxUW~4_q~J-dR27d4PA@ddwUa&4)*qn zc9hNIe>fkvaqcV5sc}7^+jQRwJ!Xd4uZQ@`v5uhtT_*fz9rG8c{y109g)}`%r=gn! zEJ)j_GjQ$J^lC=6qq<%dwT84lt(!aWBaRE{e7a8Lv-(nya7gdduTOjotfNl%={pUp zV{<^a3i4HBvmt6nMuN_#--)`(gVw%>qdchot1dxzPO~*p=PP)MKA+*4bu;*k&-$M3 zo1j{sG5XxMEH)(!Z^V{wv{pnsSWh22M?h zRupoBAPS1GDAB)e#7Hbkrmqd7k^Z_JfJAd}xNjr?JQ>%N$2EK6nzFcNeMgNxt_Khf z#|;3p!BE^9#u^j}Ul{~&8^WQX!AOlsG@?3j9ae(4ra!I|{a508tY*Qu=2BeqN|<`N zif}B?i&$d0xNT^}ALs=##>5p%@P^0#@W01xoLkB-9B*E>I3@x!Ct{Y$Us)>WGJcbp zHPO73nR8?3^_}nRo@n_ZGk0d|2QR+&;#@J?0nprv7GSRf;&bVQ^ZhFU(61?HM?wZv_-4!u+?Si{@t|98KpJ@8R<(_Q_Z8LpPX0uMS z=K+V?^x+kH%x-1BLt`@009t%pr#i3kF>xvLGaVEw-vsSW z#0FZ_bsPXJX*S38g#F{XYr$|_J2G6O6>|_FZWtQ5f<;Wjn$6l&B?F5UK+=mOk&D;F z8pLExt3&{>VzPek)nB`^mTb+=O3n(hNT64&XB!E864$fJ1#kdBYII{l7a<7#%0*Bt z;gG*C$S5X12%F)7A^hm;!(u%g0abDS{QjU{42lTwArXY_RlfSX7vp@Mujl(lLf7g- zek^h|-Z9e!IpVJ0Qs;L78_A(Jm>uopyO zZ)D_Ba9H#r0vn#a2#j-Inr#!!Z<(i@lg_Al-FVZI$^3&W-@Wqt*J7rE1yjLHTg+5- z&s4SKa!-5yaCGX`$yeWcT{0Ig+3e$ol0&G#7U4k#TNNzxwP8!p3)*@e5ep2y;s&{L z)X2E;Qt(>uVBniU+KWgLE@h>C#n}kbaGS-ljmLZgd)#hdFXh@&|0tIVaN~HB9^gXS zZq06u0vz%h1eQqiEPx2<1gz{d>*cl92cpQQQ}PG^i(&oN*|A#tw03R_dVylp=tlo% zB(W-l@A~jllwj~`DA)()=luC5V1p5HaE#GxyZ8h&y{hm4uX__gnG1OR{CR72IGDm_ zoj(s!YOw!W6Jr_xg+yNxNgu-6#{2t0V4)N_TYMc1^K7L9Ey`l`4~k$r&Y$<5KMx8r z4#4vQ1PQVjE(4HmqA+19u@1Mm?$Tf&tVgH{>!6I*@>S|g2Jv3-D>OX@{X^n5#AE|| z0D*Dtk(IL)&a_6&Wq?mx##G}(BZF0w46BwtfA59SVQ^1uF2>CYVJ?=Tx(sZ;&JKr%yj@*K znn%EAUX3U#*z=aHqsSmkxeyU**SH_hZ6>@zkGTMQNZXIFw@-sJ^)JJpK1|1Pt{Wt$ z0(2>Wt4viyRxSlY5<#k@2pl0W?NO&oC*{8CN{3N>Pm!PlZk7`@om~X>y9n^rm=`rY zOwL1}JBb!wb(La*2)gZwniIZNYl*6(md{D8whtVNPd98*vQJ5eI8*ln;hyuwlT1;xl<|6m`9IoPD5Aec+9*nkqp zHm_nClguS19k?>y4>^cG;0Le6lQ3ZyPdrnfpTY+^YOFHxpdAUzG^lYfOkd*Gu4`C8 zF1KDCjDQ6T55;q2NlR~{TD^%n$DMr_gMF8JFOGyG!~V;`xTU#O=_jPoY%ODZO-jq4T6vbSmmOE!j^myF4*h z%{^DmeAc3Cn`GYhg~_$-%!oOw?>VcZ&e{pxQc>B2Zqb~-oL}@o-FtNty0={*izlqV zwz-~Os|e88T1B|(8BDE~)+)jQf=Mt77QtF?1R&UCTwuH$1SSs1h(I?g4l&F}L~7vL z*V##S7aaB31QLkbQoCpu3TG%|%oQ00w|)kKsf|2Ee=d>LgQVdWR{5_uucrP1-InaG z6&;@UE%p@yo-0O>?kgq=nK>K3Vxf?g^Q>R7QOM32vmZDRlKUr>rzGHK=;Q$2qJMM> zHGH&!N|vuq6+tL(I1bZQr;t_Y2(l!X8Zz_{#lWskqV88+oy0Azy1F&phHh}Pn92-^ zFzl@L>4KU7CpQ|}7G>C8xdD~D5BZ|=!dzV~D+A)iUG3L1r*O+n$H@JR5A5{cwnbtxsL#7C_u z{<*u#ScEJP)8L8wX%J}oE660qn*KJnH zq!iEQ#g8=0oH(C4Ld00Ex;j(Hape=l(SJ;>3|W&s*=_ACRK6#lN_|2WeE*$COt1-d zMUE4)6>z8i-8bnsSycYqzOpHGBdBfAj%pz`*MzJuTB+w!QWIIQ>O%X$2+$~ZTLee9 zO>m|%E%h2Xx6~XJVZ275ot01dWj{-g%dYm|w zRhQaoYidm8ZOAUTyX~~v46xjUjBb19hE-YMrvj^t+Lj3oA~@OAZ3T_(Ns<}e){EL6 z^{bFc-=F-d^or6;-=P07!>RvIQvaLN_kTI3c0~n86LAU8g)2Q*L^J%tkK2w0Y4;MhbS}CMUWnl zwbp3krpVysAPKlk3guvws~k6p!O#%6U7|}^UBYn#tKzPXK?$^02Z?|EC@G#tv_;#2 zw&o^?{$)vc?}4VaBdslCc~6l^*Z6)vD4D?6T`L6VPmoTDFA#a7U%+3lEj)*u8EOqE zCy2RDqkeg|fF8M1@--$&f`e%e07$bj#hnyx14vmGO+ku7v>^*vgMD1g40b+Ly zd!qgz<#OYC8iTkYFnT!@M*oonG8i|4kQf@gKwZlc#7&8maTC5UX{4U0!niRIWC?Vj z8sa8s#{3b8+bO_a8`yZnEd+C|1SWAky;lWJ{y;eHA}DKN*~n4M$Au#-Bug4j0uW?L z1942Q5F6v%ikr(X{-eU{Ss!n`wd>D! z%?F|zcE>jCU)Zq!&i3er_VL5ZR`-;5(i^k#3s!!~Y+G{VF1fPr+Y4r{%#O|uNJZWy zdzO?_y<*U3SV14{XuP-m;IvcXw=B3C?g}d=dTrrsr4Koa%{bn?l9grHvFym4IyZSP z<|toql+QNLXWqUfIm%;>!;6l?cLPf~`8Niy5599LmQ%fuQ$3d(&DlJ0a5*#Q-77PZ z*^_gTXztcn?olcC=xzNk>_4~vrBnWOw>+BJG12msfpfNNR%$tW=G30aJyS2f`Qnl* zb6S|VI^kN;>J0Vx^=|i#(?2{tYl>!9ORj3kT>aH%N`xBx>m$&3o~g#k#+a>i!B+Z+ zGaBkaEKhH~vF-Y{w_cslFF9RP{geG~U7Q-39GMxI8=M%4I(JOykhi5C;^JGUZl1bz z=FiW(U-3chAJxuwyX% z_pa#h;s5@U_;RWNzhL7ZaTfGgS80+(Em%BS<~f^*XCT%;GS_o^H68Vbb@rv z%3iS<9NCZToGVW%*f7^L*C6H9N1dDI{gQe6zpiAW?0QiHG+ivdDVq1&X z@@b8p-gtF%-(1qRL-)7tZ3nA$e`oR@*scA$7c_YDce~3DRvJDl(%}7P#q{R03Jtxf z)X|%29&hei=>1)5Q%zfg?(Z86`0@9<%MSCp|FuMedmr(+30{}`ufe3qGnTmC)zh|} z-qG9@RW?KQv$c+uOMu@Kud8C)t_BHV^4zo2l`Oij0UZu z+*0V11evC?ADyZ+LCJ}+G)9FqIcn`8yu~h``Zi<&@nH6WO-u^@1Z~LNqY|4?z!^Z0 z=zSPe+%>oxd?q{RGlLAVD7jzM_=vqU4Ne0Dn22CB7frINazS2^0@l_(vD80b3P{EP zg8^m)KbLC=Cmipm0Ndjd-0@gFW z6UG3wwdV%|UOBh(h|LTE&aYZB$;yFQ0w4yHlE}se>7vQj29y(-8>prRiRwN#cQ4<> zUl2#GKv@i1kqi8280vAdQwWF2M5J$IG`8;eeI*xTS!Rlo{ zTBt~X`QvRY8|Vg<)QZgXi7b#MWkX{_wMn!URuM!)LiiXI$&!Ial)BVEbj5!ytP-AE zH*F>Zh6I_&r0`)3Z8tJn)hJ#Z1)+*g^s^2H@kt`B(1^&aOcZff!=_CQHLRfFU@#Eo z2SE*uTp3mjcmHhx`L!5NRam>IE^Q81<0i0xe7Fl=GlJ`v(WS z*Dep8_kL|)C_uOrp{-GW}vhrD1NsNH>v0UkqLT00GFd zom^JwY|dQp_U>rr!S5a!Z&@- zSj;H@k6qGBCw_SbE2b+0tGUfJy=&1{w4yg>ysBBzagK~fIamy5UYO0AHO~6yjI&qg zUY^gKH_QvSjq_LU=x@Iw!p4DsVWI|nnKdb7ETwFj<#^Nnf*#u6mi+5R`ieXlcaX(^s)k{pi>3O z%Ut9du~j9Ypgqh5WG+%&Mu4`t)#BiCfWQr6KPnhEg-5}6g~JENveao!Gm#PI zyLhn^k-%N)K(UJq2sI-0YO4O%I68a<_ATP~@t$>AGXkXhwf*4gcdt&HVy1QXOzUP^ zJ~;H=q1jW>!ut6aqJ=w_o$I86`g#4$t5V*MsB@=e-U+yw+A+Cf`aslHAn6N|eX9@~ zYTxp#gf)R6RWlsdgykv3#+@d^v()ncBfhIcuF=#y;BX@&8$4$E*so-@5SLvv6bv?r z30P~$6+ytW6mw~Xkt(QQ&@>`tQ=^ml9S9zvjD)5OT3vkl1N=-K7iUttN)}h*B)+ru z@d#jiZ1$<`liT0gIrZY?i&0x)%vQc&1LqL6RZIHnWGj`3r${gh=kC^=V)e^$%;?LvY~l!7%vh7$&65X&=?lKPfiK1iKYC2UH3LDNH!FUL`> zml|{-nxwP|GV^F zFm|eBbF$${@|ec`xNcOti5t}(04jXBA>C9?j{d6FLOPx@l4-5!l4;XHjxqw5wCzoD z%}2E%Rga;31wpUM^aYc=G6?2wbDA72AnR3#0$Hy;$ECs!err;HlcanUHfmJrAXj4@ z>q{satK~`E2Fg7=dO`kC!&Hp0Dt^LH+MA3(#30#B$?-zLt7KS03cfllB4aRN6%}Q|$7k;hX$d9e>7~x^gb_rygx`@fKcvxV?h-)td@FUPW3>g^cU}!~0FT@T0 zS3t=nXgFB}m5@O>$y{{kl_0G22&Ib9GjI$A4gp_)j5Blg{F#cVtpo&IeZzRmZyZ?@ z8j`aFCxg>RqR!HobG_tTKP$u>-UWv@>ew`){lb;;YkStRt7zsz)K$s|xwjjavvOxN zGqpc*M6;^q^dI-lSNvp9%G@#8vgF8^Iy-rGreQYgN4sKv3EY>XW2i^o_{U6xh1ywu(bK`UG3*(?TEZk8j0qQM%}MW7(jh2IbWC${PZN2 z_Pv5;NjMP`Aol5#&h$wy1t58G^h2Pdb6G*{S*(x`*xl2bKioduGL!Ld4@!(a`ImpP zab=^Lum?T=sdL={yZzG*g}DE%*@pWQQWE0W+1g2Hktl>U?315R<<>)=f72qXam_@r z6DZ0cW*iX*$O4H8IFBID6!{3z^3RozEFjWDlBtj#M===_r&fTb08 zMG9dAdNPGfNqI?6LaG8&VZ#X#OPx_spQa$cE)DIYVCZ+9hi=Rlvq9kma&wDJp{r=urYFg#cI{GgU3m#w*|XW=Zl&e$8U-F zifih>Hq>Lv)sMOBRV)1$L)fOX=WA;fJ-cATxF@bV*xG$lBP$y$YQ(q!_Li5!aVsJY z4l~;tGAD$cJ$`?2y)^*tZ<_Xi`_u=!vl#8>ZqVbdn)&<29xnwvIp0ySYg7US5( ztMqe>YG))7%5XruMsL_!)JLIr=teBtO}qFZh5v+ZNsN()pR@;J05&yr?+v33vH%tx z0(;*Jv~IwthXw(RM^`z=r=+_RNJs5^Cd^6+$I(O$S@&%jDSmf(*{I5(ERAM0bT z%6qO#`cTL|tQ2q=**DDB%`*j2cV*1IPjc^@%euAh=DPXy(VAWIi`y5lLO@9aYDeabp%oql1_oVVokOkJG3IQ?4GSrK#AO3vCjEv$UpnPkmAV~V5*NlInNPkK$Z{3{JD1G}>% zdi=P41R;nSm9r;2Lj;{FoxX;Ow|d)jY|X6GA8;(F(_)OqUetCI7~|BFaAH|j2p zxf>;S<81dwCqFznHxRAdCcn5{_TMs+Jrwn{N%pp86%iA=?tnB>6Iey2mp9N!cp_s*V4y$hD;6Qr!J)x zx}?$cQ>_4LNIM(SO5{~Z%2AD=59w2K)IXzNu<4&M5-QuFv^r_|rS=PGu_yXPmA!x@ z)ds<=ASc-IGPxc`1;4-mrqo0GjCy3O${Uua0`?|?g;LuE197k>HKw?=e{^W5g-H$K z7FA*ct&uE9m_iden5ab-HXCJ{lF^&gOj*(~Ng<**YmDN5z$32h4~XwmK!<$9_vkZe zK*A(8md$5m-BXrA7x&6r>3f-U_*2AU0Dl(&$fiMKE4J)Ee3 zGocdPq^eyj^t=Nj7W#R(m%Wy83b2QIsTl7~mP3jZ^ND#2srcq+NX1p0qs)hdG!_hS zlCe+PWry@9_R~yA#Hxp+7qZt9>{DbrYBhjlEfk#CH{Ahs2!qDoNA*MkB(Yyj5i&sj646+<5eGHSEjtQkTB459@dLkc zwE$59*_DNfwU;Jo1XeK<9a2;w3OqghFbjsO(?AIo`^J}XrY^WbI*C#xSq)?qE*Itf8iG5}=#Aw~L6 zbW6fX8~r4vU&hwV-cg<&c1Ch4Ay+76Gm5F)R5Umr6S-7^)zWN^Z^`pwkU$*VJ#sH-gI z+9SF4%mzNX_~FI5E77X$@{8MBB=epnn`1l;W%j#FD%vZfr`p#~thh z#jm^A`HjcZvq+`u>}(dL;9xGT7D~moPea)`^_x$#;}Go{Q804g1VMU9Dn3r1yNdUd zX=OZ3&Uh5@nNPba_v%FMXn}lz)&u0Pz66zGc{=x$y6Y%+AVfV>D;TV{fVha9WV+>7 z2w_#A!IOy>n&{Xe#s)Y_Z&_T5me?UymDtPfltH~Wp*9ki5QUARh=9BqxoN6(vUR%p ztz$7eFWLE-m!sy=*%J%qjbCwE%f=PDE!$nwnj41ehUvgNwwSwAa)SbiVt!m$uvdSj zQ=)yTM(cma5p$PG?y}j8sJ(nPvS8l;D_=tb5U9hHP9x3`ou+w6v-cZl5|XQjG0p0c zlnKVo4CG-9E*?S7Df$G~i_cp+T2VjM-f|m3I&N&~!1F%o6Yxwd9TYNiF3*aELbAOj zg=Bk6gk|)Bzh7GPs6j6@k$N} zc7`B`T|O1%gE%)UZ`0XirbGXbJ=0Fn1bGGbAzWHbfyx?SAvCb%(C8z5hynT#koxi^ zm`;d-d6l*4q+o_L-hA#ZV91s_8|7T;$2XmVeTF8qA;BHZSZJ8(@~V*ujL=8wR_I|1`cDR>k3=(E;p7Dp_UZ zthi)@?Fd}-MXs^c9^1O~q&7>D)f$O48gcjE-!APCn!A- zd4aLqGgO(gusvp`6{tLO>c)@dGk<rG`>6 zX&2wr$!y|N4A|56ACkkOnvyGn0|RsvN$%@pgL`!-<>FzF?&XEx<>2KDL2(yqD_dhD zwqj+zrhC8Ly#E{7yQ5#=55OPr#!Z)DjtYI`SOzSw$)g&dcmZovLb_vaSo=_Q6Sh48 ze$0q|q`r40ti}O$%$Q&lpGaeECMN=@_^J*;k1_9tXbJp}z{MR|eFp=(_z`#* z8CIP%Hii-o8rk=~^|C>`?B9@-qOav za+e30`G45UACtHAJ(V;8`p+6VuXqJbt;v)HbR@Tkn+C&dqRFK2S$bt)M&fb9%cJn_ zC9{k#$Rj8ll{N7`T4k9moC zEUT0+(n}ac$sY4y+yv%|I8O0@p$`Vc?j2$lta05iKG9Mf*If<|h_4|OCfY!?VNaMr z;UV*=uEcbO0$XO<{t>YmL3&wB)Z&Vpvt`nTyQnpPy!i{8lT45Ec14}LCG&2$X8FOZ z-+lE5uYd1#+L6}q7Iw6ObCtJ>6)o6`X8Pu|lC3Ce+W_UYBl`((; zuX5jX>5a+kqK3w{#3+wwP;-~-O(dHeU4-PzK*`gz^WE7G>k#h1FJ<0qpp zosv3FOF3s2-95{>`7;ggT>YWzfrE2qVsy;*iLpgf-hF50bj$R%nRT<}v)NMBi+6NC zzXI#4gPK=pgPj9+9dz&5R%g$dz6i~}t@56&a<2N;#+w`Guf=Mc?$tJ-sZw^;?A4E6 z`|!2-w&?o33sp^bGNjDrU)h@{nlbJ(nQx6q=Athv>*vjhy$qEH?^ParyL;;No2O^Y zi;hy-%W&$KXU7jLTeFs|sOo#`u^mD;J34plcFXN;cQWtz?i@mej%p54y(|Z5C;@9dm;LUWhWRF;ixSqL_dmhvE1~FY^1Ww|Gh*p#NfZZno@>6D` z@L9bQY?S8Vr`a6mFYp)oi^fxT@Ka@1U8P)7>mYF36U z^uIgfL|!x$qE|2}4Ahz>q!@pFQ`QWgQ+}5ctly#yw%@4)`?n~;@jI2^{1zp+(wESk zex%(!P6axWmek*()C|F+jD89e#aK7UoH7-nS;$g+p{xlR*+Py0htk)g_cG`6%Tlm{l-MKHal>&!+{y&$Tzm_U_M)n@-$9 z!jp7%(g%Zn@^vUOQN>t~_wfEf< zDftwtl%9EVCk`x9*25DSu%LVi)d`_YP13pp_++h`p#A}1B#d7&huR(mptw%#FpE3AR*@MR60gfDC04t!Z-jyGr<4A^;P#NI2Nm-b#&a~xZ5`cmeYsxx{P zBq)O~Yk=D^gM7vgZ9CweYzjiAkP&{))m>_`P_Ksul3l_^p{{ec@+8USE|}C zs24PyjmcO_DDAj=m2pQq&7)e@%1Z1ooA@TJNvx*%5UUc;&#vSq2DA^&X6>`U{ewm6 zgM?xv*}8hA4iG#lk!Y46-_@5Av$JXS7Z`qSQoh3HS!zCM)4qc_7F&RK8NBaWuzblGc|M-K)AdW zHBFN*6Q94ZMODF}UP|Q&R?|=j-=roZg-PB7xg1KiLkTIlj1>9EXWKTP4fpM^P;vIO zD{*?D=#npC$BBRRIfR{0S*WdA*`8# z;drQjRO(fUiCMWzX~EO<%c?rL*RE5-QEn1v?Dn~ygfZZb<4UO~esbbJsYgtAm%IG9<@~TT{^Q~i}a>VBmXdF~U zHfHJ?<9alhwl}i>Kkbd9zU*i8Mn?jQjEVf_l;$H~p}<$@D{9aQFX6Z@voFuBdc8b=-cTVXJLomR*Hs{# z=qeOWb`=Sygcs}eq5PB_PSY3)XMFicn>v!u#shk4Ir6^*OpwI;^8D{%Ii6+nKR!-kba`G$wZ5>q79@FsSUpH)9w6d1yavIAMvMDg*~oW7g!c zUSnH2$xSB@k5zlv)^*w=58Y+;h;n#T@_;6C2oB{;4JlJE28TkF542p&M^ij^4Xv4? zel;b0D~*}!>Y$LUgElpLRT5eAHD;=-V;-F1>sEayMI3X$HD;;9Ia84!Ws-#S7MmN=IzPbvRmU@GrqEkzsKJu13X&0C%b~ zS*VA*#||~)P&Ge}d4dKFo~7ulF?JRlJJ*V@W~xwTH@1c^f-qmtcQti{Yfylhyf4uK zSK=K>qogZ`D;fUYi=0~GcWSwP4y{ns|k;= z=}#mhO)7OxP29C=gyZX1y;#ek>Qi7u5Gnh5V_IkNhcr$(XZ$$W;w-$jQKw^quoLa} zj1nWn)WLG2p<5>tg{!a^fP=9V-(cjq?pUhIKGl`oXdJmD3OLaubhLFGeClX7b)N7S z{p5b1U+-Pd3RX4t3BV$!a!UPHow!FH&7OCZ-iCz#J!VV5&U!(9svkXQp;5XYT6lJF zjEr2+&Zo52YIR{iK&XXW|Bn?MC2cqQPniQKD?(5V*Bat=^g&#+aV!%`etBXc-^3JT z79JP_K$kKENl+a-Ex%WLZcPvh$Z!_mtG^*^)QmY3h9R<*#2y%VtWKN2O*oXRM&2)I z#?g+<*HO}7n>qX#D>^&Xksm~og+kK@J9s6WghaU?R* zH!>90MfyVGG~S7Ka2xZSk)0jNCM$FnB|m0OwY8__ zGPWKljClhDB&^O#KJ-OyFLSPe64_c z@Z;m-vO$8SaUe)zEbfix5(s!3hem*|!+X8zhaxaL;S$vMpMQ(WO3vedf8ws@?Kfup zv$=D%^FuLLv*c?2<%xe`=E=WbP^^J}HrD5|Z54m4otePw5l4|nT(gTAH6^W@MdD#$ z&Lqb^%VXqub>Nem33%~;qj;vo-h#2r)FR}b4vZOE;OoX)vpH_Y(WbKTh_gu?z@eDK z5k_FZpK~v~(De28#_j$=T7&K*s9CLl)$pi3xTn3;t15?Wx#%*i`w zbCGVfWl5CYk@35H%m?L}gvhwuO%F|0He8@Jisb>@RVH-N`k0u?|Pz zV_Ia+5yXE>uMKR8m$wm3P~-!2BXNTGk92cT!QZFQ8+2oP;l?O*jcz}q+aJ^Im$<>n znRu<2mU1IpnUiCN1$wLV4_~V(j%)p)xSm=rlI5L9@~pTOh`LukQz~wQL(qWoLo*C( zD0%+HHCM#~s>O$t>`&;niPD&xS zy$H)BxMzI*2d{tk_2sORS^Ya*8_U|dkhS&mtZnR2Mr?9}jgnMY7jtFcL&oV$Kbs@Qu+ z|C#j@YiwVqw6F8-g=qFmh+*ka%hg+!(cSP(GSxBJAr-XTxe&FtJ=AGytP}c0b0s`%|hk`?e6x z*cZz8eY^#VDTo$ zUo;gin;k1!jiW{TWd^ItW0M}vt~{J`{ha=mDY<4=-U)>yl$Ve5GK{ zNcOEuj>_2x*;%o0HO$iLZ`MopT@P%=Z0jR~)n$C3M~DCkE@j=B)0^LVg}`a*jju8| zi*-WtK#R{S2F_rAz#;ZmIh+}%CBZ4^w)x10bwAnGWR9q30Ll`sIq{v17l zJMHx;U9gv8Z_aq@va{&@zS)Y8ydQd_#hYWr+ZT$rM~ip<)z&-u*uJw1`_4vppNl$s z#}6*!)VHludnflsP5ChszhL5Hrm}mcGITn5yonHITBzC~Id}fbylctgh*_%eCt0c} zg7Z6z=1VAQ=Hk5JFT%IC{-W{cjnVCgV%s|wws%Cg16`epI!~hzy=9_fX8TNyRNi>I z`xiYw?^$R(a<@scb}Z_TEoWp+ci-r}-U}x?v5XB185^P*b%3nQoasm`w|XJBI+|HC ze&l{e-n4b5a@II=b@r;{+=N^`>)tPzJwA8vu4yr^GwOM1ynU&nikg%$z2W^x%v5pD zRI%hKnlvq2GN;?&CdX2B&jR;vMK|iN*U#2OJsV@5Es|%;ymrDw7KH;yWUIJmt6(F% z<95N%T~gkmsIyHnw|$9iPdnz1N8K&s2xr5;Pf6WeVE$mVq;bJqIANLIx$LZ%JvFb7 zI$vNLY~S2{-<>m4Mb3n0TuTV*m*DKDm2Pb-7GoCeN8PmZUVY1S`)uAq=7zhwrIS6< z;j=3?e4A(iI66F17DfPD3%_$a@3!sE?z`9i{<7rhS+t+Uw#2E|re2$PjU3kH6fV09 zma_7ea!aPaGi#Xb$59EVq^#}qDr?;mzy6WUnBjS-Gv_)d4nFd5I8UK(X8i{n-rF!c zG`~NZzdf3>BbKvwA!lziXJ0I*Rmy3-(-id_U>@<7b9kv7XNa9#=sY8p_C#~eO764E z_6#Yjg0_>xao(bR!;(EaW-nZ@7v9g}XUmte@@HC>5UN-L*3Mh87&1JMbS8)Mk(w^Y~Q5RKuI>L^~{d*(ERcFgOamx zk+zb$rW)zIMkHT0XTtXqEORHmGh@XW5|y)8Bsq?C68@Jixsege4OPmbH#%; zEtiq==!ljxSSQy1LC3#5?9%~{{mVZUa*iV!cJ%PC^2;yf8b8+>agRB2FBR!yH7?xa z=3eYD931ZLt{07eeA}us*ekC8s%5TI5h673mTl?@OT>>c0abmh4W)GeE&Y)3-5)G z_iN-kys13q;s4-q4z{m9F2?TL$8PLFd@O3nqIPB2!0E|6S(iuWoaFr4Z2vdL2Y8qN x&$OGdh|ccF!4WP~<~PmLnNx5!jNe}~mymzG-n4%U_t#tUnq9iTacXe?e*i?5hkF13 delta 6236 zcmb6-3v63Qa=UzwkN>a4uVj%jEladyQP#&FS(0PRvTRG1=_lp+M~kLKN@gO9FOQUC z>)c35aCgex!M412wS(Xixd>{fIKaNDEu3q6xFjta6lfxchf00%H3{yDwke8}>lkS+ zDA3uZD9cW9Ko{if?9S}&?Ci|!?7KCsxp@s+YpoU|fDAPhiC`f!EmSO1iXadC1NSyE{QctMK?4&40nlMzn4!JdTLGsg#hwSI;=lt4!jD2%s!UYZXk^j(tLN0dKklIR%5YdKE*g$?<>d~p% zT;|!-7(!;?BYSXi14xr?1u%uaL_Qq2Pr;(uw!}gGslmLYg?dRq(4(W6 z9+`qm28tPxPDS#_ew62S76-Cuz*6Q6uuAHlmTZl$OmPKnowh5!8uA-InDwWi9D(w}iBV z=S2M$IxCdzIF83P<6L}2=dvIPWWKeO+_LIP*6X6nHp)5?#1c_62uMo+9nn=6P&C4X zj-U`)CA1%o$7Rqp;{p@erVqM%8@6+6P#OnOOjsU^qE6z7mEclh^Sbo=sO++SfPC2M zDlVVGqCqqk#gz>)Fo8l5O#}LKz~{z!Xi8LX`Ds{ zfx=d@vFu=LCCs7fPNx#dl#&f&RhLVjNoB@!s!mDCqluvu^a&|N$tc+g?cRZbXi&xQ zGs5s+mv>KU8A&xzN*Co)js54dxzw4ibLpJQ4`q@mRXdtVB(q_oLf2B(QV^nG9R<|J zR8C1fr}9cFE2}y=ah~$4IHd~7R4$Po%~Jnf4<*QoLM=!&CFFEmAwOTQ>)Z`(W~vcT z{1jR;qf+mcwpW_p9Jq1pkB;5u^HnViRc&{x+U^|AS9MM8zGt*ws(+<^!5F-23@++T zixvmjU$J9StL?$}EG3r)Um0Amgzj2Gvz@m~<}INGOUo}UEq6*6-TtfTnKXHJ!CiCL zU31fucW;{R{*pu1&0iXkxn$Aim^R!q*)Fwyul4ds-c&KqSA1y$DopV8Z%mF$ZLhQ~ zm@4m@D!)J)SZmSYyy~8D6Ytf)OkmbCdE0k6kaujK7VcTS*IH&BH{93Vv-+F(=K7o7 z+m72!x2tX|ch=wW&ii)dt)1^X=Jor(UeeN;ewW<>I(Vlt+*yU*H3mDK+IKhT0e{bF z>RfAhZw*iJwH$?2fi5%mH~dbnOUM1KP7948YOe4n!_}?AQ-CY$$!KLg-a|gEZpSil zY$zkOola-cPAxUI>Yl%*@LFPx_^KK8tlkd?jAwTIH008F7po$0&hUOabMx1jTWr1O%t z@E_t{k`KSJM#c4Ej@s7qu!Y5}84b!I+9Q&u8hW)!K%;Xr4cD+y z4=5-BKd2!mO#xC@sjrXlpaGo%VF*^BhIuNHO({Cs6f(+4Je@39JRPd^U$L$`nE@;t z22cpCHX2%Ovn5Nkx;ezEJ0Vuh?~)j?5J^|G4@2qvOpsr&NLr6%ZV=1-U{cnWk!VdN z>1{Guc!&nPq=jgplLeCKa)q>qPZc9tF+Rw;7@dq2Nf**k|bzFz*$)%?73MZSKe6Y=| zSsrw}wM>X0*&rEUjtaYMT(yibW~XwI1KaJBTv2k#xb?5#8s(|1%Yv1x>|`bWN!qQ> z$GIea5ZBSWX}QR}#EZtQKSXGR&DDi_`VfkN4hlsy6<0is!{)|+r4{Xk%5`@Qkh0BHI8AnL4o`+n3e1g|sVlqbM8#SXJ~gn1Etp)x$AQg&6c0&B(mCr9dOk4Z6tf8Us06Z(?rAvmOmNDYzv$ zxW~W=$}{Bn9{=Ph-Lfv1P#{`ol`&{RXnWjX3kRxDP0(XxhYY%A2KfWp!Sus!dUi3> zz{u|JLaaAa6OR{HxC}(5z;6LEg}6nFi}2lRg}k#NZ`wrayJxFZoKvXPsk&@Bmnxou zs!7X*Aj``cB}Z(=I?Zh~w|4I+ZUje01v~r^U~oIJW9z29^hm?Ja)siA0>=9z@{4_I zg;$|{8# zDB} z)A__S7B-ES{PwhN^3gd|OXgPg_9|}bG>&x^mp=T8WGsp&TP2+xb!ql|8zI>i>ssOY zpQ4#jOOWldBSyWFmBYU+(qyOP9R63j0WmCY7?f=6xmPQ?#X51LSTETlyyRGZ_C+KI zJo`8?BGyE3CzeZM@T4jfu|cv?am7uN9blv6V8XAQeE@;o>?fa`DeGi0v?mF7{{N-- zrd6lY+{jBtsiYb7OG{A^aw$D@c+^3p1q(So+H=q)xrzoo0)gT&gWfEaenXLu>yPo%7_gVV7l<;tr&k9L`5_O#VJOwn@(4BS-79{OIUu79xnx4(@$l2BJ z6+8QV;_iOG*d^9Rc-gbczV=YPh~1J0#;(>*!Ky-&a3ikr-hvhFWn=a+EAkYrD1cVY zy1%H&060+3bRjyeqW3(ai!~t#tN}F6$(+om3%EuW(kR>;#^l>$-@+2XnSaK!x)1j5>x)m69Y|*(4IK%>V|+ZVq>{mGDwl(fEWF_0 zTNUKHbrU6hnc$v7yZf?1c;ao`IQ(34EUP>NMCDlk6Z%NueU(*c76EUkO}R$@cj#EX ztYmVTq0Fet<%VSCFeOitKgeCgFA($i-+A7`eeiIj_itFMYU&%7?`QJ)cvRDXN!!_O zZ-mZ+Ta{~cY~j+TSS5MwY&E_}-a8w{Ve+46H%+RVOjgyU#?Gdd%vd1^lmMNCra21z%Ay)!*U8jyVth3BWQNVMg+Hg9ItplFz;bcrI31`h zD8q{;p_`g)A!+&xitVLfHw7#eev~=U%v*t|gA|&bs#;lrEML_nmGg0Bd`uM*vJB4v zh2}^qml!)AE>krLS>@>-lr?l5+?1e|F;#b_aC|DBJg2-zDPVl7-Ul#g=1hEC8C6Zo zXB1Zw&nqHjDkRBtB13^jQ!Lm9QfPY4KGdkXAtjZ_rIPTm28fp^vo?1slRRHYqL^Cf zUbAHoN)h?`T(v0=2uqZ~1Rkyj3y3_0W9()A_J8&HuncKfydZ10P_b^X}-I;*De1 zkKGu&J~-dp^S+p`iUNyi(dD~(Z06X4>(IRG(3_1nTCca>=(yf7-`M;9seI^AAMh9# zJ!Mxf%v@OT^v`?x-&Af)T%Wk{!u1#CH}}m)#e8)?@R%3<6>mgdjV$scYk3g_Nf;MnR**HlLIgQc=8K_)?oa?j11O!YiL2g?tYgBIox0D)gY~Ly81hP zUp?sOj5gt`|0+k8UYzxU{y)%!PHq-H*dPG>u*Gw-jh}be0IK?UoPG(3$CWu?3;Pv# zM}zgpd#`r#O!u&%o!l7Ry+sDZCF*1CN zj31+tkCE$F$o){TV%wK%;oI`VtsE|YXvR4FP+tyM2YfquxYdH~4}*FDZg_4zEY)J+ zp`E7gOM^V(&C?UV diff --git a/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py b/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py index 9669a52..83f7ea2 100755 --- a/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py +++ b/clusters/noble/bootstrap/newt/scripts/sync_pangolin_http_resources.py @@ -6,8 +6,13 @@ Docs: https://docs.pangolin.net/manage/integration-api Walkthrough: https://docs.pangolin.net/manage/common-api-routes Environment (or --env-file) can set: - NOBLE_PANGOLIN_API_BASE, NOBLE_PANGOLIN_ORG_ID, NOBLE_PANGOLIN_API_TOKEN, - NOBLE_PANGOLIN_SITE_ID, NOBLE_PANGOLIN_TRAEFIK_IP, NOBLE_PANGOLIN_TRAEFIK_PORT (default 443) + NOBLE_PANGOLIN_API_BASE, NOBLE_PANGOLIN_ORG_ID, + NOBLE_PANGOLIN_API_TOKEN — must be **apiKeyId.apiKeySecret** (one string, dot in the middle), OR + set **NOBLE_PANGOLIN_API_KEY_ID** and put only the **secret** in **NOBLE_PANGOLIN_API_TOKEN**, + NOBLE_PANGOLIN_SITE_ID (numeric siteId **or** Pangolin site **niceId**, e.g. unruly-asian-badger), + NOBLE_PANGOLIN_TRAEFIK_IP, NOBLE_PANGOLIN_TRAEFIK_PORT (default 443) + Optional TLS: NOBLE_PANGOLIN_CA_BUNDLE (path to PEM) or NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY=true + (homelab self-signed Integration API only — insecure). CLI overrides env. FQDNs: --fqdns a.example.com,b.example.com (required). """ @@ -16,8 +21,10 @@ from __future__ import annotations import argparse import json import os +import ssl import sys import urllib.error +import urllib.parse import urllib.request from typing import Any @@ -39,16 +46,61 @@ def load_dotenv(path: str) -> dict[str, str]: return out -def api_request( +def env_truthy(raw: str | None) -> bool: + if raw is None: + return False + return raw.strip().lower() in ("1", "true", "yes", "on") + + +def normalize_api_token(raw: str) -> str: + """Strip whitespace and a single accidental ``Bearer `` prefix (`.env` / copy-paste).""" + t = str(raw).strip() + if t.lower().startswith("bearer "): + return t[7:].strip() + return t + + +def pangolin_bearer_credential(token: str, key_id: str) -> str: + """ + Pangolin's Integration API expects ``Authorization: Bearer {apiKeyId}.{apiKeySecret}`` + (see Pangolin ``verifyApiKey`` — the part after ``Bearer`` is split on the first ``.``). + """ + t = normalize_api_token(token) + kid = (key_id or "").strip() + if "." in t: + return t + if kid: + return f"{kid}.{t}" + return t + + +def tls_ssl_context(ca_bundle: str, insecure: bool) -> ssl.SSLContext | None: + """Return custom SSL context, or None for default certificate verification.""" + path = (ca_bundle or "").strip() + if path: + if not os.path.isfile(path): + raise SystemExit(f"NOBLE_PANGOLIN_CA_BUNDLE is not a readable file: {path!r}") + return ssl.create_default_context(cafile=path) + if insecure: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + return None + + +def http_exchange( method: str, url: str, token: str, body: dict[str, Any] | None = None, -) -> dict[str, Any]: - data = None if body is None else json.dumps(body).encode("utf-8") + ssl_context: ssl.SSLContext | None = None, +) -> tuple[int, Any]: + """HTTP request; returns (status_code, parsed JSON body or dict with __raw on parse error).""" + data_bytes = None if body is None else json.dumps(body).encode("utf-8") req = urllib.request.Request( url, - data=data, + data=data_bytes, method=method, headers={ "Authorization": f"Bearer {token}", @@ -57,14 +109,60 @@ def api_request( }, ) try: - with urllib.request.urlopen(req, timeout=120) as resp: + with urllib.request.urlopen(req, timeout=120, context=ssl_context) as resp: payload = resp.read().decode("utf-8") + code = int(getattr(resp, "status", None) or resp.getcode() or 200) except urllib.error.HTTPError as e: - detail = e.read().decode("utf-8", errors="replace") - raise SystemExit(f"HTTP {e.code} {method} {url}\n{detail}") from e - if not payload: - return {} - return json.loads(payload) + code = int(e.code) + raw = e.read().decode("utf-8", errors="replace") + try: + return code, (json.loads(raw) if raw.strip() else {}) + except json.JSONDecodeError: + return code, {"__raw": raw[:2000]} + except urllib.error.URLError as e: + reason = str(e.reason) if getattr(e, "reason", None) is not None else str(e) + if "CERTIFICATE_VERIFY_FAILED" in reason or "certificate verify failed" in reason.lower(): + raise SystemExit( + f"{e}\n" + "TLS verification failed. For a self-signed Integration API, set in .env:\n" + " NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY=true\n" + "or set NOBLE_PANGOLIN_CA_BUNDLE to a PEM file that trusts that API (preferred)." + ) from e + raise SystemExit(f"Request failed: {e}") from e + + if not payload.strip(): + return code, {} + try: + return code, json.loads(payload) + except json.JSONDecodeError: + return code, {"__raw": payload[:2000]} + + +def api_request( + method: str, + url: str, + token: str, + body: dict[str, Any] | None = None, + ssl_context: ssl.SSLContext | None = None, +) -> Any: + code, parsed = http_exchange(method, url, token, body, ssl_context) + if code >= 400: + detail = json.dumps(parsed) if isinstance(parsed, (dict, list)) else str(parsed) + hint = "" + if code in (401, 403): + hint = ( + "\n\nPangolin expects **Authorization: Bearer {apiKeyId}.{apiKeySecret}** (id, dot, secret " + "from **Organization → API keys** when the key is created). Put **`id.secret`** in " + "**`NOBLE_PANGOLIN_API_TOKEN`**, or set **`NOBLE_PANGOLIN_API_KEY_ID`** and only the secret in " + "**`NOBLE_PANGOLIN_API_TOKEN`**. A browser tab uses **session cookies**, not this header.\n\n" + "Also check **NOBLE_PANGOLIN_API_BASE**: this must be the **Integration API** hostname " + "(e.g. `https://api.example.com/v1`), NOT the main Pangolin UI host " + "(`https://pangolin.example.com/api/v1` is the session-based external API and always " + "returns 401 to Bearer tokens). The Integration API runs on a **separate port** (default 3003) " + "and needs its own Traefik-exposed hostname. See `flags.enable_integration_api` in Pangolin `config.yml`." + ) + raise SystemExit(f"HTTP {code} {method} {url}\n{detail}{hint}") + return parsed def unwrap(resp: dict[str, Any]) -> Any: @@ -108,13 +206,18 @@ def resolve_domain( return best[0], best[1] -def list_all_resources(api_base: str, org_id: str, token: str) -> list[dict[str, Any]]: +def list_all_resources( + api_base: str, + org_id: str, + token: str, + ssl_context: ssl.SSLContext | None, +) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] page = 1 page_size = 100 while True: url = f"{api_base.rstrip('/')}/org/{org_id}/resources?page={page}&pageSize={page_size}" - data = unwrap(api_request("GET", url, token)) + data = unwrap(api_request("GET", url, token, ssl_context=ssl_context)) if isinstance(data, list): out.extend(data) break @@ -136,6 +239,39 @@ def list_all_resources(api_base: str, org_id: str, token: str) -> list[dict[str, return out +def list_all_sites( + api_base: str, + org_id: str, + token: str, + ssl_context: ssl.SSLContext | None, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + page = 1 + page_size = 100 + while True: + url = f"{api_base.rstrip('/')}/org/{org_id}/sites?page={page}&pageSize={page_size}" + data = unwrap(api_request("GET", url, token, ssl_context=ssl_context)) + if isinstance(data, list): + out.extend(data) + break + if not isinstance(data, dict): + break + batch = data.get("sites") or data.get("items") or [] + if not isinstance(batch, list): + break + out.extend(batch) + pag = data.get("pagination") or {} + total = pag.get("total") + if isinstance(total, int) and len(out) >= total: + break + if len(batch) < page_size: + break + page += 1 + if page > 500: + raise SystemExit("Pagination safety stop (sites >500 pages)") + return out + + def resource_public_fqdn(res: dict[str, Any]) -> str | None: fd = res.get("fullDomain") if isinstance(fd, str) and fd.strip(): @@ -163,9 +299,14 @@ def find_resource_for_fqdn(resources: list[dict[str, Any]], fqdn: str) -> dict[s return None -def list_targets(api_base: str, resource_id: int, token: str) -> list[dict[str, Any]]: +def list_targets( + api_base: str, + resource_id: int, + token: str, + ssl_context: ssl.SSLContext | None, +) -> list[dict[str, Any]]: url = f"{api_base.rstrip('/')}/resource/{resource_id}/targets" - data = unwrap(api_request("GET", url, token)) + data = unwrap(api_request("GET", url, token, ssl_context=ssl_context)) if isinstance(data, list): return data if isinstance(data, dict): @@ -181,6 +322,63 @@ def target_matches(t: dict[str, Any], site_id: int, ip: str, port: int) -> bool: ) +def resolve_site_id( + api_base: str, + org_id: str, + token: str, + site_ref: str, + ssl_context: ssl.SSLContext | None, +) -> int: + """Pangolin targets need numeric siteId. Accept digits or site **niceId** (UI slug).""" + ref = site_ref.strip() + if not ref: + raise SystemExit("NOBLE_PANGOLIN_SITE_ID is empty") + if ref.isdigit(): + return int(ref) + slug = urllib.parse.quote(ref, safe="") + url = f"{api_base.rstrip('/')}/org/{org_id}/site/{slug}" + code, envelope = http_exchange("GET", url, token, None, ssl_context) + + sid: int | None = None + if code == 200 and isinstance(envelope, dict): + if not envelope.get("error") and envelope.get("success") is not False: + data = envelope.get("data") + if isinstance(data, dict) and data.get("siteId") is not None: + sid = int(data["siteId"]) + + if sid is not None: + return sid + + sites = list_all_sites(api_base, org_id, token, ssl_context) + ref_l = ref.lower() + for s in sites: + nid = s.get("niceId") or s.get("nice_id") + if isinstance(nid, str) and nid.lower() == ref_l: + out = s.get("siteId") if s.get("siteId") is not None else s.get("id") + if out is not None: + print(f"[site] resolved niceId {ref!r} via List Sites -> siteId={int(out)}") + return int(out) + + if code in (401, 403): + msg = ( + f"HTTP {code} on GET {url} and no site with niceId {ref!r} in List Sites. " + "Grant **Site → Get Site** and **Site → List Sites** on the organization API key, " + "or set **NOBLE_PANGOLIN_SITE_ID** to the numeric **siteId** from Pangolin **Sites**. " + "Remember: **Bearer** must be **apiKeyId.apiKeySecret**; a browser tab uses **session cookies** instead." + ) + if isinstance(envelope, dict) and envelope.get("message"): + msg += f" Server message: {envelope.get('message')!r}." + raise SystemExit(msg) + + detail = json.dumps(envelope) if isinstance(envelope, (dict, list)) else str(envelope) + raise SystemExit( + f"No site matched {ref!r}. GET {url} returned HTTP {code}.\n{detail}\n" + "Check **NOBLE_PANGOLIN_API_BASE** (self-hosted: often https:///api/v1 — match " + "Swagger **servers**), **NOBLE_PANGOLIN_ORG_ID**, and API key permissions. " + "Or set **NOBLE_PANGOLIN_SITE_ID** to the numeric **siteId**." + ) + + def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--env-file", help="Parse KEY=value lines (optional overrides for env)") @@ -204,17 +402,49 @@ def main() -> None: action="store_true", help="Print actions only; do not call mutating endpoints", ) + ap.add_argument( + "--ca-bundle", + default=os.environ.get("NOBLE_PANGOLIN_CA_BUNDLE", ""), + help="Path to PEM CA bundle for the Integration API (preferred over --insecure-skip-tls-verify)", + ) + ap.add_argument( + "--insecure-skip-tls-verify", + action="store_true", + help="Disable TLS certificate verification (homelab self-signed API only)", + ) args = ap.parse_args() if args.env_file: for k, v in load_dotenv(args.env_file).items(): if k.startswith("NOBLE_PANGOLIN_"): - os.environ.setdefault(k, v) - args.api_base = args.api_base or os.environ.get("NOBLE_PANGOLIN_API_BASE", "") - args.org_id = args.org_id or os.environ.get("NOBLE_PANGOLIN_ORG_ID", "") - args.token = args.token or os.environ.get("NOBLE_PANGOLIN_API_TOKEN", "") - args.site_id = args.site_id or os.environ.get("NOBLE_PANGOLIN_SITE_ID", "") - args.traefik_ip = args.traefik_ip or os.environ.get("NOBLE_PANGOLIN_TRAEFIK_IP", "") + # Assign (not setdefault): a stale or empty **NOBLE_*** in the parent environment + # must not hide values from **.env** when Ansible runs this script. + os.environ[k] = v + args.api_base = os.environ.get("NOBLE_PANGOLIN_API_BASE", args.api_base or "") + args.org_id = os.environ.get("NOBLE_PANGOLIN_ORG_ID", args.org_id or "") + args.token = os.environ.get("NOBLE_PANGOLIN_API_TOKEN", args.token or "") + args.site_id = os.environ.get("NOBLE_PANGOLIN_SITE_ID", args.site_id or "") + args.traefik_ip = os.environ.get("NOBLE_PANGOLIN_TRAEFIK_IP", args.traefik_ip or "") + args.ca_bundle = os.environ.get("NOBLE_PANGOLIN_CA_BUNDLE", args.ca_bundle or "") + + if not str(args.ca_bundle or "").strip(): + args.ca_bundle = os.environ.get("NOBLE_PANGOLIN_CA_BUNDLE", "") + if not args.insecure_skip_tls_verify: + args.insecure_skip_tls_verify = env_truthy( + os.environ.get("NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY", "") + ) + + api_key_id = os.environ.get("NOBLE_PANGOLIN_API_KEY_ID", "").strip() + args.token = pangolin_bearer_credential(str(args.token or ""), api_key_id) + # Print the key ID (never the secret) so mismatches are easy to spot. + used_key_id = args.token.split(".")[0] if "." in args.token else "(none)" + print(f"[auth] using apiKeyId={used_key_id!r} — verify this exists in Pangolin → Organization → API keys") + if "." not in args.token: + print( + "[auth] WARNING: NOBLE_PANGOLIN_API_TOKEN has no '.' — Pangolin requires **apiKeyId.apiKeySecret**. " + "Set **NOBLE_PANGOLIN_API_KEY_ID** + secret, or paste **id.secret** as a single value in **NOBLE_PANGOLIN_API_TOKEN**.", + file=sys.stderr, + ) missing = [ n @@ -237,12 +467,18 @@ def main() -> None: api_base = str(args.api_base).rstrip("/") org_id = str(args.org_id).strip() token = str(args.token).strip() - site_id = int(str(args.site_id).strip()) + ssl_ctx = tls_ssl_context(str(args.ca_bundle).strip(), bool(args.insecure_skip_tls_verify)) + if str(args.ca_bundle).strip(): + print(f"[tls] using CA bundle {args.ca_bundle!r}") + elif args.insecure_skip_tls_verify: + print("[tls] WARNING: certificate verification disabled (NOBLE_PANGOLIN_INSECURE_SKIP_TLS_VERIFY)") + site_id = resolve_site_id(api_base, org_id, token, str(args.site_id).strip(), ssl_ctx) + print(f"[site] siteId={site_id} (NOBLE_PANGOLIN_SITE_ID={args.site_id!r})") traefik_ip = str(args.traefik_ip).strip() traefik_port = int(args.traefik_port) dom_url = f"{api_base}/org/{org_id}/domains" - domains_raw = unwrap(api_request("GET", dom_url, token)) + domains_raw = unwrap(api_request("GET", dom_url, token, ssl_context=ssl_ctx)) domains: list[dict[str, Any]] = [] if isinstance(domains_raw, list): domains = domains_raw @@ -251,7 +487,7 @@ def main() -> None: if not isinstance(domains, list): raise SystemExit(f"Unexpected domains response: {domains_raw!r}") - resources = list_all_resources(api_base, org_id, token) + resources = list_all_resources(api_base, org_id, token, ssl_ctx) for fqdn in fqdns: domain_id, subdomain = resolve_domain(fqdn, domains) @@ -273,6 +509,7 @@ def main() -> None: f"{api_base}/org/{org_id}/resource", token, body, + ssl_context=ssl_ctx, ) ) rid = int(str(created.get("resourceId", "")).strip() or 0) @@ -300,7 +537,7 @@ def main() -> None: if not rid: raise SystemExit(f"Resource missing resourceId: {res!r}") - targets = list_targets(api_base, rid, token) + targets = list_targets(api_base, rid, token, ssl_ctx) if any(target_matches(t, site_id, traefik_ip, traefik_port) for t in targets): print(f" -> target OK site={site_id} {traefik_ip}:{traefik_port}") continue @@ -318,6 +555,7 @@ def main() -> None: f"{api_base}/resource/{rid}/target", token, tbody, + ssl_context=ssl_ctx, ) print(" -> target created")