Session log: Cloudflare Tunnel for azcomputerguru + Cox BGP diagnosis

Diagnosed azcomputerguru.com 521 errors: Cox's BGP route to specific
Cloudflare origin-pull prefixes (162.158.0.0/16, 172.64.0.0/13,
173.245.48.0/20, 141.101.64.0/18) is broken from 72.194.62.0/29.
Confirmed by TCP probe matrix from pfSense WAN, traceroute latency
comparison, and state-table showing 0 inbound CF connections while
direct-internet traffic still reached origin.

Deployed Cloudflare Tunnel 'acg-origin' on Jupiter Unraid as a
Docker container. Routes 4 proxied hostnames (azcomputerguru.com,
analytics., community., radio.) through the tunnel with HTTPS
backend to IX 172.16.3.10:443 with per-ingress SNI matching. All
4 hostnames return 200 OK through CF edge after the cutover.

Repo hygiene:
- Merged clients/ix-server/ into clients/internal-infrastructure/
  (IX is internal infra, not a paying-client account). Git detected
  the session-log files as renames so history is preserved. Updated
  4 stale path references in 2 files.
- Moved cox-bgp ticket draft out of projects/dataforth-dos/ (wrong
  project) to clients/internal-infrastructure/vendor-tickets/.
- Relocated tunnel-setup helper scripts from
  projects/dataforth-dos/datasheet-pipeline/implementation/ to
  clients/internal-infrastructure/scripts/cloudflared-tunnel-setup/.
  Deleted superseded/abandoned login attempts. Sanitized hardcoded
  Jupiter/pfSense SSH passwords to pull from SOPS vault at runtime;
  Cloudflare token reads from env var (tokens still in 1Password,
  vault entry is metadata-only).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

clients/internal-infrastructure/scripts/cloudflared-tunnel-setup/jupiter_tunnel_fix_https.py

@@ -0,0 +1,81 @@
+"""Switch tunnel origin from http://172.16.3.10:80 to https://172.16.3.10:443.
+
+Each ingress gets originRequest.originServerName=<hostname> so IX's Apache
+serves the right vhost cert via SNI. noTLSVerify=true to tolerate cPanel's
+self-signed or hostname-mismatch quirks (cloudflared still uses TLS).
+"""
+import socket
+import paramiko
+
+HOST, USER = "172.16.3.20", "root"
+import subprocess as _sp, yaml as _y
+PWD = _y.safe_load(_sp.run(["sops","-d","D:/vault/infrastructure/jupiter-unraid-primary.sops.yaml"],capture_output=True,text=True,timeout=30,check=True).stdout)["credentials"]["password"]
+APPDATA = '/mnt/cache/appdata/cloudflared'
+HOSTNAMES = ['azcomputerguru.com','analytics.azcomputerguru.com','community.azcomputerguru.com','radio.azcomputerguru.com']
+
+socket.setdefaulttimeout(60)
+c = paramiko.SSHClient(); c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect(HOST, username=USER, password=PWD, timeout=30, look_for_keys=False, allow_agent=False)
+
+def run(cmd, to=60):
+    _, o, e = c.exec_command(cmd, timeout=to)
+    return o.read().decode('utf-8','replace'), e.read().decode('utf-8','replace'), o.channel.recv_exit_status()
+
+# Read existing tunnel UUID from config
+out, _, _ = run(f'grep "^tunnel:" {APPDATA}/config.yml')
+UUID = out.split(':',1)[1].strip()
+print(f'tunnel UUID: {UUID}')
+
+config = f'''tunnel: {UUID}
+credentials-file: /home/nonroot/.cloudflared/{UUID}.json
+ingress:
+'''
+for h in HOSTNAMES:
+    config += (
+        f'  - hostname: {h}\n'
+        f'    service: https://172.16.3.10:443\n'
+        f'    originRequest:\n'
+        f'      originServerName: {h}\n'
+        f'      noTLSVerify: true\n'
+    )
+config += '  - service: http_status:404\n'
+
+print('\n=== new config.yml ===')
+print(config)
+
+HEREDOC = "'EOF_CFG'"
+out, err, rc = run(f"cat > {APPDATA}/config.yml <<{HEREDOC}\n{config}\nEOF_CFG")
+run(f'chown 65532:65532 {APPDATA}/config.yml')
+out, _, _ = run(f'cat {APPDATA}/config.yml')
+print('=== written ===')
+print(out)
+
+print('\n=== restart cloudflared ===')
+out, _, _ = run('docker restart cloudflared')
+print(out.rstrip())
+
+print('\n=== wait for reconnect ===')
+import time
+for i in range(15):
+    time.sleep(3)
+    out, _, _ = run('docker logs cloudflared 2>&1 | tail -30')
+    conns = out.count('Registered tunnel connection')
+    print(f'  [try {i+1}] registered: {conns}')
+    if conns >= 4: break
+
+print('\n=== external HEAD probes ===')
+c.close()
+
+# External test from this workstation
+import urllib.request, urllib.error
+for h in HOSTNAMES:
+    try:
+        req = urllib.request.Request(f'https://{h}/', method='HEAD',
+            headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'})
+        with urllib.request.urlopen(req, timeout=15) as r:
+            server = r.headers.get('Server','-')
+            print(f'  {h}: HTTP {r.status}  Server={server}')
+    except urllib.error.HTTPError as e:
+        print(f'  {h}: HTTP {e.code}')
+    except Exception as e:
+        print(f'  {h}: ERR {e}')