Session log: Cloudflare Tunnel for azcomputerguru + Cox BGP diagnosis

Diagnosed azcomputerguru.com 521 errors: Cox's BGP route to specific
Cloudflare origin-pull prefixes (162.158.0.0/16, 172.64.0.0/13,
173.245.48.0/20, 141.101.64.0/18) is broken from 72.194.62.0/29.
Confirmed by TCP probe matrix from pfSense WAN, traceroute latency
comparison, and state-table showing 0 inbound CF connections while
direct-internet traffic still reached origin.

Deployed Cloudflare Tunnel 'acg-origin' on Jupiter Unraid as a
Docker container. Routes 4 proxied hostnames (azcomputerguru.com,
analytics., community., radio.) through the tunnel with HTTPS
backend to IX 172.16.3.10:443 with per-ingress SNI matching. All
4 hostnames return 200 OK through CF edge after the cutover.

Repo hygiene:
- Merged clients/ix-server/ into clients/internal-infrastructure/
  (IX is internal infra, not a paying-client account). Git detected
  the session-log files as renames so history is preserved. Updated
  4 stale path references in 2 files.
- Moved cox-bgp ticket draft out of projects/dataforth-dos/ (wrong
  project) to clients/internal-infrastructure/vendor-tickets/.
- Relocated tunnel-setup helper scripts from
  projects/dataforth-dos/datasheet-pipeline/implementation/ to
  clients/internal-infrastructure/scripts/cloudflared-tunnel-setup/.
  Deleted superseded/abandoned login attempts. Sanitized hardcoded
  Jupiter/pfSense SSH passwords to pull from SOPS vault at runtime;
  Cloudflare token reads from env var (tokens still in 1Password,
  vault entry is metadata-only).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

clients/internal-infrastructure/scripts/cloudflared-tunnel-setup/jupiter_tunnel_complete.py

@@ -0,0 +1,153 @@
+"""Complete the tunnel setup in one pass after cert.pem is in place.
+
+Steps:
+ 1. Stop cf-login container
+ 2. Create tunnel 'acg-origin', capture UUID
+ 3. Write config.yml
+ 4. Flip DNS: A (proxied, 72.194.62.5) -> CNAME (proxied, <UUID>.cfargotunnel.com) for 4 hostnames
+ 5. Start persistent container 'cloudflared'
+ 6. Wait for 4 tunnel connections to register
+ 7. Verify site returns 200 externally
+"""
+import json, os, re, socket, subprocess, time, urllib.request
+import paramiko
+
+HOST, USER = "172.16.3.20", "root"
+import subprocess as _sp, yaml as _y
+PWD = _y.safe_load(_sp.run(["sops","-d","D:/vault/infrastructure/jupiter-unraid-primary.sops.yaml"],capture_output=True,text=True,timeout=30,check=True).stdout)["credentials"]["password"]
+APPDATA = '/mnt/cache/appdata/cloudflared'
+import os as _os
+CF_TOKEN = _os.environ.get('CF_API_TOKEN_FULL_DNS', '')
+if not CF_TOKEN:
+    raise SystemExit('[FAIL] set CF_API_TOKEN_FULL_DNS env var (token lives in 1Password)')
+ZONE = '1beb9917c22b54be32e5215df2c227ce'
+HOSTNAMES = ['azcomputerguru.com','analytics.azcomputerguru.com','community.azcomputerguru.com','radio.azcomputerguru.com']
+ORIGIN = 'http://172.16.3.10:80'
+
+socket.setdefaulttimeout(60)
+c = paramiko.SSHClient(); c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect(HOST, username=USER, password=PWD, timeout=30, look_for_keys=False, allow_agent=False)
+
+def run(cmd, to=120):
+    _, o, e = c.exec_command(cmd, timeout=to)
+    out = o.read().decode('utf-8','replace')
+    err = e.read().decode('utf-8','replace')
+    rc = o.channel.recv_exit_status()
+    return out, err, rc
+
+def cfapi(method, path, body=None):
+    req = urllib.request.Request(
+        f'https://api.cloudflare.com/client/v4{path}',
+        data=json.dumps(body).encode() if body else None,
+        method=method,
+        headers={'Authorization': f'Bearer {CF_TOKEN}', 'Content-Type':'application/json'},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        try: return json.loads(e.read())
+        except: return {'success':False,'errors':[{'message':str(e)}]}
+
+try:
+    print('=== [1] stop cf-login ===', flush=True)
+    out, _, _ = run('docker rm -f cf-login 2>&1')
+    print(out.rstrip())
+
+    print('\n=== [2] create tunnel acg-origin ===', flush=True)
+    CREATE = (
+        f'docker run --rm '
+        f'-v {APPDATA}:/home/nonroot/.cloudflared '
+        f'cloudflare/cloudflared:latest tunnel create acg-origin'
+    )
+    out, err, rc = run(CREATE)
+    print(out.rstrip())
+    if err.strip(): print(f'[stderr] {err.rstrip()}')
+    m = re.search(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', out)
+    if not m: raise SystemExit(f'[FAIL] no UUID in output; rc={rc}')
+    UUID = m.group(1)
+    print(f'[OK] tunnel UUID: {UUID}')
+
+    print('\n=== [3] write config.yml ===', flush=True)
+    config = f'''tunnel: {UUID}
+credentials-file: /home/nonroot/.cloudflared/{UUID}.json
+ingress:
+'''
+    for h in HOSTNAMES:
+        config += f'  - hostname: {h}\n    service: {ORIGIN}\n'
+    config += '  - service: http_status:404\n'
+    # Write via heredoc
+    HERE = "'EOF_CONFIG'"
+    out, err, rc = run(f"cat > {APPDATA}/config.yml <<{HERE}\n{config}\nEOF_CONFIG")
+    run(f'chown 65532:65532 {APPDATA}/config.yml')
+    out, _, _ = run(f'cat {APPDATA}/config.yml')
+    print(out.rstrip())
+
+    print('\n=== [4] DNS cutover (A -> CNAME) ===', flush=True)
+    tunnel_target = f'{UUID}.cfargotunnel.com'
+    for h in HOSTNAMES:
+        # Find existing record
+        r = cfapi('GET', f'/zones/{ZONE}/dns_records?name={h}')
+        if not r.get('success') or not r['result']:
+            print(f'  [SKIP] {h}: no record found')
+            continue
+        rec = r['result'][0]
+        print(f'  [{h}] current: type={rec["type"]} content={rec["content"]} proxied={rec["proxied"]} id={rec["id"]}')
+        if rec['type']=='CNAME' and rec['content']==tunnel_target:
+            print(f'    already pointing at tunnel, skipping')
+            continue
+        # Delete
+        d = cfapi('DELETE', f'/zones/{ZONE}/dns_records/{rec["id"]}')
+        if not d.get('success'):
+            print(f'    [FAIL delete] {d.get("errors")}')
+            continue
+        # Create CNAME
+        body = {'type':'CNAME','name':h,'content':tunnel_target,'proxied':True,'ttl':1}
+        cr = cfapi('POST', f'/zones/{ZONE}/dns_records', body)
+        if cr.get('success'):
+            print(f'    [OK] -> CNAME {tunnel_target} proxied')
+        else:
+            print(f'    [FAIL create] {cr.get("errors")}')
+
+    print('\n=== [5] start persistent cloudflared ===', flush=True)
+    run('docker rm -f cloudflared 2>&1')
+    START = (
+        'docker run -d --name cloudflared --restart=unless-stopped '
+        f'-v {APPDATA}:/home/nonroot/.cloudflared '
+        'cloudflare/cloudflared:latest '
+        'tunnel --config /home/nonroot/.cloudflared/config.yml run'
+    )
+    out, err, rc = run(START)
+    print(out.rstrip())
+    if err.strip(): print(f'[stderr] {err.rstrip()}')
+
+    print('\n=== [6] wait for tunnel connections ===', flush=True)
+    for i in range(20):
+        time.sleep(3)
+        out, _, _ = run('docker logs cloudflared 2>&1 | tail -30')
+        conns = out.count('Registered tunnel connection')
+        print(f'  [try {i+1}] connections registered: {conns}')
+        if conns >= 4:
+            print(out.rstrip()[-800:])
+            break
+
+    print('\n=== [7] verify externally ===', flush=True)
+finally:
+    c.close()
+
+# Run external curl from this workstation
+print('\n[EXTERNAL CHECK]', flush=True)
+for h in HOSTNAMES:
+    try:
+        req = urllib.request.Request(f'https://{h}/', method='HEAD',
+                                     headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'})
+        with urllib.request.urlopen(req, timeout=15) as r:
+            print(f'  {h}: HTTP {r.status}')
+    except urllib.error.HTTPError as e:
+        print(f'  {h}: HTTP {e.code}')
+    except Exception as e:
+        print(f'  {h}: ERR {e}')
+
+print(f'\n[DONE] tunnel UUID: {UUID}')
+print(f'[DONE] config: {APPDATA}/config.yml')
+print(f'[DONE] persistent container: cloudflared')