"""Paginate API to fetch every server-side SerialNumber, write to file.""" import json import time import urllib.request import urllib.parse import os, sys TOKEN_URL = os.environ.get("CF_TOKEN_URL", "https://login.dataforth.com/connect/token") API_BASE = os.environ.get("CF_API_BASE", "https://www.dataforth.com") + "/api/v1" CLIENT_ID = os.environ.get("CF_CLIENT_ID", "") CLIENT_SECRET = os.environ.get("CF_CLIENT_SECRET", "") SCOPE = os.environ.get("CF_SCOPE", "dataforth.web") if not CLIENT_ID or not CLIENT_SECRET: sys.exit("set CF_CLIENT_ID + CF_CLIENT_SECRET (vault: clients/dataforth/api-oauth.sops.yaml)") OUTFILE = r"C:\Users\guru\AppData\Local\Temp\server_inventory.txt" PAGE_SIZE = 1000 # try 1000 first; smaller if server complains def get_token(): data = urllib.parse.urlencode({ "grant_type": "client_credentials", "client_id": CLIENT_ID, "client_secret": CLIENT_SECRET, "scope": SCOPE, }).encode() with urllib.request.urlopen(urllib.request.Request(TOKEN_URL, data=data)) as r: return json.loads(r.read())["access_token"] def main(): token = get_token() headers = {"Authorization": f"Bearer {token}"} total = 0 page = 1 cursor = None t_start = time.time() with open(OUTFILE, "w") as f: while True: params = {"page": page, "pageSize": PAGE_SIZE} if cursor: params["afterSerialNumber"] = cursor url = f"{API_BASE}/TestReportDataFiles?" + urllib.parse.urlencode(params) req = urllib.request.Request(url, headers=headers) try: with urllib.request.urlopen(req, timeout=30) as r: obj = json.loads(r.read()) except Exception as e: print(f" ERROR at page {page}: {e}") # Refresh token and retry once token = get_token() headers = {"Authorization": f"Bearer {token}"} with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=30) as r: obj = json.loads(r.read()) items = obj.get("Items", []) if not items: break for it in items: f.write(it["SerialNumber"] + "\n") total += len(items) cursor = obj.get("NextCursor") if page % 50 == 0 or page == 1: rate = total / max(1, time.time() - t_start) print(f" page {page}: total={total} rate={rate:.0f}/s cursor={cursor!r}") if not cursor: break page += 1 elapsed = time.time() - t_start print(f"\nDONE: {total} serials written to {OUTFILE} in {elapsed:.1f}s") if __name__ == "__main__": main()