AD2 scheduled task for Dataforth uploader pipeline (hourly, SYSTEM)

Installed C:\ProgramData\dataforth-uploader\ on AD2 with:
  - credentials.json (SYSTEM+Administrators ACL only)
  - run-pipeline.ps1 (DFWDS-process -> enumerate For_Web -> upload-delta)
  - dfwds-process.js + upload-delta.js (copied from prior install dir)
  - logs/ with 60-day retention

Scheduled Task 'DataforthTestDatasheetUploader' registered as SYSTEM,
hourly trigger, 30-min execution limit. First SYSTEM-context run verified:
received=7061 unchanged=7061 errors=0 in 8.7s.

Initial registration via inline base64 mangled the backslashes in the -File
argument (resulted in ERROR_DIRECTORY 0x8007010B). Fixed by running the
registration PowerShell from a file rather than an encoded command string.

Also deleted throwaway tmp/list_amtransit.py + tmp/reset_cansley.py which
had hardcoded ACG admin password.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

projects/dataforth-dos/datasheet-pipeline/fix_run_pipeline.py

@@ -0,0 +1,105 @@
+"""Patch run-pipeline.ps1 to use full node.exe path and retry."""
+import base64, paramiko, subprocess, time, yaml
+
+ad2_pwd = yaml.safe_load(subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
+    capture_output=True, text=True, timeout=30, check=True).stdout)['credentials']['password'].replace('\\','')
+
+PROD_DIR = r'C:\ProgramData\dataforth-uploader'
+
+RUN_PS1 = r'''# Dataforth Test Datasheet Uploader (hourly)
+$ErrorActionPreference = 'Stop'
+$prod = 'C:\ProgramData\dataforth-uploader'
+$logDir = Join-Path $prod 'logs'
+$nodeExe = 'C:\Program Files\nodejs\node.exe'
+New-Item -ItemType Directory -Force -Path $logDir | Out-Null
+$stamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
+$log = Join-Path $logDir "pipeline-$stamp.log"
+
+function Log([string]$m) {
+    $line = "[$(Get-Date -Format o)] $m"
+    Write-Host $line
+    Add-Content -Path $log -Value $line -Encoding utf8
+}
+
+try {
+    Log "=== pipeline start (pid=$PID) ==="
+
+    # Load credentials
+    $creds = Get-Content (Join-Path $prod 'credentials.json') -Raw | ConvertFrom-Json
+    $env:CF_TOKEN_URL     = $creds.CF_TOKEN_URL
+    $env:CF_API_BASE      = $creds.CF_API_BASE
+    $env:CF_CLIENT_ID     = $creds.CF_CLIENT_ID
+    $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
+    $env:CF_SCOPE         = $creds.CF_SCOPE
+
+    # [1] DFWDS process
+    Log '[1] dfwds-process.js'
+    $dfwdsJs = Join-Path $prod 'dfwds-process.js'
+    $out = & $nodeExe $dfwdsJs 2>&1
+    $out | ForEach-Object { Log $_ }
+
+    # [2] Enumerate For_Web
+    Log '[2] enumerate For_Web'
+    $delta = Join-Path $prod 'delta_for_web_all.txt'
+    Get-ChildItem 'C:\Shares\webshare\For_Web' -File -Filter *.TXT |
+        ForEach-Object {
+            $sn = [System.IO.Path]::GetFileNameWithoutExtension($_.Name)
+            "$sn|$($_.FullName)|$($_.Length)|$($_.LastWriteTime.ToString('o'))"
+        } | Set-Content -Path $delta -Encoding ASCII
+    $count = (Get-Content $delta).Count
+    Log "    enumerated $count files"
+
+    # [3] Upload via Node
+    Log '[3] upload-delta.js'
+    $uploadJs = Join-Path $prod 'upload-delta.js'
+    $out = & $nodeExe $uploadJs --delta $delta --batch 100 2>&1
+    $out | ForEach-Object { Log $_ }
+
+    Log '=== pipeline end (OK) ==='
+} catch {
+    Log "FATAL: $_"
+    Log "StackTrace: $($_.ScriptStackTrace)"
+    throw
+} finally {
+    # Retention: keep 60 days of pipeline logs
+    Get-ChildItem $logDir -Filter 'pipeline-*.log' -ErrorAction SilentlyContinue |
+        Where-Object { $_.LastWriteTime -lt (Get-Date).AddDays(-60) } |
+        Remove-Item -Force -ErrorAction SilentlyContinue
+}
+'''
+
+c = paramiko.SSHClient(); c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect('192.168.0.6', username='sysadmin', password=ad2_pwd,
+          timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
+
+sftp = c.open_sftp()
+with sftp.open(f'{PROD_DIR.replace(chr(92),"/")}/run-pipeline.ps1', 'w') as fh:
+    fh.write(RUN_PS1)
+sftp.close()
+print('[1] run-pipeline.ps1 updated with full node.exe path')
+
+def psb64(cmd, to=120):
+    enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
+    _, o, e = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
+    return o.read().decode('utf-8','replace'), e.read().decode('utf-8','replace'), o.channel.recv_exit_status()
+
+print('\n[2] trigger scheduled task')
+out, _, _ = psb64('Start-ScheduledTask -TaskName "DataforthTestDatasheetUploader"')
+print('  triggered')
+
+print('\n[3] wait 25s for completion')
+time.sleep(25)
+out, _, _ = psb64(
+    r'Get-ScheduledTaskInfo -TaskName "DataforthTestDatasheetUploader" | '
+    r'Select LastRunTime,LastTaskResult,NextRunTime | Format-List'
+)
+print(out.strip())
+
+print('\n[4] tail latest pipeline log')
+out, _, _ = psb64(
+    f'$latest = Get-ChildItem "{PROD_DIR}\\logs" -Filter "pipeline-*.log" -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select -First 1; '
+    f'"Log: $($latest.FullName)"; Get-Content $latest.FullName -Tail 80 -ErrorAction SilentlyContinue'
+)
+print(out.strip())
+
+c.close()