sync: auto-sync from DESKTOP-0O8A1RL at 2026-05-12 06:47:00

Author: Mike Swanson
Machine: DESKTOP-0O8A1RL
Timestamp: 2026-05-12 06:47:00

projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1

@@ -0,0 +1,145 @@
+# Dataforth Test Datasheet Uploader (daily)
+$ErrorActionPreference = 'Stop'
+$prod     = 'C:\ProgramData\dataforth-uploader'
+$logDir   = Join-Path $prod 'logs'
+$uploadLogDir = Join-Path $prod 'upload-logs'
+$nodeExe  = 'C:\Program Files\nodejs\node.exe'
+New-Item -ItemType Directory -Force -Path $logDir | Out-Null
+$stamp    = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
+$log      = Join-Path $logDir "pipeline-$stamp.log"
+$smtpCred = $null
+
+function Log([string]$m) {
+    $line = "[$(Get-Date -Format o)] $m"
+    Write-Host $line
+    Add-Content -Path $log -Value $line -Encoding utf8
+}
+
+function SendEmail([string]$Subject, [string]$Body) {
+    if ($null -eq $smtpCred) { return }
+    try {
+        Send-MailMessage `
+            -From       'sysadmin@dataforth.com' `
+            -To         @('mike@azcomputerguru.com') `
+            -Subject    $Subject `
+            -Body       $Body `
+            -SmtpServer 'smtp.office365.com' `
+            -Port       587 `
+            -UseSsl `
+            -Credential $smtpCred
+        Log "Email sent: $Subject"
+    } catch {
+        Log "WARNING: Email send failed: $_"
+    }
+}
+
+try {
+    Log "=== pipeline start (pid=$PID) ==="
+
+    # Load credentials
+    $creds = Get-Content (Join-Path $prod 'credentials.json') -Raw | ConvertFrom-Json
+    $env:CF_TOKEN_URL     = $creds.CF_TOKEN_URL
+    $env:CF_API_BASE      = $creds.CF_API_BASE
+    $env:CF_CLIENT_ID     = $creds.CF_CLIENT_ID
+    $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
+    $env:CF_SCOPE         = $creds.CF_SCOPE
+
+    if ($creds.SMTP_USER -and $creds.SMTP_PASS) {
+        $secPass  = ConvertTo-SecureString $creds.SMTP_PASS -AsPlainText -Force
+        $smtpCred = New-Object System.Management.Automation.PSCredential($creds.SMTP_USER, $secPass)
+        Log "SMTP credentials loaded for $($creds.SMTP_USER)"
+    } else {
+        Log "WARNING: SMTP_USER/SMTP_PASS not in credentials.json — email notifications disabled"
+    }
+
+    # [1] DFWDS process
+    Log '[1] dfwds-process.js'
+    $dfwdsJs = Join-Path $prod 'dfwds-process.js'
+    $out = & $nodeExe $dfwdsJs 2>&1
+    $out | ForEach-Object { Log $_ }
+
+    # [2] Enumerate For_Web
+    Log '[2] enumerate For_Web'
+    $delta = Join-Path $prod 'delta_for_web_all.txt'
+    Get-ChildItem 'C:\Shares\webshare\For_Web' -File -Filter *.TXT |
+        ForEach-Object {
+            $sn = [System.IO.Path]::GetFileNameWithoutExtension($_.Name)
+            "$sn|$($_.FullName)|$($_.Length)|$($_.LastWriteTime.ToString('o'))"
+        } | Set-Content -Path $delta -Encoding ASCII
+    $count = (Get-Content $delta).Count
+    Log "    enumerated $count files"
+
+    # [3] Upload via Node
+    Log '[3] upload-delta.js'
+    $uploadJs = Join-Path $prod 'upload-delta.js'
+    $out = & $nodeExe $uploadJs --delta $delta --batch 100 2>&1
+    $out | ForEach-Object { Log $_ }
+
+    # [4] Daily summary email
+    Log '[4] sending daily summary email'
+    if ($smtpCred) {
+        try {
+            # Parse totals from most recent upload log; default to zeros if not found
+            $totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 }
+            $latestUploadLog = Get-ChildItem $uploadLogDir -Filter '*.log' -ErrorAction SilentlyContinue |
+                Sort-Object LastWriteTime -Descending | Select-Object -First 1
+            if ($latestUploadLog) {
+                $totalsLine = Get-Content $latestUploadLog.FullName -ErrorAction SilentlyContinue |
+                    Where-Object { $_ -match '^# totals ' } | Select-Object -Last 1
+                if ($totalsLine) {
+                    $totals = ($totalsLine -replace '^# totals\s*','') | ConvertFrom-Json
+                }
+            }
+            $status  = if ($totals.errors -eq 0) { 'OK' } else { 'FAIL' }
+            $dateStr = Get-Date -Format 'yyyy-MM-dd'
+            $summaryBody = @"
+Dataforth test datasheet daily pipeline completed on $env:COMPUTERNAME.
+Time: $(Get-Date -Format o)
+
+Files processed : $($totals.received)
+  Created (new on website) : $($totals.created)
+  Updated (refreshed)      : $($totals.updated)
+  Unchanged                : $($totals.unchanged)
+  Errors                   : $($totals.errors)
+
+Log: $($latestUploadLog.FullName)
+"@
+            SendEmail "[TestDataDB] Daily pipeline $status — $dateStr" $summaryBody
+        } catch {
+            Log "WARNING: Summary email failed: $_"
+        }
+    } else {
+        Log '    SMTP not configured — skipping summary email'
+    }
+
+    Log '=== pipeline end (OK) ==='
+} catch {
+    Log "FATAL: $_"
+    Log "StackTrace: $($_.ScriptStackTrace)"
+
+    $date    = Get-Date -Format 'yyyy-MM-dd'
+    $host_   = $env:COMPUTERNAME
+    $errBody = @"
+Host:       $host_
+Time:       $(Get-Date -Format o)
+Date:       $date
+
+FATAL ERROR
+-----------
+$_
+
+Stack Trace
+-----------
+$($_.ScriptStackTrace)
+
+Log file: $log
+"@
+    SendEmail "[TestDataDB] PIPELINE FAILURE — $date" $errBody
+
+    throw
+} finally {
+    # Retention: keep 60 days of pipeline logs
+    Get-ChildItem $logDir -Filter 'pipeline-*.log' -ErrorAction SilentlyContinue |
+        Where-Object { $_.LastWriteTime -lt (Get-Date).AddDays(-60) } |
+        Remove-Item -Force -ErrorAction SilentlyContinue
+}