claudetools/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1

# Dataforth Test Datasheet Uploader (daily)
$ErrorActionPreference = 'Stop'
$prod     = 'C:\ProgramData\dataforth-uploader'
$logDir   = Join-Path $prod 'logs'
$uploadLogDir = Join-Path $prod 'upload-logs'
$nodeExe  = 'C:\Program Files\nodejs\node.exe'
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
$stamp    = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
$log      = Join-Path $logDir "pipeline-$stamp.log"
$smtpCred = $null

function Log([string]$m) {
    $line = "[$(Get-Date -Format o)] $m"
    Write-Host $line
    Add-Content -Path $log -Value $line -Encoding utf8
}

function SendEmail([string]$Subject, [string]$Body) {
    if ($null -eq $smtpCred) { return }
    try {
        Send-MailMessage `
            -From       'sysadmin@dataforth.com' `
            -To         @('mike@azcomputerguru.com') `
            -Subject    $Subject `
            -Body       $Body `
            -SmtpServer 'smtp.office365.com' `
            -Port       587 `
            -UseSsl `
            -Credential $smtpCred
        Log "Email sent: $Subject"
    } catch {
        Log "WARNING: Email send failed: $_"
    }
}

try {
    Log "=== pipeline start (pid=$PID) ==="

    # Load credentials
    $creds = Get-Content (Join-Path $prod 'credentials.json') -Raw | ConvertFrom-Json
    $env:CF_TOKEN_URL     = $creds.CF_TOKEN_URL
    $env:CF_API_BASE      = $creds.CF_API_BASE
    $env:CF_CLIENT_ID     = $creds.CF_CLIENT_ID
    $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
    $env:CF_SCOPE         = $creds.CF_SCOPE

    if ($creds.SMTP_USER -and $creds.SMTP_PASS) {
        $secPass  = ConvertTo-SecureString $creds.SMTP_PASS -AsPlainText -Force
        $smtpCred = New-Object System.Management.Automation.PSCredential($creds.SMTP_USER, $secPass)
        Log "SMTP credentials loaded for $($creds.SMTP_USER)"
    } else {
        Log "WARNING: SMTP_USER/SMTP_PASS not in credentials.json — email notifications disabled"
    }

    # [1] DFWDS process
    Log '[1] dfwds-process.js'
    $dfwdsJs = Join-Path $prod 'dfwds-process.js'
    $out = & $nodeExe $dfwdsJs 2>&1
    $out | ForEach-Object { Log $_ }

    # [2] Enumerate For_Web
    Log '[2] enumerate For_Web'
    $delta = Join-Path $prod 'delta_for_web_all.txt'
    Get-ChildItem 'C:\Shares\webshare\For_Web' -File -Filter *.TXT |
        ForEach-Object {
            $sn = [System.IO.Path]::GetFileNameWithoutExtension($_.Name)
            "$sn|$($_.FullName)|$($_.Length)|$($_.LastWriteTime.ToString('o'))"
        } | Set-Content -Path $delta -Encoding ASCII
    $count = (Get-Content $delta).Count
    Log "    enumerated $count files"

    # [3] Upload via Node
    Log '[3] upload-delta.js'
    $uploadJs = Join-Path $prod 'upload-delta.js'
    $out = & $nodeExe $uploadJs --delta $delta --batch 100 2>&1
    $out | ForEach-Object { Log $_ }

    # [4] Daily summary email
    Log '[4] sending daily summary email'
    if ($smtpCred) {
        try {
            # Parse totals from most recent upload log; default to zeros if not found
            $totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 }
            $latestUploadLog = Get-ChildItem $uploadLogDir -Filter '*.log' -ErrorAction SilentlyContinue |
                Sort-Object LastWriteTime -Descending | Select-Object -First 1
            if ($latestUploadLog) {
                $totalsLine = Get-Content $latestUploadLog.FullName -ErrorAction SilentlyContinue |
                    Where-Object { $_ -match '^# totals ' } | Select-Object -Last 1
                if ($totalsLine) {
                    $totals = ($totalsLine -replace '^# totals\s*','') | ConvertFrom-Json
                }
            }
            $status  = if ($totals.errors -eq 0) { 'OK' } else { 'FAIL' }
            $dateStr = Get-Date -Format 'yyyy-MM-dd'
            $summaryBody = @"
Dataforth test datasheet daily pipeline completed on $env:COMPUTERNAME.
Time: $(Get-Date -Format o)

Files processed : $($totals.received)
  Created (new on website) : $($totals.created)
  Updated (refreshed)      : $($totals.updated)
  Unchanged                : $($totals.unchanged)
  Errors                   : $($totals.errors)

Log: $($latestUploadLog.FullName)
"@
            SendEmail "[TestDataDB] Daily pipeline $status — $dateStr" $summaryBody
        } catch {
            Log "WARNING: Summary email failed: $_"
        }
    } else {
        Log '    SMTP not configured — skipping summary email'
    }

    Log '=== pipeline end (OK) ==='
} catch {
    Log "FATAL: $_"
    Log "StackTrace: $($_.ScriptStackTrace)"

    $date    = Get-Date -Format 'yyyy-MM-dd'
    $host_   = $env:COMPUTERNAME
    $errBody = @"
Host:       $host_
Time:       $(Get-Date -Format o)
Date:       $date

FATAL ERROR
-----------
$_

Stack Trace
-----------
$($_.ScriptStackTrace)

Log file: $log
"@
    SendEmail "[TestDataDB] PIPELINE FAILURE — $date" $errBody

    throw
} finally {
    # Retention: keep 60 days of pipeline logs
    Get-ChildItem $logDir -Filter 'pipeline-*.log' -ErrorAction SilentlyContinue |
        Where-Object { $_.LastWriteTime -lt (Get-Date).AddDays(-60) } |
        Remove-Item -Force -ErrorAction SilentlyContinue
}