claudetools/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1

# Dataforth Test Datasheet Uploader (daily)
$ErrorActionPreference = 'Stop'
$prod     = 'C:\ProgramData\dataforth-uploader'
$logDir   = Join-Path $prod 'logs'
$uploadLogDir = Join-Path $prod 'upload-logs'
$nodeExe  = 'C:\Program Files\nodejs\node.exe'
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
$stamp    = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
$log      = Join-Path $logDir "pipeline-$stamp.log"
$graphCreds = $null

function Log([string]$m) {
    $line = "[$(Get-Date -Format o)] $m"
    Write-Host $line
    Add-Content -Path $log -Value $line -Encoding utf8
}

function SendEmail([string]$Subject, [string]$Body) {
    if ($null -eq $graphCreds) { return }
    try {
        # Acquire Graph token
        $tokenBody = @{
            grant_type    = 'client_credentials'
            client_id     = $graphCreds.clientId
            client_secret = $graphCreds.clientSecret
            scope         = 'https://graph.microsoft.com/.default'
        }
        $tokenResp = Invoke-RestMethod `
            -Method Post `
            -Uri    "https://login.microsoftonline.com/$($graphCreds.tenantId)/oauth2/v2.0/token" `
            -Body   $tokenBody
        $token = $tokenResp.access_token

        # Send mail via Graph
        $mailPayload = @{
            message = @{
                subject      = $Subject
                body         = @{ contentType = 'Text'; content = $Body }
                toRecipients = @(@{ emailAddress = @{ address = 'mike@azcomputerguru.com' } })
                from         = @{ emailAddress = @{ address = 'sysadmin@dataforth.com' } }
            }
        } | ConvertTo-Json -Depth 10

        Invoke-RestMethod `
            -Method  Post `
            -Uri     'https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail' `
            -Headers @{ Authorization = "Bearer $token"; 'Content-Type' = 'application/json' } `
            -Body    $mailPayload | Out-Null

        Log "Email sent: $Subject"
    } catch {
        Log "WARNING: Email send failed: $_"
    }
}

try {
    Log "=== pipeline start (pid=$PID) ==="

    # Load credentials
    $creds = Get-Content (Join-Path $prod 'credentials.json') -Raw | ConvertFrom-Json
    $env:CF_TOKEN_URL     = $creds.CF_TOKEN_URL
    $env:CF_API_BASE      = $creds.CF_API_BASE
    $env:CF_CLIENT_ID     = $creds.CF_CLIENT_ID
    $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
    $env:CF_SCOPE         = $creds.CF_SCOPE

    if ($creds.GRAPH_TENANT_ID -and $creds.GRAPH_CLIENT_ID -and $creds.GRAPH_CLIENT_SECRET) {
        $graphCreds = [PSCustomObject]@{
            tenantId     = $creds.GRAPH_TENANT_ID
            clientId     = $creds.GRAPH_CLIENT_ID
            clientSecret = $creds.GRAPH_CLIENT_SECRET
        }
        Log "Graph credentials loaded (app $($creds.GRAPH_CLIENT_ID))"
    } else {
        Log "WARNING: GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — email notifications disabled"
    }

    # [1] DFWDS process
    Log '[1] dfwds-process.js'
    $dfwdsJs = Join-Path $prod 'dfwds-process.js'
    $out = & $nodeExe $dfwdsJs 2>&1
    $out | ForEach-Object { Log $_ }

    # [2] Enumerate For_Web
    Log '[2] enumerate For_Web'
    $delta = Join-Path $prod 'delta_for_web_all.txt'
    Get-ChildItem 'C:\Shares\webshare\For_Web' -File -Filter *.TXT |
        ForEach-Object {
            $sn = [System.IO.Path]::GetFileNameWithoutExtension($_.Name)
            "$sn|$($_.FullName)|$($_.Length)|$($_.LastWriteTime.ToString('o'))"
        } | Set-Content -Path $delta -Encoding ASCII
    $count = (Get-Content $delta).Count
    Log "    enumerated $count files"

    # [3] Upload via Node
    Log '[3] upload-delta.js'
    $uploadJs = Join-Path $prod 'upload-delta.js'
    $out = & $nodeExe $uploadJs --delta $delta --batch 100 2>&1
    $out | ForEach-Object { Log $_ }

    # [4] Daily summary email
    Log '[4] sending daily summary email'
    if ($graphCreds) {
        try {
            # Parse totals from most recent upload log; default to zeros if not found
            $totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 }
            $latestUploadLog = Get-ChildItem $uploadLogDir -Filter '*.log' -ErrorAction SilentlyContinue |
                Sort-Object LastWriteTime -Descending | Select-Object -First 1
            if ($latestUploadLog) {
                $totalsLine = Get-Content $latestUploadLog.FullName -ErrorAction SilentlyContinue |
                    Where-Object { $_ -match '^# totals ' } | Select-Object -Last 1
                if ($totalsLine) {
                    $totals = ($totalsLine -replace '^# totals\s*','') | ConvertFrom-Json
                }
            }
            $status  = if ($totals.errors -eq 0) { 'OK' } else { 'FAIL' }
            $dateStr = Get-Date -Format 'yyyy-MM-dd'
            $summaryBody = @"
Dataforth test datasheet daily pipeline completed on $env:COMPUTERNAME.
Time: $(Get-Date -Format o)

Files processed : $($totals.received)
  Created (new on website) : $($totals.created)
  Updated (refreshed)      : $($totals.updated)
  Unchanged                : $($totals.unchanged)
  Errors                   : $($totals.errors)

Log: $($latestUploadLog.FullName)
"@
            SendEmail "[TestDataDB] Daily pipeline $status — $dateStr" $summaryBody
        } catch {
            Log "WARNING: Summary email failed: $_"
        }
    } else {
        Log '    Graph credentials not configured — skipping summary email'
    }

    Log '=== pipeline end (OK) ==='
} catch {
    Log "FATAL: $_"
    Log "StackTrace: $($_.ScriptStackTrace)"

    $date    = Get-Date -Format 'yyyy-MM-dd'
    $host_   = $env:COMPUTERNAME
    $errBody = @"
Host:       $host_
Time:       $(Get-Date -Format o)
Date:       $date

FATAL ERROR
-----------
$_

Stack Trace
-----------
$($_.ScriptStackTrace)

Log file: $log
"@
    SendEmail "[TestDataDB] PIPELINE FAILURE — $date" $errBody

    throw
} finally {
    # Retention: keep 60 days of pipeline logs
    Get-ChildItem $logDir -Filter 'pipeline-*.log' -ErrorAction SilentlyContinue |
        Where-Object { $_.LastWriteTime -lt (Get-Date).AddDays(-60) } |
        Remove-Item -Force -ErrorAction SilentlyContinue
}