From 470638ff8678b0dff99726a297dddf89654b98f0 Mon Sep 17 00:00:00 2001 From: Mike Swanson Date: Wed, 11 Mar 2026 20:16:24 -0700 Subject: [PATCH] sync: Dataforth sync fixes, TestDataDB stability, and client scripts Dataforth DOS: - TestDataDB: singleton DB connection fix (crash prevention), WAL mode, WinSW service config, backup script, uncaught exception handlers - Sync-FromNAS.ps1: Get-NASFileList temp file approach to avoid SSH stdout deadlock, *> $null output suppression, 8.3 filename filter for PUSH phase, backslash-escaped SCP paths, rename-to-.synced - import.js: INSERT OR REPLACE for re-tested devices - Full import run: 1,028,275 -> 1,632,793 records, indexes added - Deploy script for sync fixes to AD2 Client scripts (temp/): - BG Builders: Lesley account check, MFA phone update - Lonestar Electrical: Kyla/Russ Google Workspace setup, 2FA bypass - AD2 diagnostics and NAS connectivity tests PENDING: Investigate why newest test_date is Jan 19 despite daily tests Co-Authored-By: Claude Opus 4.6 --- .../dataforth-dos/sync-fixes/Sync-FromNAS.ps1 | 540 ++++++++++++++++++ .../sync-fixes/deploy-sync-fixes.ps1 | 302 ++++++++++ projects/dataforth-dos/sync-fixes/import.js | 395 +++++++++++++ .../testdatadb-fix/backup-db.ps1 | 61 ++ .../dataforth-dos/testdatadb-fix/deploy.ps1 | 134 +++++ .../testdatadb-fix/install-backup-task.ps1 | 55 ++ .../testdatadb-fix/install-service.js | 51 ++ .../dataforth-dos/testdatadb-fix/package.json | 18 + .../testdatadb-fix/routes/api.js | 363 ++++++++++++ .../dataforth-dos/testdatadb-fix/server.js | 103 ++++ .../testdatadb-fix/uninstall-service.js | 24 + temp/ad2-diag.ps1 | 28 + temp/bgb-lesley-check.ps1 | 51 ++ temp/bgb-lesley-mfa-phone.ps1 | 52 ++ temp/bgb-lesley-mfa-phone2.ps1 | 26 + temp/lonestar-kyla-2fa-fix.py | 52 ++ temp/lonestar-kyla-reset.py | 60 ++ temp/lonestar-kyla-reset2.py | 25 + temp/lonestar-russ-setup.py | 41 ++ temp/test-ad2-web.ps1 | 19 + temp/test-minimal.ps1 | 3 + temp/test-nas-from-ad2.ps1 | 48 ++ temp/test-nas-v2.ps1 | 32 ++ temp/testdatadb.xml | 15 + 24 files changed, 2498 insertions(+) create mode 100644 projects/dataforth-dos/sync-fixes/Sync-FromNAS.ps1 create mode 100644 projects/dataforth-dos/sync-fixes/deploy-sync-fixes.ps1 create mode 100644 projects/dataforth-dos/sync-fixes/import.js create mode 100644 projects/dataforth-dos/testdatadb-fix/backup-db.ps1 create mode 100644 projects/dataforth-dos/testdatadb-fix/deploy.ps1 create mode 100644 projects/dataforth-dos/testdatadb-fix/install-backup-task.ps1 create mode 100644 projects/dataforth-dos/testdatadb-fix/install-service.js create mode 100644 projects/dataforth-dos/testdatadb-fix/package.json create mode 100644 projects/dataforth-dos/testdatadb-fix/routes/api.js create mode 100644 projects/dataforth-dos/testdatadb-fix/server.js create mode 100644 projects/dataforth-dos/testdatadb-fix/uninstall-service.js create mode 100644 temp/ad2-diag.ps1 create mode 100644 temp/bgb-lesley-check.ps1 create mode 100644 temp/bgb-lesley-mfa-phone.ps1 create mode 100644 temp/bgb-lesley-mfa-phone2.ps1 create mode 100644 temp/lonestar-kyla-2fa-fix.py create mode 100644 temp/lonestar-kyla-reset.py create mode 100644 temp/lonestar-kyla-reset2.py create mode 100644 temp/lonestar-russ-setup.py create mode 100644 temp/test-ad2-web.ps1 create mode 100644 temp/test-minimal.ps1 create mode 100644 temp/test-nas-from-ad2.ps1 create mode 100644 temp/test-nas-v2.ps1 create mode 100644 temp/testdatadb.xml diff --git a/projects/dataforth-dos/sync-fixes/Sync-FromNAS.ps1 b/projects/dataforth-dos/sync-fixes/Sync-FromNAS.ps1 new file mode 100644 index 0000000..0ba9c61 --- /dev/null +++ b/projects/dataforth-dos/sync-fixes/Sync-FromNAS.ps1 @@ -0,0 +1,540 @@ +# Sync-AD2-NAS.ps1 (formerly Sync-FromNAS.ps1) +# Bidirectional sync between AD2 and NAS (D2TESTNAS) +# +# PULL (NAS -> AD2): Test results (LOGS/*.DAT, Reports/*.TXT) -> Database import +# PUSH (AD2 -> NAS): Software updates (ProdSW/*, TODO.BAT) -> DOS machines +# +# Run: powershell -ExecutionPolicy Bypass -File C:\Shares\test\scripts\Sync-FromNAS.ps1 +# Scheduled: Every 15 minutes via Windows Task Scheduler + +param( + [switch]$DryRun, # Show what would be done without doing it + [switch]$Verbose, # Extra output + [int]$MaxAgeMinutes = 1440 # Default: files from last 24 hours (was 60 min, too aggressive) +) + +# ============================================================================ +# Configuration +# ============================================================================ +$NAS_IP = "192.168.0.9" +$NAS_USER = "root" +$NAS_PASSWORD = "Paper123!@#-nas" +$NAS_HOSTKEY = "SHA256:5CVIPlqjLPxO8n48PKLAP99nE6XkEBAjTkaYmJAeOdA" +$NAS_DATA_PATH = "/data/test" + +$AD2_TEST_PATH = "C:\Shares\test" +$AD2_HISTLOGS_PATH = "C:\Shares\test\Ate\HISTLOGS" + +$SSH = "C:\Program Files\OpenSSH\ssh.exe" +$SCP = "C:\Program Files\OpenSSH\scp.exe" +$SSH_KEY = "C:\Users\sysadmin\.ssh\id_ed25519" + +$LOG_FILE = "C:\Shares\test\scripts\sync-from-nas.log" +$STATUS_FILE = "C:\Shares\test\_SYNC_STATUS.txt" + +$LOG_TYPES = @("5BLOG", "7BLOG", "8BLOG", "DSCLOG", "SCTLOG", "VASLOG", "PWRLOG", "HVLOG") + +# Database import configuration +$IMPORT_SCRIPT = "C:\Shares\testdatadb\database\import.js" +$NODE_PATH = "node" + +# ============================================================================ +# Functions +# ============================================================================ + +function Write-Log { + param([string]$Message) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logLine = "$timestamp : $Message" + Add-Content -Path $LOG_FILE -Value $logLine + if ($Verbose) { Write-Host $logLine } +} + +function Invoke-NASCommand { + param([string]$Command) + $result = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" $Command 2>&1 + return $result +} + +function Get-NASFileList { + param( + [string]$FindCommand, + [string]$ListLabel + ) + # Generate file list on NAS side, write to temp file, pull via SCP + # Key fix: use *> $null to discard all PowerShell output streams, preventing + # the stdout buffer deadlock that occurs with & $SSH ... 2>&1 + $remoteTemp = "/tmp/nas-file-list-${ListLabel}.txt" + + # Step 1: Run find on NAS, redirect output to temp file on NAS + # We discard ALL PowerShell output (*> $null) - we only need the side effect + & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" "$FindCommand > $remoteTemp 2>/dev/null" *> $null + + # Step 2: Pull the list file via SCP + $localTemp = "$env:TEMP\nas-file-list-${ListLabel}-$(Get-Date -Format 'yyyyMMddHHmmss').txt" + & $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "${NAS_USER}@${NAS_IP}:${remoteTemp}" "$localTemp" *> $null + + # Step 3: Read locally and clean up + $fileList = Get-Content $localTemp -ErrorAction SilentlyContinue | Where-Object { $_.Trim() -ne '' } + Remove-Item $localTemp -ErrorAction SilentlyContinue + + # Step 4: Clean up remote temp file (small output, safe with Invoke-NASCommand) + Invoke-NASCommand "rm -f $remoteTemp" | Out-Null + + return $fileList +} + +function Copy-FromNAS { + param( + [string]$RemotePath, + [string]$LocalPath + ) + + # Ensure local directory exists + $localDir = Split-Path -Parent $LocalPath + if (-not (Test-Path $localDir)) { + New-Item -ItemType Directory -Path $localDir -Force | Out-Null + } + + # Escape spaces and special chars with backslashes for SCP remote path + $escapedPath = $RemotePath -replace '([ ()\[\]{}])', '\$1' + $result = & $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "${NAS_USER}@${NAS_IP}:${escapedPath}" "$LocalPath" 2>&1 + if ($LASTEXITCODE -ne 0) { + $errorMsg = $result | Out-String + Write-Log " SCP PULL ERROR (exit $LASTEXITCODE): $errorMsg" + } + return $LASTEXITCODE -eq 0 +} + +function Rename-OnNAS { + param([string]$RemotePath) + Invoke-NASCommand "mv '$RemotePath' '${RemotePath}.synced'" | Out-Null +} + +function Copy-ToNAS { + param( + [string]$LocalPath, + [string]$RemotePath + ) + + # Ensure remote directory exists via SSH mkdir -p + $remoteDir = ($RemotePath -replace '[^/]+$', '').TrimEnd('/') + Invoke-NASCommand "mkdir -p '$remoteDir'" | Out-Null + + # Escape spaces and special chars with backslashes for SCP remote path + $escapedPath = $RemotePath -replace '([ ()\[\]{}])', '\$1' + $result = & $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "$LocalPath" "${NAS_USER}@${NAS_IP}:${escapedPath}" 2>&1 + if ($LASTEXITCODE -ne 0) { + $errorMsg = $result | Out-String + Write-Log " SCP PUSH ERROR (exit $LASTEXITCODE): $errorMsg" + } + return $LASTEXITCODE -eq 0 +} + +function Test-8dot3Name { + param([string]$Name) + # Returns $true if filename is 8.3 compatible (no spaces, name<=8, ext<=3) + if ($Name -match '[ ()\[\]{}]') { return $false } + $parts = $Name -split '\.' + if ($parts.Count -gt 2) { return $false } + if ($parts[0].Length -gt 8 -or $parts[0].Length -eq 0) { return $false } + if ($parts.Count -eq 2 -and $parts[1].Length -gt 3) { return $false } + return $true +} + +function Test-8dot3Path { + param([string]$RelativePath) + # Check every component of the path is 8.3 compatible + $segments = $RelativePath -replace '\\', '/' -split '/' + foreach ($seg in $segments) { + if ($seg -eq '') { continue } + if (-not (Test-8dot3Name $seg)) { return $false } + } + return $true +} + +function Get-FileHash256 { + param([string]$FilePath) + if (Test-Path $FilePath) { + return (Get-FileHash -Path $FilePath -Algorithm SHA256).Hash + } + return $null +} + +function Import-ToDatabase { + param([string[]]$FilePaths) + + if ($FilePaths.Count -eq 0) { return } + + Write-Log "Importing $($FilePaths.Count) file(s) to database..." + + # Build argument list + $importArgs = @("$IMPORT_SCRIPT", "--file") + $FilePaths + + try { + $output = & $NODE_PATH $importArgs 2>&1 + foreach ($line in $output) { + Write-Log " [DB] $line" + } + Write-Log "Database import complete" + } catch { + Write-Log "ERROR: Database import failed: $_" + } +} + +# ============================================================================ +# Main Script +# ============================================================================ + +Write-Log "==========================================" +Write-Log "Starting sync from NAS" +Write-Log "Max age: $MaxAgeMinutes minutes" +if ($DryRun) { Write-Log "DRY RUN - no changes will be made" } + +# Ensure .ssh directory exists for known_hosts +$sshDir = "C:\Shares\test\scripts\.ssh" +if (-not (Test-Path $sshDir)) { + New-Item -ItemType Directory -Path $sshDir -Force | Out-Null + Write-Log "Created .ssh directory: $sshDir" +} + +$errorCount = 0 +$syncedFiles = 0 +$skippedFiles = 0 +$syncedDatFiles = @() # Track DAT files for database import + +# Find all DAT files on NAS modified within the time window +# Uses temp file approach to avoid SSH output buffer hang with large file counts +Write-Log "Finding DAT files on NAS..." +$findCommand = "find $NAS_DATA_PATH/TS-*/LOGS -name '*.DAT' -type f -mmin -$MaxAgeMinutes" +$datFiles = Get-NASFileList -FindCommand $findCommand -ListLabel "dat" + +if (-not $datFiles -or $datFiles.Count -eq 0) { + Write-Log "No new DAT files found on NAS" +} else { + Write-Log "Found $($datFiles.Count) DAT file(s) to process" + + foreach ($remoteFile in $datFiles) { + $remoteFile = $remoteFile.Trim() + if ([string]::IsNullOrWhiteSpace($remoteFile)) { continue } + + # Parse the path: /data/test/TS-XX/LOGS/7BLOG/file.DAT + if ($remoteFile -match "/data/test/(TS-[^/]+)/LOGS/([^/]+)/(.+\.DAT)$") { + $station = $Matches[1] + $logType = $Matches[2] + $fileName = $Matches[3] + + Write-Log "Processing: $station/$logType/$fileName" + + # Destination 1: Per-station folder (preserves structure) + $stationDest = Join-Path $AD2_TEST_PATH "$station\LOGS\$logType\$fileName" + + # Destination 2: Aggregated HISTLOGS folder + $histlogsDest = Join-Path $AD2_HISTLOGS_PATH "$logType\$fileName" + + if ($DryRun) { + Write-Log " [DRY RUN] Would copy to: $stationDest" + $syncedFiles++ + } else { + # Copy to station folder only (skip HISTLOGS to avoid duplicates) + $success1 = Copy-FromNAS -RemotePath $remoteFile -LocalPath $stationDest + + if ($success1) { + Write-Log " Copied to station folder" + + # Rename on NAS after successful sync (preserves file as .synced) + Rename-OnNAS -RemotePath $remoteFile + Write-Log " Renamed to .synced on NAS" + + # Track for database import + $syncedDatFiles += $stationDest + + $syncedFiles++ + } else { + Write-Log " ERROR: Failed to copy from NAS" + $errorCount++ + } + } + } else { + Write-Log " Skipping (unexpected path format): $remoteFile" + $skippedFiles++ + } + } +} + +# Find and sync TXT report files +# Uses temp file approach to avoid SSH output buffer hang with large file counts +Write-Log "Finding TXT reports on NAS..." +$findReportsCommand = "find $NAS_DATA_PATH/TS-*/Reports -name '*.TXT' -type f -mmin -$MaxAgeMinutes" +$txtFiles = Get-NASFileList -FindCommand $findReportsCommand -ListLabel "txt" + +if ($txtFiles -and $txtFiles.Count -gt 0) { + Write-Log "Found $($txtFiles.Count) TXT report(s) to process" + + foreach ($remoteFile in $txtFiles) { + $remoteFile = $remoteFile.Trim() + if ([string]::IsNullOrWhiteSpace($remoteFile)) { continue } + + if ($remoteFile -match "/data/test/(TS-[^/]+)/Reports/(.+\.TXT)$") { + $station = $Matches[1] + $fileName = $Matches[2] + + Write-Log "Processing report: $station/$fileName" + + # Destination: Per-station Reports folder + $reportDest = Join-Path $AD2_TEST_PATH "$station\Reports\$fileName" + + if ($DryRun) { + Write-Log " [DRY RUN] Would copy to: $reportDest" + $syncedFiles++ + } else { + $success = Copy-FromNAS -RemotePath $remoteFile -LocalPath $reportDest + + if ($success) { + Write-Log " Copied report" + Rename-OnNAS -RemotePath $remoteFile + Write-Log " Renamed to .synced on NAS" + $syncedFiles++ + } else { + Write-Log " ERROR: Failed to copy report" + $errorCount++ + } + } + } + } +} + +# ============================================================================ +# Import synced DAT files to database +# ============================================================================ +if (-not $DryRun -and $syncedDatFiles.Count -gt 0) { + Import-ToDatabase -FilePaths $syncedDatFiles +} + +# ============================================================================ +# PUSH: AD2 -> NAS (Software Updates for DOS Machines) +# ============================================================================ +Write-Log "--- AD2 to NAS Sync (Software Updates) ---" + +$pushedFiles = 0 + +# Sync COMMON/ProdSW (batch files for all stations) +# AD2 uses _COMMON, NAS uses COMMON - handle both +$commonSources = @( + @{ Local = "$AD2_TEST_PATH\_COMMON\ProdSW"; Remote = "$NAS_DATA_PATH/COMMON/ProdSW" }, + @{ Local = "$AD2_TEST_PATH\COMMON\ProdSW"; Remote = "$NAS_DATA_PATH/COMMON/ProdSW" } +) + +foreach ($source in $commonSources) { + if (Test-Path $source.Local) { + Write-Log "Syncing COMMON ProdSW from: $($source.Local)" + $commonFiles = Get-ChildItem -Path $source.Local -File -ErrorAction SilentlyContinue + foreach ($file in $commonFiles) { + if (-not (Test-8dot3Name $file.Name)) { + Write-Log " Skipping (non-8.3): $($file.Name)" + $skippedFiles++ + continue + } + $remotePath = "$($source.Remote)/$($file.Name)" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push: $($file.Name) -> $remotePath" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath + if ($success) { + Write-Log " Pushed: $($file.Name)" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push $($file.Name)" + $errorCount++ + } + } + } + } +} + +# Sync Ate/ProdSW (shared ATE data folders - 5BDATA, 7BDATA, 8BDATA, DSCDATA, etc.) +Write-Log "Syncing Ate/ProdSW data folders..." +$ateProdSwPath = "$AD2_TEST_PATH\Ate\ProdSW" +if (Test-Path $ateProdSwPath) { + # Get all files recursively (including subdirectories like DSCDATA, 5BDATA, etc.) + $ateFiles = Get-ChildItem -Path $ateProdSwPath -File -Recurse -ErrorAction SilentlyContinue + foreach ($file in $ateFiles) { + # Calculate relative path from Ate/ProdSW folder + $relativePath = $file.FullName.Substring($ateProdSwPath.Length + 1).Replace('\', '/') + if (-not (Test-8dot3Path $relativePath)) { + Write-Log " Skipping (non-8.3): Ate/ProdSW/$relativePath" + $skippedFiles++ + continue + } + $remotePath = "$NAS_DATA_PATH/Ate/ProdSW/$relativePath" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push: Ate/ProdSW/$relativePath" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath + if ($success) { + Write-Log " Pushed: Ate/ProdSW/$relativePath" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push Ate/ProdSW/$relativePath" + $errorCount++ + } + } + } +} else { + Write-Log " Ate/ProdSW not found: $ateProdSwPath" +} + +# Sync UPDATE.BAT (root level utility) +Write-Log "Syncing UPDATE.BAT..." +$updateBatLocal = "$AD2_TEST_PATH\UPDATE.BAT" +if (Test-Path $updateBatLocal) { + $updateBatRemote = "$NAS_DATA_PATH/UPDATE.BAT" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push: UPDATE.BAT -> $updateBatRemote" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $updateBatLocal -RemotePath $updateBatRemote + if ($success) { + Write-Log " Pushed: UPDATE.BAT" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push UPDATE.BAT" + $errorCount++ + } + } +} else { + Write-Log " WARNING: UPDATE.BAT not found at $updateBatLocal" +} + +# Sync DEPLOY.BAT (root level utility) +Write-Log "Syncing DEPLOY.BAT..." +$deployBatLocal = "$AD2_TEST_PATH\DEPLOY.BAT" +if (Test-Path $deployBatLocal) { + $deployBatRemote = "$NAS_DATA_PATH/DEPLOY.BAT" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push: DEPLOY.BAT -> $deployBatRemote" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $deployBatLocal -RemotePath $deployBatRemote + if ($success) { + Write-Log " Pushed: DEPLOY.BAT" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push DEPLOY.BAT" + $errorCount++ + } + } +} else { + Write-Log " WARNING: DEPLOY.BAT not found at $deployBatLocal" +} + +# Sync per-station ProdSW folders +Write-Log "Syncing station-specific ProdSW folders..." +$stationFolders = Get-ChildItem -Path $AD2_TEST_PATH -Directory -Filter "TS-*" -ErrorAction SilentlyContinue + +foreach ($station in $stationFolders) { + # Skip station folders with non-8.3 names (e.g. "TS-1 OBS") + if (-not (Test-8dot3Name $station.Name)) { + Write-Log " Skipping station (non-8.3 name): $($station.Name)" + continue + } + + $prodSwPath = Join-Path $station.FullName "ProdSW" + + if (Test-Path $prodSwPath) { + # Get all files in ProdSW (including subdirectories) + $prodSwFiles = Get-ChildItem -Path $prodSwPath -File -Recurse -ErrorAction SilentlyContinue + + foreach ($file in $prodSwFiles) { + # Calculate relative path from ProdSW folder + $relativePath = $file.FullName.Substring($prodSwPath.Length + 1).Replace('\', '/') + if (-not (Test-8dot3Path $relativePath)) { + Write-Log " Skipping (non-8.3): $($station.Name)/ProdSW/$relativePath" + $skippedFiles++ + continue + } + $remotePath = "$NAS_DATA_PATH/$($station.Name)/ProdSW/$relativePath" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push: $($station.Name)/ProdSW/$relativePath" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath + if ($success) { + Write-Log " Pushed: $($station.Name)/ProdSW/$relativePath" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push $($station.Name)/ProdSW/$relativePath" + $errorCount++ + } + } + } + } + + # Check for TODO.BAT (one-time task file) + $todoBatPath = Join-Path $station.FullName "TODO.BAT" + if (Test-Path $todoBatPath) { + $remoteTodoPath = "$NAS_DATA_PATH/$($station.Name)/TODO.BAT" + + Write-Log "Found TODO.BAT for $($station.Name)" + + if ($DryRun) { + Write-Log " [DRY RUN] Would push TODO.BAT -> $remoteTodoPath" + $pushedFiles++ + } else { + $success = Copy-ToNAS -LocalPath $todoBatPath -RemotePath $remoteTodoPath + if ($success) { + Write-Log " Pushed TODO.BAT to NAS" + # Remove from AD2 after successful push (one-shot mechanism) + Remove-Item -Path $todoBatPath -Force + Write-Log " Removed TODO.BAT from AD2 (pushed to NAS)" + $pushedFiles++ + } else { + Write-Log " ERROR: Failed to push TODO.BAT" + $errorCount++ + } + } + } +} + +Write-Log "AD2 to NAS sync: $pushedFiles file(s) pushed" + +# ============================================================================ +# Update Status File +# ============================================================================ +$status = if ($errorCount -eq 0) { "OK" } else { "ERRORS" } +$statusContent = @" +AD2 <-> NAS Bidirectional Sync Status +====================================== +Timestamp: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss") +Status: $status + +PULL (NAS -> AD2 - Test Results): + Files Pulled: $syncedFiles + Files Skipped: $skippedFiles + DAT Files Imported to DB: $($syncedDatFiles.Count) + +PUSH (AD2 -> NAS - Software Updates): + Files Pushed: $pushedFiles + +Errors: $errorCount +"@ + +Set-Content -Path $STATUS_FILE -Value $statusContent + +Write-Log "==========================================" +Write-Log "Sync complete: PULL=$syncedFiles, PUSH=$pushedFiles, Errors=$errorCount" +Write-Log "==========================================" + +# Exit with error code if there were failures +if ($errorCount -gt 0) { + exit 1 +} else { + exit 0 +} diff --git a/projects/dataforth-dos/sync-fixes/deploy-sync-fixes.ps1 b/projects/dataforth-dos/sync-fixes/deploy-sync-fixes.ps1 new file mode 100644 index 0000000..2899e92 --- /dev/null +++ b/projects/dataforth-dos/sync-fixes/deploy-sync-fixes.ps1 @@ -0,0 +1,302 @@ +# deploy-sync-fixes.ps1 +# Deploys patched Sync-FromNAS.ps1 and import.js to AD2 (192.168.0.6) +# +# Fixes deployed: +# 1. Sync-FromNAS.ps1 - PULL hang fix (temp file approach for large find output) +# 2. Sync-FromNAS.ps1 - SCP quoting fix (handles spaces, parens, special chars) +# 3. Sync-FromNAS.ps1 - Rename-OnNAS replaces Remove-FromNAS (.synced instead of delete) +# 4. import.js - INSERT OR REPLACE instead of INSERT OR IGNORE (re-tests keep latest) +# +# Run from: Mike's workstation (where the patched files live) +# Target: AD2 (192.168.0.6) as INTRANET\sysadmin + +param( + [switch]$DryRun # Show what would be done without doing it +) + +# ============================================================================ +# Configuration +# ============================================================================ +$AD2_IP = "192.168.0.6" +$AD2_USER = "INTRANET\sysadmin" +$AD2_PASSWORD = "Paper123!@#" + +$SSH = "C:\Windows\System32\OpenSSH\ssh.exe" +$SCP = "C:\Windows\System32\OpenSSH\scp.exe" + +# Source files (local to this machine) +$SCRIPT_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path +$LOCAL_SYNC_SCRIPT = Join-Path $SCRIPT_DIR "Sync-FromNAS.ps1" +$LOCAL_IMPORT_SCRIPT = Join-Path $SCRIPT_DIR "import.js" + +# Destination paths on AD2 +$REMOTE_SYNC_PATH = "C:\Shares\test\scripts\Sync-FromNAS.ps1" +$REMOTE_IMPORT_PATH = "C:\Shares\testdatadb\database\import.js" +$REMOTE_SSH_DIR = "C:\Shares\test\scripts\.ssh" + +$DATE_SUFFIX = Get-Date -Format "yyyyMMdd" + +# ============================================================================ +# Functions +# ============================================================================ + +function Write-Status { + param( + [string]$Marker, + [string]$Message + ) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + Write-Host "$timestamp [$Marker] $Message" +} + +function Invoke-AD2Command { + param([string]$Command) + $result = & $SSH -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 "${AD2_USER}@${AD2_IP}" $Command 2>&1 + return $result +} + +function Copy-ToAD2 { + param( + [string]$LocalPath, + [string]$RemotePath + ) + # SCP to AD2 - quote remote path for Windows paths with backslashes + $result = & $SCP -O -o StrictHostKeyChecking=accept-new "$LocalPath" "${AD2_USER}@${AD2_IP}:`"${RemotePath}`"" 2>&1 + if ($LASTEXITCODE -ne 0) { + $errorMsg = $result | Out-String + Write-Status "ERROR" "SCP failed (exit $LASTEXITCODE): $errorMsg" + return $false + } + return $true +} + +# ============================================================================ +# Pre-flight Checks +# ============================================================================ + +Write-Host "" +Write-Host "============================================" +Write-Host " Dataforth Sync Fixes - Deployment Script" +Write-Host "============================================" +Write-Host "" + +if ($DryRun) { + Write-Status "INFO" "DRY RUN - no changes will be made" + Write-Host "" +} + +# Verify source files exist +Write-Status "INFO" "Checking source files..." + +if (-not (Test-Path $LOCAL_SYNC_SCRIPT)) { + Write-Status "ERROR" "Source file not found: $LOCAL_SYNC_SCRIPT" + exit 1 +} +Write-Status "OK" "Found: $LOCAL_SYNC_SCRIPT" + +if (-not (Test-Path $LOCAL_IMPORT_SCRIPT)) { + Write-Status "ERROR" "Source file not found: $LOCAL_IMPORT_SCRIPT" + exit 1 +} +Write-Status "OK" "Found: $LOCAL_IMPORT_SCRIPT" + +# Verify AD2 is reachable +Write-Status "INFO" "Testing connectivity to AD2 ($AD2_IP)..." +$pingResult = Test-Connection -ComputerName $AD2_IP -Count 1 -Quiet +if (-not $pingResult) { + Write-Status "ERROR" "Cannot reach AD2 at $AD2_IP" + exit 1 +} +Write-Status "OK" "AD2 is reachable" + +# Test SSH connection +Write-Status "INFO" "Testing SSH connection..." +$sshTest = Invoke-AD2Command "echo CONNECTION_OK" +$sshTestStr = $sshTest | Out-String +if ($sshTestStr -notmatch "CONNECTION_OK") { + Write-Status "ERROR" "SSH connection failed: $sshTestStr" + exit 1 +} +Write-Status "OK" "SSH connection established" + +# ============================================================================ +# Step 1: Create backups on AD2 +# ============================================================================ +Write-Host "" +Write-Status "INFO" "--- Step 1: Creating backups on AD2 ---" + +if ($DryRun) { + Write-Status "INFO" "[DRY RUN] Would backup $REMOTE_SYNC_PATH -> ${REMOTE_SYNC_PATH}.bak-${DATE_SUFFIX}" + Write-Status "INFO" "[DRY RUN] Would backup $REMOTE_IMPORT_PATH -> ${REMOTE_IMPORT_PATH}.bak-${DATE_SUFFIX}" +} else { + # Backup Sync-FromNAS.ps1 + $backupCmd1 = "copy `"$REMOTE_SYNC_PATH`" `"${REMOTE_SYNC_PATH}.bak-${DATE_SUFFIX}`"" + $backupResult1 = Invoke-AD2Command "cmd /c $backupCmd1" + if ($LASTEXITCODE -eq 0) { + Write-Status "OK" "Backed up Sync-FromNAS.ps1 -> .bak-${DATE_SUFFIX}" + } else { + Write-Status "WARNING" "Backup of Sync-FromNAS.ps1 may have failed (file might not exist yet): $($backupResult1 | Out-String)" + } + + # Backup import.js + $backupCmd2 = "copy `"$REMOTE_IMPORT_PATH`" `"${REMOTE_IMPORT_PATH}.bak-${DATE_SUFFIX}`"" + $backupResult2 = Invoke-AD2Command "cmd /c $backupCmd2" + if ($LASTEXITCODE -eq 0) { + Write-Status "OK" "Backed up import.js -> .bak-${DATE_SUFFIX}" + } else { + Write-Status "WARNING" "Backup of import.js may have failed (file might not exist yet): $($backupResult2 | Out-String)" + } +} + +# ============================================================================ +# Step 2: Create .ssh directory on AD2 if missing +# ============================================================================ +Write-Host "" +Write-Status "INFO" "--- Step 2: Ensuring .ssh directory exists ---" + +if ($DryRun) { + Write-Status "INFO" "[DRY RUN] Would create $REMOTE_SSH_DIR if missing" +} else { + $mkdirCmd = "cmd /c if not exist `"$REMOTE_SSH_DIR`" mkdir `"$REMOTE_SSH_DIR`"" + Invoke-AD2Command $mkdirCmd | Out-Null + Write-Status "OK" "Ensured .ssh directory exists: $REMOTE_SSH_DIR" + + # Create empty known_hosts if it does not exist + $knownHostsPath = "$REMOTE_SSH_DIR\known_hosts" + $touchCmd = "cmd /c if not exist `"$knownHostsPath`" type nul > `"$knownHostsPath`"" + Invoke-AD2Command $touchCmd | Out-Null + Write-Status "OK" "Ensured known_hosts file exists: $knownHostsPath" +} + +# ============================================================================ +# Step 3: Deploy patched files +# ============================================================================ +Write-Host "" +Write-Status "INFO" "--- Step 3: Deploying patched files ---" + +if ($DryRun) { + Write-Status "INFO" "[DRY RUN] Would SCP $LOCAL_SYNC_SCRIPT -> ${AD2_IP}:${REMOTE_SYNC_PATH}" + Write-Status "INFO" "[DRY RUN] Would SCP $LOCAL_IMPORT_SCRIPT -> ${AD2_IP}:${REMOTE_IMPORT_PATH}" +} else { + # Deploy Sync-FromNAS.ps1 + Write-Status "INFO" "Deploying Sync-FromNAS.ps1..." + $success1 = Copy-ToAD2 -LocalPath $LOCAL_SYNC_SCRIPT -RemotePath $REMOTE_SYNC_PATH + if ($success1) { + Write-Status "OK" "Deployed Sync-FromNAS.ps1" + } else { + Write-Status "ERROR" "Failed to deploy Sync-FromNAS.ps1" + exit 1 + } + + # Deploy import.js + Write-Status "INFO" "Deploying import.js..." + $success2 = Copy-ToAD2 -LocalPath $LOCAL_IMPORT_SCRIPT -RemotePath $REMOTE_IMPORT_PATH + if ($success2) { + Write-Status "OK" "Deployed import.js" + } else { + Write-Status "ERROR" "Failed to deploy import.js" + exit 1 + } +} + +# ============================================================================ +# Step 4: Verify deployment +# ============================================================================ +Write-Host "" +Write-Status "INFO" "--- Step 4: Verifying deployment ---" + +if ($DryRun) { + Write-Status "INFO" "[DRY RUN] Would verify files exist on AD2" +} else { + # Check Sync-FromNAS.ps1 exists and has content + $verifyCmd1 = "cmd /c if exist `"$REMOTE_SYNC_PATH`" (echo FILE_EXISTS) else (echo FILE_MISSING)" + $verify1 = Invoke-AD2Command $verifyCmd1 | Out-String + if ($verify1 -match "FILE_EXISTS") { + Write-Status "OK" "Verified: Sync-FromNAS.ps1 exists on AD2" + } else { + Write-Status "ERROR" "Verification failed: Sync-FromNAS.ps1 not found on AD2" + exit 1 + } + + # Check import.js exists + $verifyCmd2 = "cmd /c if exist `"$REMOTE_IMPORT_PATH`" (echo FILE_EXISTS) else (echo FILE_MISSING)" + $verify2 = Invoke-AD2Command $verifyCmd2 | Out-String + if ($verify2 -match "FILE_EXISTS") { + Write-Status "OK" "Verified: import.js exists on AD2" + } else { + Write-Status "ERROR" "Verification failed: import.js not found on AD2" + exit 1 + } +} + +# ============================================================================ +# Step 5: Quick dry-run test of the sync script +# ============================================================================ +Write-Host "" +Write-Status "INFO" "--- Step 5: Running dry-run test of sync script ---" + +if ($DryRun) { + Write-Status "INFO" "[DRY RUN] Would run: powershell -ExecutionPolicy Bypass -File $REMOTE_SYNC_PATH -DryRun" +} else { + Write-Status "INFO" "Executing sync script in dry-run mode on AD2..." + $testCmd = "powershell -ExecutionPolicy Bypass -File `"$REMOTE_SYNC_PATH`" -DryRun -Verbose" + $testResult = Invoke-AD2Command $testCmd + $testOutput = $testResult | Out-String + + # Check if the script ran without critical errors + if ($LASTEXITCODE -eq 0) { + Write-Status "OK" "Sync script dry-run completed successfully" + if ($testOutput.Trim().Length -gt 0) { + Write-Host "" + Write-Host " --- Dry-run output ---" + foreach ($line in ($testResult | Select-Object -First 20)) { + Write-Host " $line" + } + $totalLines = ($testResult | Measure-Object).Count + if ($totalLines -gt 20) { + Write-Host " ... ($($totalLines - 20) more lines)" + } + Write-Host " --- End dry-run output ---" + } + } else { + Write-Status "WARNING" "Sync script dry-run exited with code $LASTEXITCODE" + Write-Status "INFO" "This may be expected if NAS is unreachable from AD2 during test" + if ($testOutput.Trim().Length -gt 0) { + Write-Host "" + Write-Host " --- Dry-run output ---" + foreach ($line in ($testResult | Select-Object -First 10)) { + Write-Host " $line" + } + Write-Host " --- End dry-run output ---" + } + } +} + +# ============================================================================ +# Summary +# ============================================================================ +Write-Host "" +Write-Host "============================================" +Write-Host " Deployment Summary" +Write-Host "============================================" +Write-Host "" + +if ($DryRun) { + Write-Status "INFO" "DRY RUN complete - no changes were made" +} else { + Write-Status "OK" "Sync-FromNAS.ps1 deployed to AD2 (backup: .bak-${DATE_SUFFIX})" + Write-Status "OK" "import.js deployed to AD2 (backup: .bak-${DATE_SUFFIX})" + Write-Status "OK" ".ssh directory and known_hosts verified" + Write-Status "OK" "Dry-run test executed" + Write-Host "" + Write-Status "INFO" "Fixes applied:" + Write-Host " 1. PULL hang fix: find output written to temp file, pulled via SCP" + Write-Host " 2. SCP quoting fix: remote paths quoted to handle special characters" + Write-Host " 3. Rename-OnNAS: files renamed to .synced instead of deleted" + Write-Host " 4. INSERT OR REPLACE: re-tested devices keep latest result" + Write-Host "" + Write-Status "INFO" "Next sync cycle will use the patched scripts automatically" +} + +Write-Host "" +exit 0 diff --git a/projects/dataforth-dos/sync-fixes/import.js b/projects/dataforth-dos/sync-fixes/import.js new file mode 100644 index 0000000..afba25c --- /dev/null +++ b/projects/dataforth-dos/sync-fixes/import.js @@ -0,0 +1,395 @@ +/** + * Data Import Script + * Imports test data from DAT and SHT files into SQLite database + */ + +const fs = require('fs'); +const path = require('path'); +const Database = require('better-sqlite3'); + +const { parseMultilineFile, extractTestStation } = require('../parsers/multiline'); +const { parseCsvFile } = require('../parsers/csvline'); +const { parseShtFile } = require('../parsers/shtfile'); + +// Configuration +const DB_PATH = path.join(__dirname, 'testdata.db'); +const SCHEMA_PATH = path.join(__dirname, 'schema.sql'); + +// Data source paths +const TEST_PATH = 'C:/Shares/test'; +const RECOVERY_PATH = 'C:/Shares/Recovery-TEST'; +const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS'); + +// Log types and their parsers +const LOG_TYPES = { + 'DSCLOG': { parser: 'multiline', ext: '.DAT' }, + '5BLOG': { parser: 'multiline', ext: '.DAT' }, + '8BLOG': { parser: 'multiline', ext: '.DAT' }, + 'PWRLOG': { parser: 'multiline', ext: '.DAT' }, + 'SCTLOG': { parser: 'multiline', ext: '.DAT' }, + 'VASLOG': { parser: 'multiline', ext: '.DAT' }, + '7BLOG': { parser: 'csvline', ext: '.DAT' } +}; + +// Initialize database +function initDatabase() { + console.log('Initializing database...'); + const db = new Database(DB_PATH); + + // Read and execute schema + const schema = fs.readFileSync(SCHEMA_PATH, 'utf8'); + db.exec(schema); + + console.log('Database initialized.'); + return db; +} + +// Prepare insert statement +function prepareInsert(db) { + return db.prepare(` + INSERT OR REPLACE INTO test_records + (log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `); +} + +// Find all files of a specific type in a directory +function findFiles(dir, pattern, recursive = true) { + const results = []; + + try { + if (!fs.existsSync(dir)) return results; + + const items = fs.readdirSync(dir, { withFileTypes: true }); + + for (const item of items) { + const fullPath = path.join(dir, item.name); + + if (item.isDirectory() && recursive) { + results.push(...findFiles(fullPath, pattern, recursive)); + } else if (item.isFile()) { + if (pattern.test(item.name)) { + results.push(fullPath); + } + } + } + } catch (err) { + // Ignore permission errors + } + + return results; +} + +// Import records from a file +function importFile(db, insertStmt, filePath, logType, parser) { + let records = []; + const testStation = extractTestStation(filePath); + + try { + switch (parser) { + case 'multiline': + records = parseMultilineFile(filePath, logType, testStation); + break; + case 'csvline': + records = parseCsvFile(filePath, testStation); + break; + case 'shtfile': + records = parseShtFile(filePath, testStation); + break; + } + + let imported = 0; + for (const record of records) { + try { + const result = insertStmt.run( + record.log_type, + record.model_number, + record.serial_number, + record.test_date, + record.test_station, + record.overall_result, + record.raw_data, + record.source_file + ); + if (result.changes > 0) imported++; + } catch (err) { + // Duplicate or constraint error - skip + } + } + + return { total: records.length, imported }; + } catch (err) { + console.error(`Error importing ${filePath}: ${err.message}`); + return { total: 0, imported: 0 }; + } +} + +// Import from HISTLOGS (master consolidated logs) +function importHistlogs(db, insertStmt) { + console.log('\n=== Importing from HISTLOGS ==='); + + let totalImported = 0; + let totalRecords = 0; + + for (const [logType, config] of Object.entries(LOG_TYPES)) { + const logDir = path.join(HISTLOGS_PATH, logType); + + if (!fs.existsSync(logDir)) { + console.log(` ${logType}: directory not found`); + continue; + } + + const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false); + console.log(` ${logType}: found ${files.length} files`); + + for (const file of files) { + const { total, imported } = importFile(db, insertStmt, file, logType, config.parser); + totalRecords += total; + totalImported += imported; + } + } + + console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`); + return totalImported; +} + +// Import from test station logs +function importStationLogs(db, insertStmt, basePath, label) { + console.log(`\n=== Importing from ${label} ===`); + + let totalImported = 0; + let totalRecords = 0; + + // Find all test station directories (TS-1, TS-27, TS-8L, TS-10R, etc.) + const stationPattern = /^TS-\d+[LR]?$/i; + let stations = []; + + try { + const items = fs.readdirSync(basePath, { withFileTypes: true }); + stations = items + .filter(i => i.isDirectory() && stationPattern.test(i.name)) + .map(i => i.name); + } catch (err) { + console.log(` Error reading ${basePath}: ${err.message}`); + return 0; + } + + console.log(` Found stations: ${stations.join(', ')}`); + + for (const station of stations) { + const logsDir = path.join(basePath, station, 'LOGS'); + + if (!fs.existsSync(logsDir)) continue; + + for (const [logType, config] of Object.entries(LOG_TYPES)) { + const logDir = path.join(logsDir, logType); + + if (!fs.existsSync(logDir)) continue; + + const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false); + + for (const file of files) { + const { total, imported } = importFile(db, insertStmt, file, logType, config.parser); + totalRecords += total; + totalImported += imported; + } + } + } + + // Also import SHT files + const shtFiles = findFiles(basePath, /\.SHT$/i, true); + console.log(` Found ${shtFiles.length} SHT files`); + + for (const file of shtFiles) { + const { total, imported } = importFile(db, insertStmt, file, 'SHT', 'shtfile'); + totalRecords += total; + totalImported += imported; + } + + console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`); + return totalImported; +} + +// Import from Recovery-TEST backups (newest first) +function importRecoveryBackups(db, insertStmt) { + console.log('\n=== Importing from Recovery-TEST backups ==='); + + if (!fs.existsSync(RECOVERY_PATH)) { + console.log(' Recovery-TEST directory not found'); + return 0; + } + + // Get backup dates, sort newest first + const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true }) + .filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name)) + .map(i => i.name) + .sort() + .reverse(); + + console.log(` Found backup dates: ${backups.join(', ')}`); + + let totalImported = 0; + + for (const backup of backups) { + const backupPath = path.join(RECOVERY_PATH, backup); + const imported = importStationLogs(db, insertStmt, backupPath, `Recovery-TEST/${backup}`); + totalImported += imported; + } + + return totalImported; +} + +// Main import function +async function runImport() { + console.log('========================================'); + console.log('Test Data Import'); + console.log('========================================'); + console.log(`Database: ${DB_PATH}`); + console.log(`Start time: ${new Date().toISOString()}`); + + const db = initDatabase(); + const insertStmt = prepareInsert(db); + + let grandTotal = 0; + + // Use transaction for performance + const importAll = db.transaction(() => { + // 1. Import HISTLOGS first (authoritative) + grandTotal += importHistlogs(db, insertStmt); + + // 2. Import Recovery backups (newest first) + grandTotal += importRecoveryBackups(db, insertStmt); + + // 3. Import current test folder + grandTotal += importStationLogs(db, insertStmt, TEST_PATH, 'test'); + }); + + importAll(); + + // Get final stats + const stats = db.prepare('SELECT COUNT(*) as count FROM test_records').get(); + + console.log('\n========================================'); + console.log('Import Complete'); + console.log('========================================'); + console.log(`Total records in database: ${stats.count}`); + console.log(`End time: ${new Date().toISOString()}`); + + db.close(); +} + +// Import a single file (for incremental imports from sync) +function importSingleFile(filePath) { + console.log(`Importing: ${filePath}`); + + const db = new Database(DB_PATH); + const insertStmt = prepareInsert(db); + + // Determine log type from path + let logType = null; + let parser = null; + + for (const [type, config] of Object.entries(LOG_TYPES)) { + if (filePath.includes(type)) { + logType = type; + parser = config.parser; + break; + } + } + + if (!logType) { + // Check for SHT files + if (/\.SHT$/i.test(filePath)) { + logType = 'SHT'; + parser = 'shtfile'; + } else { + console.log(` Unknown log type for: ${filePath}`); + db.close(); + return { total: 0, imported: 0 }; + } + } + + const result = importFile(db, insertStmt, filePath, logType, parser); + + console.log(` Imported ${result.imported} of ${result.total} records`); + db.close(); + + return result; +} + +// Import multiple files (for batch incremental imports) +function importFiles(filePaths) { + console.log(`\n========================================`); + console.log(`Incremental Import: ${filePaths.length} files`); + console.log(`========================================`); + + const db = new Database(DB_PATH); + const insertStmt = prepareInsert(db); + + let totalImported = 0; + let totalRecords = 0; + + const importBatch = db.transaction(() => { + for (const filePath of filePaths) { + // Determine log type from path + let logType = null; + let parser = null; + + for (const [type, config] of Object.entries(LOG_TYPES)) { + if (filePath.includes(type)) { + logType = type; + parser = config.parser; + break; + } + } + + if (!logType) { + if (/\.SHT$/i.test(filePath)) { + logType = 'SHT'; + parser = 'shtfile'; + } else { + console.log(` Skipping unknown type: ${filePath}`); + continue; + } + } + + const { total, imported } = importFile(db, insertStmt, filePath, logType, parser); + totalRecords += total; + totalImported += imported; + console.log(` ${path.basename(filePath)}: ${imported}/${total} records`); + } + }); + + importBatch(); + + console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`); + db.close(); + + return { total: totalRecords, imported: totalImported }; +} + +// Run if called directly +if (require.main === module) { + // Check for command line arguments + const args = process.argv.slice(2); + + if (args.length > 0 && args[0] === '--file') { + // Import specific file(s) + const files = args.slice(1); + if (files.length === 0) { + console.log('Usage: node import.js --file [file2] ...'); + process.exit(1); + } + importFiles(files); + } else if (args.length > 0 && args[0] === '--help') { + console.log('Usage:'); + console.log(' node import.js Full import from all sources'); + console.log(' node import.js --file Import specific file(s)'); + process.exit(0); + } else { + // Full import + runImport().catch(console.error); + } +} + +module.exports = { runImport, importSingleFile, importFiles }; diff --git a/projects/dataforth-dos/testdatadb-fix/backup-db.ps1 b/projects/dataforth-dos/testdatadb-fix/backup-db.ps1 new file mode 100644 index 0000000..72d71b8 --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/backup-db.ps1 @@ -0,0 +1,61 @@ +# backup-db.ps1 +# Backs up the TestDataDB SQLite database with 7-day retention. +# Intended to run as a scheduled task via install-backup-task.ps1. + +$ErrorActionPreference = 'Stop' + +$SourceDb = 'C:\Shares\testdatadb\database\testdata.db' +$BackupDir = 'C:\Shares\testdatadb\backups' +$LogFile = 'C:\Shares\testdatadb\logs\backup.log' +$Retention = 7 + +function Write-Log { + param([string]$Message) + $timestamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + $entry = "[$timestamp] $Message" + Add-Content -Path $LogFile -Value $entry + Write-Host $entry +} + +# Ensure directories exist +foreach ($dir in @($BackupDir, (Split-Path $LogFile -Parent))) { + if (-not (Test-Path $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } +} + +# Verify source database exists +if (-not (Test-Path $SourceDb)) { + Write-Log "[ERROR] Source database not found: $SourceDb" + exit 1 +} + +# Create dated backup +$datestamp = Get-Date -Format 'yyyy-MM-dd' +$backupFile = Join-Path $BackupDir "testdata-$datestamp.db" + +try { + Copy-Item -Path $SourceDb -Destination $backupFile -Force + $sizeKb = [math]::Round((Get-Item $backupFile).Length / 1024, 1) + Write-Log "[OK] Backup created: $backupFile ($sizeKb KB)" +} catch { + Write-Log "[ERROR] Backup failed: $_" + exit 1 +} + +# Prune old backups beyond retention period +$cutoff = (Get-Date).AddDays(-$Retention) +$oldBackups = Get-ChildItem -Path $BackupDir -Filter 'testdata-*.db' | + Where-Object { $_.LastWriteTime -lt $cutoff } + +foreach ($old in $oldBackups) { + try { + Remove-Item -Path $old.FullName -Force + Write-Log "[OK] Deleted old backup: $($old.Name)" + } catch { + Write-Log "[WARNING] Could not delete old backup: $($old.Name) - $_" + } +} + +$remaining = (Get-ChildItem -Path $BackupDir -Filter 'testdata-*.db').Count +Write-Log "[INFO] Backup complete. $remaining backup(s) on disk." diff --git a/projects/dataforth-dos/testdatadb-fix/deploy.ps1 b/projects/dataforth-dos/testdatadb-fix/deploy.ps1 new file mode 100644 index 0000000..f53fa97 --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/deploy.ps1 @@ -0,0 +1,134 @@ +# deploy.ps1 +# Deploys the fixed TestDataDB application to AD2 (192.168.0.6). +# +# Copies files via SCP, installs dependencies, sets up the Windows service, +# and installs the backup scheduled task. +# +# Must be run from the directory containing the fixed files. + +$ErrorActionPreference = 'Stop' + +$RemoteHost = '192.168.0.6' +$RemoteUser = 'INTRANET\sysadmin' +$RemotePass = 'Paper123!@#' +$RemotePath = 'C:\Shares\testdatadb' +$SshExe = 'C:\Windows\System32\OpenSSH\ssh.exe' +$ScpExe = 'C:\Windows\System32\OpenSSH\scp.exe' +$LocalDir = $PSScriptRoot + +# Credentials for SSH - set up sshpass-equivalent via environment +# Note: For automated deployment, the SSH key should be pre-configured. +# This script uses password-based auth via the ssh client. + +$SshTarget = "${RemoteUser}@${RemoteHost}" + +function Invoke-RemoteCommand { + param([string]$Command) + Write-Host "[INFO] Remote: $Command" + & $SshExe $SshTarget $Command + if ($LASTEXITCODE -ne 0) { + Write-Host "[ERROR] Remote command failed with exit code $LASTEXITCODE" + exit 1 + } +} + +function Copy-ToRemote { + param([string]$LocalFile, [string]$RemoteDir) + $remoteDest = "${SshTarget}:`"${RemoteDir}`"" + Write-Host "[INFO] Copying $LocalFile -> $RemoteDir" + & $ScpExe $LocalFile $remoteDest + if ($LASTEXITCODE -ne 0) { + Write-Host "[ERROR] SCP failed for $LocalFile" + exit 1 + } +} + +Write-Host '========================================' +Write-Host 'TestDataDB Deployment to AD2' +Write-Host '========================================' +Write-Host '' + +# ----------------------------------------------------------------------- +# Step 1: Ensure remote directories exist +# ----------------------------------------------------------------------- +Write-Host '[STEP 1] Creating remote directories...' +Invoke-RemoteCommand "if not exist `"${RemotePath}\routes`" mkdir `"${RemotePath}\routes`"" +Invoke-RemoteCommand "if not exist `"${RemotePath}\logs`" mkdir `"${RemotePath}\logs`"" +Invoke-RemoteCommand "if not exist `"${RemotePath}\backups`" mkdir `"${RemotePath}\backups`"" + +# ----------------------------------------------------------------------- +# Step 2: Stop existing node process / service +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 2] Stopping existing processes...' +# Try stopping the service first (may not exist yet) +& $SshExe $SshTarget "net stop TestDataDB 2>nul & echo Service stop attempted" +# Kill any lingering node processes running server.js +& $SshExe $SshTarget "taskkill /F /FI `"IMAGENAME eq node.exe`" 2>nul & echo Process kill attempted" +Write-Host '[OK] Existing processes stopped.' + +# ----------------------------------------------------------------------- +# Step 3: Copy files to remote +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 3] Copying files to AD2...' + +$filesToCopy = @( + @{ Local = "$LocalDir\server.js"; Remote = $RemotePath }, + @{ Local = "$LocalDir\package.json"; Remote = $RemotePath }, + @{ Local = "$LocalDir\install-service.js"; Remote = $RemotePath }, + @{ Local = "$LocalDir\uninstall-service.js"; Remote = $RemotePath }, + @{ Local = "$LocalDir\backup-db.ps1"; Remote = $RemotePath }, + @{ Local = "$LocalDir\install-backup-task.ps1"; Remote = $RemotePath }, + @{ Local = "$LocalDir\routes\api.js"; Remote = "$RemotePath\routes" } +) + +foreach ($file in $filesToCopy) { + Copy-ToRemote -LocalFile $file.Local -RemoteDir $file.Remote +} + +Write-Host '[OK] All files copied.' + +# ----------------------------------------------------------------------- +# Step 4: Install npm dependencies +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 4] Installing npm dependencies...' +Invoke-RemoteCommand "cd /d `"${RemotePath}`" && npm install --production" +Write-Host '[OK] Dependencies installed.' + +# ----------------------------------------------------------------------- +# Step 5: Uninstall old service (if present) and install new one +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 5] Installing Windows service...' +# Uninstall first to ensure clean state +& $SshExe $SshTarget "cd /d `"${RemotePath}`" && node uninstall-service.js 2>nul" +Start-Sleep -Seconds 3 +Invoke-RemoteCommand "cd /d `"${RemotePath}`" && node install-service.js" +Write-Host '[OK] Windows service installed.' + +# ----------------------------------------------------------------------- +# Step 6: Install backup scheduled task +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 6] Installing backup scheduled task...' +Invoke-RemoteCommand "powershell -NoProfile -ExecutionPolicy Bypass -File `"${RemotePath}\install-backup-task.ps1`"" +Write-Host '[OK] Backup task installed.' + +# ----------------------------------------------------------------------- +# Step 7: Verify service is running +# ----------------------------------------------------------------------- +Write-Host '' +Write-Host '[STEP 7] Verifying service status...' +Start-Sleep -Seconds 5 +& $SshExe $SshTarget "sc query TestDataDB" + +Write-Host '' +Write-Host '========================================' +Write-Host '[OK] Deployment complete.' +Write-Host "[INFO] Service: TestDataDB on $RemoteHost" +Write-Host "[INFO] URL: http://${RemoteHost}:3000" +Write-Host "[INFO] Logs: ${RemotePath}\logs\" +Write-Host "[INFO] Backups: ${RemotePath}\backups\ (daily at 2 AM)" +Write-Host '========================================' diff --git a/projects/dataforth-dos/testdatadb-fix/install-backup-task.ps1 b/projects/dataforth-dos/testdatadb-fix/install-backup-task.ps1 new file mode 100644 index 0000000..407c74d --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/install-backup-task.ps1 @@ -0,0 +1,55 @@ +# install-backup-task.ps1 +# Creates a Windows Scheduled Task to run backup-db.ps1 daily at 2:00 AM. +# Must be run as Administrator. + +$ErrorActionPreference = 'Stop' + +$TaskName = 'TestDataDB-Backup' +$ScriptPath = 'C:\Shares\testdatadb\backup-db.ps1' + +# Check for admin privileges +$principal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() +if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + Write-Host '[ERROR] This script must be run as Administrator.' + exit 1 +} + +# Remove existing task if present +$existing = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue +if ($existing) { + Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false + Write-Host "[INFO] Removed existing task: $TaskName" +} + +# Build task components +$action = New-ScheduledTaskAction ` + -Execute 'powershell.exe' ` + -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$ScriptPath`"" + +$trigger = New-ScheduledTaskTrigger -Daily -At '2:00AM' + +$settings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -RunOnlyIfNetworkAvailable:$false ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 30) + +$taskPrincipal = New-ScheduledTaskPrincipal ` + -UserId 'SYSTEM' ` + -LogonType ServiceAccount ` + -RunLevel Highest + +# Register task +Register-ScheduledTask ` + -TaskName $TaskName ` + -Action $action ` + -Trigger $trigger ` + -Settings $settings ` + -Principal $taskPrincipal ` + -Description 'Daily backup of TestDataDB SQLite database with 7-day retention' | + Out-Null + +Write-Host "[OK] Scheduled task '$TaskName' created." +Write-Host "[INFO] Runs daily at 2:00 AM as SYSTEM." +Write-Host "[INFO] Script: $ScriptPath" diff --git a/projects/dataforth-dos/testdatadb-fix/install-service.js b/projects/dataforth-dos/testdatadb-fix/install-service.js new file mode 100644 index 0000000..c59788d --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/install-service.js @@ -0,0 +1,51 @@ +/** + * Install TestDataDB as a Windows Service + * + * Uses node-windows to register the server as a persistent service + * with automatic restart on crash. + * + * Run: node install-service.js + */ + +const path = require('path'); +const Service = require('node-windows').Service; + +const svc = new Service({ + name: 'TestDataDB', + description: 'Dataforth Test Data Database Server', + script: path.join(__dirname, 'server.js'), + nodeOptions: [], + workingDirectory: __dirname, + allowServiceLogon: true, + // Restart configuration: max 3 restarts with 5-second delay + maxRestarts: 3, + maxRetries: 3, + wait: 5, + grow: 0.5 +}); + +// Set log directory +svc.logpath = path.join('C:', 'Shares', 'testdatadb', 'logs'); + +svc.on('install', () => { + console.log('[OK] TestDataDB service installed successfully.'); + console.log('[INFO] Starting service...'); + svc.start(); +}); + +svc.on('start', () => { + console.log('[OK] TestDataDB service started.'); +}); + +svc.on('alreadyinstalled', () => { + console.log('[WARNING] TestDataDB service is already installed.'); + console.log('[INFO] To reinstall, run uninstall-service.js first.'); +}); + +svc.on('error', (err) => { + console.error('[ERROR] Service installation failed:', err); +}); + +console.log('[INFO] Installing TestDataDB as a Windows service...'); +console.log('[INFO] Log directory: C:\\Shares\\testdatadb\\logs\\'); +svc.install(); diff --git a/projects/dataforth-dos/testdatadb-fix/package.json b/projects/dataforth-dos/testdatadb-fix/package.json new file mode 100644 index 0000000..f4d758c --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/package.json @@ -0,0 +1,18 @@ +{ + "name": "testdatadb", + "version": "1.1.0", + "description": "Test data database and search interface", + "main": "server.js", + "scripts": { + "start": "node server.js", + "import": "node database/import.js", + "install-service": "node install-service.js", + "uninstall-service": "node uninstall-service.js" + }, + "dependencies": { + "better-sqlite3": "^9.4.3", + "cors": "^2.8.5", + "express": "^4.18.2", + "node-windows": "^1.0.0-beta.8" + } +} diff --git a/projects/dataforth-dos/testdatadb-fix/routes/api.js b/projects/dataforth-dos/testdatadb-fix/routes/api.js new file mode 100644 index 0000000..ece50d7 --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/routes/api.js @@ -0,0 +1,363 @@ +/** + * API Routes for Test Data Database + * + * Fixed version - uses a single persistent database connection instead of + * opening and closing on every request. WAL journal mode enabled for + * concurrent read support. Limit parameter capped at 1000. + */ + +const express = require('express'); +const path = require('path'); +const Database = require('better-sqlite3'); +const { generateDatasheet } = require('../templates/datasheet'); + +const router = express.Router(); + +// --------------------------------------------------------------------------- +// Singleton database connection - opened once at module load +// --------------------------------------------------------------------------- +const DB_PATH = path.join(__dirname, '..', 'database', 'testdata.db'); + +const db = new Database(DB_PATH, { readonly: false }); +db.pragma('journal_mode = WAL'); +db.pragma('busy_timeout = 5000'); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- +const MAX_LIMIT = 1000; + +function clampLimit(value) { + const parsed = parseInt(value, 10); + if (isNaN(parsed) || parsed < 1) return 100; + return Math.min(parsed, MAX_LIMIT); +} + +function clampOffset(value) { + const parsed = parseInt(value, 10); + if (isNaN(parsed) || parsed < 0) return 0; + return parsed; +} + +// --------------------------------------------------------------------------- +// GET /api/search +// Search test records +// Query params: serial, model, from, to, result, q, station, logtype, limit, offset +// --------------------------------------------------------------------------- +router.get('/search', (req, res) => { + try { + const { serial, model, from, to, result, q, station, logtype } = req.query; + const limit = clampLimit(req.query.limit || 100); + const offset = clampOffset(req.query.offset || 0); + + let sql = 'SELECT * FROM test_records WHERE 1=1'; + const params = []; + + if (serial) { + sql += ' AND serial_number LIKE ?'; + params.push(serial.includes('%') ? serial : `%${serial}%`); + } + + if (model) { + sql += ' AND model_number LIKE ?'; + params.push(model.includes('%') ? model : `%${model}%`); + } + + if (from) { + sql += ' AND test_date >= ?'; + params.push(from); + } + + if (to) { + sql += ' AND test_date <= ?'; + params.push(to); + } + + if (result) { + sql += ' AND overall_result = ?'; + params.push(result.toUpperCase()); + } + + if (station) { + sql += ' AND test_station = ?'; + params.push(station); + } + + if (logtype) { + sql += ' AND log_type = ?'; + params.push(logtype); + } + + if (q) { + // Full-text search - rebuild query with FTS + sql = `SELECT test_records.* FROM test_records + JOIN test_records_fts ON test_records.id = test_records_fts.rowid + WHERE test_records_fts MATCH ?`; + params.length = 0; + params.push(q); + + if (serial) { + sql += ' AND serial_number LIKE ?'; + params.push(serial.includes('%') ? serial : `%${serial}%`); + } + if (model) { + sql += ' AND model_number LIKE ?'; + params.push(model.includes('%') ? model : `%${model}%`); + } + if (station) { + sql += ' AND test_station = ?'; + params.push(station); + } + if (logtype) { + sql += ' AND log_type = ?'; + params.push(logtype); + } + if (result) { + sql += ' AND overall_result = ?'; + params.push(result.toUpperCase()); + } + if (from) { + sql += ' AND test_date >= ?'; + params.push(from); + } + if (to) { + sql += ' AND test_date <= ?'; + params.push(to); + } + } + + sql += ' ORDER BY test_date DESC, serial_number'; + sql += ' LIMIT ? OFFSET ?'; + params.push(limit, offset); + + const records = db.prepare(sql).all(...params); + + // Get total count + let countSql = sql.replace(/SELECT .* FROM/, 'SELECT COUNT(*) as count FROM') + .replace(/ORDER BY.*$/, ''); + countSql = countSql.replace(/LIMIT \? OFFSET \?/, ''); + + const countParams = params.slice(0, -2); + const total = db.prepare(countSql).get(...countParams); + + res.json({ + records, + total: total?.count || records.length, + limit, + offset + }); + } catch (err) { + console.error(`[${new Date().toISOString()}] [SEARCH ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// GET /api/record/:id +// Get single record by ID +// --------------------------------------------------------------------------- +router.get('/record/:id', (req, res) => { + try { + const record = db.prepare('SELECT * FROM test_records WHERE id = ?').get(req.params.id); + + if (!record) { + return res.status(404).json({ error: 'Record not found' }); + } + + res.json(record); + } catch (err) { + console.error(`[${new Date().toISOString()}] [RECORD ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// GET /api/datasheet/:id +// Generate datasheet for a record +// Query params: format (html, txt) +// --------------------------------------------------------------------------- +router.get('/datasheet/:id', (req, res) => { + try { + const record = db.prepare('SELECT * FROM test_records WHERE id = ?').get(req.params.id); + + if (!record) { + return res.status(404).json({ error: 'Record not found' }); + } + + const format = req.query.format || 'html'; + const datasheet = generateDatasheet(record, format); + + if (format === 'html') { + res.type('html').send(datasheet); + } else { + res.type('text/plain').send(datasheet); + } + } catch (err) { + console.error(`[${new Date().toISOString()}] [DATASHEET ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// GET /api/stats +// Get database statistics +// --------------------------------------------------------------------------- +router.get('/stats', (req, res) => { + try { + const stats = { + total_records: db.prepare('SELECT COUNT(*) as count FROM test_records').get().count, + by_log_type: db.prepare(` + SELECT log_type, COUNT(*) as count + FROM test_records + GROUP BY log_type + ORDER BY count DESC + `).all(), + by_result: db.prepare(` + SELECT overall_result, COUNT(*) as count + FROM test_records + GROUP BY overall_result + `).all(), + by_station: db.prepare(` + SELECT test_station, COUNT(*) as count + FROM test_records + WHERE test_station IS NOT NULL AND test_station != '' + GROUP BY test_station + ORDER BY test_station + `).all(), + date_range: db.prepare(` + SELECT MIN(test_date) as oldest, MAX(test_date) as newest + FROM test_records + `).get(), + recent_serials: db.prepare(` + SELECT DISTINCT serial_number, model_number, test_date + FROM test_records + ORDER BY test_date DESC + LIMIT 10 + `).all() + }; + + res.json(stats); + } catch (err) { + console.error(`[${new Date().toISOString()}] [STATS ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// GET /api/filters +// Get available filter options (test stations, log types, models) +// --------------------------------------------------------------------------- +router.get('/filters', (req, res) => { + try { + const filters = { + stations: db.prepare(` + SELECT DISTINCT test_station + FROM test_records + WHERE test_station IS NOT NULL AND test_station != '' + ORDER BY test_station + `).all().map(r => r.test_station), + log_types: db.prepare(` + SELECT DISTINCT log_type + FROM test_records + ORDER BY log_type + `).all().map(r => r.log_type), + models: db.prepare(` + SELECT DISTINCT model_number, COUNT(*) as count + FROM test_records + GROUP BY model_number + ORDER BY count DESC + LIMIT 500 + `).all() + }; + + res.json(filters); + } catch (err) { + console.error(`[${new Date().toISOString()}] [FILTERS ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// GET /api/export +// Export search results as CSV +// --------------------------------------------------------------------------- +router.get('/export', (req, res) => { + try { + const { serial, model, from, to, result, station, logtype } = req.query; + + let sql = 'SELECT * FROM test_records WHERE 1=1'; + const params = []; + + if (serial) { + sql += ' AND serial_number LIKE ?'; + params.push(serial.includes('%') ? serial : `%${serial}%`); + } + + if (model) { + sql += ' AND model_number LIKE ?'; + params.push(model.includes('%') ? model : `%${model}%`); + } + + if (from) { + sql += ' AND test_date >= ?'; + params.push(from); + } + + if (to) { + sql += ' AND test_date <= ?'; + params.push(to); + } + + if (result) { + sql += ' AND overall_result = ?'; + params.push(result.toUpperCase()); + } + + if (station) { + sql += ' AND test_station = ?'; + params.push(station); + } + + if (logtype) { + sql += ' AND log_type = ?'; + params.push(logtype); + } + + sql += ' ORDER BY test_date DESC, serial_number LIMIT 10000'; + + const records = db.prepare(sql).all(...params); + + // Generate CSV + const headers = ['id', 'log_type', 'model_number', 'serial_number', 'test_date', 'test_station', 'overall_result', 'source_file']; + let csv = headers.join(',') + '\n'; + + for (const record of records) { + const row = headers.map(h => { + const val = record[h] || ''; + return `"${String(val).replace(/"/g, '""')}"`; + }); + csv += row.join(',') + '\n'; + } + + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Content-Disposition', 'attachment; filename=test_records.csv'); + res.send(csv); + } catch (err) { + console.error(`[${new Date().toISOString()}] [EXPORT ERROR] ${err.message}`); + res.status(500).json({ error: err.message }); + } +}); + +// --------------------------------------------------------------------------- +// Cleanup function for graceful shutdown +// --------------------------------------------------------------------------- +function cleanup() { + try { + db.close(); + } catch (err) { + console.error(`[${new Date().toISOString()}] [CLEANUP ERROR] ${err.message}`); + } +} + +module.exports = router; +module.exports.cleanup = cleanup; diff --git a/projects/dataforth-dos/testdatadb-fix/server.js b/projects/dataforth-dos/testdatadb-fix/server.js new file mode 100644 index 0000000..406b930 --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/server.js @@ -0,0 +1,103 @@ +/** + * Test Data Database Server + * Express.js server with search API and web interface + * + * Fixed version - singleton DB connection, crash resilience, + * graceful shutdown, request logging. + */ + +const express = require('express'); +const cors = require('cors'); +const path = require('path'); + +const apiRoutes = require('./routes/api'); +const { cleanup } = require('./routes/api'); + +const app = express(); +const PORT = process.env.PORT || 3000; +const HOST = '0.0.0.0'; + +// --------------------------------------------------------------------------- +// Crash resilience - log and continue rather than dying +// --------------------------------------------------------------------------- +process.on('uncaughtException', (err) => { + console.error(`[${new Date().toISOString()}] [UNCAUGHT EXCEPTION] ${err.stack || err.message}`); +}); + +process.on('unhandledRejection', (reason) => { + console.error(`[${new Date().toISOString()}] [UNHANDLED REJECTION] ${reason}`); +}); + +// --------------------------------------------------------------------------- +// Middleware +// --------------------------------------------------------------------------- +app.use(cors()); +app.use(express.json()); +app.use(express.static(path.join(__dirname, 'public'))); + +// Request logging +app.use((req, res, next) => { + const start = Date.now(); + res.on('finish', () => { + const duration = Date.now() - start; + console.log( + `[${new Date().toISOString()}] ${req.method} ${req.originalUrl} ${res.statusCode} ${duration}ms` + ); + }); + next(); +}); + +// --------------------------------------------------------------------------- +// Routes +// --------------------------------------------------------------------------- +app.use('/api', apiRoutes); + +// Serve index.html for root +app.get('/', (req, res) => { + res.sendFile(path.join(__dirname, 'public', 'index.html')); +}); + +// --------------------------------------------------------------------------- +// Start server +// --------------------------------------------------------------------------- +const server = app.listen(PORT, HOST, () => { + console.log(`\n========================================`); + console.log(`Test Data Database Server`); + console.log(`========================================`); + console.log(`Server running on all interfaces (${HOST}:${PORT})`); + console.log(`Local: http://localhost:${PORT}`); + console.log(`LAN: http://192.168.0.6:${PORT}`); + console.log(`API endpoints:`); + console.log(` GET /api/search?serial=...&model=...`); + console.log(` GET /api/record/:id`); + console.log(` GET /api/datasheet/:id`); + console.log(` GET /api/stats`); + console.log(` GET /api/filters`); + console.log(` GET /api/export?format=csv&...`); + console.log(`========================================\n`); +}); + +// --------------------------------------------------------------------------- +// Graceful shutdown +// --------------------------------------------------------------------------- +function shutdown(signal) { + console.log(`\n[${new Date().toISOString()}] Received ${signal}. Shutting down gracefully...`); + server.close(() => { + console.log(`[${new Date().toISOString()}] HTTP server closed.`); + cleanup(); + console.log(`[${new Date().toISOString()}] Database connection closed. Goodbye.`); + process.exit(0); + }); + + // Force exit after 10 seconds if graceful shutdown stalls + setTimeout(() => { + console.error(`[${new Date().toISOString()}] Forced shutdown after timeout.`); + cleanup(); + process.exit(1); + }, 10000); +} + +process.on('SIGTERM', () => shutdown('SIGTERM')); +process.on('SIGINT', () => shutdown('SIGINT')); + +module.exports = app; diff --git a/projects/dataforth-dos/testdatadb-fix/uninstall-service.js b/projects/dataforth-dos/testdatadb-fix/uninstall-service.js new file mode 100644 index 0000000..1594dd4 --- /dev/null +++ b/projects/dataforth-dos/testdatadb-fix/uninstall-service.js @@ -0,0 +1,24 @@ +/** + * Uninstall TestDataDB Windows Service + * + * Run: node uninstall-service.js + */ + +const path = require('path'); +const Service = require('node-windows').Service; + +const svc = new Service({ + name: 'TestDataDB', + script: path.join(__dirname, 'server.js') +}); + +svc.on('uninstall', () => { + console.log('[OK] TestDataDB service uninstalled successfully.'); +}); + +svc.on('error', (err) => { + console.error('[ERROR] Service uninstall failed:', err); +}); + +console.log('[INFO] Uninstalling TestDataDB Windows service...'); +svc.uninstall(); diff --git a/temp/ad2-diag.ps1 b/temp/ad2-diag.ps1 new file mode 100644 index 0000000..9649eeb --- /dev/null +++ b/temp/ad2-diag.ps1 @@ -0,0 +1,28 @@ +# Diagnostic script for TestDataDB on AD2 +Write-Output "=== Node Process ===" +Get-CimInstance Win32_Process -Filter "Name='node.exe'" | Select-Object ProcessId, CommandLine | Format-List + +Write-Output "=== HTTP Test ===" +try { + $r = Invoke-WebRequest -Uri "http://localhost:3000/" -UseBasicParsing -TimeoutSec 10 + Write-Output "Root page status: $($r.StatusCode)" + Write-Output "Content length: $($r.Content.Length)" + Write-Output "First 200 chars: $($r.Content.Substring(0, [Math]::Min(200, $r.Content.Length)))" +} catch { + Write-Output "Root page ERROR: $($_.Exception.Message)" +} + +Write-Output "`n=== API Test ===" +try { + $r = Invoke-WebRequest -Uri "http://localhost:3000/api/stats" -UseBasicParsing -TimeoutSec 10 + Write-Output "API status: $($r.StatusCode)" + Write-Output "First 200 chars: $($r.Content.Substring(0, [Math]::Min(200, $r.Content.Length)))" +} catch { + Write-Output "API ERROR: $($_.Exception.Message)" +} + +Write-Output "`n=== Service Log Files ===" +Get-ChildItem "C:\Shares\testdatadb\logs\" -ErrorAction SilentlyContinue | Format-Table Name, Length, LastWriteTime + +Write-Output "`n=== Recent Event Log ===" +Get-EventLog -LogName Application -Newest 5 -Source "*node*" -ErrorAction SilentlyContinue | Format-List diff --git a/temp/bgb-lesley-check.ps1 b/temp/bgb-lesley-check.ps1 new file mode 100644 index 0000000..a112dd4 --- /dev/null +++ b/temp/bgb-lesley-check.ps1 @@ -0,0 +1,51 @@ +# Check Lesley's email activity since disable +$ErrorActionPreference = "Stop" +$lesleyUPN = "lesley@bgbuildersllc.com" + +Import-Module ExchangeOnlineManagement +Connect-ExchangeOnline -UserPrincipalName "sysadmin@bgbuildersllc.com" -ShowBanner:$false + +$startDate = (Get-Date).AddDays(-3) +$endDate = Get-Date + +Write-Output "=== MAILBOX STATUS ===" +$mbx = Get-Mailbox -Identity $lesleyUPN +$stats = Get-MailboxStatistics -Identity $lesleyUPN +Write-Output "Type: $($mbx.RecipientTypeDetails)" +Write-Output "LitigationHold: $($mbx.LitigationHoldEnabled)" +Write-Output "ItemCount: $($stats.ItemCount)" +Write-Output "TotalSize: $($stats.TotalItemSize)" + +Write-Output "`n=== SENT MESSAGES (last 3 days) ===" +$sent = Get-MessageTraceV2 -SenderAddress $lesleyUPN -StartDate $startDate -EndDate $endDate +if ($sent) { + $sent | Format-Table Received,RecipientAddress,Subject -AutoSize +} else { + Write-Output "None found" +} + +Write-Output "`n=== RECEIVED MESSAGES (last 3 days) ===" +$recv = Get-MessageTraceV2 -RecipientAddress $lesleyUPN -StartDate $startDate -EndDate $endDate +if ($recv) { + $recv | Select-Object -First 20 | Format-Table Received,SenderAddress,Subject -AutoSize +} else { + Write-Output "None found" +} + +Write-Output "`n=== INBOX RULES ===" +$rules = Get-InboxRule -Mailbox $lesleyUPN +if ($rules) { + $rules | Format-Table Name,Enabled,Description -AutoSize +} else { + Write-Output "No inbox rules" +} + +Write-Output "`n=== FORWARDING CONFIG ===" +Write-Output "ForwardingAddress: $($mbx.ForwardingAddress)" +Write-Output "ForwardingSmtpAddress: $($mbx.ForwardingSmtpAddress)" +Write-Output "DeliverToMailboxAndForward: $($mbx.DeliverToMailboxAndForward)" + +Write-Output "`n=== FOLDER ITEM COUNTS ===" +Get-MailboxFolderStatistics -Identity $lesleyUPN | Where-Object { $_.ItemsInFolder -gt 0 } | Sort-Object ItemsInFolder -Descending | Select-Object -First 15 | Format-Table Name,FolderType,ItemsInFolder,FolderSize -AutoSize + +Disconnect-ExchangeOnline -Confirm:$false diff --git a/temp/bgb-lesley-mfa-phone.ps1 b/temp/bgb-lesley-mfa-phone.ps1 new file mode 100644 index 0000000..b3e1b9d --- /dev/null +++ b/temp/bgb-lesley-mfa-phone.ps1 @@ -0,0 +1,52 @@ +# Update MFA phone number for Lesley Roth @ BG Builders +$ErrorActionPreference = "Stop" +$lesleyUPN = "lesley@bgbuildersllc.com" +$newPhone = "+1 4804954511" +$tenantId = "ededa4fb-f6eb-4398-851d-5eb3e11fab27" + +Import-Module Microsoft.Graph.Authentication +Import-Module Microsoft.Graph.Users + +Connect-MgGraph -TenantId $tenantId -Scopes 'UserAuthenticationMethod.ReadWrite.All','User.ReadWrite.All' -NoWelcome + +Write-Output "=== Current Auth Methods for Lesley ===" +$methods = Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods" +if ($methods.value.Count -gt 0) { + foreach ($m in $methods.value) { + Write-Output " ID: $($m.id) | Type: $($m.phoneType) | Number: $($m.phoneNumber)" + } +} else { + Write-Output " No phone methods registered" +} + +Write-Output "`n=== Updating MFA Phone ===" +# Phone method ID for mobile is always "3179e48a-750b-4051-897c-87b9720928f7" +$mobileMethodId = "3179e48a-750b-4051-897c-87b9720928f7" + +try { + # Try to update existing mobile phone method + Invoke-MgGraphRequest -Method PUT -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods/$mobileMethodId" -Body @{ + phoneNumber = $newPhone + phoneType = "mobile" + } + Write-Output "[OK] Mobile phone updated to $newPhone" +} catch { + Write-Output "[INFO] PUT failed, trying POST to create new method..." + try { + Invoke-MgGraphRequest -Method POST -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods" -Body @{ + phoneNumber = $newPhone + phoneType = "mobile" + } + Write-Output "[OK] Mobile phone created: $newPhone" + } catch { + Write-Output "[ERROR] Failed: $_" + } +} + +Write-Output "`n=== Verify Updated Methods ===" +$methods = Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods" +foreach ($m in $methods.value) { + Write-Output " ID: $($m.id) | Type: $($m.phoneType) | Number: $($m.phoneNumber)" +} + +Disconnect-MgGraph diff --git a/temp/bgb-lesley-mfa-phone2.ps1 b/temp/bgb-lesley-mfa-phone2.ps1 new file mode 100644 index 0000000..7b227d8 --- /dev/null +++ b/temp/bgb-lesley-mfa-phone2.ps1 @@ -0,0 +1,26 @@ +# Update MFA phone number for Lesley Roth @ BG Builders +$ErrorActionPreference = "Stop" +$lesleyUPN = "lesley@bgbuildersllc.com" +$newPhone = "+1 4804954511" +$tenantId = "ededa4fb-f6eb-4398-851d-5eb3e11fab27" + +Import-Module Microsoft.Graph.Authentication +Connect-MgGraph -TenantId $tenantId -Scopes 'UserAuthenticationMethod.ReadWrite.All' -NoWelcome + +$mobileMethodId = "3179e48a-750b-4051-897c-87b9720928f7" + +Write-Output "Current: +1 4802299138" +Write-Output "Changing to: $newPhone" + +$body = @{ phoneNumber = $newPhone; phoneType = "mobile" } | ConvertTo-Json +Invoke-MgGraphRequest -Method PUT -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods/$mobileMethodId" -Body $body -ContentType "application/json" + +Write-Output "[OK] Phone updated" + +# Verify +$methods = Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/users/$lesleyUPN/authentication/phoneMethods" +foreach ($m in $methods.value) { + Write-Output "Verified: $($m.phoneType) = $($m.phoneNumber)" +} + +Disconnect-MgGraph diff --git a/temp/lonestar-kyla-2fa-fix.py b/temp/lonestar-kyla-2fa-fix.py new file mode 100644 index 0000000..dbe9c1a --- /dev/null +++ b/temp/lonestar-kyla-2fa-fix.py @@ -0,0 +1,52 @@ +"""Generate backup codes for office@lonestarelectrical.net so Kyla can bypass 2FA enrollment block""" +from google.oauth2 import service_account +from googleapiclient.discovery import build + +SCOPES = [ + 'https://www.googleapis.com/auth/admin.directory.user', + 'https://www.googleapis.com/auth/admin.directory.user.security', +] + +creds = service_account.Credentials.from_service_account_file( + 'temp/acg-msp-access-8f72339997e5.json', scopes=SCOPES +) +delegated = creds.with_subject('sysadmin@lonestarelectrical.net') +service = build('admin', 'directory_v1', credentials=delegated) + +user_email = 'office@lonestarelectrical.net' + +# Check current 2SV status +print(f"=== {user_email} 2SV Status ===") +user = service.users().get(userKey=user_email).execute() +print(f"2SV Enrolled: {user.get('isEnrolledIn2Sv', False)}") +print(f"2SV Enforced: {user.get('isEnforcedIn2Sv', False)}") + +# Generate backup verification codes +print(f"\n=== Generating Backup Codes ===") +try: + codes = service.verificationCodes().generate(userKey=user_email).execute() + print("[OK] Backup codes generated") +except Exception as e: + print(f"[INFO] Generate returned: {e}") + +# List the codes +try: + result = service.verificationCodes().list(userKey=user_email).execute() + backup_codes = result.get('items', []) + if backup_codes: + print(f"\nBackup codes for Kyla to use at login:") + for code in backup_codes: + status = code.get('etag', '') + print(f" {code.get('verificationCode', 'N/A')}") + print(f"\nInstructions for Kyla:") + print(f" 1. Go to https://accounts.google.com") + print(f" 2. Enter email: {user_email}") + print(f" 3. Enter the temp password we set") + print(f" 4. When prompted for 2FA, click 'Try another way'") + print(f" 5. Select 'Enter a backup code'") + print(f" 6. Use one of the codes above") + print(f" 7. Once logged in, go to Security > 2-Step Verification to set up her phone") + else: + print("[WARNING] No codes returned") +except Exception as e: + print(f"[ERROR] Could not list codes: {e}") diff --git a/temp/lonestar-kyla-reset.py b/temp/lonestar-kyla-reset.py new file mode 100644 index 0000000..7298cee --- /dev/null +++ b/temp/lonestar-kyla-reset.py @@ -0,0 +1,60 @@ +"""Reset password for office@lonestarelectrical.net so Kyla can login and set up MFA""" +import secrets +import string +from google.oauth2 import service_account +from googleapiclient.discovery import build + +SCOPES = [ + 'https://www.googleapis.com/auth/admin.directory.user', + 'https://www.googleapis.com/auth/admin.directory.user.security', +] + +creds = service_account.Credentials.from_service_account_file( + 'temp/acg-msp-access-8f72339997e5.json', scopes=SCOPES +) +delegated = creds.with_subject('sysadmin@lonestarelectrical.net') +service = build('admin', 'directory_v1', credentials=delegated) + +user_email = 'office@lonestarelectrical.net' + +# Check current user status +print(f"=== Checking {user_email} ===") +try: + user = service.users().get(userKey=user_email).execute() + print(f"Name: {user.get('name', {}).get('fullName', 'N/A')}") + print(f"Suspended: {user.get('suspended', 'N/A')}") + print(f"Archived: {user.get('archived', 'N/A')}") + print(f"2FA Enrolled: {user.get('isEnrolledIn2Sv', 'N/A')}") + print(f"2FA Enforced: {user.get('isEnforcedIn2Sv', 'N/A')}") + print(f"Last Login: {user.get('lastLoginTime', 'N/A')}") + print(f"Creation: {user.get('creationTime', 'N/A')}") +except Exception as e: + print(f"[ERROR] Could not get user: {e}") + exit(1) + +# Generate a temp password +alphabet = string.ascii_letters + string.digits + "!@#$" +temp_pass = ''.join(secrets.choice(alphabet) for _ in range(16)) + +# Reset password, require change on next login +print(f"\n=== Resetting password ===") +try: + service.users().update( + userKey=user_email, + body={ + 'password': temp_pass, + 'changePasswordAtNextLogin': True, + 'suspended': False, + } + ).execute() + print(f"[OK] Password reset successful") + print(f"[OK] Account unsuspended (if it was)") + print(f"[OK] Must change password on first login") + print(f"\nTemporary password: {temp_pass}") + print(f"\nGive Kyla:") + print(f" Email: {user_email}") + print(f" Password: {temp_pass}") + print(f" URL: https://accounts.google.com") + print(f" She will be prompted to change password and set up MFA") +except Exception as e: + print(f"[ERROR] Password reset failed: {e}") diff --git a/temp/lonestar-kyla-reset2.py b/temp/lonestar-kyla-reset2.py new file mode 100644 index 0000000..719071e --- /dev/null +++ b/temp/lonestar-kyla-reset2.py @@ -0,0 +1,25 @@ +"""Reset password for office@lonestarelectrical.net - attempt 2, no force change""" +from google.oauth2 import service_account +from googleapiclient.discovery import build + +SCOPES = ['https://www.googleapis.com/auth/admin.directory.user'] + +creds = service_account.Credentials.from_service_account_file( + 'temp/acg-msp-access-8f72339997e5.json', scopes=SCOPES +) +delegated = creds.with_subject('sysadmin@lonestarelectrical.net') +service = build('admin', 'directory_v1', credentials=delegated) + +user_email = 'office@lonestarelectrical.net' +new_pass = 'LoneStar2026!!' + +service.users().update( + userKey=user_email, + body={ + 'password': new_pass, + 'changePasswordAtNextLogin': False, + } +).execute() + +print(f"[OK] Password reset for {user_email}") +print(f"Password: {new_pass}") diff --git a/temp/lonestar-russ-setup.py b/temp/lonestar-russ-setup.py new file mode 100644 index 0000000..7068a70 --- /dev/null +++ b/temp/lonestar-russ-setup.py @@ -0,0 +1,41 @@ +"""Reset password and generate backup codes for russ@lonestarelectrical.net""" +from google.oauth2 import service_account +from googleapiclient.discovery import build + +SCOPES = [ + 'https://www.googleapis.com/auth/admin.directory.user', + 'https://www.googleapis.com/auth/admin.directory.user.security', +] + +creds = service_account.Credentials.from_service_account_file( + 'temp/acg-msp-access-8f72339997e5.json', scopes=SCOPES +) +delegated = creds.with_subject('sysadmin@lonestarelectrical.net') +service = build('admin', 'directory_v1', credentials=delegated) + +user_email = 'russ@lonestarelectrical.net' + +# Check user +print(f"=== {user_email} ===") +user = service.users().get(userKey=user_email).execute() +print(f"Name: {user.get('name', {}).get('fullName', 'N/A')}") +print(f"2SV Enrolled: {user.get('isEnrolledIn2Sv', False)}") +print(f"2SV Enforced: {user.get('isEnforcedIn2Sv', False)}") +print(f"Last Login: {user.get('lastLoginTime', 'N/A')}") + +# Reset password +new_pass = 'LoneStar2026!!' +service.users().update( + userKey=user_email, + body={'password': new_pass, 'changePasswordAtNextLogin': False, 'suspended': False} +).execute() +print(f"\n[OK] Password reset: {new_pass}") + +# Generate backup codes +service.verificationCodes().generate(userKey=user_email).execute() +result = service.verificationCodes().list(userKey=user_email).execute() +codes = result.get('items', []) +if codes: + print(f"\nBackup codes:") + for c in codes: + print(f" {c.get('verificationCode')}") diff --git a/temp/test-ad2-web.ps1 b/temp/test-ad2-web.ps1 new file mode 100644 index 0000000..5971cac --- /dev/null +++ b/temp/test-ad2-web.ps1 @@ -0,0 +1,19 @@ +$SshExe = 'C:\Windows\System32\OpenSSH\ssh.exe' +$SshTarget = 'INTRANET\sysadmin@192.168.0.6' + +# Create a test script on AD2 +$testScript = @' +try { + $r = Invoke-WebRequest -Uri "http://localhost:3000/api/stats" -UseBasicParsing -TimeoutSec 5 + Write-Output "STATUS: $($r.StatusCode)" + Write-Output "CONTENT: $($r.Content.Substring(0, [Math]::Min(200, $r.Content.Length)))" +} catch { + Write-Output "ERROR: $($_.Exception.Message)" +} +'@ + +# Write test script to AD2 +$testScript | & $SshExe $SshTarget 'powershell -Command "Set-Content -Path C:\Shares\testdatadb\test-web.ps1 -Value (Get-Content -Raw -Path -)"' + +# Actually, simpler - just run inline +& $SshExe $SshTarget 'powershell -NoProfile -ExecutionPolicy Bypass -Command "try { $r = Invoke-WebRequest -Uri http://localhost:3000/ -UseBasicParsing -TimeoutSec 5; Write-Output STATUS:$($r.StatusCode) } catch { Write-Output ERROR:$($_.Exception.Message) }"' diff --git a/temp/test-minimal.ps1 b/temp/test-minimal.ps1 new file mode 100644 index 0000000..ad332e6 --- /dev/null +++ b/temp/test-minimal.ps1 @@ -0,0 +1,3 @@ +# Minimal test - just echo to NAS +$r = & "C:\Program Files\OpenSSH\ssh.exe" -i C:\Users\sysadmin\.ssh\id_ed25519 -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@192.168.0.9 "echo MINIMAL_TEST_OK" 2>&1 +Write-Host "Result: $r" diff --git a/temp/test-nas-from-ad2.ps1 b/temp/test-nas-from-ad2.ps1 new file mode 100644 index 0000000..37145e8 --- /dev/null +++ b/temp/test-nas-from-ad2.ps1 @@ -0,0 +1,48 @@ +# Test script to run ON AD2 - diagnoses NAS SSH hang issue +$SSH = "C:\Program Files\OpenSSH\ssh.exe" +$SSH_KEY = "C:\Users\sysadmin\.ssh\id_ed25519" +$NAS_USER = "root" +$NAS_IP = "192.168.0.9" + +Write-Host "=== Step 1: Kill any hung SSH processes ===" +Get-Process ssh -ErrorAction SilentlyContinue | ForEach-Object { + Write-Host " Killing SSH PID $($_.Id)" + Stop-Process -Id $_.Id -Force +} +Get-Process powershell -ErrorAction SilentlyContinue | Where-Object { $_.Id -ne $PID } | ForEach-Object { + Write-Host " Other PowerShell PID $($_.Id) - CommandLine: $($_.CommandLine)" +} + +Write-Host "`n=== Step 2: Basic SSH echo test ===" +$t1 = Get-Date +$r1 = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" "echo NAS_OK" 2>&1 +$d1 = (Get-Date) - $t1 +Write-Host " Result: $r1 (took $($d1.TotalSeconds)s)" + +Write-Host "`n=== Step 3: find with temp file redirect (the actual fix) ===" +$t2 = Get-Date +Write-Host " Running find with output to /tmp/test-list.txt..." +# This is exactly what Get-NASFileList does - output goes to file on NAS, stdout discarded +& $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" "find /data/test/TS-*/LOGS -name '*.DAT' -type f -mmin -1440 > /tmp/test-list.txt 2>/dev/null" *> $null +$d2 = (Get-Date) - $t2 +Write-Host " SSH returned in $($d2.TotalSeconds)s" + +Write-Host "`n=== Step 4: Count files found ===" +$r3 = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 "${NAS_USER}@${NAS_IP}" "wc -l /tmp/test-list.txt; head -3 /tmp/test-list.txt" 2>&1 +foreach ($line in $r3) { Write-Host " $line" } + +Write-Host "`n=== Step 5: Pull file list via SCP ===" +$localTemp = "$env:TEMP\test-nas-filelist.txt" +& "C:\Program Files\OpenSSH\scp.exe" -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}:/tmp/test-list.txt" "$localTemp" *> $null +if (Test-Path $localTemp) { + $lines = Get-Content $localTemp | Where-Object { $_.Trim() -ne '' } + Write-Host " Downloaded $($lines.Count) file paths" + Remove-Item $localTemp -ErrorAction SilentlyContinue +} else { + Write-Host " ERROR: SCP failed to download file" +} + +Write-Host "`n=== Step 6: Cleanup ===" +& $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 "${NAS_USER}@${NAS_IP}" "rm -f /tmp/test-list.txt" 2>&1 | Out-Null + +Write-Host "`n=== DONE ===" diff --git a/temp/test-nas-v2.ps1 b/temp/test-nas-v2.ps1 new file mode 100644 index 0000000..e2a6148 --- /dev/null +++ b/temp/test-nas-v2.ps1 @@ -0,0 +1,32 @@ +# Run ON AD2: Tests NAS SSH operations +# Deploy: scp test-nas-v2.ps1 AD2:C:\Shares\testdatadb\ +# Run: powershell -NoProfile -ExecutionPolicy Bypass -File C:\Shares\testdatadb\test-nas-v2.ps1 +$SSH = "C:\Program Files\OpenSSH\ssh.exe" +$SSH_KEY = "C:\Users\sysadmin\.ssh\id_ed25519" + +Write-Host "=== Test A: echo ===" +$t = Get-Date +$r = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@192.168.0.9 "echo NAS_OK" 2>&1 +Write-Host " Result: $r ($(((Get-Date)-$t).TotalSeconds)s)" + +Write-Host "=== Test B: ls data dir ===" +$t = Get-Date +$r = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 root@192.168.0.9 "ls /data/test/ | head -5" 2>&1 +foreach ($l in $r) { Write-Host " $l" } +Write-Host " ($(((Get-Date)-$t).TotalSeconds)s)" + +Write-Host "=== Test C: find with wc -l only ===" +$t = Get-Date +$r = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 root@192.168.0.9 "find /data/test/TS-*/LOGS -name '*.DAT' -type f -mmin -1440 2>/dev/null | wc -l" 2>&1 +Write-Host " Count: $r ($(((Get-Date)-$t).TotalSeconds)s)" + +Write-Host "=== Test D: find to temp file ===" +$t = Get-Date +& $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 root@192.168.0.9 "find /data/test/TS-*/LOGS -name '*.DAT' -type f -mmin -1440 > /tmp/test-list.txt 2>/dev/null" *> $null +Write-Host " ($(((Get-Date)-$t).TotalSeconds)s)" + +Write-Host "=== Test E: check temp file ===" +$r = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 root@192.168.0.9 "wc -l /tmp/test-list.txt; rm -f /tmp/test-list.txt" 2>&1 +Write-Host " $r" + +Write-Host "=== ALL DONE ===" diff --git a/temp/testdatadb.xml b/temp/testdatadb.xml new file mode 100644 index 0000000..c01cc2f --- /dev/null +++ b/temp/testdatadb.xml @@ -0,0 +1,15 @@ + + testdatadb + TestDataDB + Dataforth Test Data Database Server + C:\Program Files\nodejs\node.exe + C:\Shares\testdatadb\server.js + C:\Shares\testdatadb\logs + rotate + 15sec + C:\Shares\testdatadb + + + + 1 hour +