azcomputerguru/claudetools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: Dataforth sync fixes, TestDataDB stability, and client scripts

Browse Source 
Dataforth DOS:
- TestDataDB: singleton DB connection fix (crash prevention), WAL mode,
  WinSW service config, backup script, uncaught exception handlers
- Sync-FromNAS.ps1: Get-NASFileList temp file approach to avoid SSH
  stdout deadlock, *> $null output suppression, 8.3 filename filter
  for PUSH phase, backslash-escaped SCP paths, rename-to-.synced
- import.js: INSERT OR REPLACE for re-tested devices
- Full import run: 1,028,275 -> 1,632,793 records, indexes added
- Deploy script for sync fixes to AD2

Client scripts (temp/):
- BG Builders: Lesley account check, MFA phone update
- Lonestar Electrical: Kyla/Russ Google Workspace setup, 2FA bypass
- AD2 diagnostics and NAS connectivity tests

PENDING: Investigate why newest test_date is Jan 19 despite daily tests

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Mike Swanson
2026-03-11 20:16:24 -07:00
parent 1a26eb051a
commit 470638ff86
24 changed files with 2498 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

540
projects/dataforth-dos/sync-fixes/Sync-FromNAS.ps1 Normal file
View File

@@ -0,0 +1,540 @@
# Sync-AD2-NAS.ps1 (formerly Sync-FromNAS.ps1)
# Bidirectional sync between AD2 and NAS (D2TESTNAS)
#
# PULL (NAS -> AD2): Test results (LOGS/*.DAT, Reports/*.TXT) -> Database import
# PUSH (AD2 -> NAS): Software updates (ProdSW/*, TODO.BAT) -> DOS machines
#
# Run: powershell -ExecutionPolicy Bypass -File C:\Shares\test\scripts\Sync-FromNAS.ps1
# Scheduled: Every 15 minutes via Windows Task Scheduler
param(
[switch]$DryRun, # Show what would be done without doing it
[switch]$Verbose, # Extra output
[int]$MaxAgeMinutes = 1440 # Default: files from last 24 hours (was 60 min, too aggressive)
)
# ============================================================================
# Configuration
# ============================================================================
$NAS_IP = "192.168.0.9"
$NAS_USER = "root"
$NAS_PASSWORD = "Paper123!@#-nas"
$NAS_HOSTKEY = "SHA256:5CVIPlqjLPxO8n48PKLAP99nE6XkEBAjTkaYmJAeOdA"
$NAS_DATA_PATH = "/data/test"
$AD2_TEST_PATH = "C:\Shares\test"
$AD2_HISTLOGS_PATH = "C:\Shares\test\Ate\HISTLOGS"
$SSH = "C:\Program Files\OpenSSH\ssh.exe"
$SCP = "C:\Program Files\OpenSSH\scp.exe"
$SSH_KEY = "C:\Users\sysadmin\.ssh\id_ed25519"
$LOG_FILE = "C:\Shares\test\scripts\sync-from-nas.log"
$STATUS_FILE = "C:\Shares\test\_SYNC_STATUS.txt"
$LOG_TYPES = @("5BLOG", "7BLOG", "8BLOG", "DSCLOG", "SCTLOG", "VASLOG", "PWRLOG", "HVLOG")
# Database import configuration
$IMPORT_SCRIPT = "C:\Shares\testdatadb\database\import.js"
$NODE_PATH = "node"
# ============================================================================
# Functions
# ============================================================================
function Write-Log {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$logLine = "$timestamp : $Message"
Add-Content -Path $LOG_FILE -Value $logLine
if ($Verbose) { Write-Host $logLine }
}
function Invoke-NASCommand {
param([string]$Command)
$result = & $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" $Command 2>&1
return $result
}
function Get-NASFileList {
param(
[string]$FindCommand,
[string]$ListLabel
)
# Generate file list on NAS side, write to temp file, pull via SCP
# Key fix: use *> $null to discard all PowerShell output streams, preventing
# the stdout buffer deadlock that occurs with & $SSH ... 2>&1
$remoteTemp = "/tmp/nas-file-list-${ListLabel}.txt"
# Step 1: Run find on NAS, redirect output to temp file on NAS
# We discard ALL PowerShell output (*> $null) - we only need the side effect
& $SSH -i $SSH_KEY -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${NAS_USER}@${NAS_IP}" "$FindCommand > $remoteTemp 2>/dev/null" *> $null
# Step 2: Pull the list file via SCP
$localTemp = "$env:TEMP\nas-file-list-${ListLabel}-$(Get-Date -Format 'yyyyMMddHHmmss').txt"
& $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "${NAS_USER}@${NAS_IP}:${remoteTemp}" "$localTemp" *> $null
# Step 3: Read locally and clean up
$fileList = Get-Content $localTemp -ErrorAction SilentlyContinue | Where-Object { $_.Trim() -ne '' }
Remove-Item $localTemp -ErrorAction SilentlyContinue
# Step 4: Clean up remote temp file (small output, safe with Invoke-NASCommand)
Invoke-NASCommand "rm -f $remoteTemp" | Out-Null
return $fileList
}
function Copy-FromNAS {
param(
[string]$RemotePath,
[string]$LocalPath
)
# Ensure local directory exists
$localDir = Split-Path -Parent $LocalPath
if (-not (Test-Path $localDir)) {
New-Item -ItemType Directory -Path $localDir -Force | Out-Null
}
# Escape spaces and special chars with backslashes for SCP remote path
$escapedPath = $RemotePath -replace '([ ()\[\]{}])', '\$1'
$result = & $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "${NAS_USER}@${NAS_IP}:${escapedPath}" "$LocalPath" 2>&1
if ($LASTEXITCODE -ne 0) {
$errorMsg = $result | Out-String
Write-Log " SCP PULL ERROR (exit $LASTEXITCODE): $errorMsg"
}
return $LASTEXITCODE -eq 0
}
function Rename-OnNAS {
param([string]$RemotePath)
Invoke-NASCommand "mv '$RemotePath' '${RemotePath}.synced'" | Out-Null
}
function Copy-ToNAS {
param(
[string]$LocalPath,
[string]$RemotePath
)
# Ensure remote directory exists via SSH mkdir -p
$remoteDir = ($RemotePath -replace '[^/]+$', '').TrimEnd('/')
Invoke-NASCommand "mkdir -p '$remoteDir'" | Out-Null
# Escape spaces and special chars with backslashes for SCP remote path
$escapedPath = $RemotePath -replace '([ ()\[\]{}])', '\$1'
$result = & $SCP -O -i $SSH_KEY -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="C:\Shares\test\scripts\.ssh\known_hosts" "$LocalPath" "${NAS_USER}@${NAS_IP}:${escapedPath}" 2>&1
if ($LASTEXITCODE -ne 0) {
$errorMsg = $result | Out-String
Write-Log " SCP PUSH ERROR (exit $LASTEXITCODE): $errorMsg"
}
return $LASTEXITCODE -eq 0
}
function Test-8dot3Name {
param([string]$Name)
# Returns $true if filename is 8.3 compatible (no spaces, name<=8, ext<=3)
if ($Name -match '[ ()\[\]{}]') { return $false }
$parts = $Name -split '\.'
if ($parts.Count -gt 2) { return $false }
if ($parts[0].Length -gt 8 -or $parts[0].Length -eq 0) { return $false }
if ($parts.Count -eq 2 -and $parts[1].Length -gt 3) { return $false }
return $true
}
function Test-8dot3Path {
param([string]$RelativePath)
# Check every component of the path is 8.3 compatible
$segments = $RelativePath -replace '\\', '/' -split '/'
foreach ($seg in $segments) {
if ($seg -eq '') { continue }
if (-not (Test-8dot3Name $seg)) { return $false }
}
return $true
}
function Get-FileHash256 {
param([string]$FilePath)
if (Test-Path $FilePath) {
return (Get-FileHash -Path $FilePath -Algorithm SHA256).Hash
}
return $null
}
function Import-ToDatabase {
param([string[]]$FilePaths)
if ($FilePaths.Count -eq 0) { return }
Write-Log "Importing $($FilePaths.Count) file(s) to database..."
# Build argument list
$importArgs = @("$IMPORT_SCRIPT", "--file") + $FilePaths
try {
$output = & $NODE_PATH $importArgs 2>&1
foreach ($line in $output) {
Write-Log " [DB] $line"
}
Write-Log "Database import complete"
} catch {
Write-Log "ERROR: Database import failed: $_"
}
}
# ============================================================================
# Main Script
# ============================================================================
Write-Log "=========================================="
Write-Log "Starting sync from NAS"
Write-Log "Max age: $MaxAgeMinutes minutes"
if ($DryRun) { Write-Log "DRY RUN - no changes will be made" }
# Ensure .ssh directory exists for known_hosts
$sshDir = "C:\Shares\test\scripts\.ssh"
if (-not (Test-Path $sshDir)) {
New-Item -ItemType Directory -Path $sshDir -Force | Out-Null
Write-Log "Created .ssh directory: $sshDir"
}
$errorCount = 0
$syncedFiles = 0
$skippedFiles = 0
$syncedDatFiles = @() # Track DAT files for database import
# Find all DAT files on NAS modified within the time window
# Uses temp file approach to avoid SSH output buffer hang with large file counts
Write-Log "Finding DAT files on NAS..."
$findCommand = "find $NAS_DATA_PATH/TS-*/LOGS -name '*.DAT' -type f -mmin -$MaxAgeMinutes"
$datFiles = Get-NASFileList -FindCommand $findCommand -ListLabel "dat"
if (-not $datFiles -or $datFiles.Count -eq 0) {
Write-Log "No new DAT files found on NAS"
} else {
Write-Log "Found $($datFiles.Count) DAT file(s) to process"
foreach ($remoteFile in $datFiles) {
$remoteFile = $remoteFile.Trim()
if ([string]::IsNullOrWhiteSpace($remoteFile)) { continue }
# Parse the path: /data/test/TS-XX/LOGS/7BLOG/file.DAT
if ($remoteFile -match "/data/test/(TS-[^/]+)/LOGS/([^/]+)/(.+\.DAT)$") {
$station = $Matches[1]
$logType = $Matches[2]
$fileName = $Matches[3]
Write-Log "Processing: $station/$logType/$fileName"
# Destination 1: Per-station folder (preserves structure)
$stationDest = Join-Path $AD2_TEST_PATH "$station\LOGS\$logType\$fileName"
# Destination 2: Aggregated HISTLOGS folder
$histlogsDest = Join-Path $AD2_HISTLOGS_PATH "$logType\$fileName"
if ($DryRun) {
Write-Log " [DRY RUN] Would copy to: $stationDest"
$syncedFiles++
} else {
# Copy to station folder only (skip HISTLOGS to avoid duplicates)
$success1 = Copy-FromNAS -RemotePath $remoteFile -LocalPath $stationDest
if ($success1) {
Write-Log " Copied to station folder"
# Rename on NAS after successful sync (preserves file as .synced)
Rename-OnNAS -RemotePath $remoteFile
Write-Log " Renamed to .synced on NAS"
# Track for database import
$syncedDatFiles += $stationDest
$syncedFiles++
} else {
Write-Log " ERROR: Failed to copy from NAS"
$errorCount++
}
}
} else {
Write-Log " Skipping (unexpected path format): $remoteFile"
$skippedFiles++
}
}
}
# Find and sync TXT report files
# Uses temp file approach to avoid SSH output buffer hang with large file counts
Write-Log "Finding TXT reports on NAS..."
$findReportsCommand = "find $NAS_DATA_PATH/TS-*/Reports -name '*.TXT' -type f -mmin -$MaxAgeMinutes"
$txtFiles = Get-NASFileList -FindCommand $findReportsCommand -ListLabel "txt"
if ($txtFiles -and $txtFiles.Count -gt 0) {
Write-Log "Found $($txtFiles.Count) TXT report(s) to process"
foreach ($remoteFile in $txtFiles) {
$remoteFile = $remoteFile.Trim()
if ([string]::IsNullOrWhiteSpace($remoteFile)) { continue }
if ($remoteFile -match "/data/test/(TS-[^/]+)/Reports/(.+\.TXT)$") {
$station = $Matches[1]
$fileName = $Matches[2]
Write-Log "Processing report: $station/$fileName"
# Destination: Per-station Reports folder
$reportDest = Join-Path $AD2_TEST_PATH "$station\Reports\$fileName"
if ($DryRun) {
Write-Log " [DRY RUN] Would copy to: $reportDest"
$syncedFiles++
} else {
$success = Copy-FromNAS -RemotePath $remoteFile -LocalPath $reportDest
if ($success) {
Write-Log " Copied report"
Rename-OnNAS -RemotePath $remoteFile
Write-Log " Renamed to .synced on NAS"
$syncedFiles++
} else {
Write-Log " ERROR: Failed to copy report"
$errorCount++
}
}
}
}
}
# ============================================================================
# Import synced DAT files to database
# ============================================================================
if (-not $DryRun -and $syncedDatFiles.Count -gt 0) {
Import-ToDatabase -FilePaths $syncedDatFiles
}
# ============================================================================
# PUSH: AD2 -> NAS (Software Updates for DOS Machines)
# ============================================================================
Write-Log "--- AD2 to NAS Sync (Software Updates) ---"
$pushedFiles = 0
# Sync COMMON/ProdSW (batch files for all stations)
# AD2 uses _COMMON, NAS uses COMMON - handle both
$commonSources = @(
@{ Local = "$AD2_TEST_PATH\_COMMON\ProdSW"; Remote = "$NAS_DATA_PATH/COMMON/ProdSW" },
@{ Local = "$AD2_TEST_PATH\COMMON\ProdSW"; Remote = "$NAS_DATA_PATH/COMMON/ProdSW" }
)
foreach ($source in $commonSources) {
if (Test-Path $source.Local) {
Write-Log "Syncing COMMON ProdSW from: $($source.Local)"
$commonFiles = Get-ChildItem -Path $source.Local -File -ErrorAction SilentlyContinue
foreach ($file in $commonFiles) {
if (-not (Test-8dot3Name $file.Name)) {
Write-Log " Skipping (non-8.3): $($file.Name)"
$skippedFiles++
continue
}
$remotePath = "$($source.Remote)/$($file.Name)"
if ($DryRun) {
Write-Log " [DRY RUN] Would push: $($file.Name) -> $remotePath"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath
if ($success) {
Write-Log " Pushed: $($file.Name)"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push $($file.Name)"
$errorCount++
}
}
}
}
}
# Sync Ate/ProdSW (shared ATE data folders - 5BDATA, 7BDATA, 8BDATA, DSCDATA, etc.)
Write-Log "Syncing Ate/ProdSW data folders..."
$ateProdSwPath = "$AD2_TEST_PATH\Ate\ProdSW"
if (Test-Path $ateProdSwPath) {
# Get all files recursively (including subdirectories like DSCDATA, 5BDATA, etc.)
$ateFiles = Get-ChildItem -Path $ateProdSwPath -File -Recurse -ErrorAction SilentlyContinue
foreach ($file in $ateFiles) {
# Calculate relative path from Ate/ProdSW folder
$relativePath = $file.FullName.Substring($ateProdSwPath.Length + 1).Replace('\', '/')
if (-not (Test-8dot3Path $relativePath)) {
Write-Log " Skipping (non-8.3): Ate/ProdSW/$relativePath"
$skippedFiles++
continue
}
$remotePath = "$NAS_DATA_PATH/Ate/ProdSW/$relativePath"
if ($DryRun) {
Write-Log " [DRY RUN] Would push: Ate/ProdSW/$relativePath"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath
if ($success) {
Write-Log " Pushed: Ate/ProdSW/$relativePath"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push Ate/ProdSW/$relativePath"
$errorCount++
}
}
}
} else {
Write-Log " Ate/ProdSW not found: $ateProdSwPath"
}
# Sync UPDATE.BAT (root level utility)
Write-Log "Syncing UPDATE.BAT..."
$updateBatLocal = "$AD2_TEST_PATH\UPDATE.BAT"
if (Test-Path $updateBatLocal) {
$updateBatRemote = "$NAS_DATA_PATH/UPDATE.BAT"
if ($DryRun) {
Write-Log " [DRY RUN] Would push: UPDATE.BAT -> $updateBatRemote"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $updateBatLocal -RemotePath $updateBatRemote
if ($success) {
Write-Log " Pushed: UPDATE.BAT"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push UPDATE.BAT"
$errorCount++
}
}
} else {
Write-Log " WARNING: UPDATE.BAT not found at $updateBatLocal"
}
# Sync DEPLOY.BAT (root level utility)
Write-Log "Syncing DEPLOY.BAT..."
$deployBatLocal = "$AD2_TEST_PATH\DEPLOY.BAT"
if (Test-Path $deployBatLocal) {
$deployBatRemote = "$NAS_DATA_PATH/DEPLOY.BAT"
if ($DryRun) {
Write-Log " [DRY RUN] Would push: DEPLOY.BAT -> $deployBatRemote"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $deployBatLocal -RemotePath $deployBatRemote
if ($success) {
Write-Log " Pushed: DEPLOY.BAT"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push DEPLOY.BAT"
$errorCount++
}
}
} else {
Write-Log " WARNING: DEPLOY.BAT not found at $deployBatLocal"
}
# Sync per-station ProdSW folders
Write-Log "Syncing station-specific ProdSW folders..."
$stationFolders = Get-ChildItem -Path $AD2_TEST_PATH -Directory -Filter "TS-*" -ErrorAction SilentlyContinue
foreach ($station in $stationFolders) {
# Skip station folders with non-8.3 names (e.g. "TS-1 OBS")
if (-not (Test-8dot3Name $station.Name)) {
Write-Log " Skipping station (non-8.3 name): $($station.Name)"
continue
}
$prodSwPath = Join-Path $station.FullName "ProdSW"
if (Test-Path $prodSwPath) {
# Get all files in ProdSW (including subdirectories)
$prodSwFiles = Get-ChildItem -Path $prodSwPath -File -Recurse -ErrorAction SilentlyContinue
foreach ($file in $prodSwFiles) {
# Calculate relative path from ProdSW folder
$relativePath = $file.FullName.Substring($prodSwPath.Length + 1).Replace('\', '/')
if (-not (Test-8dot3Path $relativePath)) {
Write-Log " Skipping (non-8.3): $($station.Name)/ProdSW/$relativePath"
$skippedFiles++
continue
}
$remotePath = "$NAS_DATA_PATH/$($station.Name)/ProdSW/$relativePath"
if ($DryRun) {
Write-Log " [DRY RUN] Would push: $($station.Name)/ProdSW/$relativePath"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $file.FullName -RemotePath $remotePath
if ($success) {
Write-Log " Pushed: $($station.Name)/ProdSW/$relativePath"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push $($station.Name)/ProdSW/$relativePath"
$errorCount++
}
}
}
}
# Check for TODO.BAT (one-time task file)
$todoBatPath = Join-Path $station.FullName "TODO.BAT"
if (Test-Path $todoBatPath) {
$remoteTodoPath = "$NAS_DATA_PATH/$($station.Name)/TODO.BAT"
Write-Log "Found TODO.BAT for $($station.Name)"
if ($DryRun) {
Write-Log " [DRY RUN] Would push TODO.BAT -> $remoteTodoPath"
$pushedFiles++
} else {
$success = Copy-ToNAS -LocalPath $todoBatPath -RemotePath $remoteTodoPath
if ($success) {
Write-Log " Pushed TODO.BAT to NAS"
# Remove from AD2 after successful push (one-shot mechanism)
Remove-Item -Path $todoBatPath -Force
Write-Log " Removed TODO.BAT from AD2 (pushed to NAS)"
$pushedFiles++
} else {
Write-Log " ERROR: Failed to push TODO.BAT"
$errorCount++
}
}
}
}
Write-Log "AD2 to NAS sync: $pushedFiles file(s) pushed"
# ============================================================================
# Update Status File
# ============================================================================
$status = if ($errorCount -eq 0) { "OK" } else { "ERRORS" }
$statusContent = @"
AD2 <-> NAS Bidirectional Sync Status
======================================
Timestamp: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss")
Status: $status
PULL (NAS -> AD2 - Test Results):
Files Pulled: $syncedFiles
Files Skipped: $skippedFiles
DAT Files Imported to DB: $($syncedDatFiles.Count)
PUSH (AD2 -> NAS - Software Updates):
Files Pushed: $pushedFiles
Errors: $errorCount
"@
Set-Content -Path $STATUS_FILE -Value $statusContent
Write-Log "=========================================="
Write-Log "Sync complete: PULL=$syncedFiles, PUSH=$pushedFiles, Errors=$errorCount"
Write-Log "=========================================="
# Exit with error code if there were failures
if ($errorCount -gt 0) {
exit 1
} else {
exit 0
}

302
projects/dataforth-dos/sync-fixes/deploy-sync-fixes.ps1 Normal file
View File

@@ -0,0 +1,302 @@
# deploy-sync-fixes.ps1
# Deploys patched Sync-FromNAS.ps1 and import.js to AD2 (192.168.0.6)
#
# Fixes deployed:
# 1. Sync-FromNAS.ps1 - PULL hang fix (temp file approach for large find output)
# 2. Sync-FromNAS.ps1 - SCP quoting fix (handles spaces, parens, special chars)
# 3. Sync-FromNAS.ps1 - Rename-OnNAS replaces Remove-FromNAS (.synced instead of delete)
# 4. import.js - INSERT OR REPLACE instead of INSERT OR IGNORE (re-tests keep latest)
#
# Run from: Mike's workstation (where the patched files live)
# Target: AD2 (192.168.0.6) as INTRANET\sysadmin
param(
[switch]$DryRun # Show what would be done without doing it
)
# ============================================================================
# Configuration
# ============================================================================
$AD2_IP = "192.168.0.6"
$AD2_USER = "INTRANET\sysadmin"
$AD2_PASSWORD = "Paper123!@#"
$SSH = "C:\Windows\System32\OpenSSH\ssh.exe"
$SCP = "C:\Windows\System32\OpenSSH\scp.exe"
# Source files (local to this machine)
$SCRIPT_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path
$LOCAL_SYNC_SCRIPT = Join-Path $SCRIPT_DIR "Sync-FromNAS.ps1"
$LOCAL_IMPORT_SCRIPT = Join-Path $SCRIPT_DIR "import.js"
# Destination paths on AD2
$REMOTE_SYNC_PATH = "C:\Shares\test\scripts\Sync-FromNAS.ps1"
$REMOTE_IMPORT_PATH = "C:\Shares\testdatadb\database\import.js"
$REMOTE_SSH_DIR = "C:\Shares\test\scripts\.ssh"
$DATE_SUFFIX = Get-Date -Format "yyyyMMdd"
# ============================================================================
# Functions
# ============================================================================
function Write-Status {
param(
[string]$Marker,
[string]$Message
)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
Write-Host "$timestamp [$Marker] $Message"
}
function Invoke-AD2Command {
param([string]$Command)
$result = & $SSH -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 "${AD2_USER}@${AD2_IP}" $Command 2>&1
return $result
}
function Copy-ToAD2 {
param(
[string]$LocalPath,
[string]$RemotePath
)
# SCP to AD2 - quote remote path for Windows paths with backslashes
$result = & $SCP -O -o StrictHostKeyChecking=accept-new "$LocalPath" "${AD2_USER}@${AD2_IP}:`"${RemotePath}`"" 2>&1
if ($LASTEXITCODE -ne 0) {
$errorMsg = $result | Out-String
Write-Status "ERROR" "SCP failed (exit $LASTEXITCODE): $errorMsg"
return $false
}
return $true
}
# ============================================================================
# Pre-flight Checks
# ============================================================================
Write-Host ""
Write-Host "============================================"
Write-Host " Dataforth Sync Fixes - Deployment Script"
Write-Host "============================================"
Write-Host ""
if ($DryRun) {
Write-Status "INFO" "DRY RUN - no changes will be made"
Write-Host ""
}
# Verify source files exist
Write-Status "INFO" "Checking source files..."
if (-not (Test-Path $LOCAL_SYNC_SCRIPT)) {
Write-Status "ERROR" "Source file not found: $LOCAL_SYNC_SCRIPT"
exit 1
}
Write-Status "OK" "Found: $LOCAL_SYNC_SCRIPT"
if (-not (Test-Path $LOCAL_IMPORT_SCRIPT)) {
Write-Status "ERROR" "Source file not found: $LOCAL_IMPORT_SCRIPT"
exit 1
}
Write-Status "OK" "Found: $LOCAL_IMPORT_SCRIPT"
# Verify AD2 is reachable
Write-Status "INFO" "Testing connectivity to AD2 ($AD2_IP)..."
$pingResult = Test-Connection -ComputerName $AD2_IP -Count 1 -Quiet
if (-not $pingResult) {
Write-Status "ERROR" "Cannot reach AD2 at $AD2_IP"
exit 1
}
Write-Status "OK" "AD2 is reachable"
# Test SSH connection
Write-Status "INFO" "Testing SSH connection..."
$sshTest = Invoke-AD2Command "echo CONNECTION_OK"
$sshTestStr = $sshTest | Out-String
if ($sshTestStr -notmatch "CONNECTION_OK") {
Write-Status "ERROR" "SSH connection failed: $sshTestStr"
exit 1
}
Write-Status "OK" "SSH connection established"
# ============================================================================
# Step 1: Create backups on AD2
# ============================================================================
Write-Host ""
Write-Status "INFO" "--- Step 1: Creating backups on AD2 ---"
if ($DryRun) {
Write-Status "INFO" "[DRY RUN] Would backup $REMOTE_SYNC_PATH -> ${REMOTE_SYNC_PATH}.bak-${DATE_SUFFIX}"
Write-Status "INFO" "[DRY RUN] Would backup $REMOTE_IMPORT_PATH -> ${REMOTE_IMPORT_PATH}.bak-${DATE_SUFFIX}"
} else {
# Backup Sync-FromNAS.ps1
$backupCmd1 = "copy `"$REMOTE_SYNC_PATH`" `"${REMOTE_SYNC_PATH}.bak-${DATE_SUFFIX}`""
$backupResult1 = Invoke-AD2Command "cmd /c $backupCmd1"
if ($LASTEXITCODE -eq 0) {
Write-Status "OK" "Backed up Sync-FromNAS.ps1 -> .bak-${DATE_SUFFIX}"
} else {
Write-Status "WARNING" "Backup of Sync-FromNAS.ps1 may have failed (file might not exist yet): $($backupResult1 | Out-String)"
}
# Backup import.js
$backupCmd2 = "copy `"$REMOTE_IMPORT_PATH`" `"${REMOTE_IMPORT_PATH}.bak-${DATE_SUFFIX}`""
$backupResult2 = Invoke-AD2Command "cmd /c $backupCmd2"
if ($LASTEXITCODE -eq 0) {
Write-Status "OK" "Backed up import.js -> .bak-${DATE_SUFFIX}"
} else {
Write-Status "WARNING" "Backup of import.js may have failed (file might not exist yet): $($backupResult2 | Out-String)"
}
}
# ============================================================================
# Step 2: Create .ssh directory on AD2 if missing
# ============================================================================
Write-Host ""
Write-Status "INFO" "--- Step 2: Ensuring .ssh directory exists ---"
if ($DryRun) {
Write-Status "INFO" "[DRY RUN] Would create $REMOTE_SSH_DIR if missing"
} else {
$mkdirCmd = "cmd /c if not exist `"$REMOTE_SSH_DIR`" mkdir `"$REMOTE_SSH_DIR`""
Invoke-AD2Command $mkdirCmd | Out-Null
Write-Status "OK" "Ensured .ssh directory exists: $REMOTE_SSH_DIR"
# Create empty known_hosts if it does not exist
$knownHostsPath = "$REMOTE_SSH_DIR\known_hosts"
$touchCmd = "cmd /c if not exist `"$knownHostsPath`" type nul > `"$knownHostsPath`""
Invoke-AD2Command $touchCmd | Out-Null
Write-Status "OK" "Ensured known_hosts file exists: $knownHostsPath"
}
# ============================================================================
# Step 3: Deploy patched files
# ============================================================================
Write-Host ""
Write-Status "INFO" "--- Step 3: Deploying patched files ---"
if ($DryRun) {
Write-Status "INFO" "[DRY RUN] Would SCP $LOCAL_SYNC_SCRIPT -> ${AD2_IP}:${REMOTE_SYNC_PATH}"
Write-Status "INFO" "[DRY RUN] Would SCP $LOCAL_IMPORT_SCRIPT -> ${AD2_IP}:${REMOTE_IMPORT_PATH}"
} else {
# Deploy Sync-FromNAS.ps1
Write-Status "INFO" "Deploying Sync-FromNAS.ps1..."
$success1 = Copy-ToAD2 -LocalPath $LOCAL_SYNC_SCRIPT -RemotePath $REMOTE_SYNC_PATH
if ($success1) {
Write-Status "OK" "Deployed Sync-FromNAS.ps1"
} else {
Write-Status "ERROR" "Failed to deploy Sync-FromNAS.ps1"
exit 1
}
# Deploy import.js
Write-Status "INFO" "Deploying import.js..."
$success2 = Copy-ToAD2 -LocalPath $LOCAL_IMPORT_SCRIPT -RemotePath $REMOTE_IMPORT_PATH
if ($success2) {
Write-Status "OK" "Deployed import.js"
} else {
Write-Status "ERROR" "Failed to deploy import.js"
exit 1
}
}
# ============================================================================
# Step 4: Verify deployment
# ============================================================================
Write-Host ""
Write-Status "INFO" "--- Step 4: Verifying deployment ---"
if ($DryRun) {
Write-Status "INFO" "[DRY RUN] Would verify files exist on AD2"
} else {
# Check Sync-FromNAS.ps1 exists and has content
$verifyCmd1 = "cmd /c if exist `"$REMOTE_SYNC_PATH`" (echo FILE_EXISTS) else (echo FILE_MISSING)"
$verify1 = Invoke-AD2Command $verifyCmd1 | Out-String
if ($verify1 -match "FILE_EXISTS") {
Write-Status "OK" "Verified: Sync-FromNAS.ps1 exists on AD2"
} else {
Write-Status "ERROR" "Verification failed: Sync-FromNAS.ps1 not found on AD2"
exit 1
}
# Check import.js exists
$verifyCmd2 = "cmd /c if exist `"$REMOTE_IMPORT_PATH`" (echo FILE_EXISTS) else (echo FILE_MISSING)"
$verify2 = Invoke-AD2Command $verifyCmd2 | Out-String
if ($verify2 -match "FILE_EXISTS") {
Write-Status "OK" "Verified: import.js exists on AD2"
} else {
Write-Status "ERROR" "Verification failed: import.js not found on AD2"
exit 1
}
}
# ============================================================================
# Step 5: Quick dry-run test of the sync script
# ============================================================================
Write-Host ""
Write-Status "INFO" "--- Step 5: Running dry-run test of sync script ---"
if ($DryRun) {
Write-Status "INFO" "[DRY RUN] Would run: powershell -ExecutionPolicy Bypass -File $REMOTE_SYNC_PATH -DryRun"
} else {
Write-Status "INFO" "Executing sync script in dry-run mode on AD2..."
$testCmd = "powershell -ExecutionPolicy Bypass -File `"$REMOTE_SYNC_PATH`" -DryRun -Verbose"
$testResult = Invoke-AD2Command $testCmd
$testOutput = $testResult | Out-String
# Check if the script ran without critical errors
if ($LASTEXITCODE -eq 0) {
Write-Status "OK" "Sync script dry-run completed successfully"
if ($testOutput.Trim().Length -gt 0) {
Write-Host ""
Write-Host " --- Dry-run output ---"
foreach ($line in ($testResult | Select-Object -First 20)) {
Write-Host " $line"
}
$totalLines = ($testResult | Measure-Object).Count
if ($totalLines -gt 20) {
Write-Host " ... ($($totalLines - 20) more lines)"
}
Write-Host " --- End dry-run output ---"
}
} else {
Write-Status "WARNING" "Sync script dry-run exited with code $LASTEXITCODE"
Write-Status "INFO" "This may be expected if NAS is unreachable from AD2 during test"
if ($testOutput.Trim().Length -gt 0) {
Write-Host ""
Write-Host " --- Dry-run output ---"
foreach ($line in ($testResult | Select-Object -First 10)) {
Write-Host " $line"
}
Write-Host " --- End dry-run output ---"
}
}
}
# ============================================================================
# Summary
# ============================================================================
Write-Host ""
Write-Host "============================================"
Write-Host " Deployment Summary"
Write-Host "============================================"
Write-Host ""
if ($DryRun) {
Write-Status "INFO" "DRY RUN complete - no changes were made"
} else {
Write-Status "OK" "Sync-FromNAS.ps1 deployed to AD2 (backup: .bak-${DATE_SUFFIX})"
Write-Status "OK" "import.js deployed to AD2 (backup: .bak-${DATE_SUFFIX})"
Write-Status "OK" ".ssh directory and known_hosts verified"
Write-Status "OK" "Dry-run test executed"
Write-Host ""
Write-Status "INFO" "Fixes applied:"
Write-Host " 1. PULL hang fix: find output written to temp file, pulled via SCP"
Write-Host " 2. SCP quoting fix: remote paths quoted to handle special characters"
Write-Host " 3. Rename-OnNAS: files renamed to .synced instead of deleted"
Write-Host " 4. INSERT OR REPLACE: re-tested devices keep latest result"
Write-Host ""
Write-Status "INFO" "Next sync cycle will use the patched scripts automatically"
}
Write-Host ""
exit 0

395
projects/dataforth-dos/sync-fixes/import.js Normal file
View File

@@ -0,0 +1,395 @@
/**
* Data Import Script
* Imports test data from DAT and SHT files into SQLite database
*/
const fs = require('fs');
const path = require('path');
const Database = require('better-sqlite3');
const { parseMultilineFile, extractTestStation } = require('../parsers/multiline');
const { parseCsvFile } = require('../parsers/csvline');
const { parseShtFile } = require('../parsers/shtfile');
// Configuration
const DB_PATH = path.join(__dirname, 'testdata.db');
const SCHEMA_PATH = path.join(__dirname, 'schema.sql');
// Data source paths
const TEST_PATH = 'C:/Shares/test';
const RECOVERY_PATH = 'C:/Shares/Recovery-TEST';
const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS');
// Log types and their parsers
const LOG_TYPES = {
'DSCLOG': { parser: 'multiline', ext: '.DAT' },
'5BLOG': { parser: 'multiline', ext: '.DAT' },
'8BLOG': { parser: 'multiline', ext: '.DAT' },
'PWRLOG': { parser: 'multiline', ext: '.DAT' },
'SCTLOG': { parser: 'multiline', ext: '.DAT' },
'VASLOG': { parser: 'multiline', ext: '.DAT' },
'7BLOG': { parser: 'csvline', ext: '.DAT' }
};
// Initialize database
function initDatabase() {
console.log('Initializing database...');
const db = new Database(DB_PATH);
// Read and execute schema
const schema = fs.readFileSync(SCHEMA_PATH, 'utf8');
db.exec(schema);
console.log('Database initialized.');
return db;
}
// Prepare insert statement
function prepareInsert(db) {
return db.prepare(`
INSERT OR REPLACE INTO test_records
(log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`);
}
// Find all files of a specific type in a directory
function findFiles(dir, pattern, recursive = true) {
const results = [];
try {
if (!fs.existsSync(dir)) return results;
const items = fs.readdirSync(dir, { withFileTypes: true });
for (const item of items) {
const fullPath = path.join(dir, item.name);
if (item.isDirectory() && recursive) {
results.push(...findFiles(fullPath, pattern, recursive));
} else if (item.isFile()) {
if (pattern.test(item.name)) {
results.push(fullPath);
}
}
}
} catch (err) {
// Ignore permission errors
}
return results;
}
// Import records from a file
function importFile(db, insertStmt, filePath, logType, parser) {
let records = [];
const testStation = extractTestStation(filePath);
try {
switch (parser) {
case 'multiline':
records = parseMultilineFile(filePath, logType, testStation);
break;
case 'csvline':
records = parseCsvFile(filePath, testStation);
break;
case 'shtfile':
records = parseShtFile(filePath, testStation);
break;
}
let imported = 0;
for (const record of records) {
try {
const result = insertStmt.run(
record.log_type,
record.model_number,
record.serial_number,
record.test_date,
record.test_station,
record.overall_result,
record.raw_data,
record.source_file
);
if (result.changes > 0) imported++;
} catch (err) {
// Duplicate or constraint error - skip
}
}
return { total: records.length, imported };
} catch (err) {
console.error(`Error importing ${filePath}: ${err.message}`);
return { total: 0, imported: 0 };
}
}
// Import from HISTLOGS (master consolidated logs)
function importHistlogs(db, insertStmt) {
console.log('\n=== Importing from HISTLOGS ===');
let totalImported = 0;
let totalRecords = 0;
for (const [logType, config] of Object.entries(LOG_TYPES)) {
const logDir = path.join(HISTLOGS_PATH, logType);
if (!fs.existsSync(logDir)) {
console.log(` ${logType}: directory not found`);
continue;
}
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
console.log(` ${logType}: found ${files.length} files`);
for (const file of files) {
const { total, imported } = importFile(db, insertStmt, file, logType, config.parser);
totalRecords += total;
totalImported += imported;
}
}
console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`);
return totalImported;
}
// Import from test station logs
function importStationLogs(db, insertStmt, basePath, label) {
console.log(`\n=== Importing from ${label} ===`);
let totalImported = 0;
let totalRecords = 0;
// Find all test station directories (TS-1, TS-27, TS-8L, TS-10R, etc.)
const stationPattern = /^TS-\d+[LR]?$/i;
let stations = [];
try {
const items = fs.readdirSync(basePath, { withFileTypes: true });
stations = items
.filter(i => i.isDirectory() && stationPattern.test(i.name))
.map(i => i.name);
} catch (err) {
console.log(` Error reading ${basePath}: ${err.message}`);
return 0;
}
console.log(` Found stations: ${stations.join(', ')}`);
for (const station of stations) {
const logsDir = path.join(basePath, station, 'LOGS');
if (!fs.existsSync(logsDir)) continue;
for (const [logType, config] of Object.entries(LOG_TYPES)) {
const logDir = path.join(logsDir, logType);
if (!fs.existsSync(logDir)) continue;
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
for (const file of files) {
const { total, imported } = importFile(db, insertStmt, file, logType, config.parser);
totalRecords += total;
totalImported += imported;
}
}
}
// Also import SHT files
const shtFiles = findFiles(basePath, /\.SHT$/i, true);
console.log(` Found ${shtFiles.length} SHT files`);
for (const file of shtFiles) {
const { total, imported } = importFile(db, insertStmt, file, 'SHT', 'shtfile');
totalRecords += total;
totalImported += imported;
}
console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`);
return totalImported;
}
// Import from Recovery-TEST backups (newest first)
function importRecoveryBackups(db, insertStmt) {
console.log('\n=== Importing from Recovery-TEST backups ===');
if (!fs.existsSync(RECOVERY_PATH)) {
console.log(' Recovery-TEST directory not found');
return 0;
}
// Get backup dates, sort newest first
const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true })
.filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name))
.map(i => i.name)
.sort()
.reverse();
console.log(` Found backup dates: ${backups.join(', ')}`);
let totalImported = 0;
for (const backup of backups) {
const backupPath = path.join(RECOVERY_PATH, backup);
const imported = importStationLogs(db, insertStmt, backupPath, `Recovery-TEST/${backup}`);
totalImported += imported;
}
return totalImported;
}
// Main import function
async function runImport() {
console.log('========================================');
console.log('Test Data Import');
console.log('========================================');
console.log(`Database: ${DB_PATH}`);
console.log(`Start time: ${new Date().toISOString()}`);
const db = initDatabase();
const insertStmt = prepareInsert(db);
let grandTotal = 0;
// Use transaction for performance
const importAll = db.transaction(() => {
// 1. Import HISTLOGS first (authoritative)
grandTotal += importHistlogs(db, insertStmt);
// 2. Import Recovery backups (newest first)
grandTotal += importRecoveryBackups(db, insertStmt);
// 3. Import current test folder
grandTotal += importStationLogs(db, insertStmt, TEST_PATH, 'test');
});
importAll();
// Get final stats
const stats = db.prepare('SELECT COUNT(*) as count FROM test_records').get();
console.log('\n========================================');
console.log('Import Complete');
console.log('========================================');
console.log(`Total records in database: ${stats.count}`);
console.log(`End time: ${new Date().toISOString()}`);
db.close();
}
// Import a single file (for incremental imports from sync)
function importSingleFile(filePath) {
console.log(`Importing: ${filePath}`);
const db = new Database(DB_PATH);
const insertStmt = prepareInsert(db);
// Determine log type from path
let logType = null;
let parser = null;
for (const [type, config] of Object.entries(LOG_TYPES)) {
if (filePath.includes(type)) {
logType = type;
parser = config.parser;
break;
}
}
if (!logType) {
// Check for SHT files
if (/\.SHT$/i.test(filePath)) {
logType = 'SHT';
parser = 'shtfile';
} else {
console.log(` Unknown log type for: ${filePath}`);
db.close();
return { total: 0, imported: 0 };
}
}
const result = importFile(db, insertStmt, filePath, logType, parser);
console.log(` Imported ${result.imported} of ${result.total} records`);
db.close();
return result;
}
// Import multiple files (for batch incremental imports)
function importFiles(filePaths) {
console.log(`\n========================================`);
console.log(`Incremental Import: ${filePaths.length} files`);
console.log(`========================================`);
const db = new Database(DB_PATH);
const insertStmt = prepareInsert(db);
let totalImported = 0;
let totalRecords = 0;
const importBatch = db.transaction(() => {
for (const filePath of filePaths) {
// Determine log type from path
let logType = null;
let parser = null;
for (const [type, config] of Object.entries(LOG_TYPES)) {
if (filePath.includes(type)) {
logType = type;
parser = config.parser;
break;
}
}
if (!logType) {
if (/\.SHT$/i.test(filePath)) {
logType = 'SHT';
parser = 'shtfile';
} else {
console.log(` Skipping unknown type: ${filePath}`);
continue;
}
}
const { total, imported } = importFile(db, insertStmt, filePath, logType, parser);
totalRecords += total;
totalImported += imported;
console.log(` ${path.basename(filePath)}: ${imported}/${total} records`);
}
});
importBatch();
console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`);
db.close();
return { total: totalRecords, imported: totalImported };
}
// Run if called directly
if (require.main === module) {
// Check for command line arguments
const args = process.argv.slice(2);
if (args.length > 0 && args[0] === '--file') {
// Import specific file(s)
const files = args.slice(1);
if (files.length === 0) {
console.log('Usage: node import.js --file <file1> [file2] ...');
process.exit(1);
}
importFiles(files);
} else if (args.length > 0 && args[0] === '--help') {
console.log('Usage:');
console.log(' node import.js Full import from all sources');
console.log(' node import.js --file <f> Import specific file(s)');
process.exit(0);
} else {
// Full import
runImport().catch(console.error);
}
}
module.exports = { runImport, importSingleFile, importFiles };