Session log: D2TESTNAS VM build, NAS migration, rsync sync fix

Built Debian 13 VM replacement for aging ReadyNAS, deployed rsync-based
sync script to AD2, transferred data, completed IP cutover to 192.168.0.9.
Includes setup scripts, sync fixes, and comprehensive session logs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-12 18:37:54 -07:00
parent 470638ff86
commit 000ee3da5c
5 changed files with 2259 additions and 0 deletions

View File

@@ -0,0 +1,664 @@
# Sync-FromNAS-rsync.ps1
# Bidirectional sync between AD2 and NAS (D2TESTNAS) using rsync daemon
#
# PULL (NAS -> AD2): Test results (LOGS/*.DAT, Reports/*.TXT) -> Database import
# PUSH (AD2 -> NAS): Software updates (ProdSW/*, TODO.BAT) -> DOS machines
#
# Rsync daemon on NAS: port 873, module "test" maps to /data/test
# Rsync on AD2: cwRsync installed via Chocolatey (rsync.exe in PATH)
#
# Run: powershell -ExecutionPolicy Bypass -File C:\Shares\test\scripts\Sync-FromNAS-rsync.ps1
# Scheduled: Every 15 minutes via Windows Task Scheduler
param(
[switch]$DryRun, # Show what would be done without doing it
[switch]$Verbose # Extra output
)
# ============================================================================
# Configuration
# ============================================================================
$NAS_IP = "192.168.0.9"
$RSYNC_USER = "rsync"
$RSYNC_PASSWORD = "IQ203s32119"
$RSYNC_MODULE = "test"
$RSYNC_BASE = "rsync://${RSYNC_USER}@${NAS_IP}/${RSYNC_MODULE}"
$AD2_TEST_PATH = "C:\Shares\test"
$AD2_CYGDRIVE = "/cygdrive/c/Shares/test"
$LOG_FILE = "C:\Shares\test\scripts\sync-from-nas.log"
$STATUS_FILE = "C:\Shares\test\_SYNC_STATUS.txt"
$LOG_TYPES = @("5BLOG", "7BLOG", "8BLOG", "DSCLOG", "SCTLOG", "VASLOG", "PWRLOG", "HVLOG")
# Database import configuration
$IMPORT_SCRIPT = "C:\Shares\testdatadb\database\import.js"
$NODE_PATH = "node"
# Rsync timeout (seconds) - protects against NAS being unreachable
$RSYNC_TIMEOUT = 30
$RSYNC_CONTIMEOUT = 15
# ============================================================================
# Functions
# ============================================================================
function Write-Log {
param([string]$Message)
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$logLine = "$timestamp : $Message"
# Retry with brief delay if log file is locked by another process
for ($i = 0; $i -lt 3; $i++) {
try {
Add-Content -Path $LOG_FILE -Value $logLine -ErrorAction Stop
break
} catch {
Start-Sleep -Milliseconds 100
}
}
if ($Verbose) { Write-Host $logLine }
}
function Test-RsyncAvailable {
# Check that rsync.exe is available in PATH
$rsyncCmd = Get-Command "rsync" -ErrorAction SilentlyContinue
if (-not $rsyncCmd) {
Write-Log "FATAL: rsync.exe not found in PATH. Install cwRsync via Chocolatey: choco install rsync"
Write-Host "FATAL: rsync.exe not found in PATH. Install cwRsync via Chocolatey: choco install rsync" -ForegroundColor Red
return $false
}
Write-Log "Using rsync: $($rsyncCmd.Source)"
return $true
}
function ConvertTo-CygPath {
# Convert a Windows path like C:\Shares\test\TS-01\LOGS to /cygdrive/c/Shares/test/TS-01/LOGS
param([string]$WindowsPath)
$path = $WindowsPath -replace '\\', '/'
if ($path -match '^([A-Za-z]):(.*)$') {
$drive = $Matches[1].ToLower()
$rest = $Matches[2]
return "/cygdrive/$drive$rest"
}
return $path
}
function Invoke-Rsync {
# Execute rsync with the daemon password set via environment variable.
# Returns a hashtable with ExitCode and Output.
param(
[string[]]$Arguments
)
$env:RSYNC_PASSWORD = $RSYNC_PASSWORD
try {
$output = & rsync @Arguments 2>&1
$exitCode = $LASTEXITCODE
return @{
ExitCode = $exitCode
Output = $output
}
} catch {
return @{
ExitCode = 1
Output = "Exception invoking rsync: $_"
}
} finally {
$env:RSYNC_PASSWORD = $null
}
}
function Get-NASStationFolders {
# List station folders (TS-*) on the NAS using rsync --list-only.
# Returns an array of station folder names (e.g., "TS-01", "TS-02").
$result = Invoke-Rsync -Arguments @(
"--list-only",
"--timeout=$RSYNC_CONTIMEOUT",
"${RSYNC_BASE}/"
)
if ($result.ExitCode -ne 0) {
Write-Log "ERROR: Failed to list NAS root (exit $($result.ExitCode))"
$errText = $result.Output | Out-String
if ($errText.Trim()) { Write-Log " $errText" }
return @()
}
$stations = @()
foreach ($line in $result.Output) {
$lineStr = "$line".Trim()
# rsync --list-only output: "drwxrwxrwx 4,096 2026/03/10 14:30:00 TS-01"
# We want directory entries (starting with 'd') matching TS-*
if ($lineStr -match '^d' -and $lineStr -match '\s(TS-\S+)\s*$') {
$stations += $Matches[1]
}
}
return $stations
}
function Test-8dot3Name {
param([string]$Name)
# Returns $true if filename is 8.3 compatible (no spaces, name<=8, ext<=3)
if ($Name -match '[ ()\[\]{}]') { return $false }
$parts = $Name -split '\.'
if ($parts.Count -gt 2) { return $false }
if ($parts[0].Length -gt 8 -or $parts[0].Length -eq 0) { return $false }
if ($parts.Count -eq 2 -and $parts[1].Length -gt 3) { return $false }
return $true
}
function Test-8dot3Path {
param([string]$RelativePath)
# Check every component of the path is 8.3 compatible
$segments = $RelativePath -replace '\\', '/' -split '/'
foreach ($seg in $segments) {
if ($seg -eq '') { continue }
if (-not (Test-8dot3Name $seg)) { return $false }
}
return $true
}
function Import-ToDatabase {
param([string[]]$FilePaths)
if ($FilePaths.Count -eq 0) { return }
Write-Log "Importing $($FilePaths.Count) file(s) to database..."
# Build argument list
$importArgs = @("$IMPORT_SCRIPT", "--file") + $FilePaths
try {
$output = & $NODE_PATH $importArgs 2>&1
foreach ($line in $output) {
Write-Log " [DB] $line"
}
Write-Log "Database import complete"
} catch {
Write-Log "ERROR: Database import failed: $_"
}
}
# ============================================================================
# Main Script
# ============================================================================
Write-Log "=========================================="
Write-Log "Starting sync (rsync mode)"
if ($DryRun) { Write-Log "DRY RUN - no changes will be made" }
# -- Preflight: verify rsync is installed --
if (-not (Test-RsyncAvailable)) {
exit 2
}
$errorCount = 0
$syncedFiles = 0
$skippedFiles = 0
$syncedDatFiles = @() # Track DAT files for database import
$pushedFiles = 0
# ============================================================================
# PULL: NAS -> AD2 (Test Results)
# ============================================================================
Write-Log "--- NAS to AD2 Sync (Test Results) ---"
# Step 1: Enumerate station folders on NAS
Write-Log "Enumerating station folders on NAS..."
$nasStations = Get-NASStationFolders
if ($nasStations.Count -eq 0) {
Write-Log "WARNING: No station folders found on NAS (NAS may be unreachable)"
} else {
Write-Log "Found $($nasStations.Count) station folder(s): $($nasStations -join ', ')"
}
# Step 2: Pull DAT files per station per log type
foreach ($station in $nasStations) {
# Guard: if a file with this station name exists locally, skip it
$stationPath = "$AD2_TEST_PATH\$station"
if ((Test-Path $stationPath) -and -not (Test-Path $stationPath -PathType Container)) {
Write-Log "WARNING: '$station' exists as a file on AD2, not a directory - skipping (rename or delete the stray file)"
continue
}
foreach ($logType in $LOG_TYPES) {
$nasPath = "${RSYNC_BASE}/${station}/LOGS/${logType}/"
$localDir = "$AD2_TEST_PATH\$station\LOGS\$logType"
$localCygDir = "$(ConvertTo-CygPath $localDir)/"
# Ensure local directory exists (handle stray files blocking directory creation)
if (Test-Path $localDir) {
if (-not (Test-Path $localDir -PathType Container)) {
$strayName = "${localDir}.stray-file"
Write-Log " WARNING: '$station\LOGS\$logType' is a file, not directory - renaming to $(Split-Path $strayName -Leaf)"
Rename-Item -Path $localDir -NewName (Split-Path $strayName -Leaf) -Force
New-Item -ItemType Directory -Path $localDir -Force | Out-Null
}
} else {
New-Item -ItemType Directory -Path $localDir -Force | Out-Null
}
# Build rsync arguments
$rsyncArgs = @(
"--archive",
"--include=*.DAT",
"--exclude=*",
"--remove-source-files",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($DryRun) {
$rsyncArgs += "--dry-run"
}
$rsyncArgs += $nasPath
$rsyncArgs += $localCygDir
if ($Verbose) {
Write-Log " rsync: $station/LOGS/$logType/"
}
$result = Invoke-Rsync -Arguments $rsyncArgs
if ($result.ExitCode -ne 0) {
# Exit code 23 = some files vanished (not fatal for us)
# Exit code 24 = partial transfer due to vanished source files
if ($result.ExitCode -in @(23, 24)) {
Write-Log " WARNING: rsync partial for $station/LOGS/$logType/ (exit $($result.ExitCode))"
} else {
$errText = $result.Output | Out-String
# If the path simply does not exist on NAS, rsync returns error but that is normal
# (not every station has every log type)
if ($errText -match "No such file or directory|does not exist|unknown module") {
if ($Verbose) {
Write-Log " (no $logType folder on NAS for $station)"
}
} else {
Write-Log " ERROR: rsync pull failed for $station/LOGS/$logType/ (exit $($result.ExitCode)): $errText"
$errorCount++
}
}
continue
}
# Parse itemized output to count transferred files
# Lines starting with ">f" indicate a file was received
foreach ($line in $result.Output) {
$lineStr = "$line".Trim()
if ($lineStr -match '^>f.*\s(\S+\.DAT)$') {
$fileName = $Matches[1]
$localFile = Join-Path $localDir $fileName
Write-Log " Pulled: $station/LOGS/$logType/$fileName"
$syncedDatFiles += $localFile
$syncedFiles++
}
}
}
# Pull TXT reports for this station
$nasReportsPath = "${RSYNC_BASE}/${station}/Reports/"
$localReportsDir = "$AD2_TEST_PATH\$station\Reports"
$localReportsCygDir = "$(ConvertTo-CygPath $localReportsDir)/"
# Ensure local directory exists
if (-not (Test-Path $localReportsDir)) {
New-Item -ItemType Directory -Path $localReportsDir -Force | Out-Null
}
$rsyncArgs = @(
"--archive",
"--include=*.TXT",
"--exclude=*",
"--remove-source-files",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($DryRun) {
$rsyncArgs += "--dry-run"
}
$rsyncArgs += $nasReportsPath
$rsyncArgs += $localReportsCygDir
$result = Invoke-Rsync -Arguments $rsyncArgs
if ($result.ExitCode -ne 0) {
if ($result.ExitCode -in @(23, 24)) {
Write-Log " WARNING: rsync partial for $station/Reports/ (exit $($result.ExitCode))"
} else {
$errText = $result.Output | Out-String
if ($errText -match "No such file or directory|does not exist|unknown module") {
if ($Verbose) {
Write-Log " (no Reports folder on NAS for $station)"
}
} else {
Write-Log " ERROR: rsync pull failed for $station/Reports/ (exit $($result.ExitCode)): $errText"
$errorCount++
}
}
} else {
foreach ($line in $result.Output) {
$lineStr = "$line".Trim()
if ($lineStr -match '^>f.*\s(\S+\.TXT)$') {
$fileName = $Matches[1]
Write-Log " Pulled report: $station/Reports/$fileName"
$syncedFiles++
}
}
}
}
Write-Log "NAS to AD2 sync: $syncedFiles file(s) pulled"
# ============================================================================
# Import synced DAT files to database
# ============================================================================
if (-not $DryRun -and $syncedDatFiles.Count -gt 0) {
Import-ToDatabase -FilePaths $syncedDatFiles
}
# ============================================================================
# PUSH: AD2 -> NAS (Software Updates for DOS Machines)
# ============================================================================
Write-Log "--- AD2 to NAS Sync (Software Updates) ---"
# --- Helper: Push a local directory to a NAS path using rsync --update ---
# Only pushes files with 8.3-compatible paths.
# For directories, we use a filter approach: rsync the whole tree but pre-filter
# to 8.3-only names via a temp filter file or by iterating. Since rsync daemon
# does not need SSH, we can push entire directories at once for efficiency.
# However, to enforce 8.3 filtering, we use --files-from with a generated list.
function Push-DirectoryToNAS {
param(
[string]$LocalDir, # Windows path to local directory
[string]$NASPath, # rsync daemon path (e.g., rsync://rsync@.../test/COMMON/ProdSW/)
[switch]$Recurse,
[switch]$UpdateOnly # Use --update (skip files that are newer on receiver)
)
if (-not (Test-Path $LocalDir)) {
Write-Log " Path not found: $LocalDir"
return 0
}
$pushed = 0
$getChildArgs = @{
Path = $LocalDir
File = $true
ErrorAction = "SilentlyContinue"
}
if ($Recurse) { $getChildArgs["Recurse"] = $true }
$files = Get-ChildItem @getChildArgs
if (-not $files -or $files.Count -eq 0) {
if ($Verbose) { Write-Log " No files in $LocalDir" }
return 0
}
# Build a list of 8.3-compatible relative paths
$validFiles = @()
foreach ($file in $files) {
$relativePath = $file.FullName.Substring($LocalDir.Length + 1).Replace('\', '/')
if (-not (Test-8dot3Path $relativePath)) {
Write-Log " Skipping (non-8.3): $relativePath"
$script:skippedFiles++
continue
}
$validFiles += $relativePath
}
if ($validFiles.Count -eq 0) {
return 0
}
# Write valid file list to a temp file for --files-from
$tempFileList = "$env:TEMP\rsync-push-list-$(Get-Date -Format 'yyyyMMddHHmmss')-$([System.IO.Path]::GetRandomFileName()).txt"
$validFiles | Set-Content -Path $tempFileList -Encoding ASCII
$localCygDir = "$(ConvertTo-CygPath $LocalDir)/"
$tempCygPath = ConvertTo-CygPath $tempFileList
$rsyncArgs = @(
"--archive",
"--files-from=$tempCygPath",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($UpdateOnly) {
$rsyncArgs += "--update"
}
if ($DryRun) {
$rsyncArgs += "--dry-run"
}
$rsyncArgs += $localCygDir
$rsyncArgs += $NASPath
$result = Invoke-Rsync -Arguments $rsyncArgs
# Clean up temp file
Remove-Item $tempFileList -ErrorAction SilentlyContinue
if ($result.ExitCode -ne 0 -and $result.ExitCode -notin @(23, 24)) {
$errText = $result.Output | Out-String
Write-Log " ERROR: rsync push failed (exit $($result.ExitCode)): $errText"
$script:errorCount++
return 0
}
# Count transferred files from itemized output
foreach ($line in $result.Output) {
$lineStr = "$line".Trim()
# Lines starting with ">f" or "<f" indicate file transfer
if ($lineStr -match '^[><]f') {
$pushed++
}
}
# In dry-run mode, count valid files as "would push"
if ($DryRun -and $pushed -eq 0) {
# itemize-changes with dry-run still shows >f lines, so pushed should be accurate
# But if nothing was shown (all up to date), that is fine
}
return $pushed
}
# -- COMMON/ProdSW --
# AD2 uses both _COMMON and COMMON; NAS uses COMMON
$commonSources = @(
"$AD2_TEST_PATH\_COMMON\ProdSW",
"$AD2_TEST_PATH\COMMON\ProdSW"
)
foreach ($commonDir in $commonSources) {
if (Test-Path $commonDir) {
Write-Log "Syncing COMMON ProdSW from: $commonDir"
$count = Push-DirectoryToNAS -LocalDir $commonDir -NASPath "${RSYNC_BASE}/COMMON/ProdSW/" -UpdateOnly
$pushedFiles += $count
Write-Log " Pushed $count file(s) from COMMON/ProdSW"
}
}
# -- Ate/ProdSW --
Write-Log "Syncing Ate/ProdSW data folders..."
$ateProdSwPath = "$AD2_TEST_PATH\Ate\ProdSW"
if (Test-Path $ateProdSwPath) {
$count = Push-DirectoryToNAS -LocalDir $ateProdSwPath -NASPath "${RSYNC_BASE}/Ate/ProdSW/" -Recurse -UpdateOnly
$pushedFiles += $count
Write-Log " Pushed $count file(s) from Ate/ProdSW"
} else {
Write-Log " Ate/ProdSW not found: $ateProdSwPath"
}
# -- UPDATE.BAT --
Write-Log "Syncing UPDATE.BAT..."
$updateBatLocal = "$AD2_TEST_PATH\UPDATE.BAT"
if (Test-Path $updateBatLocal) {
$localCyg = ConvertTo-CygPath $updateBatLocal
$rsyncArgs = @(
"--archive",
"--update",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($DryRun) { $rsyncArgs += "--dry-run" }
$rsyncArgs += $localCyg
$rsyncArgs += "${RSYNC_BASE}/UPDATE.BAT"
$result = Invoke-Rsync -Arguments $rsyncArgs
if ($result.ExitCode -ne 0) {
$errText = $result.Output | Out-String
Write-Log " ERROR: Failed to push UPDATE.BAT (exit $($result.ExitCode)): $errText"
$errorCount++
} else {
foreach ($line in $result.Output) {
if ("$line".Trim() -match '^[><]f') {
Write-Log " Pushed: UPDATE.BAT"
$pushedFiles++
}
}
}
} else {
Write-Log " WARNING: UPDATE.BAT not found at $updateBatLocal"
}
# -- DEPLOY.BAT --
Write-Log "Syncing DEPLOY.BAT..."
$deployBatLocal = "$AD2_TEST_PATH\DEPLOY.BAT"
if (Test-Path $deployBatLocal) {
$localCyg = ConvertTo-CygPath $deployBatLocal
$rsyncArgs = @(
"--archive",
"--update",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($DryRun) { $rsyncArgs += "--dry-run" }
$rsyncArgs += $localCyg
$rsyncArgs += "${RSYNC_BASE}/DEPLOY.BAT"
$result = Invoke-Rsync -Arguments $rsyncArgs
if ($result.ExitCode -ne 0) {
$errText = $result.Output | Out-String
Write-Log " ERROR: Failed to push DEPLOY.BAT (exit $($result.ExitCode)): $errText"
$errorCount++
} else {
foreach ($line in $result.Output) {
if ("$line".Trim() -match '^[><]f') {
Write-Log " Pushed: DEPLOY.BAT"
$pushedFiles++
}
}
}
} else {
Write-Log " WARNING: DEPLOY.BAT not found at $deployBatLocal"
}
# -- Per-station ProdSW and TODO.BAT --
Write-Log "Syncing station-specific ProdSW folders..."
$stationFolders = Get-ChildItem -Path $AD2_TEST_PATH -Directory -Filter "TS-*" -ErrorAction SilentlyContinue
foreach ($station in $stationFolders) {
# Skip station folders with non-8.3 names
if (-not (Test-8dot3Name $station.Name)) {
Write-Log " Skipping station (non-8.3 name): $($station.Name)"
continue
}
$prodSwPath = Join-Path $station.FullName "ProdSW"
if (Test-Path $prodSwPath) {
$count = Push-DirectoryToNAS -LocalDir $prodSwPath -NASPath "${RSYNC_BASE}/$($station.Name)/ProdSW/" -Recurse -UpdateOnly
if ($count -gt 0) {
Write-Log " Pushed $count file(s) for $($station.Name)/ProdSW"
}
$pushedFiles += $count
}
# TODO.BAT - one-shot: push then delete from AD2
$todoBatPath = Join-Path $station.FullName "TODO.BAT"
if (Test-Path $todoBatPath) {
Write-Log "Found TODO.BAT for $($station.Name)"
$localCyg = ConvertTo-CygPath $todoBatPath
$rsyncArgs = @(
"--archive",
"--timeout=$RSYNC_TIMEOUT",
"--contimeout=$RSYNC_CONTIMEOUT",
"--itemize-changes"
)
if ($DryRun) { $rsyncArgs += "--dry-run" }
$rsyncArgs += $localCyg
$rsyncArgs += "${RSYNC_BASE}/$($station.Name)/TODO.BAT"
$result = Invoke-Rsync -Arguments $rsyncArgs
if ($result.ExitCode -ne 0) {
$errText = $result.Output | Out-String
Write-Log " ERROR: Failed to push TODO.BAT for $($station.Name) (exit $($result.ExitCode)): $errText"
$errorCount++
} else {
Write-Log " Pushed TODO.BAT to NAS for $($station.Name)"
$pushedFiles++
if (-not $DryRun) {
# Remove from AD2 after successful push (one-shot mechanism)
Remove-Item -Path $todoBatPath -Force
Write-Log " Removed TODO.BAT from AD2 (pushed to NAS)"
} else {
Write-Log " [DRY RUN] Would remove TODO.BAT from AD2 after push"
}
}
}
}
Write-Log "AD2 to NAS sync: $pushedFiles file(s) pushed"
# ============================================================================
# Update Status File
# ============================================================================
$status = if ($errorCount -eq 0) { "OK" } else { "ERRORS" }
$statusContent = @"
AD2 <-> NAS Bidirectional Sync Status (rsync)
===============================================
Timestamp: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss")
Status: $status
PULL (NAS -> AD2 - Test Results):
Files Pulled: $syncedFiles
Files Skipped: $skippedFiles
DAT Files Imported to DB: $($syncedDatFiles.Count)
PUSH (AD2 -> NAS - Software Updates):
Files Pushed: $pushedFiles
Errors: $errorCount
"@
Set-Content -Path $STATUS_FILE -Value $statusContent
Write-Log "=========================================="
Write-Log "Sync complete: PULL=$syncedFiles, PUSH=$pushedFiles, Errors=$errorCount"
Write-Log "=========================================="
# Exit with error code if there were failures
if ($errorCount -gt 0) {
exit 1
} else {
exit 0
}