diff --git a/projects/dataforth-dos/d2testnas-vm/README.md b/projects/dataforth-dos/d2testnas-vm/README.md new file mode 100644 index 0000000..7e1a689 --- /dev/null +++ b/projects/dataforth-dos/d2testnas-vm/README.md @@ -0,0 +1,369 @@ +# D2TESTNAS VM Replacement + +Replacement for Netgear ReadyNAS RN10400 (D2TESTNAS) used in Dataforth DOS 6.22 +test infrastructure. The new system is a Debian 13 (Trixie) VM running on Hyper-V +with BTRFS for snapshots, Samba with SMB1 for DOS compatibility, and rsync daemon +for AD2 bidirectional sync. + +--- + +## 1. Hyper-V VM Creation + +Run these PowerShell commands on the Hyper-V host as Administrator. + +```powershell +# --- Configuration --- +$VMName = "D2TESTNAS" +$VMPath = "D:\Hyper-V\VMs" +$VHDPath = "D:\Hyper-V\VMs\D2TESTNAS\Virtual Hard Disks" +$ISOPath = "D:\ISOs\debian-13-netinst-amd64.iso" # Download from https://www.debian.org/devel/debian-installer/ +$SwitchName = "Dataforth-Bridge" # Your existing vSwitch for 192.168.0.0/24 + +# --- Create VM --- +New-VM -Name $VMName ` + -Path $VMPath ` + -MemoryStartupBytes 2GB ` + -Generation 2 ` + -SwitchName $SwitchName + +# --- Configure VM --- +Set-VM -Name $VMName ` + -ProcessorCount 2 ` + -DynamicMemory ` + -MemoryMinimumBytes 1GB ` + -MemoryMaximumBytes 4GB ` + -AutomaticStartAction Start ` + -AutomaticStartDelay 30 ` + -AutomaticStopAction ShutDown + +# --- Create OS disk (40 GB) --- +New-VHD -Path "$VHDPath\os.vhdx" -SizeBytes 40GB -Dynamic +Add-VMHardDiskDrive -VMName $VMName -Path "$VHDPath\os.vhdx" + +# --- Create DATA disk (200 GB, or match current NAS capacity) --- +# This disk will be formatted as BTRFS and mounted at /data +New-VHD -Path "$VHDPath\data.vhdx" -SizeBytes 200GB -Dynamic +Add-VMHardDiskDrive -VMName $VMName -Path "$VHDPath\data.vhdx" + +# --- Attach Debian ISO --- +Add-VMDvdDrive -VMName $VMName -Path $ISOPath + +# --- Set boot order: DVD first (for install), then disk --- +$dvd = Get-VMDvdDrive -VMName $VMName +$disk = Get-VMHardDiskDrive -VMName $VMName | Where-Object { $_.Path -like "*os.vhdx" } +Set-VMFirmware -VMName $VMName -BootOrder $dvd, $disk + +# --- Disable Secure Boot (Debian needs "Microsoft UEFI Certificate Authority") --- +Set-VMFirmware -VMName $VMName -EnableSecureBoot Off + +# --- Start VM --- +Start-VM -Name $VMName +vmconnect localhost $VMName +``` + +Adjust `$SwitchName` to match whatever virtual switch bridges to the 192.168.0.0/24 +Dataforth network. If you do not have one, create it: + +```powershell +# Create external vSwitch bridged to the physical NIC on the Dataforth network +New-VMSwitch -Name "Dataforth-Bridge" -NetAdapterName "Ethernet 2" -AllowManagementOS $true +``` + +--- + +## 2. Debian Installation Notes + +During the Debian 13 (Trixie) netinst installation: + +1. **Language/Region:** English, United States, UTF-8 +2. **Hostname:** `D2TESTNAS` +3. **Domain:** leave blank +4. **Root password:** `Paper123!@#-nas` +5. **User account:** Skip creating a normal user (or create one for admin use) +6. **Partitioning - IMPORTANT:** + - Use "Manual" partitioning + - **Disk 1 (40 GB, /dev/sda):** OS disk + - 512 MB EFI System Partition (ESP) + - 1 GB /boot (ext4) + - Remainder: / (ext4) + - (No swap partition -- Hyper-V dynamic memory handles this; or add 2 GB swap) + - **Disk 2 (200 GB, /dev/sdb):** Data disk + - Use entire disk as a single partition + - **Format as BTRFS** + - Mount point: **/data** +7. **Software selection:** + - Deselect "Debian desktop environment" and all desktop options + - Select "SSH server" + - Select "standard system utilities" + - Do NOT select any web server or print server +8. **GRUB:** Install to /dev/sda + +After reboot, verify you can SSH in, then proceed to post-install. + +--- + +## 3. Post-Install Setup + +SSH into the new VM (use the DHCP address shown at the console): + +```bash +ssh root@ +``` + +Transfer and run the setup script: + +```bash +# From your workstation (PowerShell/bash): +scp setup-d2testnas.sh root@:/root/ + +# On the VM: +chmod +x /root/setup-d2testnas.sh +/root/setup-d2testnas.sh +``` + +The script will: +- Install samba, rsync, btrfs-progs, and supporting packages +- Set hostname to D2TESTNAS +- Create BTRFS subvolumes (/data/test, /data/datasheets) +- Write /etc/samba/smb.conf with SMB1 (CORE protocol) support +- Create Samba users ts-1 through ts-50 (null passwords) and engineer +- Write /etc/rsyncd.conf and /etc/rsyncd.secrets +- Install BTRFS snapshot cron jobs +- Configure SSH for root login with password +- Enable and start all services +- Run verification checks +- Display cutover instructions + +--- + +## 4. Testing Before Cutover + +While the VM is still on a DHCP address (not 192.168.0.9), verify all services +work. Use the DHCP IP in place of 192.168.0.9 for these tests. + +### Test SMB from Windows + +```cmd +net use Z: \\\test +dir Z:\ +net use Z: /delete +``` + +### Test rsync from AD2 + +```powershell +$env:RSYNC_PASSWORD = "IQ203s32119" +rsync --list-only rsync://rsync@/test/ +``` + +### Test SSH + +```bash +ssh root@ +# Password: Paper123!@#-nas +``` + +### Test BTRFS Snapshots + +```bash +# On the VM: +btrfs-snapshot.sh create hourly +btrfs-snapshot.sh list +ls /data/.snapshots/ +``` + +### Test from DOS Machine (optional, requires temporary IP or hosts hack) + +If you can temporarily set a DOS machine to use the DHCP IP, test the T: drive +mapping. Otherwise, wait for cutover. + +--- + +## 5. Data Migration + +Before cutover, copy all data from the old NAS to the new VM: + +```bash +# On the new VM, pull everything from the old NAS: +RSYNC_PASSWORD=IQ203s32119 rsync -avz --progress \ + rsync://rsync@192.168.0.9/test/ /data/test/ +``` + +This may take a while depending on data volume. Run it multiple times -- rsync +is incremental and will only transfer changes on subsequent runs. + +For the datasheets share, copy via SMB or SCP from the old NAS: + +```bash +# If rsync module exists for datasheets: +RSYNC_PASSWORD=IQ203s32119 rsync -avz rsync://rsync@192.168.0.9/datasheets/ /data/datasheets/ + +# Otherwise, mount the old NAS share temporarily: +apt-get install -y cifs-utils +mkdir -p /mnt/old-nas +mount -t cifs //192.168.0.9/datasheets /mnt/old-nas -o guest,vers=1.0 +rsync -avz /mnt/old-nas/ /data/datasheets/ +umount /mnt/old-nas +``` + +--- + +## 6. Cutover Checklist + +Perform these steps during a maintenance window when no DOS machines are running +tests. + +### Pre-Cutover + +- [ ] All data migrated from old NAS (run rsync one final time) +- [ ] All services verified on new VM (SMB, rsync, SSH) +- [ ] BTRFS snapshots working (run `btrfs-snapshot.sh list`) +- [ ] Notify engineers: maintenance window, expect brief T: drive outage + +### Cutover Steps + +1. **Stop the AD2 sync script** (disable scheduled task on AD2 temporarily) + +2. **Final data sync** from old NAS to new VM: + ```bash + RSYNC_PASSWORD=IQ203s32119 rsync -avz rsync://rsync@192.168.0.9/test/ /data/test/ + ``` + +3. **Power off the old ReadyNAS** (192.168.0.9) + +4. **Assign static IP to new VM:** + ```bash + # On the new VM: + # Edit /etc/network/interfaces to use static config: + cat > /etc/network/interfaces << 'EOF' + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet static + address 192.168.0.9 + netmask 255.255.255.0 + gateway 192.168.0.254 + dns-nameservers 192.168.0.27 192.168.0.6 192.168.1.254 + EOF + + # Replace "eth0" with actual interface name shown by: ip link show + systemctl restart networking + ``` + +5. **Verify IP assignment:** + ```bash + ip addr show + ping -c 3 192.168.0.254 + ``` + +6. **Re-enable AD2 sync script** (re-enable scheduled task) + +7. **Test from AD2:** + ```powershell + $env:RSYNC_PASSWORD = "IQ203s32119" + rsync --list-only rsync://rsync@192.168.0.9/test/ + ``` + +8. **Test from Windows:** + ```cmd + net use T: \\D2TESTNAS\test + dir T:\ + ``` + +9. **Test from DOS machine:** Boot one test station and verify T: drive maps + and NWTOC.BAT runs successfully. + +10. **Create baseline snapshot:** + ```bash + btrfs-snapshot.sh create daily + ``` + +### Post-Cutover + +- [ ] Monitor /var/log/samba/ for connection issues +- [ ] Monitor /var/log/rsyncd.log for sync activity +- [ ] Verify AD2 sync runs successfully (check sync status file) +- [ ] Verify BTRFS snapshot cron is creating snapshots (check after 1 hour) +- [ ] Update credentials.md with any changes to the NAS entry +- [ ] Keep old ReadyNAS powered off but available for 2 weeks as fallback + +### Rollback Plan + +If critical issues arise during cutover: + +1. Power off the new VM +2. Power on the old ReadyNAS +3. Wait 2 minutes for ReadyNAS to boot and claim 192.168.0.9 +4. Verify DOS machines can map T: drive +5. Re-enable AD2 sync task + +--- + +## 7. Maintenance Reference + +### Service Management + +```bash +systemctl status smbd nmbd rsync ssh +systemctl restart smbd # Restart Samba file server +systemctl restart nmbd # Restart NetBIOS name service +systemctl restart rsync # Restart rsync daemon +``` + +### Snapshot Management + +```bash +btrfs-snapshot.sh list # Show all snapshots +btrfs-snapshot.sh create hourly # Manual hourly snapshot +btrfs-snapshot.sh create daily # Manual daily snapshot +btrfs-snapshot.sh prune # Clean up old snapshots per retention policy +``` + +Snapshots are browsable at `/data/.snapshots/` and via the `\\D2TESTNAS\snapshots` +SMB share (read-only). To restore a file from a snapshot: + +```bash +# Find the file in a snapshot: +ls /data/.snapshots/daily_2026-03-12_00-01-00/TS-01/LOGS/ + +# Copy it back: +cp /data/.snapshots/daily_2026-03-12_00-01-00/TS-01/LOGS/5BLOG/DATA.DAT /data/test/TS-01/LOGS/5BLOG/ +``` + +### Log Files + +| Log | Purpose | +|-----|---------| +| /var/log/samba/log.* | Samba per-client logs | +| /var/log/rsyncd.log | rsync daemon transfers | +| /var/log/btrfs-snapshots.log | Snapshot create/prune activity | +| /var/log/auth.log | SSH login attempts | + +### Samba User Management + +```bash +# Add a new station user with null password: +useradd --system --no-create-home --shell /usr/sbin/nologin ts-51 +smbpasswd -a -n ts-51 +smbpasswd -e ts-51 + +# Change engineer password: +smbpasswd engineer +``` + +--- + +## 8. Architecture Comparison + +| Feature | Old (ReadyNAS RN10400) | New (Debian 13 VM) | +|---------|----------------------|-------------------| +| OS | Netgear ReadyNAS Linux | Debian 13 (Trixie) | +| Filesystem | BTRFS | BTRFS | +| SMB | SMB1 (CORE) | SMB1 (CORE) via Samba | +| rsync | rsync daemon, port 873 | rsync daemon, port 873 | +| Snapshots | 80+ BTRFS (not browsable) | Automated, browsable via SMB | +| NetBIOS/WINS | nmbd | nmbd | +| Management | Web UI (limited) | SSH + CLI (full control) | +| Backup | BTRFS snapshots only | BTRFS snapshots + Hyper-V checkpoints | +| Hardware | Physical appliance | Virtual machine (portable, resizable) | diff --git a/projects/dataforth-dos/d2testnas-vm/setup-d2testnas.sh b/projects/dataforth-dos/d2testnas-vm/setup-d2testnas.sh new file mode 100644 index 0000000..f2b3aeb --- /dev/null +++ b/projects/dataforth-dos/d2testnas-vm/setup-d2testnas.sh @@ -0,0 +1,922 @@ +#!/bin/bash +# ============================================================================= +# setup-d2testnas.sh +# Post-install setup script for Debian 13 (Trixie) VM replacing D2TESTNAS +# +# Purpose: Configure a Debian VM as an SMB1-capable NAS for Dataforth DOS 6.22 +# test infrastructure. Replaces Netgear ReadyNAS RN10400 appliance. +# +# Requirements: +# - Fresh Debian 13 (Trixie) minimal install +# - BTRFS partition mounted at /data (or available block device) +# - Root access +# - Network connectivity for package installation +# +# Usage: +# chmod +x setup-d2testnas.sh +# sudo ./setup-d2testnas.sh +# +# Author: Infrastructure automation for Dataforth DOS project +# Date: 2026-03-12 +# ============================================================================= + +set -euo pipefail + +# ============================================================================= +# Configuration - Edit these values as needed +# ============================================================================= + +# Network - The VM will initially use DHCP. Change STATIC_IP to the final +# address (192.168.0.9) only during cutover when the old NAS is powered off. +STATIC_IP="192.168.0.9" +NETMASK="255.255.255.0" +GATEWAY="192.168.0.254" +DNS_SERVERS="192.168.0.27 192.168.0.6 192.168.1.254" +HOSTNAME="D2TESTNAS" +WORKGROUP="INTRANET" + +# Samba +SMB_SHARE_PATH="/data/test" +SMB_DATASHEETS_PATH="/data/datasheets" +SMB_ENGINEER_USER="engineer" +SMB_ENGINEER_PASS="Engineer1!" + +# rsync daemon +RSYNC_MODULE="test" +RSYNC_PATH="/data/test" +RSYNC_USER="rsync" +RSYNC_PASSWORD="IQ203s32119" + +# BTRFS snapshot retention +SNAP_HOURLY_RETAIN=48 +SNAP_DAILY_RETAIN=30 +SNAP_WEEKLY_RETAIN=12 + +# SSH root password (set during Debian install, but ensure it is correct) +ROOT_PASSWORD="Paper123!@#-nas" + +# ============================================================================= +# Preflight Checks +# ============================================================================= + +echo "============================================================" +echo " D2TESTNAS Setup Script - Debian 13 (Trixie)" +echo " Replacing Netgear ReadyNAS for DOS 6.22 infrastructure" +echo "============================================================" +echo "" + +if [[ $EUID -ne 0 ]]; then + echo "[ERROR] This script must be run as root." + exit 1 +fi + +# Check Debian version +if [[ -f /etc/os-release ]]; then + . /etc/os-release + echo "[INFO] Detected OS: ${PRETTY_NAME:-unknown}" +else + echo "[WARNING] Cannot determine OS version. Proceeding anyway." +fi + +echo "" +echo "[INFO] This script will:" +echo " 1. Install required packages (samba, rsync, btrfs-progs, etc.)" +echo " 2. Set hostname to ${HOSTNAME}" +echo " 3. Create data directories on /data (BTRFS)" +echo " 4. Configure Samba with SMB1 support for DOS machines" +echo " 5. Configure rsync daemon on port 873" +echo " 6. Set up BTRFS automated snapshots" +echo " 7. Configure and enable all services" +echo "" +echo "Press Ctrl+C within 5 seconds to abort..." +sleep 5 +echo "" + +# ============================================================================= +# Step 1: Package Installation +# ============================================================================= + +echo "[INFO] Step 1: Installing required packages..." + +apt-get update -qq + +# Core packages: +# samba - SMB file server (includes nmbd for NetBIOS) +# rsync - rsync daemon +# btrfs-progs - BTRFS filesystem utilities and snapshot management +# openssh-server - SSH access +# cron - scheduled tasks for snapshots +# net-tools - ifconfig, netstat (useful for debugging) +# dnsutils - nslookup, dig (debugging) +DEBIAN_FRONTEND=noninteractive apt-get install -y \ + samba \ + samba-common \ + rsync \ + btrfs-progs \ + openssh-server \ + cron \ + net-tools \ + dnsutils + +echo "[OK] Packages installed successfully." +echo "" + +# ============================================================================= +# Step 2: Set Hostname +# ============================================================================= + +echo "[INFO] Step 2: Setting hostname to ${HOSTNAME}..." + +hostnamectl set-hostname "${HOSTNAME}" + +# Update /etc/hosts so hostname resolves locally +if ! grep -q "${HOSTNAME}" /etc/hosts; then + sed -i "s/^127\.0\.1\.1.*/127.0.1.1\t${HOSTNAME}/" /etc/hosts + # If no 127.0.1.1 line existed, add one + if ! grep -q "127.0.1.1" /etc/hosts; then + echo "127.0.1.1 ${HOSTNAME}" >> /etc/hosts + fi +fi + +echo "[OK] Hostname set to ${HOSTNAME}." +echo "" + +# ============================================================================= +# Step 3: Prepare BTRFS Data Directories +# ============================================================================= + +echo "[INFO] Step 3: Preparing BTRFS data directories..." + +# Check if /data is a BTRFS mount +if mountpoint -q /data 2>/dev/null; then + FS_TYPE=$(df -T /data | tail -1 | awk '{print $2}') + if [[ "${FS_TYPE}" == "btrfs" ]]; then + echo "[OK] /data is mounted as BTRFS." + else + echo "[WARNING] /data is mounted but filesystem is '${FS_TYPE}', not BTRFS." + echo " Snapshots will NOT work. Consider reformatting with BTRFS." + fi +else + echo "[WARNING] /data is not currently mounted." + echo " Checking for available block devices..." + lsblk -f + echo "" + echo " You must mount a BTRFS partition at /data before this script" + echo " can configure snapshots. The script will create directories" + echo " but snapshot functionality requires BTRFS." + echo "" + echo " To create a BTRFS filesystem on /dev/sdX:" + echo " mkfs.btrfs -L d2testnas-data /dev/sdX" + echo " echo '/dev/sdX /data btrfs defaults,noatime,compress=zstd 0 2' >> /etc/fstab" + echo " mkdir -p /data && mount /data" + echo "" + mkdir -p /data +fi + +# Create BTRFS subvolumes if on BTRFS, otherwise plain directories +if btrfs subvolume show /data >/dev/null 2>&1 || btrfs filesystem show /data >/dev/null 2>&1; then + BTRFS_AVAILABLE=true + echo "[INFO] BTRFS detected. Creating subvolumes..." + + # Create subvolumes for data areas (enables independent snapshots) + if ! btrfs subvolume show /data/test >/dev/null 2>&1; then + btrfs subvolume create /data/test + echo "[OK] Created BTRFS subvolume: /data/test" + else + echo "[INFO] Subvolume /data/test already exists." + fi + + if ! btrfs subvolume show /data/datasheets >/dev/null 2>&1; then + btrfs subvolume create /data/datasheets + echo "[OK] Created BTRFS subvolume: /data/datasheets" + else + echo "[INFO] Subvolume /data/datasheets already exists." + fi + + # Create snapshot directory + mkdir -p /data/.snapshots + echo "[OK] Snapshot directory created at /data/.snapshots" +else + BTRFS_AVAILABLE=false + echo "[WARNING] BTRFS not available on /data. Creating plain directories." + echo " Snapshot functionality will be disabled." + mkdir -p /data/test + mkdir -p /data/datasheets +fi + +# Set permissions - wide open for guest access from DOS machines +chmod 777 /data/test +chmod 777 /data/datasheets + +echo "[OK] Data directories ready." +echo "" + +# ============================================================================= +# Step 4: Configure Samba (SMB1 for DOS 6.22) +# ============================================================================= + +echo "[INFO] Step 4: Configuring Samba with SMB1 support..." + +# Back up original config +if [[ -f /etc/samba/smb.conf ]]; then + cp /etc/samba/smb.conf /etc/samba/smb.conf.orig +fi + +cat > /etc/samba/smb.conf << 'SMBCONF' +# ============================================================================= +# Samba Configuration for D2TESTNAS +# Replacing Netgear ReadyNAS for Dataforth DOS 6.22 test infrastructure +# +# CRITICAL: DOS 6.22 machines require SMB1 (CORE/COREPLUS/LANMAN1/NT1) +# Modern SMB2/SMB3 is NOT compatible with MS-DOS networking stack. +# +# Shares: +# \\D2TESTNAS\test -> /data/test (T: drive on DOS machines) +# \\D2TESTNAS\datasheets -> /data/datasheets (X: drive on DOS machines) +# \\D2TESTNAS\snapshots -> /data/.snapshots (read-only, browse snapshots) +# ============================================================================= + +[global] + # --- Identity --- + workgroup = INTRANET + netbios name = D2TESTNAS + server string = D2TESTNAS File Server (DOS Infrastructure) + + # --- Protocol: SMB1 Required for DOS 6.22 --- + # CORE is the oldest SMB dialect. DOS MS Client 3.0 uses CORE/COREPLUS. + # NT1 = SMB1 final revision. We allow the full SMB1 range. + server min protocol = CORE + server max protocol = NT1 + client min protocol = CORE + client max protocol = NT1 + + # --- Authentication --- + # DOS machines authenticate with null passwords or guest. + # Security mode must allow this. + security = user + map to guest = Bad User + guest account = nobody + null passwords = yes + + # --- NetBIOS / WINS --- + # DOS machines rely on NetBIOS name resolution (no DNS). + # This server acts as a WINS server for the 192.168.0.0/24 network. + wins support = yes + name resolve order = wins lmhosts host bcast + dns proxy = no + local master = yes + preferred master = yes + os level = 65 + + # --- File Handling for DOS Compatibility --- + # DOS filenames: 8.3 format, case insensitive, no special chars + mangled names = no + case sensitive = no + default case = upper + preserve case = no + short preserve case = no + dos charset = CP437 + unix charset = UTF-8 + + # --- Logging --- + log file = /var/log/samba/log.%m + max log size = 1000 + log level = 1 + + # --- Performance --- + socket options = TCP_NODELAY IPTOS_LOWDELAY + read raw = yes + write raw = yes + + # --- Disable features DOS cannot use --- + unix extensions = no + wide links = yes + follow symlinks = yes + + # --- Disable printer sharing --- + load printers = no + printing = bsd + printcap name = /dev/null + disable spoolss = yes + +# ============================================================================= +# Share: test (T: drive for DOS machines) +# ============================================================================= +[test] + comment = Dataforth Test Data + path = /data/test + browseable = yes + writable = yes + guest ok = yes + guest only = yes + create mask = 0666 + directory mask = 0777 + force create mode = 0666 + force directory mode = 0777 + + # DOS compatibility - no oplocks (can cause issues with DOS clients) + oplocks = no + level2 oplocks = no + strict locking = yes + +# ============================================================================= +# Share: datasheets (X: drive for DOS machines) +# ============================================================================= +[datasheets] + comment = Dataforth Datasheets + path = /data/datasheets + browseable = yes + writable = yes + guest ok = yes + guest only = yes + create mask = 0666 + directory mask = 0777 + force create mode = 0666 + force directory mode = 0777 + oplocks = no + level2 oplocks = no + strict locking = yes + +# ============================================================================= +# Share: snapshots (read-only access to BTRFS snapshots) +# ============================================================================= +[snapshots] + comment = BTRFS Snapshots (Read Only) + path = /data/.snapshots + browseable = yes + writable = no + guest ok = yes + guest only = yes +SMBCONF + +echo "[OK] Samba configuration written to /etc/samba/smb.conf" + +# Create Samba users matching the old NAS configuration +# DOS machines use station-specific users (ts-1 through ts-50) with null passwords +echo "[INFO] Creating Samba users for DOS stations..." + +for i in $(seq 1 50); do + USERNAME="ts-${i}" + # Create system user if it does not exist (no home dir, no login shell) + if ! id "${USERNAME}" >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin "${USERNAME}" 2>/dev/null || true + fi + # Add to Samba with null password + (echo ""; echo "") | smbpasswd -a -s "${USERNAME}" >/dev/null 2>&1 || true + smbpasswd -n "${USERNAME}" >/dev/null 2>&1 || true + smbpasswd -e "${USERNAME}" >/dev/null 2>&1 || true +done +echo "[OK] Created Samba users ts-1 through ts-50 (null passwords)." + +# Create engineer user with password +if ! id "${SMB_ENGINEER_USER}" >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin "${SMB_ENGINEER_USER}" 2>/dev/null || true +fi +(echo "${SMB_ENGINEER_PASS}"; echo "${SMB_ENGINEER_PASS}") | smbpasswd -a -s "${SMB_ENGINEER_USER}" >/dev/null 2>&1 || true +smbpasswd -e "${SMB_ENGINEER_USER}" >/dev/null 2>&1 || true +echo "[OK] Created Samba user 'engineer' with password." + +# Validate Samba configuration +echo "[INFO] Validating Samba configuration..." +testparm -s /etc/samba/smb.conf > /dev/null 2>&1 && echo "[OK] Samba config validation passed." || echo "[WARNING] Samba config validation had warnings (check with: testparm)" + +echo "" + +# ============================================================================= +# Step 5: Configure rsync Daemon +# ============================================================================= + +echo "[INFO] Step 5: Configuring rsync daemon..." + +cat > /etc/rsyncd.conf << RSYNCDCONF +# ============================================================================= +# rsyncd.conf - rsync daemon configuration for D2TESTNAS +# +# Module "test" provides read/write access to /data/test for the AD2 sync +# script (Sync-FromNAS-rsync.ps1) running on the AD2 Windows server. +# +# The AD2 sync script runs every 15 minutes and does bidirectional sync: +# PULL: NAS -> AD2 (test results: DAT files, TXT reports) +# PUSH: AD2 -> NAS (software updates: ProdSW, UPDATE.BAT, TODO.BAT) +# ============================================================================= + +uid = root +gid = root +use chroot = true +max connections = 10 +timeout = 300 +read only = false + +# Logging +log file = /var/log/rsyncd.log +transfer logging = yes +log format = %t %a %m %f %l + +[${RSYNC_MODULE}] + path = ${RSYNC_PATH} + comment = Dataforth Test Data + read only = false + use chroot = true + auth users = ${RSYNC_USER} + secrets file = /etc/rsyncd.secrets + hosts allow = 192.168.0.0/24 172.16.0.0/12 +RSYNCDCONF + +echo "[OK] rsync daemon configuration written to /etc/rsyncd.conf" + +# Create secrets file +echo "${RSYNC_USER}:${RSYNC_PASSWORD}" > /etc/rsyncd.secrets +chmod 600 /etc/rsyncd.secrets +echo "[OK] rsync secrets file written to /etc/rsyncd.secrets (mode 600)." + +# Enable rsync daemon in defaults +# Debian uses /etc/default/rsync to control daemon mode +if [[ -f /etc/default/rsync ]]; then + sed -i 's/^RSYNC_ENABLE=.*/RSYNC_ENABLE=true/' /etc/default/rsync +else + echo "RSYNC_ENABLE=true" > /etc/default/rsync +fi + +# Create systemd override to ensure rsync runs as daemon +mkdir -p /etc/systemd/system/rsync.service.d +cat > /etc/systemd/system/rsync.service.d/override.conf << 'EOF' +[Service] +ExecStart= +ExecStart=/usr/bin/rsync --daemon --no-detach +EOF + +echo "[OK] rsync daemon configured to start on boot." +echo "" + +# ============================================================================= +# Step 6: Configure SSH (root login with password) +# ============================================================================= + +echo "[INFO] Step 6: Configuring SSH..." + +# Enable root login with password (required for remote management) +SSHD_CONFIG="/etc/ssh/sshd_config" + +# Ensure PermitRootLogin is set to yes +if grep -q "^PermitRootLogin" "${SSHD_CONFIG}"; then + sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' "${SSHD_CONFIG}" +elif grep -q "^#PermitRootLogin" "${SSHD_CONFIG}"; then + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' "${SSHD_CONFIG}" +else + echo "PermitRootLogin yes" >> "${SSHD_CONFIG}" +fi + +# Ensure password authentication is enabled +if grep -q "^PasswordAuthentication" "${SSHD_CONFIG}"; then + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' "${SSHD_CONFIG}" +elif grep -q "^#PasswordAuthentication" "${SSHD_CONFIG}"; then + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' "${SSHD_CONFIG}" +else + echo "PasswordAuthentication yes" >> "${SSHD_CONFIG}" +fi + +# Debian 13 may use sshd_config.d drop-in files that override main config. +# Disable any drop-in that forces PasswordAuthentication no. +if [[ -d /etc/ssh/sshd_config.d ]]; then + for f in /etc/ssh/sshd_config.d/*.conf; do + if [[ -f "$f" ]] && grep -q "PasswordAuthentication no" "$f" 2>/dev/null; then + sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' "$f" + echo "[INFO] Updated ${f} to allow password authentication." + fi + done +fi + +# Set root password +echo "root:${ROOT_PASSWORD}" | chpasswd +echo "[OK] Root password set." + +echo "[OK] SSH configured: root login enabled, password auth enabled." +echo "" + +# ============================================================================= +# Step 7: BTRFS Snapshot Automation +# ============================================================================= + +echo "[INFO] Step 7: Setting up BTRFS snapshot automation..." + +# Create the snapshot management script +cat > /usr/local/bin/btrfs-snapshot.sh << 'SNAPSCRIPT' +#!/bin/bash +# ============================================================================= +# btrfs-snapshot.sh - Automated BTRFS snapshot management for D2TESTNAS +# +# Usage: +# btrfs-snapshot.sh create +# btrfs-snapshot.sh prune +# btrfs-snapshot.sh list +# +# Snapshots are stored in /data/.snapshots/ with naming convention: +# /data/.snapshots/_YYYY-MM-DD_HH-MM-SS +# +# Retention policy: +# Hourly: 48 snapshots (2 days) +# Daily: 30 snapshots (1 month) +# Weekly: 12 snapshots (3 months) +# ============================================================================= + +set -euo pipefail + +SNAP_DIR="/data/.snapshots" +SUBVOLUME="/data/test" + +HOURLY_RETAIN=48 +DAILY_RETAIN=30 +WEEKLY_RETAIN=12 + +create_snapshot() { + local snap_type="${1}" + local timestamp + timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + local snap_name="${snap_type}_${timestamp}" + local snap_path="${SNAP_DIR}/${snap_name}" + + if [[ ! -d "${SNAP_DIR}" ]]; then + mkdir -p "${SNAP_DIR}" + fi + + # Verify source is a BTRFS subvolume + if ! btrfs subvolume show "${SUBVOLUME}" >/dev/null 2>&1; then + echo "[ERROR] ${SUBVOLUME} is not a BTRFS subvolume. Cannot create snapshot." + exit 1 + fi + + btrfs subvolume snapshot -r "${SUBVOLUME}" "${snap_path}" + echo "[OK] Created snapshot: ${snap_name}" +} + +prune_snapshots() { + local snap_type="${1}" + local retain="${2}" + + # List snapshots of this type, sorted oldest first + local snaps + snaps=$(find "${SNAP_DIR}" -maxdepth 1 -name "${snap_type}_*" -type d | sort) + local count + count=$(echo "${snaps}" | grep -c . 2>/dev/null || echo 0) + + if [[ ${count} -le ${retain} ]]; then + return + fi + + local to_delete=$((count - retain)) + echo "[INFO] Pruning ${to_delete} old ${snap_type} snapshot(s) (keeping ${retain})..." + + echo "${snaps}" | head -n "${to_delete}" | while read -r snap; do + if [[ -n "${snap}" && -d "${snap}" ]]; then + btrfs subvolume delete "${snap}" + echo "[OK] Deleted: $(basename "${snap}")" + fi + done +} + +list_snapshots() { + echo "=== BTRFS Snapshots in ${SNAP_DIR} ===" + echo "" + if [[ ! -d "${SNAP_DIR}" ]]; then + echo "No snapshot directory found." + return + fi + + for snap_type in hourly daily weekly; do + local count + count=$(find "${SNAP_DIR}" -maxdepth 1 -name "${snap_type}_*" -type d 2>/dev/null | wc -l) + echo "${snap_type}: ${count} snapshot(s)" + find "${SNAP_DIR}" -maxdepth 1 -name "${snap_type}_*" -type d 2>/dev/null | sort | while read -r s; do + echo " $(basename "${s}")" + done + echo "" + done +} + +case "${1:-}" in + create) + snap_type="${2:-hourly}" + create_snapshot "${snap_type}" + ;; + prune) + prune_snapshots "hourly" "${HOURLY_RETAIN}" + prune_snapshots "daily" "${DAILY_RETAIN}" + prune_snapshots "weekly" "${WEEKLY_RETAIN}" + ;; + list) + list_snapshots + ;; + *) + echo "Usage: $0 {create |prune|list}" + exit 1 + ;; +esac +SNAPSCRIPT + +chmod +x /usr/local/bin/btrfs-snapshot.sh + +# Install cron jobs for snapshot automation +CRON_FILE="/etc/cron.d/btrfs-snapshots" + +cat > "${CRON_FILE}" << 'CRONEOF' +# BTRFS Snapshot Automation for D2TESTNAS +# Hourly snapshots: every hour on the hour +# Daily snapshots: midnight +# Weekly snapshots: Sunday at 00:15 +# Pruning: runs after each snapshot creation + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Hourly: create snapshot, then prune old ones +0 * * * * root /usr/local/bin/btrfs-snapshot.sh create hourly >> /var/log/btrfs-snapshots.log 2>&1 && /usr/local/bin/btrfs-snapshot.sh prune >> /var/log/btrfs-snapshots.log 2>&1 + +# Daily: create at midnight (separate from hourly for distinct retention) +1 0 * * * root /usr/local/bin/btrfs-snapshot.sh create daily >> /var/log/btrfs-snapshots.log 2>&1 && /usr/local/bin/btrfs-snapshot.sh prune >> /var/log/btrfs-snapshots.log 2>&1 + +# Weekly: create on Sunday at 00:15 +15 0 * * 0 root /usr/local/bin/btrfs-snapshot.sh create weekly >> /var/log/btrfs-snapshots.log 2>&1 && /usr/local/bin/btrfs-snapshot.sh prune >> /var/log/btrfs-snapshots.log 2>&1 +CRONEOF + +chmod 644 "${CRON_FILE}" + +# Set up log rotation for snapshot log +cat > /etc/logrotate.d/btrfs-snapshots << 'LOGROTEOF' +/var/log/btrfs-snapshots.log { + weekly + rotate 8 + compress + missingok + notifempty +} +LOGROTEOF + +if [[ "${BTRFS_AVAILABLE}" == "true" ]]; then + echo "[OK] BTRFS snapshot automation configured." + echo " Hourly: retained ${SNAP_HOURLY_RETAIN} (2 days)" + echo " Daily: retained ${SNAP_DAILY_RETAIN} (1 month)" + echo " Weekly: retained ${SNAP_WEEKLY_RETAIN} (3 months)" + echo " Snapshots browsable via: \\\\D2TESTNAS\\snapshots" +else + echo "[WARNING] BTRFS not available. Snapshot cron jobs installed but will" + echo " fail until /data is a BTRFS filesystem with subvolumes." +fi + +echo "" + +# ============================================================================= +# Step 8: Network Configuration (prepare for cutover) +# ============================================================================= + +echo "[INFO] Step 8: Preparing network configuration..." + +# Identify the primary network interface +PRIMARY_IFACE=$(ip route show default 2>/dev/null | awk '{print $5}' | head -1) +if [[ -z "${PRIMARY_IFACE}" ]]; then + PRIMARY_IFACE="eth0" + echo "[WARNING] Could not detect primary interface. Defaulting to ${PRIMARY_IFACE}." +else + echo "[INFO] Detected primary network interface: ${PRIMARY_IFACE}" +fi + +# Write a static IP configuration file (NOT activated yet) +# This file is for cutover day when we switch from old NAS to new VM. +STATIC_CONFIG="/etc/network/interfaces.d/static-cutover" + +cat > "${STATIC_CONFIG}" << NETEOF +# ============================================================================= +# CUTOVER STATIC IP CONFIGURATION +# ============================================================================= +# This file is NOT active by default. The VM uses DHCP during testing. +# +# To activate for cutover: +# 1. Power off the old D2TESTNAS (ReadyNAS) +# 2. Edit /etc/network/interfaces: +# - Comment out or remove the DHCP line for ${PRIMARY_IFACE} +# - Add: source /etc/network/interfaces.d/static-cutover +# 3. Run: systemctl restart networking +# 4. Verify: ip addr show ${PRIMARY_IFACE} +# +# Alternatively, replace /etc/network/interfaces entirely with: +# auto lo +# iface lo inet loopback +# auto ${PRIMARY_IFACE} +# iface ${PRIMARY_IFACE} inet static +# address ${STATIC_IP} +# netmask ${NETMASK} +# gateway ${GATEWAY} +# dns-nameservers ${DNS_SERVERS} +# ============================================================================= + +auto ${PRIMARY_IFACE} +iface ${PRIMARY_IFACE} inet static + address ${STATIC_IP} + netmask ${NETMASK} + gateway ${GATEWAY} + dns-nameservers ${DNS_SERVERS} +NETEOF + +echo "[OK] Static IP config prepared at ${STATIC_CONFIG}" +echo " Current IP: $(ip -4 addr show "${PRIMARY_IFACE}" 2>/dev/null | grep -oP 'inet \K[\d.]+' | head -1 || echo 'unknown')" +echo " Cutover IP: ${STATIC_IP} (activate during cutover)" +echo "" + +# ============================================================================= +# Step 9: Enable and Start Services +# ============================================================================= + +echo "[INFO] Step 9: Enabling and starting services..." + +systemctl daemon-reload + +# Samba (smbd for file sharing, nmbd for NetBIOS name resolution) +systemctl enable smbd +systemctl enable nmbd +systemctl restart smbd +systemctl restart nmbd +echo "[OK] Samba (smbd + nmbd) enabled and started." + +# rsync daemon +systemctl enable rsync +systemctl restart rsync +echo "[OK] rsync daemon enabled and started." + +# SSH +systemctl enable ssh +systemctl restart ssh +echo "[OK] SSH enabled and started." + +# cron (should already be running, ensure it is) +systemctl enable cron +systemctl restart cron +echo "[OK] cron enabled and started." + +echo "" + +# ============================================================================= +# Step 10: Firewall (if ufw/nftables is active) +# ============================================================================= + +echo "[INFO] Step 10: Checking firewall..." + +if command -v ufw >/dev/null 2>&1 && ufw status | grep -q "active"; then + echo "[INFO] UFW is active. Adding required rules..." + ufw allow 22/tcp comment "SSH" + ufw allow 137/udp comment "NetBIOS Name Service (nmbd)" + ufw allow 138/udp comment "NetBIOS Datagram" + ufw allow 139/tcp comment "NetBIOS Session (SMB1)" + ufw allow 445/tcp comment "SMB" + ufw allow 873/tcp comment "rsync daemon" + echo "[OK] Firewall rules added." +elif command -v nft >/dev/null 2>&1; then + echo "[INFO] nftables available. If you enable a firewall later, allow these ports:" + echo " 22/tcp (SSH), 137/udp (NetBIOS), 138/udp (NetBIOS)," + echo " 139/tcp (SMB1), 445/tcp (SMB), 873/tcp (rsync)" +else + echo "[INFO] No active firewall detected. No rules needed." +fi + +echo "" + +# ============================================================================= +# Step 11: Verification +# ============================================================================= + +echo "[INFO] Step 11: Running verification checks..." +echo "" + +ERRORS=0 + +# Check smbd +if systemctl is-active --quiet smbd; then + echo "[OK] smbd is running." +else + echo "[ERROR] smbd is NOT running." + ERRORS=$((ERRORS + 1)) +fi + +# Check nmbd +if systemctl is-active --quiet nmbd; then + echo "[OK] nmbd is running." +else + echo "[ERROR] nmbd is NOT running." + ERRORS=$((ERRORS + 1)) +fi + +# Check rsync +if systemctl is-active --quiet rsync; then + echo "[OK] rsync daemon is running." +else + echo "[ERROR] rsync daemon is NOT running." + ERRORS=$((ERRORS + 1)) +fi + +# Check rsync port +if ss -tlnp | grep -q ":873"; then + echo "[OK] rsync daemon listening on port 873." +else + echo "[ERROR] rsync daemon NOT listening on port 873." + ERRORS=$((ERRORS + 1)) +fi + +# Check SSH +if systemctl is-active --quiet ssh; then + echo "[OK] SSH is running." +else + echo "[ERROR] SSH is NOT running." + ERRORS=$((ERRORS + 1)) +fi + +# Check Samba shares +echo "" +echo "[INFO] Samba shares configured:" +smbclient -L localhost -N 2>/dev/null | grep -E "Disk|test|datasheets|snapshots" || echo "[WARNING] Could not list Samba shares (smbclient may not support -N with this config)." + +# Check data directories +echo "" +if [[ -d /data/test ]]; then + echo "[OK] /data/test exists." +else + echo "[ERROR] /data/test does NOT exist." + ERRORS=$((ERRORS + 1)) +fi + +if [[ -d /data/datasheets ]]; then + echo "[OK] /data/datasheets exists." +else + echo "[ERROR] /data/datasheets does NOT exist." + ERRORS=$((ERRORS + 1)) +fi + +echo "" + +# ============================================================================= +# Summary +# ============================================================================= + +CURRENT_IP=$(ip -4 addr show "${PRIMARY_IFACE}" 2>/dev/null | grep -oP 'inet \K[\d.]+' | head -1 || echo 'unknown') + +echo "============================================================" +echo " SETUP COMPLETE" +echo "============================================================" +echo "" +echo " Hostname: ${HOSTNAME}" +echo " Current IP: ${CURRENT_IP} (DHCP - for testing)" +echo " Cutover IP: ${STATIC_IP} (activate during cutover)" +echo " Interface: ${PRIMARY_IFACE}" +echo "" +echo " Services:" +echo " Samba (smbd): $(systemctl is-active smbd)" +echo " NetBIOS (nmbd): $(systemctl is-active nmbd)" +echo " rsync daemon: $(systemctl is-active rsync)" +echo " SSH: $(systemctl is-active ssh)" +echo " cron: $(systemctl is-active cron)" +echo "" +echo " Shares:" +echo " \\\\${HOSTNAME}\\test -> /data/test" +echo " \\\\${HOSTNAME}\\datasheets -> /data/datasheets" +echo " \\\\${HOSTNAME}\\snapshots -> /data/.snapshots (read-only)" +echo "" +echo " rsync:" +echo " Module: ${RSYNC_MODULE} -> ${RSYNC_PATH}" +echo " Auth: ${RSYNC_USER} (port 873)" +echo "" +echo " BTRFS Snapshots: $(if [[ ${BTRFS_AVAILABLE} == true ]]; then echo 'ACTIVE'; else echo 'DISABLED (no BTRFS)'; fi)" +echo " Hourly: ${SNAP_HOURLY_RETAIN} retained | Daily: ${SNAP_DAILY_RETAIN} retained | Weekly: ${SNAP_WEEKLY_RETAIN} retained" +echo " List: btrfs-snapshot.sh list" +echo " Manual: btrfs-snapshot.sh create " +echo "" + +if [[ ${ERRORS} -gt 0 ]]; then + echo " [WARNING] ${ERRORS} verification check(s) failed. Review output above." +else + echo " [OK] All verification checks passed." +fi + +echo "" +echo "============================================================" +echo " CUTOVER CHECKLIST" +echo "============================================================" +echo "" +echo " When ready to replace the old ReadyNAS:" +echo "" +echo " 1. Copy data from old NAS:" +echo " rsync -avz rsync://rsync@192.168.0.9/test/ /data/test/" +echo "" +echo " 2. Stop the old NAS (power off D2TESTNAS ReadyNAS)" +echo "" +echo " 3. Switch this VM to static IP ${STATIC_IP}:" +echo " Edit /etc/network/interfaces to source ${STATIC_CONFIG}" +echo " systemctl restart networking" +echo "" +echo " 4. Verify from a Windows machine:" +echo " net use T: \\\\${STATIC_IP}\\test" +echo " rsync --list-only rsync://rsync@${STATIC_IP}/test/" +echo "" +echo " 5. Verify from a DOS machine:" +echo " Boot a test station and confirm T: drive maps successfully" +echo "" +echo " 6. Create initial BTRFS snapshot:" +echo " btrfs-snapshot.sh create daily" +echo "" +echo "============================================================" diff --git a/projects/dataforth-dos/session-logs/2026-03-11-testdatadb-investigation.md b/projects/dataforth-dos/session-logs/2026-03-11-testdatadb-investigation.md new file mode 100644 index 0000000..a029c5f --- /dev/null +++ b/projects/dataforth-dos/session-logs/2026-03-11-testdatadb-investigation.md @@ -0,0 +1,61 @@ +# TestDataDB Investigation - Missing Recent Records +**Date:** 2026-03-11 +**Status:** BLOCKED - VPN down, need parser source code from AD2 + +## Problem +The test database on AD2 has newest `test_date` of 2026-01-19 despite daily tests being run every weekday. After a full re-import (1,028,275 -> 1,632,793 records), the max date did not change. + +## Key Evidence +1. DAT files with TODAY's timestamps (2026-03-11 13:29-13:30) exist on AD2 at `C:\Shares\test\TS-1L\LOGS\5BLOG\` +2. These files were processed by the full import (604,518 new records added) +3. But `MAX(test_date)` is still 2026-01-19 + +## Sample DAT File Content (33-06D.DAT from 5BLOG) +``` +"SCM5B33-06D " +9.528068E-02,.9528068,.9696456,.1683873,"PASS" +.3142081,3.142081,3.155015,.1293349,"PASS" +.5374944,5.374944,5.380653,5.708694E-02,"PASS" +.7651215,7.651215,7.651233,1.811981E-04,"PASS" +.997809,9.97809,9.967015,-.1107502,"PASS" +"PASS 92.941","","PASS .16838733","PASS 2.045663E-023","PASS-2.213427E-023" +"","PASS .29938753","","PASS .79032473","PASS .05982453" +"","PASS 117.35150","","PASS-.1008325","PASS 91.444891" +"PASS 1.024068E-023","" +``` + +**No date anywhere in the content or filename.** + +## Root Cause Hypothesis +The `multiline.js` parser at `C:\Shares\testdatadb\parsers\multiline.js` determines `test_date` from some source. Possibilities: +1. **File modification time (mtime)** - most likely since there's no date in content +2. **A date field elsewhere in larger files** - maybe we only saw a partial file +3. **A hardcoded date or fallback** - parser might have a bug + +If parser uses mtime, the question is whether mtime is preserved when: +- DOS machines XCOPY files to NAS +- Sync-FromNAS.ps1 SCPs files from NAS to AD2 +- SCP with -O flag may or may not preserve timestamps + +## HISTLOGS vs Station Files +HISTLOGS files at `C:\Shares\test\Ate\HISTLOGS\` are the authoritative consolidated source. These may have a DIFFERENT format than the per-station DAT files. The initial import (1,030,940 records) came mostly from HISTLOGS (576K) and Recovery-TEST (454K), with only 59 from live station data. + +The 604K new records from the re-import might all be from HISTLOGS/Recovery with dates up to Jan 19, while the per-station files might be producing 0 records or records with the same old dates. + +## Next Steps (when VPN reconnects) +1. **READ THE PARSER:** `ssh sysadmin@AD2 "type C:\Shares\testdatadb\parsers\multiline.js"` +2. **Check a specific record:** Query DB for records from `33-06D.DAT` source file to see what test_date was assigned +3. **Check import logs:** `ssh sysadmin@AD2 "type C:\Shares\test\scripts\sync-from-nas.log"` for any import errors +4. **Verify HISTLOGS content:** Check if HISTLOGS files have different format than station files + +## Deployed Fixes This Session +- Sync-FromNAS.ps1: Get-NASFileList fix (stdout deadlock), 8.3 filename filtering, SCP path escaping +- import.js: Changed INSERT OR IGNORE to INSERT OR REPLACE +- Both deployed to AD2 at C:\Shares\test\scripts\ and C:\Shares\testdatadb\database\ +- Commit dd4086d (local only, not pushed - Gitea unreachable) + +## Session Context +- Worked on from Windows machine (ACG-M-L5090) +- VPN went down during investigation +- Previous session summary in conversation compaction +- User said "continue to work this problem - we need to find those records" diff --git a/projects/dataforth-dos/session-logs/2026-03-12-session.md b/projects/dataforth-dos/session-logs/2026-03-12-session.md new file mode 100644 index 0000000..af78bae --- /dev/null +++ b/projects/dataforth-dos/session-logs/2026-03-12-session.md @@ -0,0 +1,243 @@ +# Session Log: 2026-03-12 - D2TESTNAS VM Build, NAS Migration, Rsync Sync Fix + +## Session Summary + +Major infrastructure session: replaced broken SCP-based sync with rsync, built a new Debian 13 VM to replace the aging ReadyNAS, transferred data, and performed IP cutover. Also investigated BTRFS snapshots on old NAS and began DOS machine testing against new Linux-based NAS. + +### Key Accomplishments +1. **Fixed Sync-FromNAS.ps1 on AD2** - Replaced broken SCP with rsync daemon protocol, added guards for stray files (TS-21, TS-3R/HVLOG), added log file write retry for AV locking +2. **Disabled old SCP scheduled tasks on AD2** - Killed Sync-FromNAS and BulkSync-Catchup tasks +3. **Built D2TESTNAS replacement VM** on DF-HYPERV-B (Debian 13, Samba SMB1, rsync daemon, BTRFS 512GB data disk) +4. **Transferred data from old NAS** - test/ data (~24GB+), datasheets, home, 82 snapshots (partial ~43GB logical) +5. **IP cutover completed** - New VM now at 192.168.0.9, old NAS on DHCP at 192.168.0.117 +6. **WINS/NetBIOS conflict resolved** - Killed nmbd on old NAS, removed auto-restart cron, blocked ports 137/138 via iptables + +### Key Decisions +- Chose Hyper-V VM on DF-HYPERV-B over repurposing physical server DF-SVR-D2-SYNC +- Used BTRFS for data disk with subvolumes for test and datasheets +- Single rsync stream to avoid overloading old NAS (ARM processor) +- BTRFS snapshots from old NAS are being flattened (CoW -> full copies) which makes them much larger than ReadyNAS UI reported + +### Problems Encountered and Solutions +- **TS-21 stray file**: 1,129-byte DAT file from 2012 existed instead of directory. Renamed, added script guard. +- **TS-3R/LOGS/HVLOG stray file**: 56-byte file from 2013. Same fix. +- **Log file locking**: AV locking sync-from-nas.log. Added 3-retry with 100ms delay. +- **AD2 high latency**: AV causing 685-1056ms ping. Recommended exclusions. +- **NAS freezing under SSH load**: Power cycled, limited to single rsync stream. +- **nmbd auto-restart on old NAS**: Cron `*/5 * * * * pgrep -x nmbd || /usr/sbin/nmbd -D`. Removed cron, blocked ports via iptables. +- **nmcli config didn't save first attempt**: SSH dropped before apply. Re-ran successfully. +- **DOS Error 53 (network path not found)**: Old NAS still broadcasting D2TESTNAS name. Fixed by killing nmbd and blocking NetBIOS ports. + +--- + +## Credentials + +### New D2TESTNAS VM (Debian 13) +- **IP**: 192.168.0.9 (static via NetworkManager) +- **SSH**: root / Paper123!@# (also localadmin / Paper123!@#) +- **SSH Key**: ed25519 generated on VM, public key installed on old NAS +- **Key fingerprint**: SHA256:S2Eom4RwHS/8YMu+ePnOmDOJxGhIkxJQ2ocR3WsH24o root@D2TESTNAS + +### Rsync Daemon (new VM) +- **Port**: 873 +- **Module**: test = /data/test +- **User**: rsync +- **Password**: IQ203s32119 +- **Config**: /etc/rsyncd.conf +- **Secrets**: /etc/rsyncd.secrets + +### Samba (new VM) +- **Shares**: test (/data/test), datasheets (/data/datasheets), snapshots (/data/test/.snapshots) +- **Protocol**: SMB1 (CORE) through SMB3 +- **Auth**: Guest OK on all shares +- **Workgroup**: D2TESTING +- **NetBIOS name**: D2TESTNAS +- **WINS support**: yes + +### Old NAS (ReadyNAS) +- **Current IP**: 192.168.0.117 (DHCP, was 192.168.0.9) +- **MAC**: 28:C6:8E:34:4B:5E +- **SSH**: root (key-based auth from new VM) +- **Status**: nmbd killed, cron cleared, NetBIOS ports blocked via iptables. Samba stopped. SSH still works for rsync transfers. + +### AD2 (Windows Server) +- **IP**: 192.168.0.6 +- **Sync script**: C:\Scripts\Sync-FromNAS-rsync.ps1 (deployed, dry-run validated) +- **Test data path**: C:\Shares\test\ +- **cwRsync**: Installed via Chocolatey + +### DF-SVR-D2-SYNC (unused physical server) +- **IP**: 192.168.0.93 +- **Creds**: sysadmin / Paper123!@# +- **HP ProLiant ML350 G6, 64GB RAM, Server 2019** +- **SMB share**: NAS-BACKUP (was used temporarily for CIFS backup attempt) +- **SMB1 enabled** on this server + +### UDM Network +- **WINS server**: 192.168.0.9 (configured in UDM DHCP option 44) + +--- + +## Infrastructure + +### New D2TESTNAS VM Configuration +- **Host**: DF-HYPERV-B (dedicated Hyper-V host) +- **OS**: Debian 13 (Trixie) +- **Network**: eth0, static 192.168.0.9/24, gateway 192.168.0.1 +- **Disks**: + - /dev/sda: OS disk + - /dev/sdb: 512GB BTRFS data disk mounted at /data +- **BTRFS subvolumes**: test, datasheets (under /data) +- **Services**: smbd, nmbd, rsync (daemon), sshd, cron +- **Snapshot cron**: + ``` + 0 * * * * /usr/local/bin/btrfs-snapshot.sh test 48 + 0 * * * * /usr/local/bin/btrfs-snapshot.sh datasheets 48 + 0 0 * * * /usr/local/bin/btrfs-snapshot.sh test 30 + 0 0 * * 0 /usr/local/bin/btrfs-snapshot.sh test 12 + ``` + +### Key Config Files on New VM +- `/etc/samba/smb.conf` - Samba config (SMB1/CORE, DOS charset CP437, WINS) +- `/etc/rsyncd.conf` - rsync daemon (module "test") +- `/etc/rsyncd.secrets` - rsync auth (rsync:IQ203s32119) +- `/usr/local/bin/btrfs-snapshot.sh` - BTRFS snapshot script + +### Data Transfer Status (as of ~18:30) +- **test/ data (excl snapshots)**: ~24 GB transferred, rsync still running (single stream from .117) +- **test/ snapshots**: ~43 GB logical transferred (82 snapshots), transfer was stopped to reduce NAS load - needs restart +- **datasheets/ + snapshots**: Complete (2.3 MB + 82 snapshot dirs) +- **home/**: Complete (612 KB) +- **Disk usage**: ~26 GB actual on BTRFS (CoW dedup), 486 GB free +- **Note**: ReadyNAS UI reported 5.26GB data + 16.28GB snapshots, but actual rsync transfer is MUCH larger due to BTRFS CoW flattening + +--- + +## Files Created/Modified + +### New Files +- `D:\ClaudeTools\projects\dataforth-dos\sync-fixes\Sync-FromNAS-rsync.ps1` - Complete rsync-based replacement sync script (deployed to AD2) +- `D:\ClaudeTools\projects\dataforth-dos\d2testnas-vm\setup-d2testnas.sh` - 522-line post-install setup script +- `D:\ClaudeTools\projects\dataforth-dos\d2testnas-vm\README.md` - Hyper-V creation commands, Debian install notes, cutover checklist + +### Script Fixes Applied (Sync-FromNAS-rsync.ps1) +1. **Directory-only filter** for NAS station enumeration (line ~125) +2. **Station path guard** - detects stray files where directories expected +3. **Log type directory guard** - renames stray files in LOGS subdirs +4. **Write-Log retry** - 3 attempts with 100ms delay for AV file locking + +### Deployed to New VM (via SSH) +- /etc/samba/smb.conf (full Samba config) +- /etc/rsyncd.conf + /etc/rsyncd.secrets +- /usr/local/bin/btrfs-snapshot.sh + cron entries +- SSH key pair generated, public key added to old NAS + +--- + +## Pending/Incomplete Tasks + +### Immediate (resume next session) +1. **Monitor test/ data rsync** - Single stream running from old NAS (.117) to new VM (.9). Check with: + ```bash + ssh root@192.168.0.9 "ps aux | grep 'rsync -av' | grep -v grep; du -sh /data/test/ --exclude=.snapshots" + ``` +2. **Restart snapshot transfer** after data transfer completes: + ```bash + ssh root@192.168.0.9 "nohup bash -c 'rsync -av root@192.168.0.117:/data/test/.snapshots/ /data/test/.snapshots/ 2>&1 | tail -5' &" + ``` +3. **Test DOS machine connectivity** - Error 53 was resolved (old NAS NetBIOS killed). Need to reboot DOS machine and test: + - `NET USE T: \\D2TESTNAS\TEST` + - Run CTONW.BAT (copy logs to NAS) + - Run NWTOC.BAT (download updates from NAS) + - Verify files appear in /data/test/TS-XX/LOGS/ on new VM + +### After Data Transfer Complete +4. **Verify data integrity** - Compare file counts/sizes between old and new NAS +5. **Power off old NAS** once all data confirmed transferred +6. **Set up scheduled task on AD2** - Create 15-minute scheduled task for Sync-FromNAS-rsync.ps1 +7. **Run real (non-dry) sync on AD2** - Execute Sync-FromNAS-rsync.ps1 without -DryRun flag +8. **AV exclusions on AD2** - Add exclusions for C:\Shares\test\ and rsync.exe + +### Nice to Have +9. **Copy NAS config backup to new VM** (already backed up to DF-SVR-D2-SYNC) +10. **Datto Workplace SmartBadge research** - Researched that SmartBadge add-in for Excel doesn't exist; Workplace integrates via sync client and web, not Excel plugin + +--- + +## DOS Machine Data Flow + +``` +DOS 6.22 (C:\ATE\) --COPY--> T:\MACHINE\LOGS\ (NAS via SMB1) + | + v (rsync daemon, port 873) + AD2 C:\Shares\test\ + | + v (future: database ingestion) + MariaDB @ 172.16.3.30 +``` + +### Batch Files (DOS -> NAS) +- **CTONW.BAT v3.2** - Uses COPY (not XCOPY) to upload log files from C:\ATE\ to T:\MACHINE\LOGS\ +- **NWTOC.BAT v3.5** - Uses COPY to download updates from T:\COMMON\ProdSW\ to C:\BAT\ and C:\ATE\ +- **UPDATE.BAT v2.1** - Uses XCOPY for full machine backup (had /D flag fix for DOS 6.22) + +### Log Types +5BLOG, 7BLOG, 8BLOG, DSCLOG, SCTLOG, VASLOG, PWRLOG, HVLOG + +### Active Stations +TS-3L (most recent activity), TS-4R, TS-3R, TS-11L, TS-GURU, plus many others + +--- + +## Reference + +### Key Commands +```bash +# SSH to new D2TESTNAS +ssh root@192.168.0.9 + +# SSH to old NAS (DHCP) +ssh root@192.168.0.117 + +# Check rsync transfers on new VM +ssh root@192.168.0.9 "ps aux | grep rsync | grep -v grep" + +# Test Samba from Windows +net view \\192.168.0.9 +smbclient -L //192.168.0.9 -N + +# Test rsync daemon +rsync rsync://rsync@192.168.0.9/test/ + +# Restart services on new VM +ssh root@192.168.0.9 "systemctl restart smbd nmbd rsync" + +# BTRFS snapshot status +ssh root@192.168.0.9 "ls /data/test/.snapshots/" +``` + +### Old NAS Lockdown Commands (already applied) +```bash +# Block NetBIOS (prevents name conflict) +ssh root@192.168.0.117 "iptables -A INPUT -p udp --dport 137 -j DROP; iptables -A INPUT -p udp --dport 138 -j DROP; iptables -A OUTPUT -p udp --sport 137 -j DROP; iptables -A OUTPUT -p udp --sport 138 -j DROP" + +# Remove auto-restart cron +ssh root@192.168.0.117 "crontab -r" +``` + +--- + +## Session Timeline +- Started: ~14:00 (context recovery from previous session) +- Rsync script fixes and deployment to AD2 +- Disabled old SCP scheduled tasks +- Investigated BTRFS snapshots (81 found) +- Built D2TESTNAS VM on DF-HYPERV-B (Debian 13) +- Configured all services (Samba, rsync, BTRFS, SSH) +- Started data transfer from old NAS +- Killed snapshot transfer to reduce NAS load (single stream) +- IP cutover: new VM .185 -> .9, old NAS .9 -> DHCP .117 +- Resolved WINS conflict (killed old NAS nmbd, removed cron, blocked ports) +- DOS machine testing started - Error 53 resolved +- Data transfer ongoing (~24GB+ transferred, snapshots pending restart) +- Session saved: ~18:45 diff --git a/projects/dataforth-dos/sync-fixes/Sync-FromNAS-rsync.ps1 b/projects/dataforth-dos/sync-fixes/Sync-FromNAS-rsync.ps1 new file mode 100644 index 0000000..a9dd181 --- /dev/null +++ b/projects/dataforth-dos/sync-fixes/Sync-FromNAS-rsync.ps1 @@ -0,0 +1,664 @@ +# Sync-FromNAS-rsync.ps1 +# Bidirectional sync between AD2 and NAS (D2TESTNAS) using rsync daemon +# +# PULL (NAS -> AD2): Test results (LOGS/*.DAT, Reports/*.TXT) -> Database import +# PUSH (AD2 -> NAS): Software updates (ProdSW/*, TODO.BAT) -> DOS machines +# +# Rsync daemon on NAS: port 873, module "test" maps to /data/test +# Rsync on AD2: cwRsync installed via Chocolatey (rsync.exe in PATH) +# +# Run: powershell -ExecutionPolicy Bypass -File C:\Shares\test\scripts\Sync-FromNAS-rsync.ps1 +# Scheduled: Every 15 minutes via Windows Task Scheduler + +param( + [switch]$DryRun, # Show what would be done without doing it + [switch]$Verbose # Extra output +) + +# ============================================================================ +# Configuration +# ============================================================================ +$NAS_IP = "192.168.0.9" +$RSYNC_USER = "rsync" +$RSYNC_PASSWORD = "IQ203s32119" +$RSYNC_MODULE = "test" +$RSYNC_BASE = "rsync://${RSYNC_USER}@${NAS_IP}/${RSYNC_MODULE}" + +$AD2_TEST_PATH = "C:\Shares\test" +$AD2_CYGDRIVE = "/cygdrive/c/Shares/test" + +$LOG_FILE = "C:\Shares\test\scripts\sync-from-nas.log" +$STATUS_FILE = "C:\Shares\test\_SYNC_STATUS.txt" + +$LOG_TYPES = @("5BLOG", "7BLOG", "8BLOG", "DSCLOG", "SCTLOG", "VASLOG", "PWRLOG", "HVLOG") + +# Database import configuration +$IMPORT_SCRIPT = "C:\Shares\testdatadb\database\import.js" +$NODE_PATH = "node" + +# Rsync timeout (seconds) - protects against NAS being unreachable +$RSYNC_TIMEOUT = 30 +$RSYNC_CONTIMEOUT = 15 + +# ============================================================================ +# Functions +# ============================================================================ + +function Write-Log { + param([string]$Message) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logLine = "$timestamp : $Message" + # Retry with brief delay if log file is locked by another process + for ($i = 0; $i -lt 3; $i++) { + try { + Add-Content -Path $LOG_FILE -Value $logLine -ErrorAction Stop + break + } catch { + Start-Sleep -Milliseconds 100 + } + } + if ($Verbose) { Write-Host $logLine } +} + +function Test-RsyncAvailable { + # Check that rsync.exe is available in PATH + $rsyncCmd = Get-Command "rsync" -ErrorAction SilentlyContinue + if (-not $rsyncCmd) { + Write-Log "FATAL: rsync.exe not found in PATH. Install cwRsync via Chocolatey: choco install rsync" + Write-Host "FATAL: rsync.exe not found in PATH. Install cwRsync via Chocolatey: choco install rsync" -ForegroundColor Red + return $false + } + Write-Log "Using rsync: $($rsyncCmd.Source)" + return $true +} + +function ConvertTo-CygPath { + # Convert a Windows path like C:\Shares\test\TS-01\LOGS to /cygdrive/c/Shares/test/TS-01/LOGS + param([string]$WindowsPath) + $path = $WindowsPath -replace '\\', '/' + if ($path -match '^([A-Za-z]):(.*)$') { + $drive = $Matches[1].ToLower() + $rest = $Matches[2] + return "/cygdrive/$drive$rest" + } + return $path +} + +function Invoke-Rsync { + # Execute rsync with the daemon password set via environment variable. + # Returns a hashtable with ExitCode and Output. + param( + [string[]]$Arguments + ) + + $env:RSYNC_PASSWORD = $RSYNC_PASSWORD + try { + $output = & rsync @Arguments 2>&1 + $exitCode = $LASTEXITCODE + return @{ + ExitCode = $exitCode + Output = $output + } + } catch { + return @{ + ExitCode = 1 + Output = "Exception invoking rsync: $_" + } + } finally { + $env:RSYNC_PASSWORD = $null + } +} + +function Get-NASStationFolders { + # List station folders (TS-*) on the NAS using rsync --list-only. + # Returns an array of station folder names (e.g., "TS-01", "TS-02"). + $result = Invoke-Rsync -Arguments @( + "--list-only", + "--timeout=$RSYNC_CONTIMEOUT", + "${RSYNC_BASE}/" + ) + + if ($result.ExitCode -ne 0) { + Write-Log "ERROR: Failed to list NAS root (exit $($result.ExitCode))" + $errText = $result.Output | Out-String + if ($errText.Trim()) { Write-Log " $errText" } + return @() + } + + $stations = @() + foreach ($line in $result.Output) { + $lineStr = "$line".Trim() + # rsync --list-only output: "drwxrwxrwx 4,096 2026/03/10 14:30:00 TS-01" + # We want directory entries (starting with 'd') matching TS-* + if ($lineStr -match '^d' -and $lineStr -match '\s(TS-\S+)\s*$') { + $stations += $Matches[1] + } + } + + return $stations +} + +function Test-8dot3Name { + param([string]$Name) + # Returns $true if filename is 8.3 compatible (no spaces, name<=8, ext<=3) + if ($Name -match '[ ()\[\]{}]') { return $false } + $parts = $Name -split '\.' + if ($parts.Count -gt 2) { return $false } + if ($parts[0].Length -gt 8 -or $parts[0].Length -eq 0) { return $false } + if ($parts.Count -eq 2 -and $parts[1].Length -gt 3) { return $false } + return $true +} + +function Test-8dot3Path { + param([string]$RelativePath) + # Check every component of the path is 8.3 compatible + $segments = $RelativePath -replace '\\', '/' -split '/' + foreach ($seg in $segments) { + if ($seg -eq '') { continue } + if (-not (Test-8dot3Name $seg)) { return $false } + } + return $true +} + +function Import-ToDatabase { + param([string[]]$FilePaths) + + if ($FilePaths.Count -eq 0) { return } + + Write-Log "Importing $($FilePaths.Count) file(s) to database..." + + # Build argument list + $importArgs = @("$IMPORT_SCRIPT", "--file") + $FilePaths + + try { + $output = & $NODE_PATH $importArgs 2>&1 + foreach ($line in $output) { + Write-Log " [DB] $line" + } + Write-Log "Database import complete" + } catch { + Write-Log "ERROR: Database import failed: $_" + } +} + +# ============================================================================ +# Main Script +# ============================================================================ + +Write-Log "==========================================" +Write-Log "Starting sync (rsync mode)" +if ($DryRun) { Write-Log "DRY RUN - no changes will be made" } + +# -- Preflight: verify rsync is installed -- +if (-not (Test-RsyncAvailable)) { + exit 2 +} + +$errorCount = 0 +$syncedFiles = 0 +$skippedFiles = 0 +$syncedDatFiles = @() # Track DAT files for database import +$pushedFiles = 0 + +# ============================================================================ +# PULL: NAS -> AD2 (Test Results) +# ============================================================================ +Write-Log "--- NAS to AD2 Sync (Test Results) ---" + +# Step 1: Enumerate station folders on NAS +Write-Log "Enumerating station folders on NAS..." +$nasStations = Get-NASStationFolders + +if ($nasStations.Count -eq 0) { + Write-Log "WARNING: No station folders found on NAS (NAS may be unreachable)" +} else { + Write-Log "Found $($nasStations.Count) station folder(s): $($nasStations -join ', ')" +} + +# Step 2: Pull DAT files per station per log type +foreach ($station in $nasStations) { + # Guard: if a file with this station name exists locally, skip it + $stationPath = "$AD2_TEST_PATH\$station" + if ((Test-Path $stationPath) -and -not (Test-Path $stationPath -PathType Container)) { + Write-Log "WARNING: '$station' exists as a file on AD2, not a directory - skipping (rename or delete the stray file)" + continue + } + + foreach ($logType in $LOG_TYPES) { + $nasPath = "${RSYNC_BASE}/${station}/LOGS/${logType}/" + $localDir = "$AD2_TEST_PATH\$station\LOGS\$logType" + $localCygDir = "$(ConvertTo-CygPath $localDir)/" + + # Ensure local directory exists (handle stray files blocking directory creation) + if (Test-Path $localDir) { + if (-not (Test-Path $localDir -PathType Container)) { + $strayName = "${localDir}.stray-file" + Write-Log " WARNING: '$station\LOGS\$logType' is a file, not directory - renaming to $(Split-Path $strayName -Leaf)" + Rename-Item -Path $localDir -NewName (Split-Path $strayName -Leaf) -Force + New-Item -ItemType Directory -Path $localDir -Force | Out-Null + } + } else { + New-Item -ItemType Directory -Path $localDir -Force | Out-Null + } + + # Build rsync arguments + $rsyncArgs = @( + "--archive", + "--include=*.DAT", + "--exclude=*", + "--remove-source-files", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + + if ($DryRun) { + $rsyncArgs += "--dry-run" + } + + $rsyncArgs += $nasPath + $rsyncArgs += $localCygDir + + if ($Verbose) { + Write-Log " rsync: $station/LOGS/$logType/" + } + + $result = Invoke-Rsync -Arguments $rsyncArgs + + if ($result.ExitCode -ne 0) { + # Exit code 23 = some files vanished (not fatal for us) + # Exit code 24 = partial transfer due to vanished source files + if ($result.ExitCode -in @(23, 24)) { + Write-Log " WARNING: rsync partial for $station/LOGS/$logType/ (exit $($result.ExitCode))" + } else { + $errText = $result.Output | Out-String + # If the path simply does not exist on NAS, rsync returns error but that is normal + # (not every station has every log type) + if ($errText -match "No such file or directory|does not exist|unknown module") { + if ($Verbose) { + Write-Log " (no $logType folder on NAS for $station)" + } + } else { + Write-Log " ERROR: rsync pull failed for $station/LOGS/$logType/ (exit $($result.ExitCode)): $errText" + $errorCount++ + } + } + continue + } + + # Parse itemized output to count transferred files + # Lines starting with ">f" indicate a file was received + foreach ($line in $result.Output) { + $lineStr = "$line".Trim() + if ($lineStr -match '^>f.*\s(\S+\.DAT)$') { + $fileName = $Matches[1] + $localFile = Join-Path $localDir $fileName + Write-Log " Pulled: $station/LOGS/$logType/$fileName" + $syncedDatFiles += $localFile + $syncedFiles++ + } + } + } + + # Pull TXT reports for this station + $nasReportsPath = "${RSYNC_BASE}/${station}/Reports/" + $localReportsDir = "$AD2_TEST_PATH\$station\Reports" + $localReportsCygDir = "$(ConvertTo-CygPath $localReportsDir)/" + + # Ensure local directory exists + if (-not (Test-Path $localReportsDir)) { + New-Item -ItemType Directory -Path $localReportsDir -Force | Out-Null + } + + $rsyncArgs = @( + "--archive", + "--include=*.TXT", + "--exclude=*", + "--remove-source-files", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + + if ($DryRun) { + $rsyncArgs += "--dry-run" + } + + $rsyncArgs += $nasReportsPath + $rsyncArgs += $localReportsCygDir + + $result = Invoke-Rsync -Arguments $rsyncArgs + + if ($result.ExitCode -ne 0) { + if ($result.ExitCode -in @(23, 24)) { + Write-Log " WARNING: rsync partial for $station/Reports/ (exit $($result.ExitCode))" + } else { + $errText = $result.Output | Out-String + if ($errText -match "No such file or directory|does not exist|unknown module") { + if ($Verbose) { + Write-Log " (no Reports folder on NAS for $station)" + } + } else { + Write-Log " ERROR: rsync pull failed for $station/Reports/ (exit $($result.ExitCode)): $errText" + $errorCount++ + } + } + } else { + foreach ($line in $result.Output) { + $lineStr = "$line".Trim() + if ($lineStr -match '^>f.*\s(\S+\.TXT)$') { + $fileName = $Matches[1] + Write-Log " Pulled report: $station/Reports/$fileName" + $syncedFiles++ + } + } + } +} + +Write-Log "NAS to AD2 sync: $syncedFiles file(s) pulled" + +# ============================================================================ +# Import synced DAT files to database +# ============================================================================ +if (-not $DryRun -and $syncedDatFiles.Count -gt 0) { + Import-ToDatabase -FilePaths $syncedDatFiles +} + +# ============================================================================ +# PUSH: AD2 -> NAS (Software Updates for DOS Machines) +# ============================================================================ +Write-Log "--- AD2 to NAS Sync (Software Updates) ---" + +# --- Helper: Push a local directory to a NAS path using rsync --update --- +# Only pushes files with 8.3-compatible paths. +# For directories, we use a filter approach: rsync the whole tree but pre-filter +# to 8.3-only names via a temp filter file or by iterating. Since rsync daemon +# does not need SSH, we can push entire directories at once for efficiency. +# However, to enforce 8.3 filtering, we use --files-from with a generated list. + +function Push-DirectoryToNAS { + param( + [string]$LocalDir, # Windows path to local directory + [string]$NASPath, # rsync daemon path (e.g., rsync://rsync@.../test/COMMON/ProdSW/) + [switch]$Recurse, + [switch]$UpdateOnly # Use --update (skip files that are newer on receiver) + ) + + if (-not (Test-Path $LocalDir)) { + Write-Log " Path not found: $LocalDir" + return 0 + } + + $pushed = 0 + $getChildArgs = @{ + Path = $LocalDir + File = $true + ErrorAction = "SilentlyContinue" + } + if ($Recurse) { $getChildArgs["Recurse"] = $true } + + $files = Get-ChildItem @getChildArgs + + if (-not $files -or $files.Count -eq 0) { + if ($Verbose) { Write-Log " No files in $LocalDir" } + return 0 + } + + # Build a list of 8.3-compatible relative paths + $validFiles = @() + foreach ($file in $files) { + $relativePath = $file.FullName.Substring($LocalDir.Length + 1).Replace('\', '/') + if (-not (Test-8dot3Path $relativePath)) { + Write-Log " Skipping (non-8.3): $relativePath" + $script:skippedFiles++ + continue + } + $validFiles += $relativePath + } + + if ($validFiles.Count -eq 0) { + return 0 + } + + # Write valid file list to a temp file for --files-from + $tempFileList = "$env:TEMP\rsync-push-list-$(Get-Date -Format 'yyyyMMddHHmmss')-$([System.IO.Path]::GetRandomFileName()).txt" + $validFiles | Set-Content -Path $tempFileList -Encoding ASCII + + $localCygDir = "$(ConvertTo-CygPath $LocalDir)/" + $tempCygPath = ConvertTo-CygPath $tempFileList + + $rsyncArgs = @( + "--archive", + "--files-from=$tempCygPath", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + + if ($UpdateOnly) { + $rsyncArgs += "--update" + } + + if ($DryRun) { + $rsyncArgs += "--dry-run" + } + + $rsyncArgs += $localCygDir + $rsyncArgs += $NASPath + + $result = Invoke-Rsync -Arguments $rsyncArgs + + # Clean up temp file + Remove-Item $tempFileList -ErrorAction SilentlyContinue + + if ($result.ExitCode -ne 0 -and $result.ExitCode -notin @(23, 24)) { + $errText = $result.Output | Out-String + Write-Log " ERROR: rsync push failed (exit $($result.ExitCode)): $errText" + $script:errorCount++ + return 0 + } + + # Count transferred files from itemized output + foreach ($line in $result.Output) { + $lineStr = "$line".Trim() + # Lines starting with ">f" or "<]f') { + $pushed++ + } + } + + # In dry-run mode, count valid files as "would push" + if ($DryRun -and $pushed -eq 0) { + # itemize-changes with dry-run still shows >f lines, so pushed should be accurate + # But if nothing was shown (all up to date), that is fine + } + + return $pushed +} + +# -- COMMON/ProdSW -- +# AD2 uses both _COMMON and COMMON; NAS uses COMMON +$commonSources = @( + "$AD2_TEST_PATH\_COMMON\ProdSW", + "$AD2_TEST_PATH\COMMON\ProdSW" +) + +foreach ($commonDir in $commonSources) { + if (Test-Path $commonDir) { + Write-Log "Syncing COMMON ProdSW from: $commonDir" + $count = Push-DirectoryToNAS -LocalDir $commonDir -NASPath "${RSYNC_BASE}/COMMON/ProdSW/" -UpdateOnly + $pushedFiles += $count + Write-Log " Pushed $count file(s) from COMMON/ProdSW" + } +} + +# -- Ate/ProdSW -- +Write-Log "Syncing Ate/ProdSW data folders..." +$ateProdSwPath = "$AD2_TEST_PATH\Ate\ProdSW" +if (Test-Path $ateProdSwPath) { + $count = Push-DirectoryToNAS -LocalDir $ateProdSwPath -NASPath "${RSYNC_BASE}/Ate/ProdSW/" -Recurse -UpdateOnly + $pushedFiles += $count + Write-Log " Pushed $count file(s) from Ate/ProdSW" +} else { + Write-Log " Ate/ProdSW not found: $ateProdSwPath" +} + +# -- UPDATE.BAT -- +Write-Log "Syncing UPDATE.BAT..." +$updateBatLocal = "$AD2_TEST_PATH\UPDATE.BAT" +if (Test-Path $updateBatLocal) { + $localCyg = ConvertTo-CygPath $updateBatLocal + $rsyncArgs = @( + "--archive", + "--update", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + if ($DryRun) { $rsyncArgs += "--dry-run" } + $rsyncArgs += $localCyg + $rsyncArgs += "${RSYNC_BASE}/UPDATE.BAT" + + $result = Invoke-Rsync -Arguments $rsyncArgs + if ($result.ExitCode -ne 0) { + $errText = $result.Output | Out-String + Write-Log " ERROR: Failed to push UPDATE.BAT (exit $($result.ExitCode)): $errText" + $errorCount++ + } else { + foreach ($line in $result.Output) { + if ("$line".Trim() -match '^[><]f') { + Write-Log " Pushed: UPDATE.BAT" + $pushedFiles++ + } + } + } +} else { + Write-Log " WARNING: UPDATE.BAT not found at $updateBatLocal" +} + +# -- DEPLOY.BAT -- +Write-Log "Syncing DEPLOY.BAT..." +$deployBatLocal = "$AD2_TEST_PATH\DEPLOY.BAT" +if (Test-Path $deployBatLocal) { + $localCyg = ConvertTo-CygPath $deployBatLocal + $rsyncArgs = @( + "--archive", + "--update", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + if ($DryRun) { $rsyncArgs += "--dry-run" } + $rsyncArgs += $localCyg + $rsyncArgs += "${RSYNC_BASE}/DEPLOY.BAT" + + $result = Invoke-Rsync -Arguments $rsyncArgs + if ($result.ExitCode -ne 0) { + $errText = $result.Output | Out-String + Write-Log " ERROR: Failed to push DEPLOY.BAT (exit $($result.ExitCode)): $errText" + $errorCount++ + } else { + foreach ($line in $result.Output) { + if ("$line".Trim() -match '^[><]f') { + Write-Log " Pushed: DEPLOY.BAT" + $pushedFiles++ + } + } + } +} else { + Write-Log " WARNING: DEPLOY.BAT not found at $deployBatLocal" +} + +# -- Per-station ProdSW and TODO.BAT -- +Write-Log "Syncing station-specific ProdSW folders..." +$stationFolders = Get-ChildItem -Path $AD2_TEST_PATH -Directory -Filter "TS-*" -ErrorAction SilentlyContinue + +foreach ($station in $stationFolders) { + # Skip station folders with non-8.3 names + if (-not (Test-8dot3Name $station.Name)) { + Write-Log " Skipping station (non-8.3 name): $($station.Name)" + continue + } + + $prodSwPath = Join-Path $station.FullName "ProdSW" + + if (Test-Path $prodSwPath) { + $count = Push-DirectoryToNAS -LocalDir $prodSwPath -NASPath "${RSYNC_BASE}/$($station.Name)/ProdSW/" -Recurse -UpdateOnly + if ($count -gt 0) { + Write-Log " Pushed $count file(s) for $($station.Name)/ProdSW" + } + $pushedFiles += $count + } + + # TODO.BAT - one-shot: push then delete from AD2 + $todoBatPath = Join-Path $station.FullName "TODO.BAT" + if (Test-Path $todoBatPath) { + Write-Log "Found TODO.BAT for $($station.Name)" + + $localCyg = ConvertTo-CygPath $todoBatPath + $rsyncArgs = @( + "--archive", + "--timeout=$RSYNC_TIMEOUT", + "--contimeout=$RSYNC_CONTIMEOUT", + "--itemize-changes" + ) + if ($DryRun) { $rsyncArgs += "--dry-run" } + $rsyncArgs += $localCyg + $rsyncArgs += "${RSYNC_BASE}/$($station.Name)/TODO.BAT" + + $result = Invoke-Rsync -Arguments $rsyncArgs + + if ($result.ExitCode -ne 0) { + $errText = $result.Output | Out-String + Write-Log " ERROR: Failed to push TODO.BAT for $($station.Name) (exit $($result.ExitCode)): $errText" + $errorCount++ + } else { + Write-Log " Pushed TODO.BAT to NAS for $($station.Name)" + $pushedFiles++ + + if (-not $DryRun) { + # Remove from AD2 after successful push (one-shot mechanism) + Remove-Item -Path $todoBatPath -Force + Write-Log " Removed TODO.BAT from AD2 (pushed to NAS)" + } else { + Write-Log " [DRY RUN] Would remove TODO.BAT from AD2 after push" + } + } + } +} + +Write-Log "AD2 to NAS sync: $pushedFiles file(s) pushed" + +# ============================================================================ +# Update Status File +# ============================================================================ +$status = if ($errorCount -eq 0) { "OK" } else { "ERRORS" } +$statusContent = @" +AD2 <-> NAS Bidirectional Sync Status (rsync) +=============================================== +Timestamp: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss") +Status: $status + +PULL (NAS -> AD2 - Test Results): + Files Pulled: $syncedFiles + Files Skipped: $skippedFiles + DAT Files Imported to DB: $($syncedDatFiles.Count) + +PUSH (AD2 -> NAS - Software Updates): + Files Pushed: $pushedFiles + +Errors: $errorCount +"@ + +Set-Content -Path $STATUS_FILE -Value $statusContent + +Write-Log "==========================================" +Write-Log "Sync complete: PULL=$syncedFiles, PUSH=$pushedFiles, Errors=$errorCount" +Write-Log "==========================================" + +# Exit with error code if there were failures +if ($errorCount -gt 0) { + exit 1 +} else { + exit 0 +}