diff --git a/.claude/skills/msp360/SKILL.md b/.claude/skills/msp360/SKILL.md index a9ea369d..48b7d568 100644 --- a/.claude/skills/msp360/SKILL.md +++ b/.claude/skills/msp360/SKILL.md @@ -35,7 +35,7 @@ py "$MSP" create-company --name "New Client LLC" --confirm py "$MSP" create-user --email backup@client.com --company "New Client LLC" \ --first Jane --last Doe --notify admin@acg.com --confirm py "$MSP" grant-license --user-id --license-id --confirm -py "$MSP" release-license --user-id --confirm +py "$MSP" release-license --user-id --license-id --confirm py "$MSP" delete-user --id [--keep-data] --confirm py "$MSP" delete-company --id --confirm py "$MSP" raw POST /api/Accounts/CreateDestination --data '{...}' --confirm @@ -91,6 +91,11 @@ RequestCustomBuilds), **Administrators** (GET/POST/PUT/DELETE). Reference: - **`/api/Computers` returns HTTP 400 "Remote Management API methods are not enabled"** — that feature is off on this account. `computers` prints a clean [INFO] and points you at `users` + `monitoring`, which cover endpoint/backup data without it. +- **Licensing API is also feature-gated:** `POST /api/Licenses/Grant|Release|Revoke` returns + HTTP 400 "Licensing API methods are not enabled for your account" — license moves must be + done in the mspbackups.com console (Mike's admin account) until that's enabled. +- **Release/Revoke require `--license-id`:** the API rejects a Release/Revoke payload without + `LicenseID` (HTTP 400 ModelState). The CLI now enforces it; get the ID from `licenses --json`. - **Git-Bash path mangling:** a leading-slash `raw` path like `/api/Users` gets rewritten to `C:/Program Files/Git/api/Users` by MSYS. The `raw` command auto-recovers the `/api/...` tail, so pass the path normally. diff --git a/.claude/skills/msp360/scripts/msp360.py b/.claude/skills/msp360/scripts/msp360.py index 8a8701f6..2e206b1f 100644 --- a/.claude/skills/msp360/scripts/msp360.py +++ b/.claude/skills/msp360/scripts/msp360.py @@ -392,12 +392,14 @@ def build_parser() -> argparse.ArgumentParser: sp.add_argument("--confirm", action="store_true") sp.set_defaults(func=cmd_delete_user) - for name, fn, verb in (("grant-license", cmd_grant_license, "grant"), - ("release-license", cmd_release_license, "release"), - ("revoke-license", cmd_revoke_license, "revoke")): + # API requires LicenseID for Release/Revoke (HTTP 400 without it); Grant can + # auto-pick a license, so it stays optional there. + for name, fn, verb, lic_required in (("grant-license", cmd_grant_license, "grant", False), + ("release-license", cmd_release_license, "release", True), + ("revoke-license", cmd_revoke_license, "revoke", True)): sp = sub.add_parser(name, help=f"{verb} a license [--confirm]") sp.add_argument("--user-id", required=True) - sp.add_argument("--license-id") + sp.add_argument("--license-id", required=lic_required) sp.add_argument("--confirm", action="store_true") sp.set_defaults(func=fn) diff --git a/.lic.json b/.lic.json index aced1aae..9643d77b 100644 --- a/.lic.json +++ b/.lic.json @@ -142,6 +142,45 @@ "User": null, "UserID": null }, + { + "ID": "7d26ae45-8af9-4fd0-8bed-011d18e82631", + "Number": 28, + "ComputerName": null, + "OperatingSystem": null, + "IsTrial": false, + "IsTaken": false, + "LicenseType": "Server", + "DateExpired": "2026-07-01T00:00:00Z", + "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", + "User": null, + "UserID": null + }, + { + "ID": "0a120352-aa4a-4785-bfdb-4e0c2980dd3c", + "Number": 18, + "ComputerName": null, + "OperatingSystem": null, + "IsTrial": false, + "IsTaken": false, + "LicenseType": "Server", + "DateExpired": "2026-07-01T00:00:00Z", + "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", + "User": null, + "UserID": null + }, + { + "ID": "93b3412b-9746-4949-a592-5369cd29d1c7", + "Number": 6, + "ComputerName": null, + "OperatingSystem": null, + "IsTrial": false, + "IsTaken": false, + "LicenseType": "Server", + "DateExpired": "2026-07-01T00:00:00Z", + "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", + "User": null, + "UserID": null + }, { "ID": "dfb1b1b9-bf99-49e9-ad28-6fca59bd86be", "Number": 4, @@ -259,32 +298,6 @@ "User": "jimmyco333@gmail.com", "UserID": "d94b286b-d22e-49c3-8bb1-415848f448f4" }, - { - "ID": "0a120352-aa4a-4785-bfdb-4e0c2980dd3c", - "Number": 18, - "ComputerName": "VWP-FILES", - "OperatingSystem": "Microsoft Windows Server 2019 Standard ", - "IsTrial": false, - "IsTaken": true, - "LicenseType": "Server", - "DateExpired": "2026-07-01T00:00:00Z", - "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", - "User": "VWP-FILES-BDC660BB0EB7", - "UserID": "634e86db-b3e5-4f4e-ae9b-74d83eeb21de" - }, - { - "ID": "93b3412b-9746-4949-a592-5369cd29d1c7", - "Number": 6, - "ComputerName": "WINFileSvr", - "OperatingSystem": "Microsoft Windows Server 2019 Standard ", - "IsTrial": false, - "IsTaken": true, - "LicenseType": "Server", - "DateExpired": "2026-07-01T00:00:00Z", - "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", - "User": "WINFileSvr-BDC660835B68", - "UserID": "30d7edce-1d60-4657-abc1-58ad30210e81" - }, { "ID": "52e2b4a4-da50-4980-bfad-61f3b397e559", "Number": 115, @@ -299,17 +312,17 @@ "UserID": "06109a61-2ba2-4547-8294-f88d5e1799ba" }, { - "ID": "7d26ae45-8af9-4fd0-8bed-011d18e82631", - "Number": 28, - "ComputerName": "CS-SERVER", - "OperatingSystem": "Microsoft Windows Server 2019 Standard ", + "ID": "b6c96a4c-cdfd-4035-a218-06d965315296", + "Number": 137, + "ComputerName": null, + "OperatingSystem": null, "IsTrial": false, - "IsTaken": true, + "IsTaken": false, "LicenseType": "Server", - "DateExpired": "2026-07-01T00:00:00Z", - "Transaction": "dbc23ae9-2010-4e27-863c-37290bcb0db1", - "User": "CS-SERVER-9163AEDE7357", - "UserID": "08d55d69-bf76-40b5-ae45-b1bbe0064c14" + "DateExpired": "2026-08-01T00:00:00Z", + "Transaction": "33df8a30-ef00-42c8-9c37-2b8b207c845f", + "User": null, + "UserID": null }, { "ID": "8d60d555-ad23-40e8-8962-29c7d0c143ce", @@ -470,10 +483,10 @@ { "ID": "7454b8e3-5aa7-43a1-b3c1-eb7605bad6c2", "Number": 6, - "ComputerName": "SERVER", - "OperatingSystem": "Microsoft Windows NT 6.3.9600.0", + "ComputerName": null, + "OperatingSystem": null, "IsTrial": false, - "IsTaken": true, + "IsTaken": false, "LicenseType": "Server", "DateExpired": "2026-08-01T00:00:00Z", "Transaction": "9fff9d71-0db7-4b66-9c9c-b0fafff28cc9", @@ -545,6 +558,32 @@ "User": "vwp", "UserID": "38905235-0bb7-48a9-8728-de469d544fed" }, + { + "ID": "fcc6c069-b745-4234-b399-b492b6d61bce", + "Number": 137, + "ComputerName": "VWP-FILES", + "OperatingSystem": "Microsoft Windows Server 2019 Standard ", + "IsTrial": false, + "IsTaken": true, + "LicenseType": "Server", + "DateExpired": "2026-08-01T00:00:00Z", + "Transaction": "33df8a30-ef00-42c8-9c37-2b8b207c845f", + "User": "VWP-FILES-BDC660BB0EB7", + "UserID": "634e86db-b3e5-4f4e-ae9b-74d83eeb21de" + }, + { + "ID": "5bf8d40c-430b-4108-b4a6-c4001e2dabdd", + "Number": 137, + "ComputerName": "WINFileSvr", + "OperatingSystem": "Microsoft Windows Server 2019 Standard ", + "IsTrial": false, + "IsTaken": true, + "LicenseType": "Server", + "DateExpired": "2026-08-01T00:00:00Z", + "Transaction": "33df8a30-ef00-42c8-9c37-2b8b207c845f", + "User": "WINFileSvr-BDC660835B68", + "UserID": "30d7edce-1d60-4657-abc1-58ad30210e81" + }, { "ID": "7c1a1758-52a7-4c4e-8caa-426ebb776ab4", "Number": 136, @@ -662,6 +701,19 @@ "User": "lulu.camacho@khalsaschools.org", "UserID": "3f80c65f-ea43-4e40-bd38-e4a5854d2e1d" }, + { + "ID": "db511bb2-0f59-41f4-bbff-da6db1c814c2", + "Number": 137, + "ComputerName": "CS-SERVER", + "OperatingSystem": "Microsoft Windows Server 2019 Standard ", + "IsTrial": false, + "IsTaken": true, + "LicenseType": "Server", + "DateExpired": "2026-08-01T00:00:00Z", + "Transaction": "33df8a30-ef00-42c8-9c37-2b8b207c845f", + "User": "CS-SERVER-9163AEDE7357", + "UserID": "08d55d69-bf76-40b5-ae45-b1bbe0064c14" + }, { "ID": "53f1df9f-72f8-4b89-b7e8-5e717184d645", "Number": 82, diff --git a/clients/prairie-schooner/cutover-runbook.md b/clients/prairie-schooner/cutover-runbook.md index 3b73b9a7..20bf3194 100644 --- a/clients/prairie-schooner/cutover-runbook.md +++ b/clients/prairie-schooner/cutover-runbook.md @@ -1,97 +1,64 @@ -# The Prairie Schooner — Cutover Runbook (tonight + tomorrow morning) +# The Prairie Schooner — Cutover Runbook -Prepared 2026-07-14 for the after-hours data move + morning UDM Pro cable swap. +REVISED 2026-07-14 evening (Howard's call): **decouple the two migrations.** +Tomorrow morning = NETWORK SWAP ONLY (SonicWall/USG out, UDM Pro in). Nothing else +changes — Q: keeps pointing at \\tps-server, QB keeps running off the OLD server. +If QB works after the cable move, the network is proven. The QB/file-server move to +TPS-SVR happens LATER as its own step with its own test. Companion file: `udm-dhcp-plan.md` (LAN/DHCP/reservations detail). -## Facts this plan is built on (verified today via RMM/SC) +## State as of tonight (all verified) -- Data to move: `C:\Share\Quickbooks` (154.8 GB, 55,915 files) + `C:\Share\Deployment` - (24.0 GB, 10,623 files). Everything else is empty Essentials defaults (Company, Users, - Folder Redirection: 0 bytes) or role data (SYSVOL/NETLOGON/CertEnroll — replicates via AD, - do NOT copy). -- TPS-SVR: C: 5.3 TB, 4.83 TB free. Already shares `Share -> C:\Share` — copy into - `C:\Share\Quickbooks` / `C:\Share\Deployment` and paths mirror the old server. -- The ONLY client-side dependency is drive **Q: -> \\tps-server\quickbooks**, delivered as a - **GPP drive map in Default Domain Policy** (User Prefs > Drives, action=U, persistent=0, - bypassErrors=1). Same on all 3 workstations (TINA, JAYMI, MEL-PC). No logon scripts, no - other maps, DFS root unused ("Shared Folders" — legacy, empty). -- QB ACLs show QBDataServiceUser27/28/31/34 (QB 2017/2018/2021/2024 DB managers installed - over the years on the old box). robocopy /COPYALL carries these SIDs. -- 3x Yealink phones (.141/.142/.146) + Dahua NVR (.145) — outbound-only, unaffected except - brief outage during the swap. +- QuickBooks CLOSED everywhere: MEL-PC killed clean; TINA + JAYMI had kernel-zombie + QBW (unkillable, hung SMB handle) -> users logged off, both machines rebooted, + QBCFMonitorService stopped on all 3. Zero open handles on the server's Quickbooks share. +- Data ALREADY STAGED on TPS-SVR (held in reserve for the later server move): + robocopy delta completed 18:40 with FAILED=0 — Quickbooks 144.2 GB / Deployment 22.4 GB + in C:\Share\*. Company file: schoonerQB2025.QBW (2.9 GB) + consistent ND/DSN/TLG set. +- Q: -> \\tps-server\quickbooks via GPP in Default Domain Policy — **NOT touched** (stays + on the old server for tomorrow's test). +- Client-visible network today: gateway .1, DHCP from SonicWall, DNS handed = .125. -## TONIGHT — after users leave +## TONIGHT — UDM Pro config only (UniFi app, Pro LAN ISOLATED) -### 1. Data copy (run in ScreenConnect session on TPS-SVR as TPS\guru) +Values in `udm-dhcp-plan.md`. Summary: +- LAN 192.168.1.1/24 (Pro takes the SonicWall's IP), domain tps.local +- DHCP pool 192.168.1.100–199 +- DNS handed to clients: **.125, .135** — SAME primary as today. Zero observable change. + (Flip to .135-first later, after the network is proven — that control is the point + of this swap.) +- 7 fixed-IP reservations: 3 PCs, 3 Yealink phones, Dahua NVR (MACs in the plan file) +- Content filtering / DNS Shield / ad-block OFF +- WAN already live on the Cox modem (public DHCP) -QuickBooks must be CLOSED on all workstations (QBW/TLG lock + change under copy). +## TOMORROW MORNING — cables + test -```bat -robocopy \\TPS-SERVER\C$\Share\Quickbooks C:\Share\Quickbooks /E /COPYALL /DCOPY:DAT /R:1 /W:1 /MT:16 /NP /TEE /LOG:C:\Backups\robocopy-quickbooks.log -robocopy \\TPS-SERVER\C$\Share\Deployment C:\Share\Deployment /E /COPYALL /DCOPY:DAT /R:1 /W:1 /MT:16 /NP /TEE /LOG:C:\Backups\robocopy-deployment.log -``` +1. Pull SonicWall (.1) and USG; patch the office switch into the Pro's LAN port. +2. On one workstation: `ipconfig /renew` -> expect SAME IP, gw .1, DNS .125/.135, + internet up. +3. Log on as a user (Howard has passwords): Q: maps to \\tps-server\Quickbooks + (unchanged), open the company file — QB works exactly as yesterday. + -> proves the network move broke nothing. +4. Phones: dial tone / registration on all 3 Yealinks. NVR: local UI + remote view. +5. Run `bash clients/prairie-schooner/verify-cutover.sh` from ClaudeTools for the + scripted pass/fail readout. -- ~179 GB over gigabit: expect roughly 30–60 min total. -- Verify: tail of each log — `FAILED` column must be 0. Re-run the same command for a - delta/repair pass (robocopy is idempotent here; do NOT add /MIR on the first night — - nothing to mirror-delete anyway). -- Run as guru (SYSTEM/RMM cannot auth to \\TPS-SERVER\C$ — that is why this step is - interactive, not dispatched). +**Rollback:** plug the SonicWall back in — nothing else was changed. -### 2. Create the shares on TPS-SVR (match old server exactly) +## LATER (separate visit) — QB/file-server move to TPS-SVR -```powershell -New-SmbShare -Name Quickbooks -Path C:\Share\Quickbooks -Description "QuickBooks data" -FullAccess Everyone -New-SmbShare -Name Deployment -Path C:\Share\Deployment -FullAccess "NT AUTHORITY\Authenticated Users" -ReadAccess Everyone -``` +Data is already staged; refresh + cut when ready: +1. QB closed everywhere -> re-run the robocopy delta (seconds). +2. On TPS-SVR: create shares + `New-SmbShare -Name Quickbooks -Path C:\Share\Quickbooks -FullAccess Everyone` + `New-SmbShare -Name Deployment -Path C:\Share\Deployment -FullAccess "NT AUTHORITY\Authenticated Users" -ReadAccess Everyone` +3. Install QB Database Server Manager (match their QB Desktop year) -> scan C:\Share\Quickbooks. +4. GPMC: Default Domain Policy -> User Prefs -> Drive Maps -> Q: -> \\tps-svr\Quickbooks. +5. Relogon test on one PC, open company file multi-user. +6. Flip Pro DHCP DNS order to .135, .125. -- NTFS perms arrive with /COPYALL — do not re-ACL. -- Old empty shares (Company, Users, "Folder Redirection", "Shared Folders" DFS root): - intentionally NOT recreated — zero bytes, no map/script references them. +## Then the AD retirement (unchanged) -### 3. QuickBooks Database Server Manager on TPS-SVR - -- Install the QB DBSM matching the client's current QB year (ACL evidence: 27/28/31/34 = - 2017/2018/2021/2024 — 2024 is presumably live; confirm the .QBW's version). -- Run DBSM -> scan `C:\Share\Quickbooks` -> confirm company file(s) found; it fixes - service-user ACLs itself. -- Firewall: DBSM installer opens its ports; verify QBDBMgrN service running. - -### 4. Repoint the Q: drive GPP (GPMC on TPS-SVR) - -Default Domain Policy -> User Configuration -> Preferences -> Windows Settings -> -Drive Maps -> Q: -> change path `\\tps-server\quickbooks` -> **`\\tps-svr\Quickbooks`**. -(persistent=0 + action=Update means every morning logon re-applies it — no per-PC work.) - -Optional immediate test tonight: `gpupdate /force` + logoff/logon on one workstation, -open the company file on Q:, confirm multi-user mode works — while the old server is -still fully up. Zero-risk rollback window. - -### 5. Enter the UDM Pro config (UniFi app, Pro LAN still ISOLATED) - -Full detail in `udm-dhcp-plan.md`: -- LAN 192.168.1.1/24, domain tps.local -- DHCP pool 192.168.1.100–199, DNS handed to clients: **.135, .125** -- 7 fixed-IP reservations (3 PCs, 3 Yealink phones, Dahua NVR) -- Content filtering / DNS Shield OFF - -## TOMORROW MORNING — cable swap + test - -1. Pull SonicWall (.1) and USG; patch the office switch into the UDM Pro LAN port. -2. On one workstation: `ipconfig /renew` -> expect SAME IP, gateway .1, DNS .135/.125, - internet up. (`nslookup tps.local` should answer from .135.) -3. Log users on normally -> Q: now maps to \\tps-svr\Quickbooks (GPP applies at logon). -4. Open QuickBooks on 2 workstations -> company file opens multi-user. -5. Phones: confirm dial tone / registration on all 3 Yealinks (reboot any that sulk). -6. NVR: confirm local UI (https://192.168.1.145) and remote/P2P view. -7. Spot-check internet + printing. - -**Rollback (if the Pro misbehaves):** plug the SonicWall back in — nothing on the old -path was changed. Q:/shares work under either gateway since both servers are on the LAN. - -## Explicitly OUT of scope for tomorrow (old server stays ON) - -TPS-SERVER remains DC/DNS/CA until the AD migration completes: FSMO transfer, CA -migration (test `Backup-CARoleService` key export first), DNS role checks, demote + -power off, then shrink the DHCP DNS list to .135 only (or add .1). Do NOT power off -the old server after the file cutover — .125 is still second DNS and a live DC. +FSMO transfer, CA migration (test Backup-CARoleService key export first), QB checks, +demote TPS-SERVER + power off, real backup. Old server stays ON until this completes — +it is still a DC, DNS (.125), CA, and (for now) the QB host. diff --git a/clients/prairie-schooner/udm-dhcp-plan.md b/clients/prairie-schooner/udm-dhcp-plan.md index b8726a8d..8c9d6f9e 100644 --- a/clients/prairie-schooner/udm-dhcp-plan.md +++ b/clients/prairie-schooner/udm-dhcp-plan.md @@ -9,8 +9,10 @@ against the 2026-07-14 onsite ping sweep. Goal: every device keeps its current I - Domain name: `tps.local` - DHCP pool (superset of today's clients): **192.168.1.100 – 192.168.1.199** - Statics .125 and .135 are OUTSIDE reservations but INSIDE this range — see exclusions below. -- DHCP DNS handed to clients: **192.168.1.135, 192.168.1.125** (new DC first — this is the - migration unblock) +- DHCP DNS handed to clients: **192.168.1.125, 192.168.1.135** for the swap (SAME primary as + the SonicWall hands today — the gateway swap must change nothing observable). AFTER the + network is proven, flip the order to .135 first on the Pro (the migration unblock — now a + 10-second change we control). - Disable content filtering / DNS Shield / ad-block (DCs do DNS) ## Fixed-IP reservations (MAC -> IP) diff --git a/clients/prairie-schooner/verify-cutover.sh b/clients/prairie-schooner/verify-cutover.sh index 3957f908..7a421224 100644 --- a/clients/prairie-schooner/verify-cutover.sh +++ b/clients/prairie-schooner/verify-cutover.sh @@ -9,9 +9,9 @@ eval "$(bash .claude/scripts/rmm-auth.sh 2>/dev/null)" || { echo "[ERROR] RMM au AGENTS=$(curl -s "$RMM/api/agents" -H "Authorization: Bearer $TOKEN") resolve(){ echo "$AGENTS" | jq -r --arg h "$1" '[.[]|select(.hostname|ascii_downcase==($h|ascii_downcase))][0].id // empty'; } -TINA=$(resolve TPS-Tina); SVR=$(resolve TPS-SVR) -[ -z "$TINA" ] || [ -z "$SVR" ] && { echo "[ERROR] could not resolve agents (TINA=$TINA SVR=$SVR)"; exit 1; } -echo "[INFO] TPS-Tina=$TINA TPS-SVR=$SVR" +TINA=$(resolve TPS-Tina); SVR=$(resolve TPS-SERVER) +[ -z "$TINA" ] || [ -z "$SVR" ] && { echo "[ERROR] could not resolve agents (TINA=$TINA SERVER=$SVR)"; exit 1; } +echo "[INFO] TPS-Tina=$TINA TPS-SERVER=$SVR" WS_SYS=' $ErrorActionPreference="Continue" @@ -20,9 +20,10 @@ Write-Output ("IP: " + $ip.IPv4Address.IPAddress + " GW: " + $ip.IPv4DefaultGat Write-Output ("DNS: " + (($ip.DNSServer | ForEach-Object{$_.ServerAddresses}) -join ", ")) Write-Output ("GW ping: " + (Test-Connection 192.168.1.1 -Count 1 -Quiet)) Write-Output ("Internet ping 1.1.1.1: " + (Test-Connection 1.1.1.1 -Count 1 -Quiet)) -try{$r=Resolve-DnsName tps.local -Server 192.168.1.135 -ErrorAction Stop; Write-Output ("DNS via .135 tps.local: OK -> " + (($r|ForEach-Object{$_.IPAddress}) -join ","))}catch{Write-Output "DNS via .135: FAIL"} -Write-Output ("UNC \\tps-svr\Quickbooks: " + (Test-Path "\\tps-svr\Quickbooks")) -Write-Output ("UNC \\tps-svr\Deployment: " + (Test-Path "\\tps-svr\Deployment")) +try{$r=Resolve-DnsName tps.local -Server 192.168.1.125 -ErrorAction Stop; Write-Output ("DNS via .125 tps.local: OK -> " + (($r|ForEach-Object{$_.IPAddress}) -join ","))}catch{Write-Output "DNS via .125: FAIL"} +try{$r=Resolve-DnsName tps.local -Server 192.168.1.135 -ErrorAction Stop; Write-Output "DNS via .135: OK"}catch{Write-Output "DNS via .135: FAIL"} +Write-Output ("UNC \\tps-server\Quickbooks (LIVE QB path): " + (Test-Path "\\tps-server\Quickbooks")) +Write-Output ("UNC \\tps-svr\Share (staging): " + (Test-Path "\\tps-svr\Share")) foreach($d in @("192.168.1.141","192.168.1.142","192.168.1.146","192.168.1.145")){ Write-Output ("ping " + $d + ": " + (Test-Connection $d -Count 1 -Quiet)) } $c=New-Object Net.Sockets.TcpClient; $h=$c.BeginConnect("192.168.1.145",443,$null,$null) Write-Output ("NVR 443: " + ($h.AsyncWaitHandle.WaitOne(1500) -and $c.Connected)); $c.Close() @@ -30,12 +31,12 @@ $ext=(Resolve-DnsName myip.opendns.com -Server resolver1.opendns.com -ErrorActio Write-Output ("Public egress: " + $ext + " (SonicWall was 184.176.147.183 - should be DIFFERENT now)")' WS_USER='net use | Select-String "Q:" -Get-ItemProperty "Registry::HKEY_CURRENT_USER\Network\Q" 2>$null | ForEach-Object{ Write-Output ("HKCU Q: -> " + $_.RemotePath) } -Write-Output ("Q:\ accessible: " + (Test-Path Q:\))' +Write-Output ("Q:\ accessible: " + (Test-Path Q:\)) +Write-Output ("company file visible: " + (Test-Path "Q:\schoonerQB2025.QBW"))' SVR_SYS=' -Get-Service | Where-Object{$_.Name -like "QBDBMgr*" -or $_.Name -eq "DNS" -or $_.Name -eq "NTDS"} | ForEach-Object{ Write-Output ($_.Name + ": " + $_.Status) } -Get-SmbShare | Where-Object{$_.Name -in @("Quickbooks","Deployment")} | ForEach-Object{ Write-Output ("share " + $_.Name + " -> " + $_.Path) } +Get-Service | Where-Object{$_.Name -like "QBDBMgr*" -or $_.Name -like "QBCFMonitor*" -or $_.Name -eq "DNS" -or $_.Name -eq "NTDS"} | ForEach-Object{ Write-Output ($_.Name + ": " + $_.Status) } +Get-SmbShare | Where-Object{$_.Name -eq "Quickbooks"} | ForEach-Object{ Write-Output ("share " + $_.Name + " -> " + $_.Path) } Get-SmbSession | Group-Object ClientComputerName | ForEach-Object{ Write-Output ("SMB client: " + $_.Name + " (" + $_.Count + ")") } Write-Output ("Internet: " + (Test-Connection 1.1.1.1 -Count 1 -Quiet))' @@ -63,11 +64,11 @@ echo "[INFO] dispatched: tina-net=$C1 tina-user=$C2 svr=$C3" echo; echo "##### TPS-TINA network/UNC #####"; poll "$C1" echo; echo "##### TPS-TINA Q: mapping (user session — needs someone logged in) #####"; poll "$C2" -echo; echo "##### TPS-SVR services/shares #####"; poll "$C3" +echo; echo "##### TPS-SERVER (old, still the QB host) services/shares #####"; poll "$C3" bash .claude/scripts/post-bot-alert.sh "[RMM] Howard ran post-cutover verification on TPS-Tina + TPS-SVR (Prairie Schooner UDM swap) -> cmd:${C1:0:8},${C2:0:8},${C3:0:8}" >/dev/null 2>&1 echo -echo "[INFO] PASS looks like: GW ping True, internet True, DNS via .135 OK, both UNC True," -echo "[INFO] Q: -> \\\\tps-svr\\Quickbooks + Q:\\ True, QBDBMgr Running, phones+NVR ping True," -echo "[INFO] public egress != 184.176.147.183. Phones/NVR checks need the swap completed first." \ No newline at end of file +echo "[INFO] NETWORK-SWAP-ONLY pass criteria: GW ping True, internet True, DNS .125 + .135 OK," +echo "[INFO] UNC \\\\tps-server\\Quickbooks True, Q: -> \\\\tps-server\\Quickbooks + company file True," +echo "[INFO] QBDBMgr Running on TPS-SERVER, phones+NVR ping True, egress != 184.176.147.183." \ No newline at end of file diff --git a/session-logs/2026-07/2026-07-14-howard-msp360-license-fix-triage.md b/session-logs/2026-07/2026-07-14-howard-msp360-license-fix-triage.md new file mode 100644 index 00000000..1f1058e7 --- /dev/null +++ b/session-logs/2026-07/2026-07-14-howard-msp360-license-fix-triage.md @@ -0,0 +1,79 @@ +# 2026-07-14 — MSP360 expired-license fix + backup triage worklist + +## User +- **User:** Howard Enos (howard) +- **Machine:** Howard-Home +- **Role:** tech + +## Session Summary + +Started from a license-expired error on VWP-FILES ("The license (LicenseID: 0a120352-...) assigned to the computer has expired"). Diagnosis: the 2026-07-01 monthly renewal left 4 Server seats expired, stranding VWP-FILES + WINFileSvr (Valleywide), CS-SERVER (Cascades of Tucson), and DESKTOP-BMBTQLI (Len's Auto), all in automatic license mode. The Licensing API (Grant/Release/Revoke) is feature-gated off (HTTP 400), so the fix was console-side: Howard bought 4 Server licenses in the MSP360 console. + +Executed the swap on the 3 reachable machines: restart Online Backup service + force plan run via `cbb.exe plan -r` (service restart alone does NOT grab a new seat; the plan run does). All three verified backing up clean on new 8/1 seats (VWP-FILES 0.47GB, WINFileSvr 2.0GB, CS-SERVER 2.61GB). DESKTOP-BMBTQLI left to self-heal at next scheduled run (verify tomorrow via `msp360 licenses`). + +Merged the triage backlog from the parallel session (`2026-07-14-howard-msp360-backup-list-discord.md`) and worked it by priority: fixed the msp360 skill CLI (`--license-id` now required on release/revoke — API rejects without it), diagnosed Blaster2 disk-full (decision package for Mike), restarted the crashed backup service on Hartman DESKTOP-EVA4H1A and re-ran its plan, and Howard fixed FSG-SRV-02's schedule in console. + +Ran the stale-P2 sweep: root-caused IMC1's 628-day-stale Images plan (schedule `Enabled=false` in the plan XML — plan intact, Files/SQL plans fine), found LAB-SVR's RMM agent dead since 06-18 (SC service restarts did not reconnect it; Howard confirmed the box is offline — hands-on item), and confirmed GTI-INV-VMHOST, Tedards DESKTOP-SUFJR0J, TSAM-SERVER, and Desert RV DESKTOP-9T0UAON are unreachable by any remote path (not in RMM; SC absent or agent offline). + +## Key Decisions + +- License fix executed per Mike's "fix all"; Howard purchased 4 Server seats (console) because the Licensing API is feature-gated off. +- Plan force-run (`cbb.exe plan -r ""`), not just service restart, is the trigger that makes an auto-mode agent grab a fresh seat. +- Monitoring feed lag pattern confirmed: a fresh LastStart with prior FAILED status + no error means a run is in flight — verify by re-poll, not by first read. +- Blaster2 deletion/retention decision deferred to Mike at end of session (Howard: "we will pass that decision to mike when we are done"). +- No-plan clients ON HOLD per Howard: "we wont be deleting anyone just yet" — excluded from the Mike package. +- IMC1 Images plan NOT silently re-enabled — 224GB job, deliberate-looking disable; flagged for decision (going over it next). +- LAB-SVR marked needs-hands-on rather than chasing further blind SC commands. + +## Problems Encountered + +- `release-license --user-id` alone → HTTP 400 "LicenseID field is required" — fixed in CLI (`--license-id` now required for release/revoke); SKILL.md updated, errorlog'd. +- Licensing API gate: POST /api/Licenses/* → HTTP 400 "not enabled for your account" — console-only; errorlog'd. +- RMM dispatch: raw heredoc JSON with `\\` escapes rejected ("invalid escape") — build payloads with `jq -n --arg`. +- RMM quote-stripping: bash `'...''...'` concatenation collapsed the PS single quotes → `C:\Program` not recognized; use bash double-quoted strings with embedded PS single quotes. +- Service restart on stranded machines did not trigger license swap — force plan run did. +- Foreground `sleep` blocked by harness — poll loops in background Bash instead. + +## Configuration Changes + +- `.claude/skills/msp360/scripts/msp360.py` (+ user copy `C:\Users\Howard\.claude\skills\msp360\scripts\msp360.py`): `--license-id` required for release-license/revoke-license. +- `.claude/skills/msp360/SKILL.md` (+ user copy): usage line + gotchas (Licensing API gated, Release/Revoke need LicenseID). +- `errorlog.md`: Licensing API gate entry. +- Scratch (removed/ignorable): `./.lic.json`, `./.mon.json`, `./.bill.json`. + +## Credentials & Secrets + +- None new. MSP360 API creds from vault `msp-tools/msp360-api.sops.yaml`; RMM from `infrastructure/gururmm-server.sops.yaml`; SC from `msp-tools/screenconnect.sops.yaml`. + +## Infrastructure & Servers + +- New Server seats (bought 2026-07-14): VWP-FILES=fcc6c069..., WINFileSvr=5bf8d40c..., CS-SERVER=db511bb2... (all exp 2026-08-01); 4th reserved for DESKTOP-BMBTQLI self-heal. +- Blaster2 (Jimmy Company) E: 7.45TB FULL — MSP360 repo `E:\MSPBackup\MBS-d94b286b-d22e-49c3-8bb1-415848f448f4` = 6,728.9 GB; reclaimable: `E:\VeeamBackup` 542.9 GB (newest file 3/29/2024), `E:\My backups` 98 GB, FileHistory 80.6 GB. +- IMC1 (Instrumental Music, RMM id 88cbf7c0-abfa-4f12-846c-96274f718bff, online): plans Files dd1f1d8d / Images 2438483c (schedule Enabled=false, OnceDate 2020-03-18, last ran 2024-10-24) / SQL 2ede2a63 (Warning). Plan XMLs: `C:\ProgramData\Online Backup\.cbb`. +- LAB-SVR (Len's Auto): RMM agent 8afb1430-c8f2-4fa2-8335-35b13d8a732f last_seen 2026-06-18, v0.6.66; SC session 8d111067-7246-4c91-8027-d0fbb8c3a2de. Box offline per Howard. +- Hartman DESKTOP-EVA4H1A (RMM 54ff7d66-8f3f-44a6-98a8-1419ffa78f6d): service crash mid-scan (net.pipe CBBPlanStatusMonitor gone); after restart the re-run was live at save time (log 2eff7410-70b5-468c-b17a-0440e3969283.log writing 20:55 local, deep repo sync). +- FSG-SRV-02: Image plan OK (8.9GB); Files plan FAILED 2026-07-15T01:46 UTC 0.0B after Howard's console fix — re-check pending. +- Unreachable stale machines: GTI-INV-VMHOST (no RMM/SC; SC has GTI-INV-DC/DC1/SQL only), DESKTOP-SUFJR0J (Tedards, none), TSAM-SERVER (SC a5845c99... agent offline since 2021 events), DESKTOP-9T0UAON (Desert RV, SC d2ecf021... offline). + +## Commands & Outputs + +- `py .claude/skills/msp360/scripts/msp360.py monitoring --company X` / `--stale 30` / `licenses --json` — primary reads. +- RMM dispatch pattern: `PAYLOAD=$(jq -n --arg cmd "$SCRIPT" '{command_type:"powershell", command:$cmd, timeout_seconds:120}')` then POST `/api/agents//command`; PS paths in single quotes inside a bash double-quoted string. +- cbb: `& 'C:\Program Files\Arizona Computer Guru\Online Backup\cbb.exe' plan -l` / `plan -r ""`; logs `C:\ProgramData\Online Backup\Logs\.log`. +- Plan schedule check: `[xml]$x = Get-Content '.cbb'; $x.DocumentElement.Schedule` → IMC1 Images `Enabled: false`. +- SC blind restart: `sc.py send-command --session --command "powershell ... Get-Service *guru*|Restart-Service" --confirm` → returns `{}`; verify by RMM last_seen poll (LAB-SVR: NOT reconnected). + +## Pending / Incomplete Tasks + +- Verify DESKTOP-BMBTQLI (Len's Auto) grabs the 4th new seat at next scheduled run (`msp360 licenses` tomorrow). +- Re-check Hartman DESKTOP-EVA4H1A run completion (was in-flight at save). +- Re-check FSG-SRV-02 Files plan (Howard fixed console-side; last poll FAILED 0.0B — may need log dig; box not in RMM, use SC). +- IMC1 Images plan re-enable decision — being discussed next. +- End-of-session Mike package: Blaster2 disk-full options (delete VeeamBackup 543GB / My backups 98GB / FileHistory 81GB + tighten MSP360 retention). No-plan clients EXCLUDED (on hold per Howard). +- Hands-on list: LAB-SVR (offline), GTI-INV-VMHOST, Tedards DESKTOP-SUFJR0J, TSAM-SERVER, Desert RV DESKTOP-9T0UAON. Deferred per Howard: Air Pros, Tucson Coin. + +## Reference Information + +- Stranded-machine user/license IDs + status/error codes: see `session-logs/2026-07/2026-07-14-howard-msp360-backup-list-discord.md` (companion session). +- MSP360 error codes: 1407 MBSLicense, 1506 dest-full, 1537 retention-without-full. +- "Consistency check" plans stuck Running = UI artifact, ignore. diff --git a/wiki/clients/dataforth.md b/wiki/clients/dataforth.md index a114dfdd..6804057e 100644 --- a/wiki/clients/dataforth.md +++ b/wiki/clients/dataforth.md @@ -1,587 +1,583 @@ ---- -type: client -name: dataforth -display_name: Dataforth Corporation -last_compiled: 2026-07-04 -compiled_by: Howard-Home/claude-main -sources: - - clients/dataforth/docs/overview.md - - clients/dataforth/docs/active-directory.md - - clients/dataforth/docs/workstations.md - - clients/dataforth/docs/manufacturing.md - - clients/dataforth/docs/billing-log.md - - clients/dataforth/docs/SYNC_SCRIPT_UPDATE_SUMMARY.md - - clients/dataforth/docs/incident-2026-03-27-abuse-report-virtuo.md - - clients/dataforth/docs/incident-2026-03-27-abuse-report-connectwise.md - - clients/dataforth/docs/cloud/m365.md - - clients/dataforth/docs/issues/log.md - - clients/dataforth/docs/network/topology.md - - clients/dataforth/docs/network/vlans.md - - clients/dataforth/docs/network/firewall.md - - clients/dataforth/docs/rmm/rmm.md - - clients/dataforth/docs/security/antivirus.md - - clients/dataforth/docs/security/backup.md - - clients/dataforth/docs/servers/ad1.md - - clients/dataforth/docs/servers/ad2.md - - clients/dataforth/docs/servers/d2testnas.md - - clients/dataforth/docs/servers/df-hyperv-b.md - - clients/dataforth/docs/servers/files-d1.md - - clients/dataforth/docs/servers/sage-sql.md - - clients/dataforth/docs/projects/shares-permissions/roadmap.md - - clients/dataforth/docs/projects/shares-permissions/current-state-2026-06-10.md - - clients/dataforth/docs/projects/shares-permissions/acl-audit-detail-2026-06-10.md - - clients/dataforth/docs/projects/shares-permissions/discovery-email-draft.md - - clients/dataforth/docs/projects/shares-permissions/target-structure-draft-2026-06-22.md - - clients/dataforth/session-logs/2026-06/2026-06-23-howard-dataforth-share-plan-recovery.md - - clients/dataforth/docs/aoi-xp-vlan-backup-runbook.md - - clients/dataforth/session-logs/2026-03-23-galactic-advisors-report.md - - clients/dataforth/session-logs/2026-03-27-security-incident-mfa-datasheets.md - - clients/dataforth/session-logs/SESSION-SUMMARY.md - - clients/dataforth/session-logs/MEMORY.md - - clients/dataforth/session-logs/2026-04-12-session.md - - clients/dataforth/session-logs/2026-04-13-session.md - - clients/dataforth/session-logs/2026-04-14-session.md - - clients/dataforth/session-logs/2026-04-23-session.md - - clients/dataforth/session-logs/2026-05-03-session.md - - clients/dataforth/session-logs/2026-05-04-lobby-phone-vlan-fix.md - - clients/dataforth/session-logs/2026-05-06-session.md - - clients/dataforth/session-logs/2026-05-12-session.md - - clients/dataforth/session-logs/2026-06-01-aoi-xp-vlan-share.md - - clients/dataforth/session-logs/2026-06-01-cbell-m365-bobbi-outlook.md - - clients/dataforth/session-logs/2026-06-02-session.md - - clients/dataforth/session-logs/2026-06-04-session.md - - clients/dataforth/session-logs/project_ad2_context.md - - clients/dataforth/session-logs/project_pipeline_rebuilt.md - - clients/dataforth/session-logs/project_test_datasheet_pipeline.md - - clients/dataforth/session-logs/project_new_product_lines.md - - clients/dataforth/migration-gap-diff-RESUME.md - - clients/dataforth/CLAUDE.dataforth.md - - projects/dataforth-dos/CONTEXT.md - - session-logs/2026-06-05-session.md - - session-logs/2026-06/2026-06-09-mike-dataforth-freepbx-safesite-forensics.md - - session-logs/2026-06/2026-06-18-mike-testdatadb-render-and-security-app.md - - .claude/memory/project_dataforth_incident_2026-03-27.md - - .claude/memory/project_datasheet_pipeline.md - - .claude/memory/project_neptune_sbr_email_routing.md - - .claude/memory/reference_dataforth_contact.md - - .claude/memory/reference_neptune_access_d2testnas.md - - .claude/memory/feedback_d2testnas_ssh.md - - .claude/memory/infra_office_network.md - - .claude/memory/project_dataforth.md - - .claude/memory/project_dataforth_history.md - - .claude/memory/project_ad2_dataforth_fork.md - - .claude/memory/ad2-ssh-mtu-blackhole.md - - .claude/memory/ad2-comms-via-sync-only.md - - clients/dataforth/session-logs/2026-06/2026-06-23-mike-pbx-no-inbound-calls-fix.md - - clients/dataforth/session-logs/2026-06/2026-06-25-howard-dforth-ship-tdr-bsod.md - - clients/dataforth/session-logs/2026-07/2026-07-01-mike-dataforth-test-data-chain-audit.md - - clients/dataforth/session-logs/2026-07/2026-07-04-howard-mydata-tpsys-smt-controller-access.md - - clients/dataforth/docs/audits/2026-07-01-test-data-chain-audit-AD2.md - - .claude/memory/reference_rmm_spawn_headless_claude.md -backlinks: - - projects/dataforth-dos - - systems/jupiter ---- - -# Dataforth Corporation - -Signal conditioning / data acquisition manufacturer in Tucson, AZ. Long-standing ACG client. Active managed relationship — monthly prepaid block. Notable for 64 MS-DOS 6.22 test stations, a major security incident in March 2026, an ongoing test datasheet pipeline modernization project, an incomplete 2025 post-ransomware recovery restore that silently dropped files across multiple shares (active audit underway), a shares/permissions remediation project (Phase 1 still pending client input), and — newly documented as of 2026-07-04 — a previously-undocumented legacy Linux SMT line controller (MYDATA TPSys, Fedora Core 3) discovered on the manufacturing VLAN. - ---- - -## Profile - -- **Contract type:** Prepaid hour block (monthly replenishment invoice $2,098.87) -- **Key contacts:** - -| Name | Username | Role | Email | -|---|---|---|---| -| Dan Center | dcenter | Operations (primary IT contact) | dcenter@dataforth.com | -| John Lehman | jlehman | Engineering, QB code, test specs | jlehman@dataforth.com | -| Peter Iliya | pIliya | Applications Engineer | pIliya@dataforth.com | -| Georg Haubner | ghaubner | Engineering; D: drive on HGHAUBNER has pre-ransomware-attack backup of all DF shares | ghaubner@dataforth.com | -| Kevin Wackerly | kwackerly | IT/Admin, handles calibration@ account | kwackerly@dataforth.com | -| Logan Tobey | ltobey | Support/Sales | ltobey@dataforth.com | -| Ben Wadzinski | bwadzinski | Engineering | — | -| Lee Payne | lpayne | Engineering | — | -| Theresa Dean | tdean | Admin | tdean@dataforth.com | -| Joel Lohr | jlohr | **RETIRED 2026-03-31** — account intentionally kept enabled; inbox rule forwards ntirety.com notifications to mike@azcomputerguru.com | jlohr@dataforth.com | -| Ken Hoffman | khoffman / oemdata | TestDataSheetUploader author, external; also owns Dataforth product API | — | -| Winter | — | Dataforth contact who requested Syncro asset cleanup 2026-06-02 | — | - -- **External distributor:** Ginger (gy@quatronix-cn.com) — Quatronix China; receives datasheets -- **Billing rate:** Prepaid block; all invoices show $0.00 — hours drawn from block -- **Hours remaining:** 30.0 hrs as of 2026-07-04 (live-check Syncro before billing — `GET /customers/578095`) -- **Syncro customer ID:** 578095 -- **Syncro managed assets:** 50 -- **Open Syncro tickets:** 0 as of 2026-07-04 -- **Invoice CC:** jantar@dataforth.com - ---- - -## 2026-07-08 GPS-RMM reconciliation - -GPS billing = 49 workstation + 3 server = 52 (matches RMM 52) (Winter/Mike, 2026-07-08). - ---- - -## Infrastructure - -### Servers & Services - -| Host | IP | Role | OS | Notes | -|---|---|---|---|---| -| AD1 | 192.168.0.27 | Primary DC, DNS, FSMO roles, Engineering share | Windows Server 2016 | C:\ at **90%** capacity (C:\Engineering = 787 GB) — critical risk. FSMO roles (assumed all). GuruRMM agent `bf7bc5ee-4167-4a62-912a-c88b11a5943d`. Image plan (`Image2025`) + Files plan (NBF, daily 2 AM, 180-day retention — created 2026-06-05). | -| AD2 | 192.168.0.6 | Secondary DC, TestDataDB service host, NAS mirror, WebShare | Windows Server 2019 (10.0.17763) | Hosts testdatadb Node.js service on :3000. Wiped by crypto attack 2025 — rebuilt. Windows Firewall disabled (all profiles). Shares: `C:\Shares\{c-drive,e-drive,webshare,test}`. Old `D:\c-drive` data volume is GONE — D: is now a mounted Windows install ISO. MSP360 agent at `C:\Program Files\Arizona Computer Guru\Online Backup\cbb.exe`; storage account `ACG-Dataforth`. GuruRMM agent `cfa93bb6-0cdc-4d4e-a29e-1609cda6f047`. No shadow copies. Runs ClaudeTools on `ad2` branch (coord-API isolated; comms via git sync only). **New (2026-07-01): its GuruRMM agent can also be used to spawn a headless `claude -p` for live read-only ground truth** — see [RMM-Spawned Claude](#rmm-spawned-claude-on-ad2) below; console user `sysadmin`, `claude.exe` v2.1.181 at `C:\Users\sysadmin\.local\bin\`, node v20.10.0. OS build corrected 2026-07-01 (was previously logged as 2022; live audit confirmed Server 2019). | -| FILES-D1 | 192.168.0.189 | File server | Windows Server 2016 | Shares: `E:\Shares\{sales,archive}`. GuruRMM agent `8566a19d-49a9-4f8b-9c6c-012cc934484b`. **NOTE: `staff` share is missing** on FILES-D1 — separate issue. | -| SAGE-SQL | 192.168.0.153 | Sage ERP (S:), RDS Session Host/Connection Broker/Web Access | Windows Server 2016 | RDS licensing grace period was expired (reset 2026-05-06). TSGateway disabled (server not externally exposed). New self-signed RDS cert installed. Bitdefender GravityZone managed AV. Share: `C:\sage`. GuruRMM agent `120ba7bf-8544-48a0-98a1-40ed5cdd3e1f`. | -| 3CX | 192.168.0.125 | Phone system (possibly inactive) | — | Last logon Oct 2025. Production phones live on VLAN 100 under the Sangoma/FreePBX PBX — 3CX role likely superseded. | -| DF-HYPERV-B | 192.168.0.123 | Hyper-V hypervisor | Windows Server 2025 | GuruRMM enrolled. Newest server in environment. VM inventory not captured. | -| DF-SVR-D2-Sync | — | (role TBD) | — | GuruRMM enrolled | -| ENG-DEV-SERVER | 192.168.0.126 | Engineering dev server | Windows 11 Pro | GuruRMM enrolled | -| D2TESTNAS | 192.168.0.9 | SMB1 bridge for DOS test stations + AOI XP backup; Neptune Exchange colocation routing; rsync source/target for the test-data pipeline | Debian 13 (trixie), Samba 4.22.6 | **Repurposed Netgear ReadyNAS.** SMB1 enabled globally (CORE..SMB3, NTLMv1) — required for DOS 6.22 stations. rsync daemon on port 873 (module `test`, user `rsync`, hosts allow 192.168.0.0/24 + 172.16.0.0/12). SSH: `root@192.168.0.9`. Tailscale route for 172.16.0.0/22. **Shares:** `test`/`datasheets`/`snapshots` (guest; `hosts deny 192.168.1.175`), `aoibackup` (XP-only — see Access). Acts as jump host for UDM SSH (D2TESTNAS direct-tcpip channel to 192.168.0.254). **2026-07-01 audit:** root SSH key auth from AD2 (`root@192.168.0.9`) is broken (publickey denied) — no operational impact since the live sync uses the rsync daemon, not SSH, but the dormant SCP fallback script would fail if re-enabled (F9). | -| ESXi hosts | 192.168.0.122, 192.168.0.124 | VMware ESXi hypervisors | ESXi | — | -| UDM Firewall | 192.168.0.254 | Perimeter firewall/router | UniFi OS 5.1.15 | MAC d0:21:f9:6c:11:02. Also responds on 192.168.0.1. SSH: `azcomputerguru@192.168.0.254`, root SSH key added 2026-06-08, 2FA push required. Vault: `clients/dataforth/udm.sops.yaml`. C2 IPs blocked via iptables (NOT permanent — need to add to UniFi UI). Boot scripts in `/data/on_boot.d/`: `10-neptune-snat.sh` (Neptune outbound SNAT), `30-freepbx-sip-forward.sh` (SIP DNAT, WAN UDP 5060 source-locked to 66.7.123.0/24 → 192.168.100.2; SIP-only — do NOT add RTP forward). **[WARNING] Confirmed 2026-06-23: the SIP DNAT rule can be silently flushed by a UniFi controller provision/update, not only a reboot** — the on_boot.d script only re-applies at boot, so a mid-uptime provision event leaves inbound calls dead until the script is manually re-run. Recommend adding a persistent UI port-forward rule as a belt-and-suspenders measure (still not done as of 2026-07-04). | -| PBX (Sangoma FreePBX) | 192.168.100.2 | VoIP PBX — production phones on 192.168.100.0/24 | Sangoma FreePBX 17 / Asterisk 22.5.2, Debian 12 | FirstDigital PJSIP trunk; SBC 66.7.123.215:5060 (Sonus), match 66.7.123.0/24; IP-auth (no registration). `qualify_frequency=0` (FD SBC ignores OPTIONS — do NOT revert). TFTP provisioning for Cisco SPA502G phones. Extensions 201-343. SSH: `sangoma@192.168.100.2` (ACG SSH key also authenticates). Vault: `clients/dataforth/pbx.sops.yaml` — password corrected 2026-06-23 (prior entry had a backslash-escaping corruption in the stored value; re-verify the vault entry byte-for-byte before use rather than assuming it is stale). [WARNING] Re-apply `PJSip.class.php` line-504 patch after any `fwconsole ma updateall`. **NOTE:** `sangoma` user is in the `sudo` group but sudo authorization on this box appears not actually granted — verify before assuming privileged ops will work via sudo. | -| **MYDATA TPSys SMT Controller** (`myserver`) | **192.168.1.1** (VLAN2, SSH-confirmed 2026-07-04) | MYDATA/Mycronic TPSys pick-and-place SMT production-line controller | Fedora Core 3 "Heidelberg" (Nov 2004), kernel 2.6.16.20, glibc ~2.3.5, bash 3.00, **LILO** bootloader, **SysV init** (no systemd) | **NEW, discovered 2026-07-04.** On VLAN 2 "mydata" (192.168.1.0/24). **[WARNING] The box's own IP 192.168.1.1 collides with the address recorded as the VLAN 2 gateway — reconcile: either this box IS the gateway/router for the SMT line, or there is an IP conflict with the UDM. Confirm before relying on either.** TPSys operator UI runs under X started from tpsys's autologin `.profile` -> `startx` on tty1 (default runlevel is **3**, not 5); local PostgreSQL backs TPSys. Accounts: `root`, `tpsys` (TPSys app user), `tpspool` (TPSys spool), `postgres`. **No credential existed prior to discovery** — root was RESET via physical-console LILO recovery, vaulted at `clients/dataforth/mydata-smt.sops.yaml` (initial entry had a password typo, corrected 2026-07-04 — re-read the vault value byte-for-byte). **[CRITICAL] root was intentionally PASSWORDLESS on this appliance** — TPSys's launcher `/home/tpsys/bin/go` escalates repeatedly via `su -c` with no tty, so *setting a root password broke every escalation* -> PostgreSQL/controller never start -> **X comes up empty (fvwm2 only, no TPSys UI).** **Resolved 2026-07-04:** `gpasswd -a tpsys wheel` + uncommented `auth sufficient .../pam_wheel.so trust use_uid` in `/etc/pam.d/su` (backup `su.bak-20260704`), restoring passwordless `su` while keeping a real root password; verified across a reboot (TPSys `mmi` came up). **Do NOT blank root, remove tpsys from wheel, or re-comment the trust line — the SMT line goes down.** Remote access: ACG RSA key `clients/dataforth/mydata-smt-ssh-key.sops.yaml` (FC3 OpenSSH 3.9 needs legacy `-o KexAlgorithms=+diffie-hellman-group14-sha1 -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa -o MACs=+hmac-sha1 -o Ciphers=+aes128-cbc`), or root pw. **DR backup (2026-07-04):** hybrid live image — partition table + MBR/LILO + raw `dd` of OS partitions (hda1/2/5/6) + `/home` tar + `pg_dumpall`, ~640 MB, SHA256-verified, with `README-RESTORE.md` runbook. Copies: **on-site** `D2TESTNAS:/root/dr-backups/mydata-tpsys/2026-07-04/`; **offsite** `AD1 C:\Shares\ITSvc\DR\mydata-tpsys\2026-07-04\` (inside AD1's MSP360 Files plan → B2 bucket `ACG-Dataforth`, nightly 2 AM). TODO: full offline 80 GB `dd`/Clonezilla image in a maintenance window; install a backup agent on D2TESTNAS so the on-site DR store self-protects to B2. Documented for the client in Syncro #32501. **GuruRMM agent CANNOT run on this box** — see Patterns below. | - -**Neptune Exchange (ACG infrastructure, physically at Dataforth D2):** -- `neptune.acghosting.com` | internal `172.16.3.11` | external inbound `67.206.163.124` / outbound `67.206.163.122` -- Exchange Server 2016, active ACG-hosted mail server for multiple clients -- Physically colocated at Dataforth's D2 facility — NOT on ACG office LAN despite 172.16.x.x IP -- Access requires routing through D2TESTNAS (192.168.0.9): Dataforth UDM has a 172.16.x.x subnet that overlaps ACG office LAN, making direct routing ambiguous -- SNAT rule on Dataforth UDM at `/data/on_boot.d/10-neptune-snat.sh` should force Neptune outbound to use `.124` (not always active — verify) -- Vault: `clients/dataforth/neptune-exchange.sops.yaml` -- [WARNING] TODO: Resubnet Dataforth UDM to a non-overlapping range to permanently fix Neptune routing - -### RMM-Spawned Claude on AD2 - -- **New capability proven 2026-07-01:** a headless `claude -p` can be launched on AD2 through its GuruRMM agent (`cfa93bb6-...`, `context:user_session`, since `sysadmin` is logged into the console and the RMM elevated token works), detached (`Start-Process ... -WindowStyle Hidden`) so it survives the RMM command-timeout window, writing a deliverable file + `DONE.txt` marker that a background poller watches. This is a live read-only ground-truth channel that works despite AD2 being isolated from the ACG coord API (172.16.3.30 unreachable from Dataforth LAN) — it does NOT replace the git-sync handoff for anything that needs to write back to the shared repo. -- **Gotcha:** a stale machine-level `ANTHROPIC_API_KEY` (108 chars, invalid) on AD2 shadows `sysadmin`'s working OAuth credentials (`C:\Users\sysadmin\.claude\.credentials.json`). Must `Remove-Item Env:\ANTHROPIC_API_KEY` before invoking `claude -p`, or it fails with `Invalid API key`. -- Reference: `.claude/memory/reference_rmm_spawn_headless_claude.md`; full pattern + transcript in `clients/dataforth/session-logs/2026-07/2026-07-01-mike-dataforth-test-data-chain-audit.md`. - -### Share -> Server -> Physical Path Map - -| Drive/Share | Server | Physical path | Notes | -|---|---|---|---| -| Q: / `c-drive` | AD2 | `C:\Shares\c-drive` | Old `D:\c-drive` is gone (D: = mounted install ISO) | -| T: / `e-drive` | AD2 | `C:\Shares\e-drive` | On AD2 itself, `T:` is `\\ad2\e-drive` — NOT the NAS. DOS stations separately map their own `T:` to `\\D2TESTNAS\test` (see DOS Test Station Data Pipeline pattern). | -| X: / `webshare` | AD2 | `C:\Shares\webshare` | On AD2, `X:` = `\\ad2\webshare`. DOS stations separately map their own `X:` to `\\D2TESTNAS\datasheets`. | -| S: / `sage` | SAGE-SQL | `C:\sage` | — | -| W: / `sales` | FILES-D1 | `E:\Shares\sales` | — | -| Y: / `archive` | FILES-D1 | `E:\Shares\archive` | — | -| B: / `Engineering` | AD1 | `C:\Engineering` | — | -| B: / `itsvc` | AD1 | `C:\Shares\ITSvc` | — | -| `staff` | FILES-D1 | — | **MISSING** — share does not exist on FILES-D1 | - -### Workstations (summary) - -| Category | Count | OS | Notable | -|---|---|---|---| -| Engineering | ~12 | Win 10/11 Pro | HGHAUBNER (192.168.0.148) — Georg's PC; `D:` = full pre-attack backup of all 7 DF shares (`DF C-Drive`, `DF E-Drive`, `DF WebShare`, `DF Sage`, `DF Server Sales/Archive/Engineering`, + personal). GuruRMM agent `2aefe0d5-2357-4bdd-965a-abfccb4767a5`. D1-PWRM for PWRM10 test. | -| Manufacturing/Assembly | ~14 | Win 10/11 Pro | AS24, AS26 + various assembly/hi-pot stations | -| Office/Admin | ~12 | Win 10/11 Pro | DF-GAGETRAK (192.168.0.102) — GAGEtrak calibration host. DF-JOEL2 (192.168.0.174) — compromised 2026-03-27, remediated. | -| Shipping | (part of Office/Admin count) | Win 10/11 Pro | **DFORTH-Ship** — GuruRMM agent `db17e069-2948-4cbc-97ea-1da721edcaf5`, HP EliteDesk 800 G1 USDT (BIOS 2014-12-10, ~11.5 yrs old). Recurring BSOD `0x116 VIDEO_TDR_FAILURE` on integrated Intel HD Graphics 4600 — see Patterns. Do not confuse with near-twin host **DForth-Shipp** (`95991b45-d843-4586-8275-9996d0d9ae17`), a separate machine. | -| End-of-Life (Win 7) | 3 | Windows 7 Pro | LABELPC (192.168.0.100), LABELPC2 (192.168.0.98), D2-RCVG-003 (192.168.0.47) — EOL, on network | -| AOI Optical Inspection (XP) | 1 | Windows XP | WinXPBE-724667 @ **192.168.1.175** on VLAN 2 (mydata/SMT). Holds the AOI machine's external drive; backs up to `\\192.168.0.9\aoibackup` (SMB1, XP-only). EOL. See AOI runbook + 2026-06-01 session log. | -| SMT Line Controller (legacy Linux appliance) | 1 | Fedora Core 3 "Heidelberg" (2004) | **MYDATA TPSys** (`myserver`), on VLAN 2 (mydata/SMT), ~20 years old. Full detail in Infrastructure Servers table above. No GuruRMM agent possible (glibc/kernel/init all below the agent's floor) — agentless monitoring planned. | -| DOS Test Stations | 64 | MS-DOS 6.22 | TS-1 through TS-30 + variants (62 `TS-*` folders confirmed on NAS 2026-07-01). Not domain-joined. SMB1 via D2TESTNAS. See DOS Test Station Data Pipeline pattern for the 2026-07-01 audit findings. | - -### Email & Identity - -- **M365 tenant:** dataforth.com | Tenant ID: `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` -- **Entra ID Sync:** Yes — Azure AD Connect. Synced OUs include **OU=SyncedUsers** and **OU=Azure_Users** (cbell confirmed in OU=Azure_Users and syncing, 2026-06-01) — the earlier "SyncedUsers only" note was incomplete. -- **M365 licenses:** 50x Business Premium (39 used), 19x Exchange Online Plan 1 (5 used), 5x SPB (4 used) -- **SMTP settings:** smtp.office365.com, port 587, STARTTLS — use `sysadmin@dataforth.com` -- **SMTP AUTH status:** Tenant-level not disabled; per-mailbox varies. `calibration@dataforth.com` had SmtpClientAuthentication=true re-enabled 2026-04-23. `sysadmin@dataforth.com` SMTP AUTH is blocked by Exchange Online default — testdatadb uses Graph API for email (Mail.Send permission granted to Claude-Code-M365 app 2026-05-12). -- **Mail security stack (layered):** - 1. **INKY PhishFence** — active transport rule `B859327F-3FBD-4BE7-A47A-97D02F1558A7` fires first (StopProcessingRules=true). Use inbox rules for per-user mail routing, NOT transport rules. - 2. **Mailprotector CloudFilter** — outbound delivery gateway (`dataforth-com.outbound.emailservice.io`, 52.3.213.180). Active outbound connector "Outbound-Mailprotector" (recipientDomains `*`). Mail may be held here. If a message shows "Delivered" in Dataforth outbound trace but never arrives, check Mailprotector (/mailprotector skill). Discovered 2026-06-05 when ghaubner email was held by "INKY - Annotation - Recipient Not Group Member" transport rule. -- **DKIM:** Both selector1 and selector2 published. Rotated 2026-05-12; cutover to selector2 on 2026-05-16. - - `selector1._domainkey.dataforth.com` → selector1-dataforth-com._domainkey.dataforthcom.onmicrosoft.com - - `selector2._domainkey.dataforth.com` → selector2-dataforth-com._domainkey.dataforthcom.onmicrosoft.com -- **DNS Host:** ntirety.com — Dataforth's public DNS zone managed through ntirety's portal (not a standard registrar). DNS change requests go to ntirety, not a domain control panel. Joel Lohr's account retained to receive ntirety.com infrastructure notifications (inbox rule → mike@azcomputerguru.com). -- **AutoForwarding blocked by default** (tenant outbound spam policy). If per-user forwarding needed, create scoped HostedOutboundSpamFilterPolicy for that sender with AutoForwardingMode=On. -- **MFA:** 3 Conditional Access policies created 2026-03-27 (initially report-only; enforced 2026-04-04): - - "ACG - Require MFA for All Users" — skip from office IP 67.206.163.122 - - "ACG - Block Foreign Sign-Ins" — US-only; MFA-Travel-Bypass group for exceptions - - "ACG - Block Legacy Authentication" -- **Named locations:** Dataforth Office - Tucson (67.206.163.122/32, trusted), Allowed Countries - US Only -- **MFA-Excluded-BreakGlass group:** Brian Faires, Dataforth Calibration, Dataforth Notifications, Endcap, Tablet 01 -- **MFA enrollment (as of 2026-03-27):** 19/38 ready, 19 needed setup — deadline April 4, 2026 - -### Network - -- **Domain:** intranet.dataforth.com | Forest/Domain Level: Windows Server 2016 -- **ISP:** fdtnet.net | Public IP: 67.206.163.122 (outbound), 67.206.163.124 (Neptune inbound) -- **Firewall/Router:** UniFi Dream Machine Pro at 192.168.0.254 (also 192.168.0.1), UniFi OS 5.1.15 -- **Network:** Flat (no VLANs on main LAN — 192.168.0.0/24). Voice/PBX VLAN: 192.168.100.0/24 — production phones live here. **VLAN 2 "mydata" (192.168.1.0/24)** = SMT production-line network (gateway 192.168.1.1); members on the *D2-SMT Switch* (USW Enterprise 8) + *D2-Breakroom* port 12. Supersedes the earlier note that 192.168.1.0/24 was an unused UDM default voice VLAN — it is in active use by SMT. Inter-VLAN routing from mydata → main LAN is currently OPEN. - - **mydata members (2026-06-01):** WinXPBE-724667 (AOI XP, .175), goldstar19, DESKTOP-FT0T4MK, My9-PC, + 3 unnamed industrial/SMT devices (MAC 00:90:fb:80:f0:c6, 00:80:79:05:23:f2, 00:80:79:04:47:e7). - - **mydata addition (2026-07-04):** **MYDATA TPSys controller** (`myserver`) confirmed present on this VLAN — hostname distinct from the named members above; exact IP still to confirm (verify). Likely corresponds to one of the previously "unnamed industrial" devices or is a device not yet captured in the member list; reconcile on next VLAN sweep. -- **VPN:** OpenVPN for ACG remote access. Client subnet 192.168.6.x (GURU-5070 gets 192.168.6.2). [WARNING] GURU-5070 OpenVPN adapter "Local Area Connection" (ifIndex 12) MTU must be set to 1400 — default 1500 causes PMTU blackhole (tunnel path MTU ~1424; bulk SSH/SCP silently drops). Verify/re-apply: `Set-NetIPInterface -InterfaceIndex 12 -AddressFamily IPv4 -NlMtuBytes 1400`. Permanent fix: add `mssfix 1360` server-side on the Dataforth OpenVPN server. -- **Drive mappings (GPO):** B: (\\ad1\itsvc), Q: (\\ad2\c-drive), S: (\\SAGE-SQL\sage), T: (\\ad2\e-drive), W: (\\files-d1\sales), X: (\\ad2\webshare), Y: (\\files-d1\archive). DOS test stations: T: (\\D2TESTNAS\test), X: (\\D2TESTNAS\datasheets) - -### GuruRMM Enrollment - -- **Site name:** Dataforth D1 | Site ID: `3a2f6866-26cd-452c-9806-a8df21475c3c` -- **Site API key:** vault `clients/dataforth/...` [check vault for current entry] -- **Fleet size:** 45 agents enrolled as of 2026-06-04; Syncro managed count 50 as of 2026-07-04 -- **[WARNING] GuruRMM enrollment workaround:** WebSocket auth in `ws/mod.rs` does not validate `enrolled_agents.agent_key_hash`. New agent installs must overwrite registry AgentKey with the site API key (not the enrollment AgentKey) and restart service. See Gitea issue #8. -- **[WARNING] Agent floor confirmed 2026-07-04 (MYDATA TPSys case):** the Linux installer (`agent/scripts/install.sh`) requires glibc ~2.17+, kernel >=2.6.32, and a systemd host. Legacy Linux appliances below that floor (e.g. Fedora Core 3, SysV init) cannot run the agent — plan agentless monitoring (ICMP/TCP probe, SSH heartbeat from a reachable host) for such boxes instead of attempting install. - -**Known enrolled agents:** - -| Host | Agent ID | Notes | -|---|---|---| -| DF-GAGETRAK | `7626d82c-0736-47a6-8bc6-68e39859caed` | Enrolled 2026-04-23 (auth workaround applied) | -| HGHAUBNER | `2aefe0d5-2357-4bdd-965a-abfccb4767a5` | Georg's PC; pre-attack backup on D: | -| AD2 | `cfa93bb6-0cdc-4d4e-a29e-1609cda6f047` | Enrolled 2026-06-04; also used 2026-07-01 to spawn headless Claude for the test-data-chain audit | -| AD1 | `bf7bc5ee-4167-4a62-912a-c88b11a5943d` | Enrolled 2026-06-04 | -| FILES-D1 | `8566a19d-49a9-4f8b-9c6c-012cc934484b` | Enrolled 2026-06-04 | -| SAGE-SQL | `120ba7bf-8544-48a0-98a1-40ed5cdd3e1f` | Enrolled 2026-06-04 | -| DF-HYPERV-B | (see RMM dashboard) | Enrolled 2026-06-04 | -| DF-SVR-D2-Sync | (see RMM dashboard) | Enrolled 2026-06-04 | -| ENG-DEV-SERVER | (see RMM dashboard) | Enrolled 2026-06-04 | -| DFORTH-Ship | `db17e069-2948-4cbc-97ea-1da721edcaf5` | Shipping-station PC; recurring TDR BSOD — see Patterns | -| (36 additional agents) | — | Mix of workstations; full list in GuruRMM dashboard | - -**Cannot be enrolled:** - -| Host | Reason | -|---|---| -| MYDATA TPSys (`myserver`) | Fedora Core 3: glibc ~2.3.5 (<2.17 floor), kernel 2.6.16 (<2.6.32 floor), SysV init (no systemd unit target) — three independent hard blockers. Confirmed 2026-07-04. | - -### Backup Architecture - -- **MSP360 ("ACG-Online Backup", `cbb.exe`):** Backup provider. Storage account: `ACG-Dataforth` (account ID `0b49ca5e-...`). -- **AD2:** Two plans — `AD2 Image` (image plan, bunch `35a5c3d2`, running daily), `Files` plan (180-day retention, NBF, daily 2 AM, covers `C:\Shares` tree; GFS off, synthetic full, compression, fast-NTFS). No shadow copies on AD2. -- **AD1:** `Image2025` image plan + **Files plan created 2026-06-05** (NBF, daily 2 AM, 180-day retention, `ACG-Dataforth`, covers `C:\Engineering` + `C:\Shares\ITSvc`; initial run at 2:00 AM, not manually triggered). Both image and file plans now in place, matching AD2. -- **Pre-attack backup (offline, not MSP360):** HGHAUBNER `D:` drive holds a full pre-attack snapshot of all 7 mapped DF shares, captured before the 2025 ransomware event. This is the only recovery source predating the attack. Accessible via GuruRMM `user_session` on HGHAUBNER. Cross-machine writes use existing GPO-mapped drives only (fresh UNC blocked by WTS-impersonation — see Patterns). -- **Historical file-level backup:** NBF bunch `faad5a67` ("Backup plan on 8/29/2025") in `ACG-Dataforth` storage contains restore points 8/29–9/29/2025, archived at old physical path `D:\c-drive\...` (pre-migration layout). Used successfully 2026-06-04 to confirm SP1366 file contents. -- **WizTree backup CSV (2026-06-04):** Full-drive WizTree export of HGHAUBNER's `D:` stored at AD2 `C:\ClaudeTools\clients\dataforth\WizTree_20260604184904.zip` (sensitive — kept OFF shares). ~8.7M files / 5.7 TB across 7 shares documented. - -### Key Applications - -| Application | Host | URL/Port | Notes | -|---|---|---|---| -| TestDataDB | AD2 | http://192.168.0.6:3000 | Node.js + Express, PostgreSQL 18.3. Internal LAN only. Redesigned UI deployed 2026-06-18 (cert-fit, publish chips, push toasts, full-screen results). Row counts per the 2026-07-01 audit: `test_records` 475,553, `work_orders` 34,149, `work_order_lines` 64,051; 99.3% of test_records have `api_uploaded_at` set (HTTP API uploader is the live web-delivery path — the older `For_Web`/ASP.NET path has been dead since 2026-05-11). | -| Sage ERP | SAGE-SQL | \\SAGE-SQL\sage (S:) | RDS-served RemoteApp | -| GageTrak | DF-GAGETRAK (192.168.0.102) | — | Calibration tracking. Sends email via calibration@dataforth.com (SMTP). GuruRMM enrolled. | -| Dataforth Product API | Hoffman's servers | https://www.dataforth.com/api/v1/TestReportDataFiles | OAuth2 client_credentials. Vault: `clients/dataforth/api-oauth.sops.yaml`. Used actively to recover DSCA33/45 and 8B/5B/SCM spec templates. | -| QuickBASIC 4.5 ATE | 64 DOS stations | T:\ (\\D2TESTNAS\test) | Automated test equipment programs. 1,470+ product model specs. Inbound spec/software distribution to stations is currently broken (root-caused 2026-07-01 — see DOS Test Station Data Pipeline pattern and Syncro #32489). | -| Power Monitor SPA | Georg's dev / TBD | — | Vanilla-JS SPA for Dataforth power meters (built by Georg/Antigravity AI). Demo at PWM.dataforth.com proposed; gateway architecture designed. Parked pending Mike↔Georg conversation. `clients/dataforth/power-monitor-demo/` | -| MYDATA TPSys | MYDATA TPSys controller (`myserver`, VLAN 2) | local X UI / local PostgreSQL | SMT pick-and-place line control software. Newly documented 2026-07-04 — see Infrastructure Servers table. | - ---- - -## Syncro Asset Inventory (2026-06-02 Reconciliation) - -Pulled full Syncro asset list for customer_id `578095`: **78 assets** across 2 pages. Syncro currently shows 50 managed assets (confirmed again 2026-07-04 live pull); reconciliation/cleanup ongoing. - -### Reconciliation Result - -| Bucket | Count | Meaning | -|---|---|---| -| KEEP | 20 | Active in Syncro (<150 days since last check-in) | -| SAVE + FLAG | 21 | Alive in ScreenConnect or Bitdefender but Syncro agent broken; do NOT delete — reinstall agent | -| REMOVE | 28 | Dead in all three systems (Syncro + ScreenConnect + Bitdefender) | -| VERIFY | 9 | Servers with no agent anywhere; could be live console-only; confirm before removing | - -**Governing rule (Howard's 3-system OR):** A machine is saved if it has been online within 150 days in ANY of Syncro, ScreenConnect, or Bitdefender. Removal only if dead in all three. - -### SAVE + FLAG — alive but Syncro agent broken (21 machines) - -AD1, AD2, SAGE-SQL, FILES-D1, ENG-DEV-SERVER, D2-MFG-001, D1-ENGI-012, MY9-PC, D1-CUST-003, DANC0619, DFORTH-SHIP, DF-LEE11-I9, DFASLB0519, D2-AS-26, HGHAUBNER, D1-PWRM, D1-ENGI-EMCLAB1, D1-CONF-002, D2-HIPOT-SURFAC, D2-AS-34, TS-41 (shows as STATION_41 in ScreenConnect) - -### VERIFY — servers with no agent (9 machines) - -APPS, EXCHANGE, EXCHANGE16, AD-3, AD-4, OLD-AD2, SAGETS-1, EPICOR, D2-ASSY-001 - -Likely dead: OLD-AD2, EXCHANGE16, SAGETS-1. Confirm before removing: APPS, AD-3, AD-4, EXCHANGE, EPICOR, D2-ASSY-001. - -### REMOVE — confirmed dead in all systems (28 asset IDs) - -Syncro asset IDs: 23845, 149614, 9708445, 9357407, 9276901, 9212922, 9078651, 8824875, 8824867, 8726494, 8726485, 8657233, 8606209, 8572160, 8523941, 8411908, 8410614, 8632009, 8726495, 8421223, 9081717, 8726493, 8423782, 8726481, 8525650, 8622969, 8361459, 8670944 - -**Deletion method:** Syncro GUI only (`https://computerguru.syncromsp.com/customer_assets?customer_id=578095`). API route `DELETE /customer_assets/{id}` returns HTML 404 for this integration token — not exposed. - -### Root Cause — Fleet-wide Syncro Agent Break ~2025-10-06 - -57 of 78 assets show `updated_at` frozen at or before 2025-10-06, while the remaining 21 show recent check-ins. This is a hard cutoff, not gradual attrition — indicating a fleet-wide Syncro agent failure around that date. The machines stayed online (visible in ScreenConnect); only the Syncro agent stopped reporting. Root cause not yet investigated. - -### Pending Actions (Coord todo tree, parent `103c48ad-7b31-4967-9388-065a91888e7c`, assigned to Howard) - -1. Delete the 28 confirmed-dead assets in Syncro GUI. -2. Decide the 9 VERIFY servers. -3. Reinstall Syncro agent on the 21 SAVE+FLAG machines. -4. Switch Dataforth to metered Syncro asset billing once clean. -5. Reply to Winter; flag the ~2025-10-06 fleet-wide agent break for investigation. - ---- - -## Third-Party Tool Inventory - -### Bitdefender GravityZone - -- **Company ID:** `64c94ef310db128bfa0d908f` (suffix `_578095` confirms Dataforth mapping) -- **Status:** Dataforth is being **phased off Bitdefender**. Only 4 of 57 GravityZone endpoints remain in "Custom Groups" (actively managed); 53 are in the "Deleted" folder (mostly unmanaged). -- **[WARNING] Bitdefender absence is NOT a decommission signal for Dataforth.** A machine missing from BD may simply have had its BD agent uninstalled as part of the phase-off. Use Syncro or ScreenConnect as liveness indicators. -- GravityZone company owner field: Lee Payne. - -### ScreenConnect - -- **Host:** `https://computerguru.screenconnect.com` -- **Extension GUID:** `2d558935-686a-4bd0-9991-07539f5fe749` -- **Vault:** `msp-tools/screenconnect.sops.yaml` (fields `credentials.username`, `credentials.api_secret`) -- **Working API auth (determined 2026-06-02):** `CTRLAuthHeader: ` (NO "Basic " prefix) + `Origin: https://computerguru.screenconnect.com`. Basic-auth or "Basic " in CTRLAuthHeader both return 401. -- **Only exposed method:** `POST /App_Extensions//Service.ashx/GetSessionsByName` with body `{"sessionName":""}`. All other Get* method names return 500. Agent `Name` fields are blank for unattended sessions — this API cannot enumerate the full Dataforth fleet; name-based lookup only. -- Custom session properties: CP1=Company, CP2=Site, CP3=Tag. - ---- - -## Access - -### Domain / Server Access -- **AD2 SSH:** `ssh sysadmin@192.168.0.6` (port 22) — vault: `clients/dataforth/ad2.sops.yaml` → `credentials.password` — NOTE: stale backslash escape in vault entry; strip with `sed 's/\\//g'`. MTU-sensitive: GURU-5070 OpenVPN adapter ifIndex 12 must be MTU 1400 for reliable bulk transfers. -- **AD1 SSH:** `ssh sysadmin@192.168.0.27` — vault: `clients/dataforth/ad1.sops.yaml` -- **D2TESTNAS SSH:** `ssh root@192.168.0.9` — vault: `clients/dataforth/d2testnas.sops.yaml`. Use root, NOT sysadmin (sysadmin SSH fails on D2TESTNAS). SSH key from acg-guru-5070 authorized. **From AD2, root SSH key auth to D2TESTNAS is currently broken** (publickey denied, confirmed 2026-07-01) — the live rsync-daemon sync path does not depend on it, but fix or retire the dormant SCP script that does. -- **D2TESTNAS `aoibackup` share (AOI XP backup):** `\\192.168.0.9\aoibackup` — Samba user `admin` (password matches the XP's local login), `hosts allow = 192.168.1.175` only, `browseable = no`. Other NAS shares explicitly deny 192.168.1.175. Creds in vault: `clients/dataforth/d2testnas.sops.yaml → credentials.smb.aoi-user` / `.aoi-password` / `.aoi-share`. -- **UDM SSH:** `ssh azcomputerguru@192.168.0.254` (2FA push) or `ssh root@192.168.0.254` (root SSH key installed 2026-06-08). Jump via D2TESTNAS: paramiko `direct-tcpip` channel or ProxyJump. Vault: `clients/dataforth/udm.sops.yaml` (corrected 2026-06-09). -- **SAGE-SQL SSH:** `ssh sysadmin@192.168.0.153` — SSH key (`C:\ProgramData\ssh\administrators_authorized_keys` on SAGE-SQL) -- **All server passwords:** vault (individual vault entries per server — `clients/dataforth/.sops.yaml`) -- **WinRM (AD2/AD1):** port 5985 — pywinrm with NTLM, user `INTRANET\sysadmin` -- **HGHAUBNER:** No SSH. Reached via GuruRMM agent `2aefe0d5`. Logged-in user `intranet\ghaubner`. Cross-machine file writes use existing GPO-mapped drives only (Q: → \\ad2\c-drive, T: → \\ad2\e-drive, etc.). - -### MYDATA TPSys SMT Controller (new, 2026-07-04) -- **Root:** password — vault `clients/dataforth/mydata-smt.sops.yaml`. Console/network access method beyond physical console not yet documented (verify — likely SSH once IP is confirmed). Exact IP on VLAN 2 "mydata" (192.168.1.0/24) not yet confirmed (verify). -- **Recovery method if locked out again:** at the LILO `boot:` prompt, boot `linux init=/bin/bash rw` to land in a passwordless root shell (bypasses `sulogin`, which on this Red-Hat-family box would otherwise demand the root password even in single-user mode), then `mount -o remount,rw /` and `passwd root`. Reboot with `reboot -f` or, if that hangs, `echo b > /proc/sysrq-trigger`. -- **Accounts:** `root`, `tpsys` (TPSys application user, being added to `wheel` + scoped `NOPASSWD` sudo for the app-launch command), `tpspool` (TPSys spool user), `postgres` (uid 500, local TPSys database). -- **No prior credential existed** for this machine in vault or wiki before 2026-07-04 — it was undocumented until discovered at the physical console. - -### M365 / Entra -- **M365 admin:** sysadmin@dataforth.com — vault: `clients/dataforth/m365.sops.yaml` -- **Tenant ID:** `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` -- **Claude-Code-M365 Entra App:** App ID `7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29`, secret expires 2027-12-22 — vault: `clients/dataforth/m365.sops.yaml → credentials.entra-app` -- **MSP remediation app suite:** MSP tenant `ce61461e-81a0-4c84-bb4a-7b354a9a356d` — tiered ComputerGuru apps (Exchange Operator `b43e7342` etc.), vault `msp-tools/computerguru-*.sops.yaml`. *(Old single app `fabb3421`/Claude-MSP-Access DELETED 2026-06-14 — AADSTS700016, do not use.)* -- **ComputerGuru tiered apps:** All 5 apps consented 2026-04-23. Exchange Operator SP (b43e7342) had Exchange Admin role added manually (gap in onboard-tenant.sh — not auto-assigned for Exch Operator). - -### MSP360 Managed Backup API -- **Vault:** `msp-tools/msp360-api.sops.yaml` (api.mspbackups.com, /api/Provider/Login) -- `cbb.exe` path on AD2: `C:\Program Files\Arizona Computer Guru\Online Backup\cbb.exe` -- Browse file backup: `cbb.exe list -a "ACG-Dataforth" -b -rp -path ""` - -### Dataforth Product API (Hoffman) -- **Vault:** `clients/dataforth/api-oauth.sops.yaml` -- Token URL: `https://login.dataforth.com/connect/token` -- Grant: `client_credentials`, Client ID: `dataforth.onprem.sync`, Scope: `dataforth.web` -- Token TTL: 1 hour -- Swagger: `https://www.dataforth.com/swagger/index.html` -- Endpoints: `GET /api/v1/TestReportDataFiles/{serial}` (per-model cert), `/bulk`, `/stats` - -### ESXi / Hypervisors -- ESXi-122: 192.168.0.122 — vault: `clients/dataforth/esxi-122.sops.yaml` -- ESXi-124: 192.168.0.124 — vault: `clients/dataforth/esxi-124.sops.yaml` - -### PBX -- Vault: `clients/dataforth/pbx.sops.yaml` (password corrected 2026-06-23 — see Infrastructure Servers table) -- SSH: `sangoma@192.168.100.2` - ---- - -## Patterns & Known Issues - -### Active Directory -- **No custom security groups** — only default Windows groups. Service accounts in OU=ServiceAccounts. -- **ClaudeTools-ReadOnly AD account** — purpose unclear. Investigate. -- **Ken Hoffman has two accounts** (khoffman + oemdata) — not consolidated. -- **jlohr account retained** — post-retirement (2026-03-31), kept enabled specifically to receive ntirety.com infrastructure notifications. Inbox rule forwards to mike@azcomputerguru.com. Do NOT disable. -- **Entra sync scope:** OU=SyncedUsers **and OU=Azure_Users** sync to Entra (cbell confirmed in OU=Azure_Users, synced — 2026-06-01; the prior "SyncedUsers only" note was incomplete). CompanyUsers OU does NOT sync. 38 stale TS-* test station accounts were cleaned from Entra 2026-03-27. - -### RDS / SAGE-SQL -- **RDS licensing:** Grace period reset 2026-05-06 by deleting GracePeriod registry key. Grace period expires again without proper CALs. Purchase RDS CALs (Per User mode, LicensingType=4). -- **TSGateway:** Disabled on SAGE-SQL (server not externally exposed at firewall). Do NOT re-enable without reason. -- **SSL cert:** Self-signed, subject `CN=sage-sql.intranet.dataforth.com`. Non-domain machines must manually import to Trusted Root + Trusted Publishers. -- **GPO cert distribution:** Not completed (AD2 SYSVOL write blocked from non-domain workstation). Pending. -- **Bitdefender GravityZone:** Managed AV on SAGE-SQL. Can block PowerShell execution — may need temporary disable for admin work. - -### Voice / Phones / FreePBX -- **Production phones VLAN:** 192.168.100.0/24. PBX at .196 / .2. All production phones live here. -- **Unifi default voice VLAN (192.168.1.0/24):** NOT used for production voice — phones landing here cannot reach PBX. Switch port misconfiguration symptom: phone shows wrong date/time (NTP failure) and no dial tone. (This subnet is however in active use for the mydata/SMT VLAN — see Network section; do not confuse the two purposes.) -- **D1-Server-Room port 1:** Controls lobby drop → must stay on VLAN 100. Reverted to default once before (2026-05-04 incident). -- **FirstDigital trunk — `qualify_frequency=0`:** FD's Sonus SBC ignores SIP OPTIONS keepalives. Setting `qualify=0` in the `pjsip` DB (id=1) prevents trunk from going Unavailable. **Do NOT revert to a non-zero qualify.** (Total phone outage 2026-06-08 was caused by FD SBC not answering OPTIONS, making trunk go Unavailable and blocking all INVITEs.) -- **PJSip.class.php line 504 patch must be re-applied** after any `fwconsole ma updateall`. It is wiped by FreePBX updates. Backup before each update (`PJSip.class.php.bak.`). -- **Do NOT port-forward the RTP range (10000-20000)** on the UDM for this trunk. A static RTP DNAT creates a conntrack collision with the PBX's outbound RTP — inbound works but outbound audio dies. SIP 5060 forward only (source-locked to 66.7.123.0/24). Current on_boot.d script (`30-freepbx-sip-forward.sh`) is SIP-only, correct. -- **Inbound SIP relies on `/data/on_boot.d/30-freepbx-sip-forward.sh`** — not a persistent UniFi UI rule. Must survive UDM reboot via the script. **Confirmed 2026-06-23: the risk window is broader than "survives reboot" — a mid-uptime UniFi controller provision/update can also flush the SIP DNAT** while the box never reboots, silently killing inbound calls until the script is manually re-run (`sh /data/on_boot.d/30-freepbx-sip-forward.sh`). Diagnostic: `iptables -t nat -S | grep -iE '5060|192.168.100.2'` on the UDM — empty output means the rule is gone. Recommend Mike add a UI port-forward as a belt-and-suspenders measure (not yet done as of 2026-07-04). -- **PBX sudo authorization gap:** `sangoma` is in the `sudo` group but sudo authorization on this distro appears not actually granted (a sudo-based liveness test can fail on authorization, not auth — do not use it to judge whether a password/credential is stale). - -### Exchange Online / Email -- **INKY PhishFence StopProcessingRules:** Kills all subsequent transport rules. Use inbox rules for per-mailbox forwarding, NOT transport rules. -- **Mailprotector CloudFilter:** Outbound delivery goes through Mailprotector. If a message is "Delivered" per Dataforth's outbound trace but never arrives, check Mailprotector (`/mailprotector skill`, `py mp.py messages ...`) — it may be held. The INKY "Annotation - Recipient Not Group Member" transport rule can route mail to Mailprotector's hold queue. -- **AutoForwarding blocked by default** (tenant outbound spam policy). If per-user forwarding needed, create scoped HostedOutboundSpamFilterPolicy for that sender with AutoForwardingMode=On. -- **Get-MessageTrace deprecated Sept 2025:** Use Get-MessageTraceV2 and Get-MessageTraceDetailV2 in Exchange PowerShell. - -### GuruRMM Agent Deployment -- **WebSocket auth bug (Issue #8):** enrolled_agents.agent_key_hash is never checked by ws/mod.rs. Workaround: after MSI install, overwrite registry `HKLM:\SOFTWARE\GuruRMM\AgentKey` with the site API key (not enrollment AgentKey), then restart service. -- **rmm-api.azcomputerguru.com must be grey-clouded** (DNS-only, not proxied) — Cloudflare proxy blocks WebSocket. Do NOT re-enable orange cloud. Gitea Issue #9. -- **No RMM agent possible on FC3-class legacy appliances (confirmed 2026-07-04, MYDATA TPSys):** the Linux agent installer requires glibc ~2.17+, kernel >=2.6.32, and installs a systemd unit. A box on glibc ~2.3.5 / kernel 2.6.16 / SysV init (Fedora Core 3 "Heidelberg", Nov 2004) fails all three floors simultaneously — do not attempt install; plan agentless monitoring (ICMP/TCP probe or SSH heartbeat from a host that can reach the appliance's VLAN, e.g. D2TESTNAS or the RMM server, since inter-VLAN routing from mydata to main LAN is open) instead. If this becomes a recurring need across other legacy appliances, file a `/feature-request` for legacy/appliance Linux monitoring support in GuruRMM. -- **Legacy Red-Hat-family recovery pattern:** if a box like this is ever locked out again with no vaulted credential, LILO `boot:` → `linux init=/bin/bash rw` gives a passwordless root shell and bypasses `sulogin` (which on this OS family would otherwise demand the root password even in single-user mode). Always check BOTH vault and wiki before assuming a machine has no documented credential — a prior session missed the wiki check here and had to be corrected. - -### Cross-Machine File Operations (Windows Domain) -- **Double-hop / WTS-impersonation blocks fresh UNC paths.** When running commands in GuruRMM `user_session` (or via SSH-through-another-server), the impersonated token carries no network credentials. `net use` and fresh `\\server\share` paths fail with Access Denied. -- **Workaround that works:** Run on the SOURCE machine in `user_session` and write to an **existing GPO-mapped drive** (e.g. Q: → `\\ad2\c-drive`). The existing mapping survives impersonation; fresh UNC does not. -- **Proven 2026-06-04 on HGHAUBNER:** local `D:\DF C-Drive` read + `Q:` write succeeded; AD2-side `user_session` copy and SSH-from-AD2 both failed. - -### AD2 SSH / VPN MTU -- **PMTU blackhole on GURU-5070 → AD2 SSH:** GURU-5070's OpenVPN adapter "Local Area Connection" (ifIndex 12, IP 192.168.6.2) defaults to MTU 1500. Tunnel path MTU is ~1424 (FD ping confirms). Over-MTU bulk TCP segments (SSH transfers, SCP) are silently dropped. Small interactive commands pass, creating a false appearance of "flaky VPN" or "SSH ban." -- **Fix (applied 2026-06-18):** `Set-NetIPInterface -InterfaceIndex 12 -AddressFamily IPv4 -NlMtuBytes 1400` on GURU-5070 via SYSTEM RMM agent. Registry-persistent but may reset on OpenVPN reconnect — verify with `Get-NetIPInterface -InterfaceIndex 12`. -- **Durable fix:** server-side `mssfix 1360` on the Dataforth OpenVPN server (or `push "tun-mtu 1400"`) — would auto-clamp all fleet clients, not just GURU-5070. -- **AD2 is NOT the target for SSH diagnosis** when SSH is the failing channel — use RMM instead. - -### AD2 Branch / Coordination -- **AD2 operates on the `ad2` git branch.** Fork is rebased from main + thin Dataforth-specific commits. Do NOT edit shared fleet files on `ad2` — conflicts on every sync. Dataforth context lives in `clients/dataforth/CLAUDE.dataforth.md`. -- **AD2 is coord-API isolated:** 172.16.3.30 is unreachable from Dataforth LAN. Coord messages, locks, and todos NEVER reach AD2. All inter-session coordination goes through git sync: committed handoff docs + `## Note for ` blocks. Do NOT use the coord skill for AD2. -- **sync.sh on AD2:** not fork-aware on the push step (always tries `main`); force-push manually: `git push --force-with-lease origin ad2` after rebasing. -- **New (2026-07-01): AD2's GuruRMM agent can be used as a live read-only ground-truth channel** by spawning a headless `claude -p` through it (`context:user_session`, detached, poll for a `DONE` marker) — bypasses the coord isolation for READ-ONLY investigation without waiting on git-sync round trips. Unset the stale machine-level `ANTHROPIC_API_KEY` env var first or auth fails. See [RMM-Spawned Claude](#rmm-spawned-claude-on-ad2) above. - -### Post-Ransomware Recovery Restore (2025) — Incomplete File Migration -- **The 10/1/2025 recovery restore was incomplete.** The `Restore plan 10/1/2025` (~3.4M files) migrated each share from the old `D:\` layout to the current `C:\Shares\...` layout on AD2 and dropped files in the process. Proven case: SP1366 MAQ20 Communications Module — each `PRINTOUTS FOR MANUFACTURING` folder for revisions E–H received only one file (the drill panel) when the backup contained ~6 files per revision. The 9/29/2025 file-level backup confirms the files existed before the restore. -- **Scope unknown.** Other folders across the 7 shares may have similar gaps. A full migration-gap audit is underway (WizTree both sides — see Active Work). The audit is **review-only** — no automatic restore, because some deletions were intentional and the HGHAUBNER backup is additive-only (includes Georg's personal files alongside corporate data). -- **Backup-side CSV** for diffing stored at AD2 `C:\ClaudeTools\clients\dataforth\WizTree_20260604184904.zip` (sensitive file list — keep off shares and off any publicly accessible directory). -- **AD2 D: drive is gone.** The old `D:\c-drive` data volume was repurposed as a mounted Windows install ISO during the rebuild. All share data now lives under `C:\Shares`. The historical file-level backup (bunch `faad5a67`) archived the data under `D:\c-drive\...` (pre-migration path) — reconcile paths accordingly. - -### Shares ACL State — All Open to All Staff -- **All 8 business shares grant access to every employee** via `Everyone`/`Domain Users` (FullControl on 4 shares, Modify on 3). No department-based security groups exist. Sensitive data — Payroll, OSHA records, Purchase Orders, Accounting/QuickBooks, Sage financials — is fully readable and writable by all domain users. -- **Remediation project in progress** (Shares & Permissions, started 2026-06-10). Phase 0 (discovery) complete. Phase 1 (client input/department matrix) pending email to Dan Center. Do not apply ACL changes until after client sign-off on the target model. Details: `clients/dataforth/docs/projects/shares-permissions/`. -- **Special shares excluded from remediation:** `test` (DOS/SMB1 guest — leave open); `webshare` (preserve `svc_testdatadb:Full`); `ITSvc` (Domain Computers needs Read); Sage app data path (restrict by group at the share, but keep the live UNC stable for the ERP/SQL). -- **Phase 2 target-state strawman (drafted 2026-06-22, pre-client-input):** `target-structure-draft-2026-06-22.md`. Inferred from the existing share/folder layout (which is already department-shaped) plus a client-facing render at `Dataforth-Shared-Drives-Plan.html`. Target = one logical tree: `Company\Departments\` (Engineering [+Test-Engineering], Manufacturing, Quality, Sales-Marketing, Shipping-Receiving, Purchasing, IT), a `Restricted\` branch with **broken inheritance / no Domain Users** (Accounting-Finance, Payroll, HR, OSHA, Purchase-Orders), a read-mostly `Company-Wide\`, per-user `Users\`, and read-only `Archive\`. ABE on. Groups named `SG--`; users get **Modify** via the RW group (never Full), SYSTEM/Administrators keep Full. -- **Drive-letter strategy — Option A recommended:** keep current Q/S/T/W/Y/B mappings and realize the tree *logically* (reorg folders within each share + apply groups) for the first rollout — lowest disruption, no app/UNC breakage, no retraining. Hold physical consolidation to one `Company` drive (Option B) as a later optional phase after a hard-coded-UNC-path audit (DOS, Sage, datasheet pipeline, GageTrak/Epicor). The permission model is identical either way. -- **Strawman is NOT a build order — six items still gate Phase 2 sign-off (need the client):** confirm the inferred department list; the per-department RW/RO/none access matrix; named access for sensitive data (Payroll/OSHA/POs/Accounting — likely HR/Finance sign-off, not just Dan); department rosters to populate groups; legacy cleanup approval (person-named / "Do not use" folders); and an Engineering destination volume (AD1 C: ~90% full blocks any ENGR restructure). - -### DOS Test Station Data Pipeline (new, 2026-07-01 ground-truth audit) -- **Root cause of Syncro #32489 confirmed (F1, HIGH):** the deployed inbound spec downloader `T:\COMMON\ProdSW\NWTOC.BAT` v5.0 (mirrored to NAS `COMMON/ProdSW/NWTOC.BAT`) copies only `*.BAT` and `*.EXE` from the NAS to stations — **zero `.DAT` files**. Its own changelog header says so verbatim: "Added EXE copy, removed DATA folder copies (avoid cyclic overwrites)". No version of NWTOC, past or present, has ever distributed the shared `COMMON` master specs — the fix must ADD a data copy, not "restore" one. Engineering masters (e.g. `5BMAIN.DAT`, updated 2026-06-26) reach the NAS fine; they simply never reach the stations. -- **New risk found (F2, HIGH, needs on-station confirmation):** the deployed `NWTOC.BAT` v5.0 and `CTONWTXT.BAT` v2.3 use `COPY /Y`. `/Y` is **not a valid MS-DOS 6.22 switch** (introduced in MS-DOS 7.0/Windows 95); 6.22's `COPY` supports only `/A /B /V`. If the stations run genuine 6.22, `COPY /Y` returns `Invalid switch - /Y` and copies nothing — meaning NWTOC has been silently copying NOTHING (not even the .BAT/.EXE files it's supposed to) since it was deployed 2026-03-16. This was independently confirmed by Grok in verify mode. The pivotal unresolved question — whether the stations run true 6.22 or MS-DOS 7.x — is empirical and can only be checked on a station itself (`VER`, then `COPY /Y NUL C:\TEST.TXT`); stations have no RMM agent. The upload path (`CTONW.BAT` v5.0) is unaffected — it uses plain `COPY` — which is why test data has kept flowing even if NWTOC is dead. -- **Do NOT judge DOS-6.22 compatibility from the root-level `test\{NWTOC,CTONW,CHECKUPD,STAGE}.BAT` v1.x files** — those are abandoned drafts riddled with NT-only constructs (`FOR /F`, `SET /A`, `CALL :label`, `2>NUL` stderr redirection, tilde path modifiers). The scripts that actually run on stations live under `COMMON\ProdSW\` and were deliberately cleaned of those constructs (except for the `/Y` issue above). -- **F3 (MED):** `C:\Shares\test\TS-21\ProdSW` is a stray **file** (a misplaced `7BMAIN4`-type EXE), not the directory rsync expects — this makes the 15-minute AD2↔NAS sync report `ERRORS` on every single run (`exit 3`, "Not a directory"), masking any genuine new failure. Fix: remove/relocate the file, harden the push loop to skip non-directory `ProdSW` paths. -- **F4 (MED):** Server-side datasheet generation (`spec-reader.js`) reads specs from `testdatadb\specdata\`, which is a **frozen 2026-03-27 snapshot** — not the live engineering masters. Any spec limit changed after that date is not reflected in generated datasheets even though the outbound data pipeline itself is healthy. -- **F5 (MED, security):** Plaintext credentials found hard-coded in scripts on AD2 — rsync daemon password in `Sync-FromNAS-rsync.ps1`, NAS root SSH password in the dormant `Sync-FromNAS.ps1`, and the Postgres `testdatadb_app` password in `testdatadb\database\db.js`. Flagged for rotation + proper vaulting; not yet remediated as of 2026-07-04. -- **Stale-assumption corrections established by this audit** (do not carry forward the old versions): datastore is **PostgreSQL 18**, not SQLite (the SQLite file is a 4.4 GB archive from the 2026-04-03 migration cutover); the scheduled sync task runs **`Sync-FromNAS-rsync.ps1`**, not the dormant per-file SCP script; web delivery is a live **HTTP API uploader** (`upload-to-api.js`, 472,290 records flagged as of 2026-07-01), not the dead `For_Web`/ASP.NET path (dead since 2026-05-11); `CTONWTXT.BAT` IS actively invoked (called from `CTONW.BAT` line 30), contradicting an earlier assumption that it was a gap. -- Full report: `clients/dataforth/docs/audits/2026-07-01-test-data-chain-audit-AD2.md`. Recommendations in that report are proposals only — nothing has been applied (the audit was strictly read-only). - -### Hardware / Endpoint — Aging Fleet -- **DFORTH-Ship recurring TDR BSOD (`0x116 VIDEO_TDR_FAILURE`), diagnosed 2026-06-25:** integrated Intel HD Graphics 4600 on driver 20.19.15.5126 (Intel's final driver for that part, dated 2020-01-20) hitting the GPU-reset timeout on an 11.5-year-old HP EliteDesk 800 G1 USDT (BIOS 2014-12-10). Five minidumps span 2025-11-03 through 2026-06-24 with an **accelerating cadence** — treat as a degrading-hardware trend, not a one-off. Mitigation applied: Edge hardware acceleration disabled via machine policy (`HKLM\SOFTWARE\Policies\Microsoft\Edge\HardwareAccelerationModeEnabled = 0`). No durable fix is possible for integrated graphics (nothing to reseat/replace) — **PC replacement is the real fix**; thermal cleaning of the USDT chassis is a secondary mitigation worth doing regardless. Do not confuse with near-twin host **DForth-Shipp** — verify the exact agent ID before acting. - -### Security -- **C2 IP blocks are iptables only** — do not survive UDM reboot. Must add to permanent UniFi block list via UI. C2 IPs: 80.76.49.18, 45.88.91.99 (AS399486 Virtuo, Montreal). -- **AD1 disk 90% full** — C:\Engineering = 787 GB of 1023 GB. Risk of replication failures. -- **Windows Firewall disabled on AD2** (all profiles) — known risk, not yet remediated. -- **3 Windows 7 machines on network** (LABELPC, LABELPC2, D2-RCVG-003) — EOL, unpatched. -- **AD1/AD2 on Windows Server 2016 / 2019 respectively** — approaching/at end of mainstream support. Plan upgrade. -- **Entra ID P2 not licensed** — IdentityRiskyUser risk check returns 403 even with scope consented. Would need P2 upgrade to enable Identity Protection. -- **IdentityRiskyUser.Read.All scope:** Consented to Security Investigator app but unusable (no P2 license). -- **Plaintext credentials in Dataforth test-data-chain scripts (F5, 2026-07-01):** rsync daemon, NAS root, and Postgres app passwords are hard-coded in scripts under `C:\Shares\test\scripts\` and `testdatadb\database\db.js`, world-readable via the `test` SMB share. Rotation + vaulting recommended, not yet done. -- **Legacy appliance with no prior credential (MYDATA TPSys, 2026-07-04):** an entire production SMT-line controller existed with no vault or wiki entry until physically discovered. Root password reset via LILO recovery; now vaulted. Worth a broader sweep for other undocumented devices on VLAN 2 "mydata" given 3 unnamed industrial MACs were already known but unresolved as of 2026-06-01. - -### Syncro Asset Management -- **Fleet-wide Syncro agent break ~2025-10-06:** ~half of Dataforth machines stopped reporting to Syncro on or around that date while remaining online in ScreenConnect. Do NOT auto-remove machines frozen at that date without cross-checking ScreenConnect. Root cause unknown — needs investigation. -- **Bitdefender is NOT a liveness signal:** Dataforth is being phased off BD; 53 of 57 GravityZone endpoints are in the "Deleted" folder. Missing from BD = BD agent uninstalled, not machine dead. -- **API delete not available:** `DELETE /customer_assets/{id}` returns HTML 404 for the current integration token. All asset deletions must go through the Syncro GUI. - -### `staff` Share Missing -- The `staff` network share is absent from FILES-D1 (only `archive` and `sales` exist). HGHAUBNER's backup includes a `DF Staff` folder, suggesting the share existed pre-attack. Not in scope for the current migration-gap diff — separate issue requiring investigation. - ---- - -## Active Work - -As of 2026-07-04 (0 open Syncro tickets per live pull): - -- **DOS Test Station Data Pipeline (Syncro #32489, active):** Root cause confirmed 2026-07-01 via a read-only ground-truth audit run through a headless Claude spawned on AD2's GuruRMM agent (new capability — see [RMM-Spawned Claude](#rmm-spawned-claude-on-ad2)). F1 (NWTOC v5.0 never copies master `.DAT` specs to stations) is confirmed; F2 (`COPY /Y` may not be valid on true MS-DOS 6.22) needs a station-side check before scoping the fix. **Next steps:** (1) confirm station DOS version (`VER` + `COPY /Y NUL C:\TEST.TXT` on a station), (2) draft a DOS-6.22-safe `NWTOC v5.1` that adds a one-way pull of master `.DAT`s (plain `COPY`, no `/Y` if 6.22 confirmed) without reintroducing the cyclic-overwrite problem v5.0 was avoiding, (3) Grok-review the new script before it touches a station, (4) update ticket #32489 with the confirmed root cause and plan. Secondary cleanup items from the same audit (not urgent): remove the stray `TS-21\ProdSW` file (F3), feed `testdatadb\specdata\` from live engineering masters (F4), rotate/vault the plaintext creds found in scripts (F5), retire dead `For_Web` output and abandoned v1.x script drafts (F6/F7/F8). - -- **MYDATA TPSys SMT controller (new, discovered 2026-07-04):** Root password reset via LILO recovery and **vaulted 2026-07-04** at `clients/dataforth/mydata-smt.sops.yaml` (host/VLAN/OS/accounts/recovery-method documented; decrypt-verified). **Outstanding:** (1) **IP confirmed 192.168.1.1 (2026-07-04) — but this collides with the recorded VLAN 2 gateway; reconcile whether this box is the SMT-line gateway/router or there is an IP conflict with the UDM;** (2) verify the `tpsys` wheel-group + scoped `NOPASSWD` sudoers change actually landed (`id tpsys`, `sudo -l` as tpsys); (3) get the exact TPSys app-launch command from Howard/Mike to finalize the sudoers scope; (4) confirm the controller booted cleanly into TPSys after the forced reboot (it is a live production SMT line); (5) decide and stand up agentless monitoring (ICMP/TCP probe or SSH heartbeat from D2TESTNAS or the RMM server — inter-VLAN routing to mydata is open) since a GuruRMM agent is impossible on this OS; formalize via `/feature-request` if Mike wants legacy/appliance Linux monitoring as a standing GuruRMM capability. - -- **DFORTH-Ship BSOD (ongoing monitoring):** Edge hardware-acceleration mitigation applied 2026-06-25; needs on-site Edge restart/reboot to take effect, verify at `edge://policy`. Monitor for recurrence — if it bugchecks again, pull and analyze the four older dump signatures to confirm whether it is drifting toward a hard hardware fault. Schedule thermal cleaning of the USDT chassis. Recommend/plan replacement of the 11.5-year-old EliteDesk 800 G1 USDT shipping station as the durable fix. - -- **UDM inbound SIP DNAT (recurring risk, unresolved):** Confirmed again 2026-06-23 that the SIP 5060 DNAT can be flushed by a UniFi controller provision mid-uptime, not only at reboot. Coord to-do `45572ee1` tracks the durable fix (persistent UI port-forward rule or a cron/watcher re-running the idempotent on_boot.d script) — needs a maintenance window. Still SIP-only; never forward the RTP range. - -- **Shares & Permissions project (Phase 1 — BLOCKING, pending client input):** Phase 0 (discovery) completed 2026-06-10 — read-only ACL audit confirmed all 8 business shares open to all employees; Domain Users has FullControl on 4 shares. Discovery email to Dan Center drafted (`clients/dataforth/docs/projects/shares-permissions/discovery-email-draft.md`); **not yet sent — recipients/sender not locked** (Dan Center primary; CC Kevin Wackerly?; Mike or Howard sending?). Phase 1 blocked on client responses: department list, access matrix, sensitive-data rules, staff rosters. A **Phase 2 target-state strawman was drafted 2026-06-22** (`target-structure-draft-2026-06-22.md` + client-facing `Dataforth-Shared-Drives-Plan.html`) from the existing layout — see [Shares ACL State](#shares-acl-state--all-open-to-all-staff); it still needs the Phase 1 client matrix to finalize. Next-step options: polish the client HTML, finalize + send the discovery email to unblock Phase 1, or refine the internal strawman. Full roadmap: `clients/dataforth/docs/projects/shares-permissions/roadmap.md`. - -- **8B/5B/SCM render completion (parked with AD2):** Root-caused a `parseRawData` bug (PASS/FAIL line consumed as step-response for families that omit `"0","0",v` line). 136 8B/5B/SCM templates mined from Hoffman API (2026-06-18). Completion — wiring templates into the live renderer with correct slotmaps, QB rounding, and frequency/AAC accuracy — handed to AD2 (its now-proven machinery from DSCA33/45 work). Sync handoff at `projects/dataforth-dos/8B5BSCM-RENDER-VERIFY-2026-06-18.md`. ~9,624 records remain unpublished; this is a render-coverage gap (null renders correctly skipped), not a backlog. - -- **Migration-gap audit (parked):** WizTree CSV of HGHAUBNER's pre-attack backup captured (AD2 `C:\ClaudeTools\clients\dataforth\WizTree_20260604184904.zip`). WizTree runs on live servers deferred — no diff yet. Plan: run WizTree on AD2, FILES-D1, SAGE-SQL, AD1 → diff CSV-to-CSV per share → `clients/dataforth/migration-gap-catalog-2026-06-04.md`. Full plan in `clients/dataforth/migration-gap-diff-RESUME.md`. No auto-restore — review-only catalog. - -- **Syncro asset cleanup (with Howard):** 78-asset reconciliation complete. 28 confirmed-dead assets pending GUI deletion; 21 alive-but-broken machines need Syncro agent reinstall; 9 servers in VERIFY bucket. Move to metered billing once clean. Coord todo tree assigned to Howard (parent `103c48ad-7b31-4967-9388-065a91888e7c`). See [Syncro Asset Inventory](#syncro-asset-inventory-2026-06-02-reconciliation) above. - -- **AOI XP backup + isolation (ongoing):** AOI optical-inspection XP PC on VLAN 2 (mydata/SMT) @ 192.168.1.175; locked-down SMB1 share `aoibackup` on D2TESTNAS (XP-only, user `admin`). Other NAS shares now deny the XP. **Optional EOL hardening pending:** block XP → company LAN (except NAS 192.168.0.9) + Internet on the UDM, scoped to .175. Todo `37543f7f`. - -- **AD2 Claude capability updates (parked):** AD2 runs its own Claude from `C:\ClaudeTools` on the `ad2` branch. Needs: (a) syncro + coord commands, (b) DF wiki read-write, (c) Dataforth client data access. Python 3.12.8 and identity.json installed 2026-06-17. Coord API unreachable from Dataforth LAN — comms via git sync only, though the 2026-07-01 RMM-spawn pattern now offers a read-only side channel for investigation. - -- **Power Monitor SPA demo (parked):** Georg Haubner developed a vanilla-JS power-meter SPA (AI-built, `clients/dataforth/ExternalCodeReview.zip`). ACG designed a gateway architecture for a gated demo at `PWM.dataforth.com` (inbound tunnel, no meter publicly exposed, magic-link auth). Spec at `clients/dataforth/power-monitor-demo/GATEWAY-SPEC.md`. Parked pending Mike↔Georg conversation. - -- **Test Datasheet Pipeline:** - - Production pipeline healthy — outbound (station → NAS → AD2 → Postgres → web) confirmed current as of 2026-07-01 (import as recent as 13:41 UTC same day). 475K+ records, DSCA33/45 recovery complete (1,452 new certs published 2026-06-18 via Hoffman API). - - Inbound spec/software distribution to stations is broken — see DOS Test Station Data Pipeline pattern above and Syncro #32489. - - Email notifications deployed (Graph API via `sysadmin@dataforth.com`). - - 8B/5B/SCM render gap — parked with AD2 (see above). - - 2 niche DSCA models (DSCA33-1948, DSCA45-1746) and their 8B equivalents have no Hoffman original — no template, cannot auto-publish. - - DKIM: cutover to selector2 on 2026-05-16 — no action needed; verify signing after that date. - -- **GAGEtrak email (ticket #32142):** calibration@ SMTP re-enabled 2026-04-23. GAGEtrak configured (smtp.office365.com:587, calibration@dataforth.com). Kevin Wackerly verifying schedule — expected Monday run appears to run Tuesday. - -- **jlohr forwarding:** ntirety.com inbox rule active as of 2026-05-12; confirmed delivering to mike@azcomputerguru.com. Defunct transport rule pending cleanup. - -- **RDS / SAGE-SQL:** RDS grace period reset. GPO cert distribution pending. RDS CALs purchase needed long-term. - -- **MFA enforcement ongoing** — 19 users were not enrolled as of April 4 enforcement date; current enrollment count unverified. - -- **C2 IP blocks need permanence:** Iptables rules on UDM (80.76.49.18, 45.88.91.99) need to be added to permanent UniFi UI block list. - ---- - -## History Highlights - -| Date | Event | -|---|---| -| 2025 | Crypto/ransomware attack — AD2 wiped and rebuilt, many files lost. Test datasheet pipeline broken. | -| 2025-08-29 – 2025-09-29 | MSP360 file-level backup (`faad5a67`) covering DF shares at old `D:\c-drive\...` path. Last snapshot before the recovery restore. | -| 2025-10-01 – 2025-10-02 | Post-ransomware recovery restore (`Restore plan 10/1/2025`, ~3.4M files) migrated shares from `D:\` to `C:\Shares\...` on AD2. Restore was incomplete — files dropped in multiple folders (root cause: restore tool gap, not user deletion). AD2 `C:\Shares` tree NTFS creation timestamp confirms this date. | -| ~2025-10-06 | Fleet-wide Syncro agent break — ~half of Dataforth machines freeze in Syncro while remaining online in ScreenConnect. Root cause unknown. | -| 2026-01-19 | DOS Update System built and deployed — NWTOC/CTONW/UPDATE/DEPLOY BAT files, 39 deployments. Sync-FromNAS updated (DEPLOY.BAT). | -| 2026-03-20 | Galactic Advisors security assessment — AD1 C: at 90%, legacy SQL 2008 R2 client noted, 3 computers scanned. | -| 2026-03-23 | Galactic Advisors assessment analyzed by ACG. | -| 2026-03-27 | **Major security incident:** DF-JOEL2 compromised via social engineering/ScreenConnect (attacker "Angel Raya", C2 on Virtuo hosting). M365 sign-in from Turkey. Full remediation. 3 CA policies deployed. MFA notice sent. IC3 filed (1c32ade367084be9acd548f23705736f). | -| 2026-03-27–29 | Test datasheet pipeline rebuilt — 72/73 Quatronix datasheets generated, new Node.js pipeline replaces VB6 DFWDS + VB.NET uploader. | -| 2026-03-31 | Joel Lohr retirement. Brian Faires mailbox converted to shared (5,711 messages preserved). 38 stale Entra TS-* accounts deleted. | -| 2026-04-04 | MFA CA policies enforced (switched from report-only). | -| 2026-04-11–12 | SCMVAS/SCMHVAS pipeline extension — 27,503 records backfilled, 434 Engineering-Tested .txt files imported. | -| 2026-04-12 | TestDataDB PostgreSQL migration verified (2.89M records). Hoffman API discovered (Swagger). | -| 2026-04-13 | API architecture discussion with Hoffman — client_credentials grant confirmed for dataforth.onprem.sync client. | -| 2026-04-14 | DFWDS logic ported to Node.js (dfwds-process.js). 897 staged datasheets drained. 803 new records created on Hoffman API. | -| 2026-04-15 | Major release — DB dedup (2.89M→469K rows), FAIL→PASS retest rule, For_Web filesystem dependency eliminated, 170,984 records bulk-pushed to Hoffman. Dashboard UI upgrades. | -| 2026-04-23 | Full Dataforth tenant onboarded to all 5 ComputerGuru tiered apps. calibration@ SMTP AUTH fixed. DF-GAGETRAK GuruRMM agent enrolled (with auth workaround). Syncro ticket #32142 billed. | -| 2026-05-03 | jantar@dataforth.com darkweb breach check — no indicators of compromise. eM Client OAuth grant and SP revoked/disabled. 1 hr billed. | -| 2026-05-04 | Howard onsite — lobby phone offline (VLAN misconfiguration on D1-Server-Room port 1 → fixed to VLAN 100). | -| 2026-05-06 | SAGE-SQL RDS issues resolved — grace period reset, SSL cert replaced, TSGateway disabled, RemoteApp permission prompts fixed. | -| 2026-05-12 | Pipeline audit + email notifications implemented (Graph API). jlohr forwarding configured (ntirety.com → mike@). DKIM keys rotated. | -| 2026-06-01 | AOI optical-inspection XP PC isolated onto VLAN 2 (mydata/SMT) @ 192.168.1.175; `aoibackup` SMB1 share created on D2TESTNAS locked to the XP only; other NAS shares set to deny the XP. D2TESTNAS confirmed Debian 13 / Samba 4.22.6 (repurposed Netgear ReadyNAS); vault + wiki OS corrected. | -| 2026-06-01 | Chauncey Bell (cbell) M365 verified — active mailbox, licensed M365 Business Standard; AD password reset on AD2 (synced user, OU=Azure_Users), signed into Office. Bobbi's Outlook printing fixed. Ticket #32364 (0.5 hr onsite). | -| 2026-06-02 | Syncro asset reconciliation (78 assets): 20 keep / 21 save+flag / 28 remove / 9 verify. Root cause identified: fleet-wide Syncro agent break ~2025-10-06 silenced ~half the fleet while boxes stayed online (visible in ScreenConnect). Dataforth confirmed phasing off Bitdefender. Cleanup list handed to Howard. | -| 2026-06-04 | SP1366 MAQ20 manufacturing print recovery — 19/20 PDFs for revisions E–H restored to AD2 from HGHAUBNER's pre-attack backup via GuruRMM user_session + GPO-mapped Q: drive. Root cause of loss: incomplete 10/1/2025 recovery restore. Syncro #32385, 1.0 hr remote, prepaid $0, resolved. GuruRMM fleet grew 13 → 45 agents. WizTree backup-side CSV captured for migration-gap diff (deferred). | -| 2026-06-05 | AD1 Files backup plan created via GuruRMM remote command (cbb.exe, NBF, 180-day retention, daily 2 AM, covers C:\Engineering + C:\Shares\ITSvc). AD1 now has both image and file plans matching AD2. | -| 2026-06-05 | **Mailprotector CloudFilter discovered** as Dataforth's outbound delivery layer (atop INKY + Exchange Online). Email from Georg Haubner was held by Mailprotector due to INKY "Annotation" transport rule. Released manually. New `/mailprotector` skill built and committed. | -| 2026-06-05 | Georg Haubner's Power Monitor SPA analyzed (vanilla-JS, AI-built). Gateway architecture designed for PWM.dataforth.com demo. Parked pending Mike↔Georg conversation. | -| 2026-06-08–09 | **Total Dataforth phone outage.** Outbound failed (FirstDigital SBC ignoring OPTIONS → trunk Unavailable); inbound never worked (no SIP port-forward existed). Fixed: `qualify_frequency=0` in pjsip DB; `PJSip.class.php` line 504 re-patched; `/data/on_boot.d/30-freepbx-sip-forward.sh` added (SIP-only DNAT, source-locked 66.7.123.0/24). Two-way audio verified. UDM vault corrected. Syncro #32392, 1.0 hr emergency (×1.5 rate) remote, prepaid. | -| 2026-06-10 | **Shares & Permissions Phase 0 complete.** Read-only ACL audit of all 8 business shares: all grant Domain Users/Everyone Full or Modify; no department security groups exist; Payroll/OSHA/PO/accounting data open to all employees. Phase 1 (client input) pending discovery email to Dan Center. | -| 2026-06-17 | AD2 identity.json + Python 3.12.8 installed. `CLAUDE.dataforth.md` created for AD2 context file (relocated from in-line `.claude/CLAUDE.md` edits to maintain clean fork). | -| 2026-06-18 | **DSCA33/45 certs recovered via Hoffman API** — 56 model templates mined, 1,452 new DSCA33/45 certs published on AD2 (0 overwrites). Root-caused `parseRawData` bug affecting 8B/5B/SCM families. 136 8B/5B/SCM templates mined from Hoffman and handed to AD2 for wiring. TestDataDB UI redesigned and deployed on AD2 (cert-fit, publish chips, push toasts, full-screen inspector). AD2 SSH PMTU blackhole diagnosed (GURU-5070 adapter MTU 1500 vs tunnel ~1424) and fixed (MTU 1400). Syncro #32441. | -| 2026-06-22 | **Shares & Permissions Phase 2 target-state strawman drafted** — proposed `Company\Departments\…Restricted\…Company-Wide\…Users\…Archive\` tree with `SG--` groups, current→target migration map, and Option-A (keep drive letters) rollout, all inferred from the existing layout. Internal draft + client-facing HTML render. Phase 1 client input still gates sign-off. | -| 2026-06-23 | **PBX no-inbound-calls emergency fixed.** UDM SIP 5060 DNAT was completely absent — flushed by a UniFi controller provision (not a reboot), confirming the on_boot.d re-apply script's coverage gap is broader than previously understood. Re-ran `/data/on_boot.d/30-freepbx-sip-forward.sh`, verified DNAT + forward-accept restored, inbound confirmed working end-to-end. Vault `pbx.sops.yaml` password corrected (was backslash-corrupted from shell-escaping leaking into storage). Syncro #32450, 1.5 hr emergency (×1.5 rate) remote, invoiced, block debited 31.5 → 30.0 hrs. Durable-fix coord to-do `45572ee1` filed. | -| 2026-06-24–25 | **DFORTH-Ship recurring BSOD diagnosed.** Stop code `0x116 VIDEO_TDR_FAILURE` on integrated Intel HD Graphics 4600 (final 2020 driver) — an 11.5-year-old HP EliteDesk 800 G1 USDT with an accelerating crash cadence (5 dumps since 2025-11-03). Mitigated by disabling Edge hardware acceleration via machine policy; PC replacement recommended as the durable fix; thermal cleaning flagged as a secondary measure. | -| 2026-07-01 | **Test-data-chain ground-truth audit** via a headless Claude spawned through AD2's GuruRMM agent (new capability, bypassing AD2's coord-API isolation for read-only investigation). Confirmed root cause of Syncro #32489: deployed `NWTOC.BAT` v5.0 never copies master spec `.DAT` files to stations (removed by design in the v5.0 changelog). New HIGH finding: `NWTOC.BAT`/`CTONWTXT.BAT` use `COPY /Y`, not a valid MS-DOS 6.22 switch — pending station-side DOS-version confirmation before scoping the fix (independently confirmed by Grok). Corrected several stale assumptions: datastore is PostgreSQL 18 (475,553 test_records), the sync task runs `Sync-FromNAS-rsync.ps1`, web delivery is a live HTTP API uploader (472,290 records flagged). Also found: a stray file breaking the AD2↔NAS rsync push on every run (masking real errors), a frozen 2026-03-27 server-side spec snapshot, and plaintext credentials hard-coded in three scripts (flagged for rotation, not yet fixed). | -| 2026-07-04 | MYDATA TPSys SMT controller (myserver, FC3/VLAN2) discovered + root recovered via LILO single-user; vaulted; RMM agent ruled out (legacy glibc/kernel/no-systemd). | - ---- - -## Backlinks - -- [[projects/dataforth-dos]] — Active test datasheet pipeline project on AD2 -- [[systems/jupiter]] — Neptune Exchange physically colocated at Dataforth D2 facility; D2TESTNAS provides Tailscale routing +--- +type: client +name: dataforth +display_name: Dataforth Corporation +last_compiled: 2026-07-14 +compiled_by: HOWARD-HOME/claude-main +sources: + - clients/dataforth/docs/overview.md + - clients/dataforth/docs/active-directory.md + - clients/dataforth/docs/workstations.md + - clients/dataforth/docs/manufacturing.md + - clients/dataforth/docs/billing-log.md + - clients/dataforth/docs/SYNC_SCRIPT_UPDATE_SUMMARY.md + - clients/dataforth/docs/incident-2026-03-27-abuse-report-virtuo.md + - clients/dataforth/docs/incident-2026-03-27-abuse-report-connectwise.md + - clients/dataforth/docs/cloud/m365.md + - clients/dataforth/docs/issues/log.md + - clients/dataforth/docs/network/topology.md + - clients/dataforth/docs/network/vlans.md + - clients/dataforth/docs/network/firewall.md + - clients/dataforth/docs/rmm/rmm.md + - clients/dataforth/docs/security/antivirus.md + - clients/dataforth/docs/security/backup.md + - clients/dataforth/docs/servers/ad1.md + - clients/dataforth/docs/servers/ad2.md + - clients/dataforth/docs/servers/d2testnas.md + - clients/dataforth/docs/servers/df-hyperv-b.md + - clients/dataforth/docs/servers/files-d1.md + - clients/dataforth/docs/servers/sage-sql.md + - clients/dataforth/docs/projects/shares-permissions/roadmap.md + - clients/dataforth/docs/projects/shares-permissions/current-state-2026-06-10.md + - clients/dataforth/docs/projects/shares-permissions/acl-audit-detail-2026-06-10.md + - clients/dataforth/docs/projects/shares-permissions/discovery-email-draft.md + - clients/dataforth/docs/projects/shares-permissions/target-structure-draft-2026-06-22.md + - clients/dataforth/docs/aoi-xp-vlan-backup-runbook.md + - clients/dataforth/session-logs/2026-03-23-galactic-advisors-report.md + - clients/dataforth/session-logs/2026-03-27-security-incident-mfa-datasheets.md + - clients/dataforth/session-logs/SESSION-SUMMARY.md + - clients/dataforth/session-logs/MEMORY.md + - clients/dataforth/session-logs/2026-04-12-session.md + - clients/dataforth/session-logs/2026-04-13-session.md + - clients/dataforth/session-logs/2026-04-14-session.md + - clients/dataforth/session-logs/2026-04-23-session.md + - clients/dataforth/session-logs/2026-05-03-session.md + - clients/dataforth/session-logs/2026-05-04-lobby-phone-vlan-fix.md + - clients/dataforth/session-logs/2026-05-06-session.md + - clients/dataforth/session-logs/2026-05-12-session.md + - clients/dataforth/session-logs/2026-06-01-aoi-xp-vlan-share.md + - clients/dataforth/session-logs/2026-06-01-cbell-m365-bobbi-outlook.md + - clients/dataforth/session-logs/2026-06-02-session.md + - clients/dataforth/session-logs/2026-06-04-session.md + - clients/dataforth/session-logs/project_ad2_context.md + - clients/dataforth/session-logs/project_pipeline_rebuilt.md + - clients/dataforth/session-logs/project_test_datasheet_pipeline.md + - clients/dataforth/session-logs/project_new_product_lines.md + - clients/dataforth/migration-gap-diff-RESUME.md + - clients/dataforth/CLAUDE.dataforth.md + - projects/dataforth-dos/CONTEXT.md + - session-logs/2026-06-05-session.md + - session-logs/2026-06/2026-06-09-mike-dataforth-freepbx-safesite-forensics.md + - session-logs/2026-06/2026-06-18-mike-testdatadb-render-and-security-app.md + - session-logs/2026-06/2026-06-29-mike-dataforth-nwtoc-pst-deletion-scope-birthbio-corruption.md + - session-logs/2026-07/2026-07-13-howard-reliant-phones-dataforth-rdp-signing.md + - .claude/memory/project_dataforth_incident_2026-03-27.md + - .claude/memory/project_datasheet_pipeline.md + - .claude/memory/project_neptune_sbr_email_routing.md + - .claude/memory/reference_dataforth_contact.md + - .claude/memory/reference_neptune_access_d2testnas.md + - .claude/memory/feedback_d2testnas_ssh.md + - .claude/memory/infra_office_network.md + - .claude/memory/project_dataforth.md + - .claude/memory/project_dataforth_history.md + - .claude/memory/project_ad2_dataforth_fork.md + - .claude/memory/ad2-ssh-mtu-blackhole.md + - .claude/memory/ad2-comms-via-sync-only.md + - clients/dataforth/session-logs/2026-06/2026-06-23-mike-pbx-no-inbound-calls-fix.md + - clients/dataforth/session-logs/2026-06/2026-06-25-howard-dforth-ship-tdr-bsod.md + - clients/dataforth/session-logs/2026-07/2026-07-01-mike-dataforth-test-data-chain-audit.md + - clients/dataforth/session-logs/2026-07/2026-07-04-howard-mydata-tpsys-smt-controller-access.md + - clients/dataforth/docs/audits/2026-07-01-test-data-chain-audit-AD2.md + - clients/dataforth/session-logs/2026-07/2026-07-05-mike-dos-spec-fix-telemetry-udm.md + - clients/dataforth/session-logs/2026-07/2026-07-06-mike-sage-sql-peter-session-reset.md + - clients/dataforth/session-logs/2026-07/2026-07-06-winter-delfina-lopez-new-hire-account.md + - clients/dataforth/session-logs/2026-07/2026-07-10-mike-haubner-ltobey-address-invalid.md + - clients/dataforth/session-logs/2026-07/2026-07-14-howard-sage-printing-guru-user.md + - clients/dataforth/session-logs/2026-07/2026-07-14-howard-pbx-inbound-dnat-flush-fix.md + - .claude/memory/reference_rmm_spawn_headless_claude.md +backlinks: + - projects/dataforth-dos + - systems/jupiter +--- + +# Dataforth Corporation + +Signal conditioning / data acquisition manufacturer in Tucson, AZ. Long-standing ACG client. Active managed relationship — monthly prepaid block. Notable for 64 MS-DOS 6.22 test stations, a major security incident in March 2026, an ongoing test datasheet pipeline modernization project (DOS spec distribution fixed fleet-wide 2026-07-05), an incomplete 2025 post-ransomware recovery restore that silently dropped files across multiple shares (active audit underway), a shares/permissions remediation project (Phase 1 still pending client input), a legacy Fedora Core 3 SMT line controller discovered and recovered in July 2026, and Sage Pro 200 ERP running as an RDS RemoteApp on SAGE-SQL with RDP-redirected printing. + +--- + +## Profile + +- **Contract type:** Prepaid hour block (monthly replenishment invoice $2,098.87) +- **Hours remaining:** 24.75 hrs as of 2026-07-14 (live Syncro `GET /customers/578095` before billing) +- **Syncro customer ID:** 578095 +- **Syncro managed assets:** 50 +- **Billing rate:** All invoices $0.00 — hours drawn from prepaid block +- **Invoice CC:** jantar@dataforth.com + +**Key contacts:** + +| Name | Username | Role | Email | +|---|---|---|---| +| Dan Center | dcenter | Operations (primary IT contact) | dcenter@dataforth.com | +| John Lehman | jlehman | Engineering, QB code, test specs | jlehman@dataforth.com | +| Peter Iliya | pIliya | Applications Engineer | pIliya@dataforth.com | +| Georg Haubner | ghaubner | Engineering; D: drive on HGHAUBNER has pre-ransomware backup of all DF shares | ghaubner@dataforth.com | +| Kevin Wackerly | kwackerly | IT/Admin, handles calibration@ account | kwackerly@dataforth.com | +| Logan Tobey | ltobey | Support/Sales | ltobey@dataforth.com | +| Ben Wadzinski | bwadzinski | Engineering | — | +| Lee Payne | lpayne | Engineering | — | +| Theresa Dean | tdean | Admin | tdean@dataforth.com | +| Joel Lohr | jlohr | **RETIRED 2026-03-31** — account kept enabled; inbox rule forwards ntirety.com notifications to mike@azcomputerguru.com | jlohr@dataforth.com | +| Ken Hoffman | khoffman / oemdata | TestDataSheetUploader author (external); owns Dataforth product API | — | +| Delfina Lopez | dlopez | New hire (account created 2026-07-06, OU=Azure_Users) | DLopez@dataforth.com | +| Winter Williams | — | Dataforth contact for Syncro cleanup requests | — | + +- **External distributor:** Ginger (gy@quatronix-cn.com) — Quatronix China; receives datasheets +- **Address:** 3331 E. Hemisphere Loop, Tucson, AZ 85706 | (520) 741-1404 + +**Open Syncro tickets (as of 2026-07-14):** + +| Ticket | Subject | Status | Date | +|---|---|---|---| +| #32505 | Microsoft Access/Log-in request recent hire | Customer Reply | 2026-07-06 | +| #32489 | DOS test stations not pulling updated spec files (NWTOC v5.0 regression) | Customer Reply | 2026-06-29 | +| #32441 | Test datasheet generation defects + Phytec recovery | New | 2026-06-17 | +| #32399 | 10 x Flex Mini Switch installation | New | 2026-06-09 | + +--- + +## Syncro Asset Reconciliation (2026-06-02) + +Pulled 78 assets for customer_id `578095`. Syncro managed count 50 (live, 2026-07-14). Fleet-wide Syncro agent break ~2025-10-06 silenced ~half the fleet while machines stayed online in ScreenConnect. + +| Bucket | Count | +|---|---| +| KEEP | 20 | +| SAVE + FLAG (alive in SC/BD, Syncro agent broken — reinstall) | 21 | +| REMOVE (dead in all three systems) | 28 | +| VERIFY (servers, no agent anywhere) | 9 | + +**Governing rule:** machine is saved if online within 150 days in ANY of Syncro, ScreenConnect, or Bitdefender. + +**API delete not available** — `DELETE /customer_assets/{id}` returns HTML 404 for current integration token. All deletions via Syncro GUI only. + +**Coord todo tree** (assigned Howard, parent `103c48ad-7b31-4967-9388-065a91888e7c`): delete 28 dead assets in GUI, decide 9 VERIFY servers, reinstall Syncro agent on 21 SAVE+FLAG machines, switch to metered billing once clean. + +--- + +## 2026-07-08 GPS-RMM Reconciliation + +GPS billing = 49 workstation + 3 server = 52 (matches RMM 52) per Winter/Mike. + +--- + +## Infrastructure + +### Servers & Services + +| Host | IP | Role | OS | Notes | +|---|---|---|---|---| +| AD1 | 192.168.0.27 | Primary DC, DNS, FSMO roles, Engineering share | Windows Server 2016 | C:\ at **90%** capacity (C:\Engineering = 787 GB) — critical risk. Image plan (`Image2025`) + Files plan (NBF, daily 2 AM, 180-day retention, `C:\Engineering` + `C:\Shares\ITSvc`). GuruRMM agent `bf7bc5ee-4167-4a62-912a-c88b11a5943d`. | +| AD2 | 192.168.0.6 | Secondary DC, TestDataDB service host, NAS mirror, WebShare | Windows Server 2019 (10.0.17763) | Node.js TestDataDB on :3000. Wiped by 2025 crypto attack — rebuilt. Windows Firewall disabled (all profiles). Shares: `C:\Shares\{c-drive,e-drive,webshare,test}`. Old `D:\c-drive` volume gone — D: = mounted Windows install ISO. MSP360 agent `C:\Program Files\Arizona Computer Guru\Online Backup\cbb.exe`; storage account `ACG-Dataforth`. GuruRMM agent **re-enrolled 2026-07-05: new id `4b9fe588-2a86-4855-912e-d90cf5f5e3e4`** (old `cfa93bb6-…` stale). No shadow copies. Runs ClaudeTools on `ad2` branch; coord-API isolated — comms via git sync only. | +| FILES-D1 | 192.168.0.189 | File server | Windows Server 2016 | Shares: `E:\Shares\{sales,archive}`. `staff` share **MISSING** (see Patterns). GuruRMM agent `8566a19d-49a9-4f8b-9c6c-012cc934484b`. | +| SAGE-SQL | 192.168.0.153 | Sage ERP (S:), RDS Session Host / Connection Broker / Web Access | Windows Server 2016 | Hosts Sage Pro 200 ERP 2010 (`S:\sbt\sbtw.exe`, Visual FoxPro). ~10 concurrent RDP users. RDS licensing grace period reset 2026-05-06. TSGateway disabled (not externally exposed). Self-signed cert `CN=sage-sql.intranet.dataforth.com`. Bitdefender GravityZone managed AV. Share: `C:\sage`. GuruRMM agent `120ba7bf-8544-48a0-98a1-40ed5cdd3e1f`. **Sage printers = RDP-redirected (Easy Print) only** — native Brother/HP drivers NOT installed server-side (event 1111s); every session default = Microsoft Print to PDF. Sage app credential vault: `clients/dataforth/sage`. | +| 3CX | 192.168.0.125 | Phone system (possibly inactive) | — | Last logon Oct 2025. Production phones live on VLAN 100 under Sangoma/FreePBX — 3CX role likely superseded. | +| DF-HYPERV-B | 192.168.0.123 | Hyper-V hypervisor | Windows Server 2025 | GuruRMM enrolled. Newest server in environment. VM inventory not captured. | +| DF-SVR-D2-Sync | — | (role TBD) | — | GuruRMM enrolled. | +| ENG-DEV-SERVER | 192.168.0.126 | Engineering dev server | Windows 11 Pro | GuruRMM enrolled. | +| D2TESTNAS | 192.168.0.9 | SMB1 bridge for DOS test stations + AOI XP backup; rsync source/target for the test-data pipeline | Debian 13 (trixie), Samba 4.22.6 | Repurposed Netgear ReadyNAS. SMB1 enabled globally (CORE..SMB3, NTLMv1) — required for DOS 6.22 stations. rsync daemon port 873 (module `test`, user `rsync`, hosts allow 192.168.0.0/24 + 172.16.0.0/12). SSH: `root@192.168.0.9`. Tailscale: subnet router, **advertises 192.168.100.0/24 ONLY** (not 192.168.0.0/24). Shares: `test`/`datasheets`/`snapshots` (guest; `hosts deny 192.168.1.175`), `aoibackup` (XP-only). Jump host for UDM SSH (direct-tcpip to 192.168.0.254). **2026-07-01 audit:** root SSH key auth from AD2 is broken (publickey denied) — no impact on live rsync-daemon sync, but dormant SCP fallback script would fail. | +| ESXi hosts | 192.168.0.122, 192.168.0.124 | VMware ESXi hypervisors | ESXi | — | +| UDM Firewall | 192.168.0.254 | Perimeter firewall/router | UniFi OS 5.1.15 | MAC d0:21:f9:6c:11:02; also responds on 192.168.0.1. SSH: `azcomputerguru@192.168.0.254` (2FA push) or root (keyboard-interactive password, vault `clients/dataforth/udm.sops.yaml`). Boot scripts `/data/on_boot.d/`: `10-neptune-snat.sh`, `20-acg-network-isolation.sh`, `30-freepbx-sip-forward.sh`. **[WARNING] UniFi Protect, Access, and Talk were uninstalled 2026-07-05** (were consuming ~1 GB RAM on a 4 GB device; Dataforth uses FreePBX, none were in use). UniFi Protect 7.1.83, Access 4.2.29, Talk 5.2.7 removed. C2 IPs blocked via iptables (NOT permanent — add to UniFi UI). | +| PBX (Sangoma FreePBX) | 192.168.100.2 | VoIP PBX — production phones on 192.168.100.0/24 | Sangoma FreePBX 17 / Asterisk 22.5.2, Debian 12 | FirstDigital PJSIP trunk; SBC 66.7.123.215:5060 (Sonus), match 66.7.123.0/24; IP-auth. `qualify_frequency=0` (FD SBC ignores OPTIONS — do NOT revert). TFTP provisioning for Cisco SPA502G phones. Extensions 201–343. SSH: `sangoma@192.168.100.2`. `sudo -S` with password grants root (older "sudo not authorized" note was wrong — corrected 2026-07-14). Vault: `clients/dataforth/pbx.sops.yaml`. [WARNING] Re-apply `PJSip.class.php` line-504 patch after any `fwconsole ma updateall`. **Durable fix applied 2026-07-14:** native UniFi UI Port Forwarding policy "FreePBX-SIP" (WAN1 udp/5060, source-limited 66.7.123.0/24 → 192.168.100.2:5060) — provision-persistent, supersedes the on_boot.d script as primary; script retained as idempotent backup. | +| **MYDATA TPSys SMT Controller** (`myserver`) | 192.168.1.1 (VLAN 2, SSH-confirmed 2026-07-04) | MYDATA/Mycronic TPSys SMT pick-and-place line controller | Fedora Core 3 "Heidelberg" (Nov 2004), kernel 2.6.16.20, glibc ~2.3.5, bash 3.00, LILO bootloader, SysV init | Discovered + root recovered 2026-07-04. On VLAN 2 "mydata" (192.168.1.0/24). **[WARNING] 192.168.1.1 collides with VLAN 2 gateway address — reconcile whether this box IS the gateway or there is an IP conflict with the UDM.** TPSys UI runs under X (autologin, `.profile` → `startx` on tty1, default runlevel 3 not 5); local PostgreSQL backs TPSys. Root password reset via LILO recovery and vaulted: `clients/dataforth/mydata-smt.sops.yaml`. ACG RSA key: `clients/dataforth/mydata-smt-ssh-key.sops.yaml` (FC3 OpenSSH 3.9 needs legacy `-o KexAlgorithms=+diffie-hellman-group14-sha1 -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa -o MACs=+hmac-sha1 -o Ciphers=+aes128-cbc`). **[CRITICAL] root must NOT be passwordless AND `tpsys` must remain in `wheel` with `pam_wheel.so trust use_uid` uncommented in `/etc/pam.d/su`** — TPSys launcher `/home/tpsys/bin/go` escalates via `su -c` with no tty; a root password without the wheel trust breaks all escalation → X comes up empty (fvwm2 only, no TPSys UI). DR backup (2026-07-04): partition table + MBR/LILO + raw `dd` of OS partitions + `/home` tar + `pg_dumpall`, ~640 MB, SHA256-verified. On-site copy: `D2TESTNAS:/root/dr-backups/mydata-tpsys/2026-07-04/`. Offsite: `AD1 C:\Shares\ITSvc\DR\mydata-tpsys\2026-07-04\` (inside AD1's MSP360 Files plan → B2 `ACG-Dataforth`, nightly 2 AM). **GuruRMM agent cannot run on this box** (glibc ~2.3.5 <2.17, kernel 2.6.16 <2.6.32, SysV init, no systemd). | + +**Neptune Exchange (ACG infrastructure, physically at Dataforth D2):** +- `neptune.acghosting.com` | internal `172.16.3.11` | external inbound `67.206.163.124` / outbound `67.206.163.122` +- Exchange Server 2016, ACG-hosted mail server. Physically colocated at Dataforth D2 — NOT on ACG office LAN despite 172.16.x.x IP. +- Access requires routing through D2TESTNAS (192.168.0.9). +- SNAT rule: `/data/on_boot.d/10-neptune-snat.sh` on UDM forces Neptune outbound to `.124`. +- Vault: `clients/dataforth/neptune-exchange.sops.yaml` +- [WARNING] TODO: Resubnet Dataforth UDM to a non-overlapping range to permanently fix Neptune routing. + +### RMM-Spawned Claude on AD2 + +Proven capability (2026-07-01): a headless `claude -p` can be launched on AD2 through its GuruRMM agent (`context:user_session`, detached), writing a deliverable file + `DONE.txt` marker for a background poller. Bypasses AD2's coord-API isolation for read-only investigation. Must `Remove-Item Env:\ANTHROPIC_API_KEY` before invoking — a stale 108-char machine-level key shadows the working OAuth creds at `C:\Users\sysadmin\.claude\.credentials.json`. Reference: `.claude/memory/reference_rmm_spawn_headless_claude.md`. + +### Share -> Server -> Physical Path Map + +| Drive/Share | Server | Physical path | Notes | +|---|---|---|---| +| Q: / `c-drive` | AD2 | `C:\Shares\c-drive` | Old `D:\c-drive` is gone (D: = mounted install ISO) | +| T: / `e-drive` | AD2 | `C:\Shares\e-drive` | DOS stations map T: to `\\D2TESTNAS\test` (separate) | +| X: / `webshare` | AD2 | `C:\Shares\webshare` | DOS stations map X: to `\\D2TESTNAS\datasheets` (separate) | +| S: / `sage` | SAGE-SQL | `C:\sage` | — | +| W: / `sales` | FILES-D1 | `E:\Shares\sales` | — | +| Y: / `archive` | FILES-D1 | `E:\Shares\archive` | — | +| B: / `Engineering` | AD1 | `C:\Engineering` | — | +| B: / `itsvc` | AD1 | `C:\Shares\ITSvc` | — | +| `staff` | FILES-D1 | — | **MISSING** — share does not exist on FILES-D1 | + +### Workstations (summary) + +| Category | Count | OS | Notable | +|---|---|---|---| +| Engineering | ~12 | Win 10/11 Pro | HGHAUBNER (192.168.0.148) — Georg's PC; D: = full pre-attack backup of all 7 DF shares. GuruRMM agent `2aefe0d5-2357-4bdd-965a-abfccb4767a5`. D1-PWRM for PWRM10 test. | +| Manufacturing/Assembly | ~14 | Win 10/11 Pro | AS24, AS26 + various assembly/hi-pot stations | +| Office/Admin | ~12 | Win 10/11 Pro | DF-GAGETRAK (192.168.0.102) — GAGEtrak calibration host. DF-JOEL2 (192.168.0.174) — compromised 2026-03-27, remediated. DESKTOP-U8PHS72 (192.168.0.48) — dlopez/kellynwackerly; local Brother DCP-L2550DW + HP LaserJet. | +| Shipping | (part of Office/Admin) | Win 10/11 Pro | **DFORTH-Ship** — GuruRMM agent `db17e069-2948-4cbc-97ea-1da721edcaf5`, HP EliteDesk 800 G1 USDT. Recurring BSOD `0x116 VIDEO_TDR_FAILURE` — see Patterns. Do not confuse with near-twin **DForth-Shipp** (`95991b45-d843-4586-8275-9996d0d9ae17`). | +| End-of-Life (Win 7) | 3 | Windows 7 Pro | LABELPC (192.168.0.100), LABELPC2 (192.168.0.98), D2-RCVG-003 (192.168.0.47) — EOL, on network | +| AOI Optical Inspection (XP) | 1 | Windows XP | WinXPBE-724667 @ 192.168.1.175 on VLAN 2. Backs up to `\\192.168.0.9\aoibackup` (SMB1, XP-only). EOL. | +| SMT Line Controller | 1 | Fedora Core 3 "Heidelberg" (2004) | MYDATA TPSys (`myserver`), VLAN 2. Full detail in Servers table. No GuruRMM agent possible. | +| DOS Test Stations | 64 | MS-DOS 6.22 | TS-1 through TS-30 + variants. Not domain-joined. SMB1 via D2TESTNAS. 14 remain active (confirmed 2026-07-05). | + +### Email & Identity + +- **M365 tenant:** dataforth.com | Tenant ID: `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` +- **Entra ID Sync:** Yes — Azure AD Connect on AD1 (ADSync service). Synced OUs: **OU=SyncedUsers** and **OU=Azure_Users**. CompanyUsers OU does NOT sync. +- **M365 licenses:** 50x Business Premium (40 used as of 2026-07-06), 19x Exchange Online Plan 1 (5 used), 5x SPB (4 used). Note: `O365_BUSINESS_PREMIUM` SKU (f245ecc8) is the legacy part number for M365 Business Standard, distinct from the 5-seat true Business Premium (`SPB`). +- **SMTP settings:** smtp.office365.com, port 587, STARTTLS — use `sysadmin@dataforth.com` +- **SMTP AUTH status:** Tenant-level not disabled; per-mailbox varies. `calibration@dataforth.com` SmtpClientAuthentication=true (re-enabled 2026-04-23). `sysadmin@dataforth.com` SMTP AUTH blocked by Exchange Online default — testdatadb uses Graph API for email (Mail.Send permission granted to Claude-Code-M365 app 2026-05-12). +- **Mail security stack (layered):** + 1. **INKY PhishFence** — transport rule `B859327F-3FBD-4BE7-A47A-97D02F1558A7`, StopProcessingRules=true. Use inbox rules for per-mailbox routing, NOT transport rules. + 2. **Mailprotector CloudFilter** — outbound delivery gateway (`dataforth-com.outbound.emailservice.io`, 52.3.213.180). Active outbound connector "Outbound-Mailprotector". If a message shows "Delivered" in outbound trace but never arrives, check Mailprotector (`/mailprotector skill`). +- **DKIM:** Both selectors published. Rotated 2026-05-12; cutover to selector2 on 2026-05-16. +- **DNS Host:** ntirety.com — Dataforth's public DNS zone managed through ntirety's portal. DNS change requests go to ntirety, not a registrar panel. Joel Lohr's account retained to receive ntirety.com notifications (inbox rule → mike@azcomputerguru.com). +- **AutoForwarding:** blocked by default (tenant outbound spam policy). If per-user forwarding needed, create scoped HostedOutboundSpamFilterPolicy with AutoForwardingMode=On. +- **MFA:** 3 Conditional Access policies enforced 2026-04-04: + - "ACG - Require MFA for All Users" — skip from office IP 67.206.163.122 + - "ACG - Block Foreign Sign-Ins" — US-only; MFA-Travel-Bypass group for exceptions + - "ACG - Block Legacy Authentication" +- **Named locations:** Dataforth Office - Tucson (67.206.163.122/32, trusted), Allowed Countries - US Only +- **MFA-Excluded-BreakGlass group:** Brian Faires, Dataforth Calibration, Dataforth Notifications, Endcap, Tablet 01 +- **Get-MessageTrace deprecated Sept 2025:** Use Get-MessageTraceV2 and Get-MessageTraceDetailV2. + +### Network + +- **Domain:** intranet.dataforth.com | Forest/Domain Level: Windows Server 2016 +- **ISP:** fdtnet.net | Public IP: 67.206.163.122 (outbound), 67.206.163.124 (Neptune inbound) +- **Firewall/Router:** UniFi Dream Machine Pro at 192.168.0.254 (also 192.168.0.1), UniFi OS 5.1.15 +- **Network layout:** + - Main LAN: 192.168.0.0/24 (flat — servers, workstations, DOS stations) + - Voice/PBX: VLAN 100 (192.168.100.0/24) — production phones. PBX at .2/.196. + - VLAN 2 "mydata" (192.168.1.0/24): SMT production-line network; gateway 192.168.1.1. Members: WinXPBE-724667 (.175), goldstar19, DESKTOP-FT0T4MK, My9-PC, MYDATA TPSys (`myserver`, .1), + 3 unnamed industrial MACs (00:90:fb:80:f0:c6, 00:80:79:05:23:f2, 00:80:79:04:47:e7). Inter-VLAN routing mydata → main LAN currently OPEN. +- **VPN:** OpenVPN for ACG remote access. Client subnet 192.168.6.x (GURU-5070 gets 192.168.6.2). [WARNING] GURU-5070 OpenVPN adapter "Local Area Connection" (ifIndex 12) MTU must be 1400 — default 1500 causes PMTU blackhole (tunnel path MTU ~1424; bulk SSH/SCP silently drops). Verify/re-apply: `Set-NetIPInterface -InterfaceIndex 12 -AddressFamily IPv4 -NlMtuBytes 1400`. Permanent fix: `mssfix 1360` server-side. +- **Drive mappings (GPO):** B: (\\ad1\itsvc), Q: (\\ad2\c-drive), S: (\\SAGE-SQL\sage), T: (\\ad2\e-drive), W: (\\files-d1\sales), X: (\\ad2\webshare), Y: (\\files-d1\archive). DOS stations: T: (\\D2TESTNAS\test), X: (\\D2TESTNAS\datasheets). + +### GuruRMM Enrollment + +- **Site name:** Dataforth D1 | Site ID: `3a2f6866-26cd-452c-9806-a8df21475c3c` +- **Site API key:** vault `clients/dataforth/...` [check vault] +- **Fleet size:** 45+ agents enrolled; Syncro managed count 50 as of 2026-07-14 + +**[WARNING] GuruRMM enrollment workaround:** WebSocket auth in `ws/mod.rs` does not validate `enrolled_agents.agent_key_hash`. New agent installs must overwrite registry AgentKey with the site API key and restart service. See Gitea issue #8. + +**[WARNING] `rmm-api.azcomputerguru.com` must be grey-clouded** (DNS-only, not Cloudflare-proxied) — proxy blocks WebSocket. Gitea issue #9. + +**Known enrolled agents:** + +| Host | Agent ID | Notes | +|---|---|---| +| AD1 | `bf7bc5ee-4167-4a62-912a-c88b11a5943d` | Enrolled 2026-06-04 | +| AD2 | `4b9fe588-2a86-4855-912e-d90cf5f5e3e4` | Re-enrolled 2026-07-05 (old `cfa93bb6-…` stale) | +| FILES-D1 | `8566a19d-49a9-4f8b-9c6c-012cc934484b` | Enrolled 2026-06-04 | +| SAGE-SQL | `120ba7bf-8544-48a0-98a1-40ed5cdd3e1f` | Enrolled 2026-06-04 | +| DF-HYPERV-B | (see RMM dashboard) | Enrolled 2026-06-04 | +| DF-SVR-D2-Sync | (see RMM dashboard) | Enrolled 2026-06-04 | +| ENG-DEV-SERVER | (see RMM dashboard) | Enrolled 2026-06-04 | +| DF-GAGETRAK | `7626d82c-0736-47a6-8bc6-68e39859caed` | Enrolled 2026-04-23 (auth workaround applied) | +| HGHAUBNER | `2aefe0d5-2357-4bdd-965a-abfccb4767a5` | Georg's PC; pre-attack backup on D: | +| DFORTH-Ship | `db17e069-2948-4cbc-97ea-1da721edcaf5` | Recurring TDR BSOD — see Patterns | +| DESKTOP-U8PHS72 | `25a75932-31cf-4b33-b94b-f4c404830877` | dlopez/kellynwackerly workstation | +| (additional agents) | — | Full list in GuruRMM dashboard | + +**Cannot be enrolled:** + +| Host | Reason | +|---|---| +| MYDATA TPSys (`myserver`) | Fedora Core 3: glibc ~2.3.5 (<2.17), kernel 2.6.16 (<2.6.32), SysV init — three hard blockers. Plan agentless monitoring (ICMP/TCP or SSH heartbeat from D2TESTNAS). | + +### Backup Architecture + +- **MSP360 ("ACG-Online Backup", `cbb.exe`):** Storage account `ACG-Dataforth` (account ID `0b49ca5e-…`). +- **AD2:** `AD2 Image` image plan + `Files` plan (180-day retention, NBF, daily 2 AM, `C:\Shares` tree; GFS off, synthetic full, compression, fast-NTFS). No shadow copies on AD2. +- **AD1:** `Image2025` image plan + Files plan (NBF, daily 2 AM, 180-day retention, covers `C:\Engineering` + `C:\Shares\ITSvc`; created 2026-06-05). Both image and file plans now match AD2. +- **Pre-attack backup (offline):** HGHAUBNER `D:` drive holds a full pre-attack snapshot of all 7 mapped DF shares. The only recovery source predating the 2025 attack. Access via GuruRMM `user_session` on HGHAUBNER. Cross-machine writes use existing GPO-mapped drives only. +- **Historical file-level backup:** NBF bunch `faad5a67` in `ACG-Dataforth` storage — restore points 8/29–9/29/2025, archived at old `D:\c-drive\...` path (pre-migration layout). +- **WizTree backup CSV (2026-06-04):** Full-drive export of HGHAUBNER's D: at AD2 `C:\ClaudeTools\clients\dataforth\WizTree_20260604184904.zip` (sensitive — keep off shares). +- **MYDATA TPSys DR (2026-07-04):** hybrid image on D2TESTNAS + offsite via AD1 MSP360 Files plan → B2 `ACG-Dataforth`. See Servers table. TODO: full offline `dd`/Clonezilla image in a maintenance window. + +### Key Applications + +| Application | Host | URL/Port | Notes | +|---|---|---|---| +| TestDataDB | AD2 | http://192.168.0.6:3000 | Node.js + Express, PostgreSQL 18.3. Internal LAN only. UI redesigned 2026-06-18. 475,553 test_records as of 2026-07-01 audit; HTTP API uploader is the live web-delivery path. | +| Sage Pro 200 ERP 2010 | SAGE-SQL | RDS RemoteApp / S:\sbt\sbtw.exe | Visual FoxPro-based. Users RDP to SAGE-SQL and run Sage there — printers = RDP-redirected (Easy Print). Native drivers not on server; default printer is Microsoft Print to PDF. Credential vault: `clients/dataforth/sage` (username: guru). | +| GageTrak | DF-GAGETRAK (192.168.0.102) | — | Calibration tracking. Sends email via calibration@dataforth.com (SMTP AUTH re-enabled 2026-04-23). GuruRMM enrolled. | +| Dataforth Product API | Hoffman's servers | https://www.dataforth.com/api/v1/TestReportDataFiles | OAuth2 client_credentials. Vault: `clients/dataforth/api-oauth.sops.yaml`. Token URL: `https://login.dataforth.com/connect/token`. | +| QuickBASIC 4.5 ATE | 64 DOS stations | T:\ (\\D2TESTNAS\test) | Automated test equipment. 1,470+ product model specs. 14 stations remain active (2026-07-05). Spec distribution pipeline fixed fleet-wide 2026-07-05 (NWTOC v5.1). | +| Power Monitor SPA | Georg's dev / TBD | — | Vanilla-JS SPA for Dataforth power meters. Gateway architecture designed for PWM.dataforth.com demo. Parked pending Mike↔Georg conversation. `clients/dataforth/power-monitor-demo/`. | +| MYDATA TPSys | MYDATA TPSys controller (`myserver`, VLAN 2) | Local X UI / local PostgreSQL | SMT pick-and-place line control. Discovered + documented 2026-07-04. See Servers table. | + +--- + +## Access + +### Domain / Server Access + +- **AD2 SSH:** `ssh sysadmin@192.168.0.6` (port 22) — vault: `clients/dataforth/ad2.sops.yaml` → `credentials.password` — strip backslash escape: `sed 's/\\//g'`. MTU-sensitive: GURU-5070 OpenVPN adapter ifIndex 12 must be MTU 1400. +- **AD1 SSH:** `ssh sysadmin@192.168.0.27` — vault: `clients/dataforth/ad1.sops.yaml` +- **D2TESTNAS SSH:** `ssh root@192.168.0.9` — vault: `clients/dataforth/d2testnas.sops.yaml` (password corrected 2026-07-14; was backslash-corrupted). Use `root`, NOT `sysadmin`. Tailscale: `root@100.85.152.90`. SSH key from GURU-5070 authorized; **from BEAST: not authorized**. From AD2, root SSH key auth is broken (publickey denied) — live rsync-daemon path unaffected. +- **D2TESTNAS `aoibackup` share:** `\\192.168.0.9\aoibackup` — Samba user `admin`, `hosts allow = 192.168.1.175` only. Creds: vault `clients/dataforth/d2testnas.sops.yaml → credentials.smb.aoi-*`. +- **UDM SSH:** `ssh azcomputerguru@192.168.0.254` (2FA push) or root (keyboard-interactive password; GURU-5070 id_ed25519 key also authorized). Jump via D2TESTNAS: paramiko `direct-tcpip` or ProxyJump. Vault: `clients/dataforth/udm.sops.yaml` (`credentials.ssh_root_password` = keyboard-interactive only — `plink -pw` and paramiko `connect(password=)` refused; use paramiko `auth_interactive`). From BEAST: no route to 192.168.0.x — jump through D2TESTNAS (advertises VLAN 100 only, not 192.168.0.x). +- **SAGE-SQL SSH:** `ssh sysadmin@192.168.0.153` — SSH key at `C:\ProgramData\ssh\administrators_authorized_keys` on SAGE-SQL. Times out from BEAST (no VPN route); use GuruRMM instead. +- **PBX SSH:** `sangoma@192.168.100.2` — vault: `clients/dataforth/pbx.sops.yaml`. `sudo -S` with same password grants root (note corrected 2026-07-14 — prior "sudo not authorized" entry was wrong). +- **WinRM (AD2/AD1):** port 5985 — pywinrm with NTLM, user `INTRANET\sysadmin` +- **HGHAUBNER:** No SSH. Reached via GuruRMM agent `2aefe0d5`. Cross-machine file writes use existing GPO-mapped drives only. +- **All server passwords:** vault (individual `clients/dataforth/.sops.yaml` per server) + +### MYDATA TPSys SMT Controller + +- Root password: vault `clients/dataforth/mydata-smt.sops.yaml`. ACG RSA key: vault `clients/dataforth/mydata-smt-ssh-key.sops.yaml` (requires legacy SSH options). +- **Recovery if locked out again:** LILO `boot:` → `linux init=/bin/bash rw` → passwordless root shell (bypasses `sulogin`). Then `mount -o remount,rw /` + `passwd root`. Reboot: `reboot -f` or `echo b > /proc/sysrq-trigger`. +- **DO NOT blank root, remove tpsys from wheel, or re-comment the `pam_wheel.so trust` line** — any of these kills TPSys on next boot (all su escalations fail, X comes up empty). +- Accounts: `root`, `tpsys` (app user, in wheel), `tpspool` (spool), `postgres` (uid 500, local DB). + +### M365 / Entra + +- **M365 admin:** sysadmin@dataforth.com — vault: `clients/dataforth/m365.sops.yaml` +- **Tenant ID:** `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` +- **Claude-Code-M365 Entra App:** App ID `7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29`, secret expires 2027-12-22 — vault: `clients/dataforth/m365.sops.yaml → credentials.entra-app` +- **MSP remediation app suite:** MSP tenant `ce61461e-81a0-4c84-bb4a-7b354a9a356d`. Old single app `fabb3421`/Claude-MSP-Access DELETED 2026-06-14 — do not use. ComputerGuru tiered apps (Exchange Operator `b43e7342`, etc.) all consented 2026-04-23. Vault: `msp-tools/computerguru-*.sops.yaml`. +- **Contacts.Read not held** by the investigator app in Dataforth tenant (ErrorAccessDenied, confirmed 2026-07-10). Exchange REST returns 401 (Exchange Admin role not assigned to investigator SP in this tenant). + +### MSP360 Managed Backup API + +- **Vault:** `msp-tools/msp360-api.sops.yaml` +- `cbb.exe` path on AD2: `C:\Program Files\Arizona Computer Guru\Online Backup\cbb.exe` + +### Dataforth Product API (Hoffman) + +- **Vault:** `clients/dataforth/api-oauth.sops.yaml` +- Token URL: `https://login.dataforth.com/connect/token` | Grant: `client_credentials` | Client ID: `dataforth.onprem.sync` | Scope: `dataforth.web` | Token TTL: 1 hour +- Endpoints: `GET /api/v1/TestReportDataFiles/{serial}`, `/bulk`, `/stats` + +### ESXi / Hypervisors + +- ESXi-122: 192.168.0.122 — vault: `clients/dataforth/esxi-122.sops.yaml` +- ESXi-124: 192.168.0.124 — vault: `clients/dataforth/esxi-124.sops.yaml` + +--- + +## Patterns & Known Issues + +### RDS / SAGE-SQL / Sage Printing + +- **Sage Pro 200 ERP 2010 is an RDS RemoteApp only** — users RDP to SAGE-SQL (192.168.0.153, Session Host) and run `S:\sbt\sbtw.exe` there. Sage is NOT installed on workstations. +- **Printers = RDP-redirected (Easy Print)** — native Brother/HP drivers are NOT installed on SAGE-SQL (event 1111 "driver unknown"). Every session default = Microsoft Print to PDF. If a user sees no printers in Sage, first check for stale disconnected RDP sessions on SAGE-SQL. +- **Stale session printing fix (dlopez, 2026-07-14):** user had two stale sessions (one active since prior day, one disconnected); fresh reconnects kept re-entering the stale sessions with zero printers. Fix: `logoff /v` for all stale sessions via GuruRMM; new session brought printer redirection up correctly. Note: `logoff` exits nonzero on success — use `query user` output as truth, not exit code. +- **Registry change for RDP behavior** was applied prior to 2026-07-14 (before that session); fresh logon required to take effect. Watch-item: if Sage printing complaints recur with the Easy Print queues, install native Brother DCP-L2550DW + HP Universal PCL 6 drivers on SAGE-SQL. +- **RDS licensing:** Grace period reset 2026-05-06 by deleting GracePeriod registry key. Purchase RDS CALs (Per User mode, LicensingType=4). +- **TSGateway:** Disabled on SAGE-SQL (server not externally exposed). Do NOT re-enable without reason. +- **SSL cert:** Self-signed, `CN=sage-sql.intranet.dataforth.com`. Non-domain machines must import to Trusted Root + Trusted Publishers manually. +- **Bitdefender GravityZone** managed AV on SAGE-SQL — may block PowerShell execution; temporary disable may be needed for admin work. +- **Sage credential vault:** `clients/dataforth/sage` (username: guru). Password reset by Dataforth staff 2026-07-14; verified working. Never inline the password. + +### Active Directory + +- **No custom security groups** — only default Windows groups. Service accounts in OU=ServiceAccounts. +- **ClaudeTools-ReadOnly AD account** — purpose unclear. Investigate. +- **Ken Hoffman has two accounts** (khoffman + oemdata) — not consolidated. +- **jlohr account retained** — post-retirement (2026-03-31); kept enabled for ntirety.com notifications. Inbox rule forwards to mike@azcomputerguru.com. Do NOT disable. +- **Entra sync scope:** OU=SyncedUsers **and OU=Azure_Users** sync to Entra. CompanyUsers OU does NOT sync. 38 stale TS-* test station accounts cleaned from Entra 2026-03-27. +- **New hire pattern (dlopez, 2026-07-06):** create on-prem in OU=Azure_Users, add to RDWeb-Users, trigger ADSync delta. ADSync service runs on AD1 itself. +- **SKU disambiguation:** `O365_BUSINESS_PREMIUM` (f245ecc8) = M365 Business Standard (legacy part number), NOT true Business Premium (`SPB`). This is what standard user accounts hold. + +### Voice / Phones / FreePBX + +- **Production phones VLAN:** 192.168.100.0/24. PBX at .2/.196. All production phones live here. +- **VLAN 2 (192.168.1.0/24) is the mydata/SMT VLAN** — do NOT confuse with voice. Phones landing on 192.168.1.x cannot reach PBX. +- **D1-Server-Room port 1:** Controls lobby drop → must stay on VLAN 100. Reverted to default once before (2026-05-04 incident). +- **FirstDigital trunk — `qualify_frequency=0`:** FD's Sonus SBC ignores SIP OPTIONS keepalives. Non-zero qualify causes trunk to go Unavailable and blocks all INVITEs. **Do NOT revert.** +- **PJSip.class.php line 504 patch must be re-applied** after any `fwconsole ma updateall` — wiped by FreePBX updates. Backup before each update. +- **Do NOT port-forward the RTP range (10000-20000)** on the UDM. Static RTP DNAT causes a conntrack collision with the PBX's outbound RTP — inbound works but outbound audio dies. SIP 5060 forward only, source-locked to 66.7.123.0/24. +- **Inbound SIP DNAT (recurring failure, now durably fixed):** Previously relied only on `/data/on_boot.d/30-freepbx-sip-forward.sh`, which is flushed by any UniFi controller provision mid-uptime (not just reboots). Outage 2026-07-13 11:05 AM → 2026-07-14 09:39 AM fixed by: (1) re-running the on_boot.d script (immediate fix), (2) creating a native UniFi UI Port Forwarding policy "FreePBX-SIP" (WAN1 udp/5060, source 66.7.123.0/24 → 192.168.100.2:5060) that survives provisions. Coord to-do `45572ee1` (durable fix) is now closed. on_boot.d script retained as an idempotent backup. Diagnostic: `iptables -t nat -S | grep -iE '5060|192.168.100.2'` on UDM — empty means rule is gone. +- **`sangoma` sudo clarification:** `sangoma` is in the `sudo` group and `sudo -S` with the password DOES grant root (prior "sudo not authorized" note was wrong — corrected 2026-07-14). `asterisk -rx` still denied for sangoma (socket is owner-writable only). +- Prior outages: 2026-06-08 (#32392), 2026-06-23 (#32450), 2026-07-14 (#32553). + +### Exchange Online / Email + +- **INKY PhishFence StopProcessingRules:** Kills all subsequent transport rules. Use inbox rules for per-mailbox forwarding, NOT transport rules. +- **Mailprotector CloudFilter:** Outbound delivery goes through Mailprotector. If "Delivered" per outbound trace but message never arrives, check Mailprotector (`/mailprotector skill`). +- **AutoForwarding blocked by default** — create scoped HostedOutboundSpamFilterPolicy with AutoForwardingMode=On if needed per sender. +- **Autocomplete stale-cache trap (Haubner→Tobey, 2026-07-10):** Outlook displays the correct SMTP address from cache but routes via a stored EX/LegacyExchangeDN that no longer resolves. Fix: user deletes the stale autocomplete entry in Outlook and re-picks from GAL. Remote cache-clear more disruptive (file locked while Outlook is open, nukes all cached names). +- **Contacts.Read / Exchange REST gaps:** The investigator app does not have Contacts.Read in Dataforth (ErrorAccessDenied). Exchange REST returns 401 (Exchange Admin role not assigned to investigator SP in this tenant). Use GuruRMM for workstation-level inspection when these APIs are blocked. + +### GuruRMM Agent Deployment + +- **WebSocket auth bug (Issue #8):** `enrolled_agents.agent_key_hash` not validated. Workaround: overwrite registry `HKLM:\SOFTWARE\GuruRMM\AgentKey` with site API key, restart service. +- **rmm-api.azcomputerguru.com must be grey-clouded** — Cloudflare proxy blocks WebSocket. Gitea Issue #9. +- **AD2 re-enrolled 2026-07-05:** new agent id `4b9fe588-2a86-4855-912e-d90cf5f5e3e4`. Old id `cfa93bb6-…` is stale — do not dispatch to it. +- **No RMM agent possible on FC3-class legacy appliances:** confirmed on MYDATA TPSys (glibc ~2.3.5, kernel 2.6.16, SysV init — three hard blockers). Plan agentless monitoring (ICMP/TCP probe or SSH heartbeat from D2TESTNAS). +- **SYSTEM token cross-machine writes:** GuruRMM `user_session` commands use an impersonated token with no network credentials. Fresh UNC paths fail with Access Denied. Use existing GPO-mapped drives (Q:, T:, etc.) for cross-machine writes — proven on HGHAUBNER 2026-06-04. +- **Legacy Red-Hat-family recovery pattern:** if locked out, LILO `boot:` → `linux init=/bin/bash rw` → passwordless root shell (bypasses sulogin on Red-Hat family). +- **RMM dispatch JSON quoting:** inline JSON with `\\` escapes fails the API's JSON parse. Use `ps-encoded.sh` for PowerShell dispatches. `query session` exits nonzero on success — use stdout content as truth. + +### AD2 SSH / VPN MTU + +- **PMTU blackhole on GURU-5070 → AD2 SSH:** OpenVPN adapter ifIndex 12 defaults MTU 1500; tunnel path MTU is ~1424. Bulk TCP segments silently dropped; interactive commands pass. Presents as "flaky VPN" or "random lockouts." +- **Fix (applied 2026-06-18, registry-persistent):** `Set-NetIPInterface -InterfaceIndex 12 -AddressFamily IPv4 -NlMtuBytes 1400` via GURU-5070 RMM agent. May reset on OpenVPN reconnect — re-apply if stalling resumes. +- **Durable fix:** `mssfix 1360` server-side on the Dataforth OpenVPN server (fleet-wide). +- **AD2 is NOT the target for SSH diagnosis** when SSH is the failing channel — use RMM. + +### AD2 Branch / Coordination + +- **AD2 operates on the `ad2` git branch.** Fork = main + thin Dataforth-specific commits. Never edit shared fleet files on `ad2`. Dataforth context: `clients/dataforth/CLAUDE.dataforth.md`. +- **AD2 is coord-API isolated:** 172.16.3.30 unreachable from Dataforth LAN. All inter-session coordination via git sync: committed handoff docs + `## Note for ` blocks. Do NOT use the coord skill for AD2. +- **sync.sh push gotcha:** always fails at Phase 4 on `ad2` branch (pushes `main`). Force-push manually: `git push --force-with-lease origin ad2`. +- **Stale machine-level ANTHROPIC_API_KEY on AD2** (108 chars, invalid) shadows working OAuth creds. Must `Remove-Item Env:\ANTHROPIC_API_KEY` before invoking `claude -p`. +- **AD2 RMM side channel:** GuruRMM agent (`4b9fe588-…`, current) can spawn a headless `claude -p` for live read-only ground-truth investigation (proven 2026-07-01). Unset the stale API key first. + +### Post-Ransomware Recovery Restore (2025) — Incomplete File Migration + +- **The 10/1/2025 recovery restore was incomplete.** `Restore plan 10/1/2025` (~3.4M files) migrated each share from old `D:\` to `C:\Shares\...` on AD2 and dropped files in the process. Proven case: SP1366 MAQ20 — each `PRINTOUTS FOR MANUFACTURING` folder for revisions E–H received only one file when the backup contained ~6 per revision. +- **Scope unknown.** A full migration-gap audit is underway (WizTree both sides). Review-only — no automatic restore (some deletions were intentional; HGHAUBNER backup is additive-only and includes personal files). +- **AD2 D: drive is gone.** Old `D:\c-drive` data volume repurposed as mounted Windows install ISO. Historical file-level backup (bunch `faad5a67`) archived data under `D:\c-drive\...` (pre-migration path) — reconcile paths accordingly. + +### Shares ACL State — All Open to All Staff + +- **All 8 business shares grant access to every employee** (Everyone/Domain Users Full or Modify on all). No department-based security groups. Payroll, OSHA, Purchase Orders, Accounting/QuickBooks, Sage financials fully readable/writable by all domain users. +- **Remediation project in progress** (Shares & Permissions, started 2026-06-10). Phase 0 (discovery) complete. Phase 1 (client input) pending discovery email to Dan Center. Do not apply ACL changes until after client sign-off. +- **Special shares excluded from department-model remediation:** `test` (DOS/SMB1 guest — leave open), `webshare` (preserve `svc_testdatadb:Full`), `ITSvc` (Domain Computers needs Read), Sage app data path. +- **Phase 2 target-state strawman (drafted 2026-06-22):** `Company\Departments\` tree with `SG--` groups, `Restricted\` branch with broken inheritance, ABE on. Option A (keep current drive letters, reorganize folders within) recommended for least disruption. Full detail: `clients/dataforth/docs/projects/shares-permissions/`. +- **Six items gate Phase 2 sign-off:** confirmed department list; department → share access matrix; named access for sensitive data; department rosters; legacy cleanup approval; Engineering destination volume (AD1 C: ~90% full blocks any ENGR restructure). + +### DOS Test Station Data Pipeline + +- **DOS spec distribution fixed fleet-wide 2026-07-05** (Syncro #32489 root-caused 2026-07-01, fix deployed 2026-07-05). All 14 active stations confirmed carrying current `5BMAIN.DAT` (83,200, 06-26-26). +- **Root cause (F1, resolved):** deployed `NWTOC.BAT` v5.0 copied only `*.BAT` and `*.EXE` — zero `.DAT` files — by design (its own changelog said so). No version of NWTOC ever distributed shared `COMMON` master specs; v5.1 added that copy. +- **F2 (resolved):** `COPY /Y` is not a valid MS-DOS 6.22 switch. Mike confirmed all stations are genuine 6.22 — `/Y` caused NWTOC to copy nothing fleet-wide since 2026-03-16. Fixed in v5.1 (plain `COPY`). +- **NWTOC v5.1 deployed 2026-07-05** (plain `COPY`, adds one-way `.DAT` pull from `COMMON/ProdSW` into `C:\ATE\DATA`, `ATTRIB -R` guard, CRLF-forced). Validated on TS-3L, promoted fleet-wide. v5.0 backed up to `NWTOC50.BAK`. `CTONWTXT` updated to v2.4 (drops `/Y`). `CTONW` at v5.2 (flat-file telemetry, no network MD) then v5.3 pending (CTONW v5.3 dispatch went to stale AD2 id and needs redoing). +- **Tier-A telemetry deployed:** `SYSTAT.BAT` (plain paused status page) and flat-file telemetry to `T:\STATUS\%MACHINE%.{CFG,AUT,SPV,MEM}` — 14 stations collected cleanly 2026-07-05. +- **Rollout is per-station opt-in** (broken v5.0 cannot bootstrap its own replacement — each station needs one manual `COPY` seed, then self-maintains). +- **Secondary cleanup items (not yet done):** remove stray `TS-21\ProdSW` file (F3 — breaks AD2↔NAS rsync push every run); update `testdatadb\specdata\` from live engineering masters (F4 — frozen 2026-03-27 snapshot); rotate/vault plaintext creds in scripts (F5); retire dead `For_Web` output and abandoned v1.x script drafts. +- Full 2026-07-01 audit report: `clients/dataforth/docs/audits/2026-07-01-test-data-chain-audit-AD2.md`. + +### Hardware / Endpoint — Aging Fleet + +- **DFORTH-Ship recurring TDR BSOD (`0x116 VIDEO_TDR_FAILURE`, diagnosed 2026-06-25):** integrated Intel HD Graphics 4600, driver 20.19.15.5126 (Intel's final, 2020-01-20), on an 11.5-year-old HP EliteDesk 800 G1 USDT (BIOS 2014-12-10). Five minidumps 2025-11-03 through 2026-06-24, accelerating cadence. Mitigation: Edge hardware acceleration disabled via machine policy (`HKLM\SOFTWARE\Policies\Microsoft\Edge\HardwareAccelerationModeEnabled = 0`). No durable fix for integrated graphics — **PC replacement is the real fix**; thermal cleaning is a secondary measure. Verify Edge policy at `edge://policy` after restart. Do not confuse with near-twin **DForth-Shipp** (`95991b45-…`) — verify agent ID before acting. +- **MYDATA TPSys controller (~20 years old, FC3 Nov 2004):** documented and DR-backed 2026-07-04 after being completely undocumented until discovered at the physical console. Worth a broader sweep for other undocumented devices on VLAN 2 — 3 unnamed industrial MACs already known but unresolved as of 2026-06-01. +- **End-of-Life machines on network:** 3x Windows 7 (LABELPC, LABELPC2, D2-RCVG-003), 1x Windows XP (WinXPBE-724667/AOI). All EOL, all unpatched. + +### Security + +- **C2 IP blocks are iptables only** — do not survive UDM reboot. Must add 80.76.49.18 and 45.88.91.99 (AS399486 Virtuo, Montreal) to permanent UniFi UI block list. +- **AD1 disk 90% full** — C:\Engineering = 787 GB of 1023 GB. Risk of replication failures. Blocks Engineering share restructure. +- **Windows Firewall disabled on AD2** (all profiles) — known risk, not remediated. +- **AD1/AD2 on Server 2016/2019** — approaching end of mainstream support. +- **Entra ID P2 not licensed** — IdentityRiskyUser risk check returns 403 (no P2). +- **Plaintext credentials in Dataforth test-data-chain scripts (F5, 2026-07-01):** rsync daemon password in `Sync-FromNAS-rsync.ps1`, Postgres `testdatadb_app` password in `testdatadb\database\db.js` — world-readable via the `test` SMB share. Rotation + vaulting recommended, not yet done. +- **UniFi Protect/Access/Talk removed 2026-07-05** — were consuming ~1 GB RAM on 4 GB UDM while unused (Dataforth uses FreePBX). All three uninstalled. UDM management plane briefly down during post-uninstall reboot; data plane (forwarding) remained live. + +### Syncro Asset Management + +- **Fleet-wide Syncro agent break ~2025-10-06:** ~half of Dataforth machines stopped reporting to Syncro while remaining online in ScreenConnect. Hard cutoff, not gradual attrition. Do NOT auto-remove machines frozen at that date without cross-checking ScreenConnect. +- **Bitdefender is NOT a liveness signal:** Dataforth phasing off BD; 53 of 57 GravityZone endpoints in "Deleted" folder. Missing from BD = BD agent uninstalled. +- **API delete not available** — all asset deletions via Syncro GUI (`/customer_assets?customer_id=578095`). + +### `staff` Share Missing + +The `staff` network share is absent from FILES-D1 (only `archive` and `sales` exist). HGHAUBNER's backup includes a `DF Staff` folder, suggesting the share existed pre-attack. Not in scope for the current migration-gap diff — separate issue requiring investigation. + +--- + +## Active Work + +As of 2026-07-14 (4 open Syncro tickets per live pull): + +- **DOS Test Station Data Pipeline (Syncro #32489, active):** Root cause confirmed and fix deployed fleet-wide 2026-07-05 (NWTOC v5.1). All 14 active stations confirmed with current specs. Ticket status is Customer Reply — awaiting Dataforth to confirm on their end. Pending: (1) promote `CTONW v5.3` (dispatch to stale AD2 agent id needs redoing with new id `4b9fe588-…`); (2) secondary cleanup items F3–F5 from the 2026-07-01 audit (stray TS-21 file, frozen specdata, plaintext creds in scripts); (3) close ticket once client confirms resolution. + +- **Test Datasheet Pipeline (Syncro #32441, open):** + - Production outbound pipeline healthy — 475K+ records, DSCA33/45 recovery complete (1,452 certs published 2026-06-18 via Hoffman API). + - Email notifications deployed (Graph API via `sysadmin@dataforth.com`). + - 8B/5B/SCM render gap (~9,624 records unpublished) — 136 templates mined, wiring handed to AD2 (parked). + - `testdatadb\specdata\` frozen at 2026-03-27 snapshot — any spec limit changed after that date not reflected in server-side generated datasheets even though the outbound pipeline is healthy (F4). + - 2 niche DSCA models (DSCA33-1948, DSCA45-1746) have no Hoffman original — cannot auto-publish. + +- **Flex Mini Switch installation (Syncro #32399, open):** 10 x Flex Mini Switch installation. No further detail captured in reviewed session logs. + +- **New hire account/access (Syncro #32505, Customer Reply):** Delfina Lopez account created 2026-07-06 in AD + M365. Ticket is in Customer Reply — follow up with Winter. + +- **DFORTH-Ship BSOD (ongoing monitoring):** Edge hardware-acceleration mitigation applied 2026-06-25; needs on-site Edge restart/reboot to take effect. Monitor for recurrence. If it bugchecks again, pull older dump signatures to confirm hardware fault trajectory. Schedule thermal cleaning. Recommend PC replacement as durable fix. + +- **UDM inbound SIP DNAT (closed 2026-07-14):** Durable fix applied — native UI Port Forwarding rule "FreePBX-SIP" created. Coord to-do `45572ee1` closed. Syncro ticket #32553 (0.5 hr, block debited 25.25 → 24.75 hrs). Monitor stability. + +- **Shares & Permissions project (Phase 1 — BLOCKING, pending client input):** Phase 0 complete 2026-06-10. Discovery email to Dan Center drafted (`docs/projects/shares-permissions/discovery-email-draft.md`) — **not yet sent** (recipients/sender not locked). Phase 1 blocked on client responses: department list, access matrix, sensitive-data rules, rosters. Phase 2 strawman drafted 2026-06-22. Next step: polish client HTML and send discovery email. + +- **MYDATA TPSys SMT controller:** Root recovered and documented 2026-07-04. **Outstanding:** (1) reconcile 192.168.1.1 IP vs VLAN 2 gateway conflict; (2) verify tpsys wheel-group + NOPASSWD sudoers change survived the reboot (`id tpsys`, `sudo -l` as tpsys); (3) stand up agentless monitoring (ICMP/TCP probe or SSH heartbeat from D2TESTNAS — inter-VLAN routing to mydata is open); (4) full offline `dd`/Clonezilla image in a maintenance window. + +- **Migration-gap audit (parked):** WizTree CSV of HGHAUBNER's pre-attack backup captured. WizTree on live servers deferred — no diff yet. Plan: run WizTree on AD2, FILES-D1, SAGE-SQL, AD1 → diff per share → `clients/dataforth/migration-gap-catalog-2026-06-04.md`. Full plan: `clients/dataforth/migration-gap-diff-RESUME.md`. + +- **Syncro asset cleanup (with Howard):** 28 confirmed-dead assets pending GUI deletion; 21 machines need Syncro agent reinstall; 9 servers in VERIFY bucket. Coord todo tree `103c48ad-…` assigned to Howard. Switch to metered billing once clean. + +- **AOI XP backup + isolation (ongoing):** AOI XP @ 192.168.1.175 on VLAN 2; `aoibackup` SMB1 share locked to XP only. Optional EOL hardening pending: block XP → company LAN (except NAS 192.168.0.9) + Internet on UDM. Todo `37543f7f`. + +- **AD2 Claude capability updates (parked):** AD2 needs syncro + coord commands, DF wiki read-write, Dataforth client data access. Coord API unreachable from Dataforth LAN — comms via git sync + RMM-spawn side channel. + +- **Power Monitor SPA demo (parked):** Georg Haubner's vanilla-JS power-meter SPA. Gateway spec at `clients/dataforth/power-monitor-demo/GATEWAY-SPEC.md`. Parked pending Mike↔Georg conversation. + +- **MFA enrollment:** 19 users not enrolled at April 4 enforcement date; current enrollment count unverified. + +- **C2 IP blocks need permanence:** iptables rules on UDM (80.76.49.18, 45.88.91.99) need to be added to permanent UniFi UI block list. + +- **jlohr forwarding:** ntirety.com inbox rule active as of 2026-05-12. Defunct transport rule pending cleanup. + +- **RDS / SAGE-SQL:** GPO cert distribution pending. RDS CALs purchase needed long-term. + +- **Haubner → Tobey autocomplete fix (2026-07-10):** Diagnosed as stale cached EX/LegacyExchangeDN in Outlook autocomplete. Recommended fix: Haubner deletes the Logan entry in Outlook and re-picks from GAL. Awaiting Mike's direction on delivery method (instructions to Haubner vs scheduled remote clear). + +--- + +## History Highlights + +| Date | Event | +|---|---| +| 2025 | Crypto/ransomware attack — AD2 wiped and rebuilt, many files lost. Test datasheet pipeline broken. | +| 2025-08-29 – 2025-09-29 | MSP360 file-level backup (`faad5a67`) covering DF shares at old `D:\c-drive\...` path. Last snapshot before the recovery restore. | +| 2025-10-01 – 2025-10-02 | Post-ransomware recovery restore (`Restore plan 10/1/2025`, ~3.4M files) migrated shares from `D:\` to `C:\Shares\...` on AD2. Restore was incomplete — files dropped in multiple folders. | +| ~2025-10-06 | Fleet-wide Syncro agent break — ~half of Dataforth machines freeze in Syncro while remaining online in ScreenConnect. Root cause unknown. | +| 2026-01-19 | DOS Update System built and deployed — NWTOC/CTONW/UPDATE/DEPLOY BAT files, 39 deployments. | +| 2026-03-20 | Galactic Advisors security assessment — AD1 C: at 90%, legacy SQL 2008 R2 client noted. | +| 2026-03-27 | **Major security incident:** DF-JOEL2 compromised via social engineering/ScreenConnect (attacker "Angel Raya", C2 on Virtuo hosting). M365 sign-in from Turkey. Full remediation. 3 CA policies deployed. MFA notice sent. IC3 filed (`1c32ade367084be9acd548f23705736f`). | +| 2026-03-27–29 | Test datasheet pipeline rebuilt — new Node.js pipeline replaces VB6 DFWDS + VB.NET uploader. 72/73 Quatronix datasheets generated. | +| 2026-03-31 | Joel Lohr retirement. Brian Faires mailbox converted to shared. 38 stale Entra TS-* accounts deleted. | +| 2026-04-04 | MFA CA policies enforced (switched from report-only). | +| 2026-04-11–12 | SCMVAS/SCMHVAS pipeline extension — 27,503 records backfilled, 434 Engineering-Tested .txt files imported. | +| 2026-04-12 | TestDataDB PostgreSQL migration verified (2.89M records). Hoffman API discovered. | +| 2026-04-13 | API architecture discussion with Hoffman — client_credentials grant confirmed. | +| 2026-04-14 | DFWDS logic ported to Node.js. 897 staged datasheets drained. 803 new records pushed to Hoffman API. | +| 2026-04-15 | Major release — DB dedup (2.89M→469K rows), FAIL→PASS retest rule, 170,984 records bulk-pushed to Hoffman. Dashboard upgrades. | +| 2026-04-23 | Full Dataforth tenant onboarded to all 5 ComputerGuru tiered apps. calibration@ SMTP AUTH fixed. DF-GAGETRAK GuruRMM enrolled. Syncro #32142 billed. | +| 2026-05-03 | jantar@dataforth.com darkweb breach check — no indicators of compromise. eM Client OAuth grant revoked. | +| 2026-05-04 | Howard onsite — lobby phone offline (VLAN misconfiguration on D1-Server-Room port 1 → fixed to VLAN 100). | +| 2026-05-06 | SAGE-SQL RDS issues resolved — grace period reset, SSL cert replaced, TSGateway disabled. | +| 2026-05-12 | Pipeline audit + email notifications implemented (Graph API). jlohr forwarding configured. DKIM keys rotated. | +| 2026-06-01 | AOI optical-inspection XP PC isolated onto VLAN 2 @ 192.168.1.175; `aoibackup` SMB1 share created on D2TESTNAS locked to the XP only. D2TESTNAS confirmed Debian 13 / Samba 4.22.6. | +| 2026-06-01 | Chauncey Bell (cbell) M365 verified; Bobbi's Outlook printing fixed. Ticket #32364 (0.5 hr onsite). | +| 2026-06-02 | Syncro asset reconciliation (78 assets): 20 keep / 21 save+flag / 28 remove / 9 verify. | +| 2026-06-04 | SP1366 MAQ20 manufacturing print recovery — 19/20 PDFs for revisions E–H restored from HGHAUBNER pre-attack backup. GuruRMM fleet grew 13 → 45 agents. WizTree backup-side CSV captured. | +| 2026-06-05 | AD1 Files backup plan created (cbb.exe, NBF, 180-day retention, `C:\Engineering` + `C:\Shares\ITSvc`). | +| 2026-06-05 | Mailprotector CloudFilter discovered as Dataforth's outbound delivery layer. Email from Georg held by Mailprotector; released manually. `/mailprotector` skill built. | +| 2026-06-05 | Georg Haubner's Power Monitor SPA analyzed. Gateway architecture designed for PWM.dataforth.com demo. Parked. | +| 2026-06-08–09 | **Total Dataforth phone outage.** Outbound failed (`qualify_frequency` → trunk Unavailable); inbound never worked (no SIP port-forward existed). Fixed: `qualify_frequency=0` in pjsip DB; PJSip.class.php line 504 re-patched; `30-freepbx-sip-forward.sh` added. Two-way audio verified. Syncro #32392, 1.0 hr emergency (×1.5 rate). | +| 2026-06-10 | Shares & Permissions Phase 0 complete. All 8 business shares open to all employees. Phase 1 pending. | +| 2026-06-17 | AD2 identity.json + Python 3.12.8 installed. `CLAUDE.dataforth.md` created. | +| 2026-06-18 | DSCA33/45 certs recovered via Hoffman API — 1,452 new certs published. `parseRawData` bug root-caused. AD2 SSH PMTU blackhole diagnosed and fixed (MTU 1400). TestDataDB UI redesigned. Syncro #32441. | +| 2026-06-22 | Shares & Permissions Phase 2 target-state strawman drafted — `Company\Departments\…Restricted\…` tree with `SG--` groups. Internal draft + client-facing HTML render. | +| 2026-06-23 | **PBX no-inbound-calls emergency fixed.** UDM SIP 5060 DNAT flushed by UniFi controller provision (not a reboot). Re-ran on_boot.d script; verified inbound restored. Vault `pbx.sops.yaml` password corrected. Syncro #32450, 1.5 hr emergency (×1.5 rate), block 31.5 → 30.0 hrs. | +| 2026-06-24–25 | DFORTH-Ship recurring BSOD diagnosed (0x116, Intel HD 4600, 11.5-yr-old EliteDesk). Edge hardware-acceleration mitigation applied. PC replacement recommended. | +| 2026-07-01 | Test-data-chain ground-truth audit via RMM-spawned headless Claude on AD2. Confirmed NWTOC v5.0 never copies .DAT files (F1); `COPY /Y` invalid on 6.22 (F2); three additional findings. Stale assumptions corrected (PostgreSQL 18, rsync-daemon sync path, HTTP API uploader is live web-delivery). | +| 2026-07-04 | MYDATA TPSys SMT controller (myserver, FC3/VLAN2) discovered + root recovered via LILO single-user; vaulted; RMM agent ruled out. DR backup taken and stored on-site + offsite. | +| 2026-07-05 | **DOS spec distribution fixed fleet-wide.** NWTOC v5.1 deployed (adds .DAT pull, plain COPY, no /Y). All 14 active DOS stations confirmed with current specs. Tier-A station telemetry added. UDM: UniFi Protect/Access/Talk uninstalled (memory savings). `unifi.service` restarted (0-clients issue — RabbitMQ/WebSocket memory starvation). | +| 2026-07-06 | Delfina Lopez (dlopez) new hire account created in AD (OU=Azure_Users) + M365 (Business Standard). Ticket #32505. | +| 2026-07-06 | SAGE-SQL session reset for piliya (Peter Iliya, disconnected session 13). Ticket #32506, no billing. | +| 2026-07-08 | GPS-RMM billing reconciliation: 49 workstation + 3 server = 52 (matches RMM). | +| 2026-07-10 | Haubner→Tobey "address invalid" diagnosed — stale autocomplete EX/LegacyExchangeDN on Haubner's Outlook. Awaiting fix delivery decision. | +| 2026-07-13 – 2026-07-14 | **PBX inbound outage** (2nd UDM SIP DNAT flush, 07-13 ~11:05 AM → 07-14 09:39 AM AZ). Fixed by re-running on_boot.d script; **durable fix:** native UI Port Forwarding policy "FreePBX-SIP" created (provision-persistent). Coord to-do `45572ee1` closed. Vault: UDM root password corrected (keyboard-interactive only); d2testnas password de-corrupted. Sangoma sudo-works note corrected. Syncro #32553, 0.5 hr, block 25.25 → 24.75 hrs. | +| 2026-07-14 | **Sage printing fix (dlopez).** Diagnosed: stale RDP sessions on SAGE-SQL (no redirected printers until logged off). Logged off stale sessions via GuruRMM; fresh RDP brought printer redirection up. Sage app credential (guru user) vaulted at `clients/dataforth/sage`. Syncro #32552, 0.5 hr, block 25.75 → 25.25 hrs (then 25.25 → 24.75 from #32553 same day; final live block = 24.75). | + +--- + +## Backlinks + +- [[projects/dataforth-dos]] — Active test datasheet pipeline project on AD2 diff --git a/wiki/index.md b/wiki/index.md index 0050fe17..0e504aca 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -20,7 +20,7 @@ Run `/wiki-lint` to check for stale entries and broken backlinks. | Article | Summary | Last Compiled | |---|---|---| | [Cascades of Tucson](clients/cascades-tucson.md) | Prepaid block $175/hr, **37.5 hrs remaining** (live 2026-07-10); senior living; active domain migration + HIPAA caregiver-lockdown project (GPOs deployed; Entra Hybrid Join + CA allow-list + ALIS SSO model proven); single DC (CS-SERVER) on aging R610 -- RAID **live-verified HEALTHY 2026-06-24** (the 6/15 "degraded" self-recovered; both mirrors Ok, 1:0:4 = global hot spare; consumer 320GB drives + lost-PSU-redundancy are planned follow-ups, NOT an emergency); cloud backup verified running; **Planned power outage 2026-06-23** clean self-shutdown executed + verified (bring-up ~09:00, John onsite); **Voice VLAN 30 migration COMPLETE 2026-06-19** (~38 devices: 29 Poly + 8 AudioCodes + desktop; awaiting Vertical to set Poly 5GHz-only); **UniFi RF optimized 2026-06-19** (77 U7-Pro APs/~587 clients: 2.4GHz power->Medium on 47 radios + 5GHz clean-DFS 40MHz channel plan -> 5GHz retry halved; 6GHz blocked by WPA3 on PPSK SSID); Syncro 0 open tickets (live EOD 2026-06-25), device-readiness audit done (5 PCs on Win Home need Home->Pro before join); **Alma Montt offboarded 2026-06-25** (Tenant Admin SP left holding a standing PAA role -- removal pending Mike); **CARF Technology & System Plan** deliverable in progress for Ashley Jensen; **endpoint security migration started 2026-06-25** (Datto EDR/AV replacing Bitdefender; 34 agents enrolled); **CS-SERVER: all Datto software removed 2026-06-26**, and the CS-SERVER "SMB error 67" proved to be an RMM-test artifact -- server is healthy, Karen share access verified interactively; **caregiver phone login LIVE 2026-07-01: roster reconciled to 35 (8 offboarded incl. Lassey dup, 4 new hires), two CA blockers fixed (MFA-exclude bug + compliance-block disabled), interim posture = desktops+phones on-network only; ALIS Email=UPN match + 3 new-hire ALIS records pending**; **CSC ENT->VLAN 20 migration (live 2026-07-01): 22 machines on VLAN 20 (only CS-SERVER + ~6 stragglers left on old LAN); printer shares lag at 4/15 repointed; P&P GPO still pilot-scoped**; `PSO-Caregivers` never-expire FGPP created 2026-07-02 (stops the 42-day cohort re-break); Shelby "Company Web Docs" share built 2026-07-02; Chris Knight granted FullAccess+SendAs on `accounting@` 2026-07-09; remaining-work plan: docs/REMAINING-WORK-PLAN.md | 2026-07-10 | -| [Dataforth Corporation](clients/dataforth.md) | Prepaid block ~$2,099/mo, **30.0 hrs remaining** (live 2026-07-04, 0 open tickets); signal-conditioning manufacturer; 64 DOS test stations; **new 2026-07-04: undocumented MYDATA TPSys SMT line controller (`myserver`, Fedora Core 3, VLAN2) discovered — root recovered via LILO single-user + vaulted; RMM agent ruled out (legacy glibc/kernel/no-systemd)**; test-data-chain audit 2026-07-01 (NWTOC v5.0 spec-copy gap = Syncro #32489); PBX inbound-SIP DNAT emergency fixed 2026-06-23; DFORTH-Ship 0x116 BSOD (aging G1 USDT); 2025 ransomware recovery + incomplete file restore; 2026-03 phishing + MFA; test-datasheet pipeline (Hoffman API + testdatadb on AD2); mail stack INKY->Mailprotector->EXO; shares-ACL project (all open to staff; Phase 2 strawman 2026-06-22); GuruRMM fleet ~45; Bitdefender phase-off | 2026-07-04 | +| [Dataforth Corporation](clients/dataforth.md) | Prepaid block ~$2,099/mo, **24.75 hrs remaining** (live 2026-07-14, 4 open tickets); signal-conditioning manufacturer; DOS test stations (NWTOC v5.1 deployed fleet-wide 2026-07-05); MYDATA TPSys SMT line controller (`myserver`, Fedora Core 3, VLAN2 — root recovered + DR backup); **2026-07-14: PBX SIP DNAT durable fix (native UniFi Port Forwarding rule, #32553) + Sage printing fix (stale RDP sessions on SAGE-SQL, #32552; Sage `guru` credential vaulted at `clients/dataforth/sage`)**; Sage Pro 200 ERP runs on SAGE-SQL RDS only (Easy Print redirection — native drivers not installed, watch item); 2025 ransomware recovery + incomplete file restore; 2026-03 phishing + MFA; test-datasheet pipeline (Hoffman API + testdatadb on AD2); mail stack INKY->Mailprotector->EXO; shares-ACL project (Phase 2 strawman 2026-06-22); GuruRMM fleet ~50; Bitdefender phase-off | 2026-07-14 | | [Desert RV Center](clients/desert-rv-center.md) | Break-fix + $213.35/mo recurring; RV dealership, Huachuca City AZ; 6 managed devices; DESERTRV.local domain (LAN 10.0.0.0/24); remote users RDP into on-site VMs over **ZeroTier** ("Desert RV" net 172.29.0.0/16); **2026-07-08: restored Alicia's RDP to DRV-BECKY** (stopped ZeroTierOneService = RDP 0x204; started + auto-restart; BECKY + DESKTOP-LBO396F enrolled in GuruRMM); DRV-BECKY firewall locked to RDP-only (no silent RMM push); BECKY slowness pending; 0 open tickets | 2026-07-08 | | [Instrumental Music Center](clients/instrumental-music-center.md) | Prepaid block $175/hr; music retail/repair; AIMsi POS on IMC1\SQLEXPRESS (SQL 2019 Std, misleading name); phantom DC (ServerIMC) slow logons; **two locations split into GuruRMM Main + MOO sites 2026-07-07** (Main 14 agents / MOO 3: BigMom, GHG12G3, IMC-LUIS); full fleet enrolled; AV = Datto EDR (not Bitdefender); IMC1 SQL max-memory caps applied 2026-07-07 | 2026-07-10 | | [Jimmy Company](clients/jimmy.md) | Break-fix, $150/hr; single aging workstation BLASTER2 (Win10 22H2 EOL, i5-3470/3.8GB — replace); backups the recurring theme (QuickBooks data); onboarded to GuruRMM 2026-06-19 (RDP NLA + Kaseya removal + cleanup); MSP360 local backup drive full, 90-day retention set, space reclaim pending in console (cloud B2 healthy) | 2026-06-19 |