diff --git a/clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1 b/clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1 new file mode 100644 index 0000000..9863062 --- /dev/null +++ b/clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1 @@ -0,0 +1,258 @@ +<# +.SYNOPSIS + Offline (WinPE / WinRE) neutralization of Sophos Endpoint tamper protection + so that SophosZap can complete removal after a single reboot. + +.DESCRIPTION + Inherited-MSP Sophos installs with NO Sophos Central access cannot be removed + from inside Windows: tamper protection is enforced by a boot-start kernel + driver (SophosED.sys / SophosEL.sys), and SophosZap refuses to run while the + registry flag SEDEnabled = 1. + + Run this from a PowerShell prompt in WinPE / WinRE (NOT normal Windows), + pointed at the OFFLINE Windows volume. It performs every edit needed so that + after ONE reboot, SophosZap --confirm runs cleanly: + + 1. Renames Sophos*.sys driver files -> .old (cannot load at boot) + 2. Sets the "Sophos Endpoint Defense" service Start = 4 (Disabled) + 3. Clears the tamper flags SEDEnabled = 0 and IgnoreSAV = 0 + + It asks for the Windows drive letter, proves the volume is really Windows + (not the ~600 MB recovery partition), shows you the current values before + changing anything, and confirms at every destructive step. + +.NOTES + Origin : Built from the Lone Star Electrical LS-1 removal, 2026-06-02. + Run from : WinPE / WinRE -> Command Prompt -> powershell (or a PE with PS). + Requires : the target Windows volume must be UNLOCKED. If BitLocker is on, + System32\config\SYSTEM is unreadable -- unlock with the recovery + key first (manage-bde -unlock X: -RecoveryPassword ), or + confirm BitLocker OFF from normal Windows before booting to PE. + + AFTER this script: + a. Remove the PE USB. + b. Reboot into normal Windows. + c. Run: SophosZap.exe --confirm (pass 1 -- bulk removal) + d. Reboot when it says "reboot and re-execute". + e. Run: SophosZap.exe --confirm (pass 2 -- finishes the job) + f. Verify: no Sophos services, drivers, folders, or Add/Remove entries; + Windows Defender real-time protection ON. +#> + +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +$HiveMount = 'HKLM\OFFSYS' # temporary mount point for the offline SYSTEM hive + +function Write-Head([string]$t) { Write-Host ""; Write-Host "==== $t ====" -ForegroundColor Cyan } +function Write-Ok ([string]$t) { Write-Host " [OK] $t" -ForegroundColor Green } +function Write-Warn([string]$t) { Write-Host " [WARN] $t" -ForegroundColor Yellow } +function Write-Err ([string]$t) { Write-Host " [ERROR] $t" -ForegroundColor Red } + +function Confirm-Step([string]$Message) { + $ans = Read-Host "$Message [y/N]" + return ($ans.Trim() -match '^(y|yes)$') +} + +Write-Host @" +============================================================ + Sophos Offline Removal (PE) - tamper-protection neutralizer +============================================================ + This edits an OFFLINE Windows volume. Make sure you are in + WinPE/WinRE, NOT the live Windows you want to clean. +"@ -ForegroundColor White + +try { + +# --------------------------------------------------------------------------- +# 1. Identify and validate the Windows drive letter +# --------------------------------------------------------------------------- +Write-Head "Step 1 - Identify the offline Windows volume" +Write-Host "Volumes visible in this PE session:" +Get-Volume -ErrorAction SilentlyContinue | + Where-Object DriveLetter | + Select-Object DriveLetter, FileSystemLabel, + @{n='Size(GB)';e={[math]::Round($_.Size/1GB,1)}}, + @{n='Free(GB)';e={[math]::Round($_.SizeRemaining/1GB,1)}} | + Format-Table -AutoSize | Out-String | Write-Host + +$drive = $null +do { + $entry = (Read-Host "Enter the Windows drive letter as shown HERE in PE (e.g. C, D, E)").Trim().TrimEnd(':') + if ($entry -notmatch '^[A-Za-z]$') { Write-Warn "Enter a single letter."; continue } + $win = "${entry}:\Windows" + $hive = "${entry}:\Windows\System32\config\SYSTEM" + if (-not (Test-Path $win)) { Write-Warn "$win not found -- that is not the Windows volume."; continue } + if (-not (Test-Path $hive)) { Write-Warn "$hive not found -- volume locked by BitLocker? Unlock it first."; continue } + $drive = $entry.ToUpper() +} while (-not $drive) + +# Prove it is the real OS volume, not the recovery partition +Write-Host "" +Write-Host "Evidence that ${drive}: is the real Windows volume:" +foreach ($p in 'Windows','Windows\System32','Windows\System32\config','Users','Program Files') { + $present = Test-Path "${drive}:\$p" + "{0,-28} {1}" -f $p, $(if ($present) {'present'} else {'MISSING'}) | Write-Host +} +Write-Host "" +if (-not (Confirm-Step "Is ${drive}: definitely the Windows install you want to clean?")) { + Write-Err "Aborted by user. No changes made."; return +} +$driversDir = "${drive}:\Windows\System32\drivers" +$systemHive = "${drive}:\Windows\System32\config\SYSTEM" + +# --------------------------------------------------------------------------- +# 2. Find Sophos kernel driver files +# --------------------------------------------------------------------------- +Write-Head "Step 2 - Sophos driver files on disk" +$sophosDrivers = @(Get-ChildItem $driversDir -Filter 'Sophos*.sys' -ErrorAction SilentlyContinue) +if ($sophosDrivers.Count -eq 0) { + Write-Warn "No Sophos*.sys driver files found (already removed, or different names)." +} else { + $sophosDrivers | Select-Object Name, Length, LastWriteTime | Format-Table -AutoSize | Out-String | Write-Host +} +# Note: *.man files are ETW manifests, not drivers -- SophosZap removes them. Ignore here. + +# --------------------------------------------------------------------------- +# 3. Load the offline SYSTEM hive and resolve the active ControlSet +# --------------------------------------------------------------------------- +Write-Head "Step 3 - Load the offline registry hive" +# Clean up a stale mount from a previous aborted run, if any. +reg unload $HiveMount 2>$null | Out-Null +$loaded = $false +try { + & reg load $HiveMount $systemHive | Out-Null + if ($LASTEXITCODE -ne 0) { throw "reg load failed (exit $LASTEXITCODE). Is the hive in use / volume locked?" } + $loaded = $true + Write-Ok "Loaded $systemHive as $HiveMount" + + # Offline hives have ControlSet001/002 + Select\Current -- NOT CurrentControlSet. + $controlSet = 'ControlSet001' + $sel = & reg query "$HiveMount\Select" /v Current 2>$null + if ($sel -match 'Current\s+REG_DWORD\s+0x([0-9a-fA-F]+)') { + $controlSet = "ControlSet{0:D3}" -f [Convert]::ToInt32($matches[1], 16) + } + Write-Ok "Active control set: $controlSet" + + $svcKey = "$HiveMount\$controlSet\Services\Sophos Endpoint Defense" + $tpKey = "$svcKey\TamperProtection\Config" + + # ----------------------------------------------------------------------- + # 4. Show current values BEFORE changing anything + # ----------------------------------------------------------------------- + Write-Head "Step 4 - Current Sophos tamper state (offline hive)" + $svcExists = $false + & reg query $svcKey 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + $svcExists = $true + Write-Host "Service 'Sophos Endpoint Defense' -> Start:" + & reg query $svcKey /v Start 2>$null | Where-Object { $_ -match 'Start' } | Write-Host + Write-Host "TamperProtection flags:" + & reg query $tpKey /v SEDEnabled 2>$null | Where-Object { $_ -match 'SEDEnabled' } | Write-Host + & reg query $tpKey /v IgnoreSAV 2>$null | Where-Object { $_ -match 'IgnoreSAV' } | Write-Host + } else { + Write-Warn "Service key 'Sophos Endpoint Defense' not found under $controlSet (already removed?)." + } + + Write-Host "" + Write-Host "Planned changes:" -ForegroundColor White + Write-Host " - rename $($sophosDrivers.Count) Sophos*.sys driver file(s) to .old" + Write-Host " - set service 'Sophos Endpoint Defense' Start = 4 (Disabled)" + Write-Host " - set SEDEnabled = 0 and IgnoreSAV = 0" + Write-Host "" + if (-not (Confirm-Step "Apply these changes to ${drive}: now?")) { + Write-Err "Aborted by user before changes. Unloading hive, no edits made." + return + } + + # ----------------------------------------------------------------------- + # 5. Apply registry edits (hive still loaded) + # ----------------------------------------------------------------------- + Write-Head "Step 5 - Apply registry edits" + if ($svcExists) { + & reg add $svcKey /v Start /t REG_DWORD /d 4 /f | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Ok "Service Start set to 4 (Disabled)" } else { Write-Err "Failed to set Start" } + + & reg add $tpKey /v SEDEnabled /t REG_DWORD /d 0 /f | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Ok "SEDEnabled set to 0" } else { Write-Warn "Could not set SEDEnabled (key may not exist on this version)" } + + & reg add $tpKey /v IgnoreSAV /t REG_DWORD /d 0 /f | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Ok "IgnoreSAV set to 0" } else { Write-Warn "Could not set IgnoreSAV" } + + Write-Host "" + Write-Host "Read-back after edit:" + & reg query $svcKey /v Start 2>$null | Where-Object { $_ -match 'Start' } | Write-Host + & reg query $tpKey /v SEDEnabled 2>$null | Where-Object { $_ -match 'SEDEnabled' } | Write-Host + } else { + Write-Warn "No SED service key to edit -- skipping registry changes." + } +} +finally { + if ($loaded) { + [gc]::Collect(); Start-Sleep -Milliseconds 300 + & reg unload $HiveMount 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { Write-Ok "Unloaded offline hive ($HiveMount)" } + else { Write-Warn "reg unload reported a non-zero exit -- if it stayed mounted, close regedit/handles and run: reg unload $HiveMount" } + } +} + +# --------------------------------------------------------------------------- +# 6. Rename the driver files (after the hive is unloaded) +# --------------------------------------------------------------------------- +Write-Head "Step 6 - Rename Sophos driver files" +if ($sophosDrivers.Count -gt 0) { + if (Confirm-Step "Rename $($sophosDrivers.Count) Sophos*.sys file(s) to .old so they cannot load?") { + foreach ($f in $sophosDrivers) { + $target = "$($f.FullName).old" + try { + if (Test-Path $target) { Remove-Item $target -Force } + Rename-Item -LiteralPath $f.FullName -NewName "$($f.Name).old" -Force + Write-Ok "Renamed $($f.Name) -> $($f.Name).old" + } catch { + Write-Err "Could not rename $($f.Name): $($_.Exception.Message)" + } + } + } else { + Write-Warn "Skipped driver rename (service Start=4 alone should still stop it loading)." + } +} else { + Write-Host " (nothing to rename)" +} + +# --------------------------------------------------------------------------- +# 7. Next steps +# --------------------------------------------------------------------------- +Write-Head "DONE - offline edits complete" +Write-Host @" +Next, in NORMAL Windows (not PE): + + 1. Remove the PE USB so the box boots to Windows. + 2. Reboot into Windows. + 3. Run: SophosZap.exe --confirm (pass 1) + 4. Reboot when it reports 'reboot and re-execute'. + 5. Run: SophosZap.exe --confirm (pass 2) + 6. Verify clean: + Get-Service *sophos* -> nothing + dir C:\Windows\System32\drivers\Sophos* -> nothing (or only *.old) + 'C:\Program Files\Sophos','C:\ProgramData\Sophos' -> gone + Get-MpComputerStatus -> RealTimeProtectionEnabled = True + +If SophosZap still says 'tamper protection on', the SEDEnabled flag did not +clear -- re-check HKLM\SYSTEM\CurrentControlSet\services\Sophos Endpoint Defense\ +TamperProtection\Config\SEDEnabled in live Windows and set it to 0. +"@ -ForegroundColor White + +} +catch { + Write-Host "" + Write-Err "Script stopped on an error:" + Write-Host " $($_.Exception.Message)" -ForegroundColor Red + if ($_.InvocationInfo) { Write-Host " at line $($_.InvocationInfo.ScriptLineNumber): $($_.InvocationInfo.Line.Trim())" -ForegroundColor DarkGray } + # Best-effort: make sure we never leave the offline hive mounted after a crash. + reg unload $HiveMount 2>$null | Out-Null +} +finally { + Write-Host "" + [void](Read-Host "Press Enter to close this window") +} diff --git a/clients/lonestar-electrical/session-logs/2026-06-02-session.md b/clients/lonestar-electrical/session-logs/2026-06-02-session.md index dbb9b38..3883e1a 100644 --- a/clients/lonestar-electrical/session-logs/2026-06-02-session.md +++ b/clients/lonestar-electrical/session-logs/2026-06-02-session.md @@ -115,3 +115,76 @@ Separately, a `/sync` exposed a fleet repo-coordination problem: the `.claude/sk - Mike's deletion commit: `c759f04` "chore(memory): re-apply consolidation deletions + lift additive-only constraint". - HEAD after sync: `dd414c4`. - Full LS-1/LS-2 offline procedure: `clients/lonestar-electrical/session-logs/2026-05-29-sophos-removal.md`. + +--- + +## Update: 17:39 PT — Sophos removal COMPLETE (LS-1 + LS-2) + Unraid ticket + +## User +- **User:** Howard Enos (howard) +- **Machine:** Howard-Home +- **Role:** tech + +### Session Summary + +Completed the long-pending Sophos Endpoint removal on both Lone Star Norris workstations (LS-1 and LS-2), then created/closed the Syncro ticket for the earlier Unraid boot-USB replacement. Both removal jobs were driven remotely through GuruRMM once each machine was back in Windows. + +LS-1 was resumed from the offline-PE prep done earlier (driver renamed, BitLocker confirmed off). The blocker turned out to be more than the kernel driver: SophosZap refuses to run while the registry flag `HKLM\SYSTEM\CurrentControlSet\services\Sophos Endpoint Defense\TamperProtection\Config\SEDEnabled = 1`. Because the SophosED tamper driver was not loaded this boot (renamed offline), the flag could be cleared live as SYSTEM (`SEDEnabled=0`). SophosZap v1.9.158.0 then ran two passes (with a reboot between) via RMM and reported clean — no Sophos services, drivers, folders, or Add/Remove entries; Windows Defender real-time protection active. + +LS-2 was done via the manual offline (WinRE) procedure since it was offline in RMM at the start. Howard loaded the offline SYSTEM hive and set the SED service `Start=4` + `SEDEnabled=0`, then renamed the Sophos driver files. On reboot the machine dropped into Automatic Repair. The `SrtTrail.txt` root cause was explicit: "Boot critical file C:\WINDOWS\system32\DRIVERS\SophosEL.sys is corrupt" — i.e. missing because it was renamed. `SophosEL.sys` is the **Sophos ELAM** (Early Launch Anti-Malware) driver: `Start=0` (Boot), `ErrorControl=3` (Critical), so its absence aborts boot. Recovery: booted back to PE and renamed `SophosEL.sys.old` back to `SophosEL.sys`; the machine then booted. + +Once LS-2 was back in Windows, an RMM read of the service config showed the earlier offline edits had actually landed correctly: `Select\Current = 0x1` (ControlSet001 IS active), the SED tamper driver service (`Sophos Endpoint Defense`, SophosED.sys) was already `Start=4`, and `SEDEnabled` was 0 — so tamper protection was already neutralized. SophosZap then ran two passes via RMM (with a verified-safe reboot between — `SophosEL.sys` confirmed present on disk and no pending Sophos file-renames before rebooting) and reported clean. Defender active on LS-2. + +Billing: created/closed two Syncro tickets against the Lone Star prepaid block (customer 33809612). #32347 (Sophos removal LS-1+LS-2): 2.0h in-shop, invoiced $0.00, block 17.0 -> 15.0, Closed. #32372 (Unraid boot-USB replacement, documenting the earlier 2026-06-02 server fix): 1.5h in-shop, invoiced $0.00, block 15.0 -> 13.5, Closed. + +### Key Decisions + +- Cleared `SEDEnabled=0` (the SophosZap tamper gate) rather than only relying on the driver rename — the registry flag, not the driver presence, is what SophosZap checks. +- LS-2: after the boot failure, did NOT re-rename the boot-critical `SophosEL.sys`. Restored it and relied on the (already-correct) SED service `Start=4` + `SEDEnabled=0` to neutralize tamper, letting SophosZap remove the ELAM driver itself the boot-safe way. +- Verified `SophosEL.sys` present + no pending Sophos file-renames BEFORE the pass-2 reboot on LS-2, to avoid repeating the boot failure. +- Drove both machines via GuruRMM (read service config, set registry, run SophosZap, reboot) rather than hands-on once each was in Windows. + +### Problems Encountered + +- **LS-2 boot failure (Automatic Repair).** Root cause (SrtTrail.txt): boot-critical `SophosEL.sys` (Sophos ELAM, Start=0/ErrorControl=3) was renamed and thus "corrupt"/missing. Resolved by booting to PE and renaming `SophosEL.sys.old` back to `SophosEL.sys`. +- **SophosZap blocked by tamper flag, not driver.** First LS-1 run errored "SophosZap does not run with tamper protection on" with the driver already renamed — the `SEDEnabled=1` registry flag was the gate. Resolved by setting `SEDEnabled=0`. +- **Offline ControlSet correctness.** The offline edit used `ControlSet001`; this only worked because `Select\Current=0x1`. Documented that the active control set must be read from `HKLM\OFFSYS\Select\Current` before editing; `CurrentControlSet` does not exist in an offline hive. +- **PE PowerShell script closed on error.** The first-draft `Remove-Sophos-Offline-PE.ps1` exited (window closed) on an unhandled error. Hardened with a top-level try/catch + guaranteed `Read-Host` pause; abandoned in favor of the manual walkthrough for this job. + +### Configuration Changes + +- LS-1, LS-2: Sophos Endpoint Protection fully removed (services, drivers, `C:\Program Files\Sophos`, `C:\Program Files (x86)\Sophos`, `C:\ProgramData\Sophos`, Add/Remove entries, catalogs, certs). Windows Defender now the active AV on both. +- LS-2 registry (offline, ControlSet001): `Sophos Endpoint Defense` service `Start=4`; `...\TamperProtection\Config\SEDEnabled=0`, `IgnoreSAV=0`. +- Created `clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1` (offline PE removal helper; hardened error handling). + +### Credentials & Secrets + +- None created or changed. (Lone Star Unraid root password still not vaulted — pre-existing TODO.) + +### Infrastructure & Servers + +- **LS-1** GuruRMM agent id `6b9617fa-5c77-40e1-8b64-a1545e730895` (windows). +- **LS-2** GuruRMM agent id `97fe5582-aa3d-4132-94a6-f4c8582bca31` (windows). +- Sophos drivers (LS-2): `SophosED.sys` (2,561,552 B) = "Sophos Endpoint Defense" tamper driver, Type 2, ended at Start=4; `SophosEL.sys` (28,616 B) = "Sophos ELAM", Type 1, Start=0/ErrorControl=3 (BOOT-CRITICAL). +- SophosZap: v1.9.158.0; log at `C:\WINDOWS\SystemTemp\SophosZap log.txt`; staged to `C:\Windows\Temp\SophosZap.exe` for pass 2. + +### Commands & Outputs + +- Tamper gate (per SophosZap log): `Value 'SEDEnabled' ... is set to 1. Tamper-protected by SED. ERROR: SophosZap does not run with tamper protection on`. +- Clear it (live, SYSTEM): `reg add "HKLM\SYSTEM\CurrentControlSet\services\Sophos Endpoint Defense\TamperProtection\Config" /v SEDEnabled /t REG_DWORD /d 0 /f`. +- Offline (PE): `reg load HKLM\OFFSYS X:\Windows\System32\config\SYSTEM` -> edit under `HKLM\OFFSYS\ControlSet001\Services\...` -> `reg unload HKLM\OFFSYS`. Active set from `reg query HKLM\OFFSYS\Select /v Current`. +- LS-2 boot root cause: `Boot critical file C:\WINDOWS\system32\DRIVERS\SophosEL.sys is corrupt` (SrtTrail.txt). Fix: `ren X:\Windows\System32\drivers\SophosEL.sys.old SophosEL.sys`. +- Removal run: `SophosZap.exe --confirm` x2 (reboot between); final outcome `error flag: 0`, services/drivers/folders NONE, Defender RTP True. + +### Pending / Incomplete Tasks + +- Vault the Lone Star Unraid root password + document the server (hostname, IP, Unraid 7.1.4, license type) in the wiki — still open. +- Keep the old failing Unraid USB stick as backup until the new stick is confirmed stable, then retire. +- Optional: delete leftover `SophosEL.sys.old` on LS-2 if any remained (cleanup attempted in pass 2). + +### Reference Information + +- Syncro: #32347 (Sophos removal, id 111423954, invoice 1650552617) and #32372 (Unraid USB, id 112022651, invoice 1650552739) — both Closed, prepaid, customer 33809612. Block now 13.5 hrs. +- RMM API base `http://172.16.3.30:3001`. +- PE removal script: `clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1`. +- Offline procedure reference: `clients/lonestar-electrical/session-logs/2026-05-29-sophos-removal.md`. diff --git a/wiki/clients/lonestar-electrical.md b/wiki/clients/lonestar-electrical.md index a426e0c..ffc0ea8 100644 --- a/wiki/clients/lonestar-electrical.md +++ b/wiki/clients/lonestar-electrical.md @@ -3,7 +3,7 @@ type: client name: lonestar-electrical display_name: Lone Star Electrical Systems LLC last_compiled: 2026-06-02 -compiled_by: HOWARD-HOME/claude-main +compiled_by: Howard-Home/claude-main sources: - clients/lonestar-electrical/session-logs/2026-06-02-session.md - clients/lonestar-electrical/session-logs/2026-06-01-session.md @@ -29,7 +29,7 @@ Electrical contractor in Tucson, AZ. ACG-managed client. Distinctive in the flee - **Company type:** Electrical contractor (field service) - **Contract type:** Prepaid hour block -- **Hours remaining:** 17.0 hrs as of 2026-06-01 (Syncro live). Always live-check `GET /customers/33809612` before billing. +- **Hours remaining:** 13.5 hrs as of 2026-06-02 (Syncro live — always re-check `GET /customers/33809612` before billing). - **Billing rate:** (verify — check recent Syncro invoices; not captured in available sources) - **Syncro customer ID:** `33809612` (Lone Star Electrical Systems LLC) - **Address:** 3774 North Warren Avenue, Tucson, AZ @@ -42,7 +42,7 @@ Electrical contractor in Tucson, AZ. ACG-managed client. Distinctive in the flee - James — account compromised 2026-03-10 (Syncro #32010); [verify current name/role] - Kyla, Russ — GWS user accounts touched via provisioning/2FA scripts (temp/); [verify roles] - Main phone on file (Syncro): 520-730-3642 -- **Active ticket:** None open in Syncro as of 2026-06-01 (see Active Work) +- **Active ticket:** None open in Syncro as of 2026-06-02 --- @@ -63,7 +63,10 @@ Electrical contractor in Tucson, AZ. ACG-managed client. Distinctive in the flee ### Workstations -- **LS-1, LS-2** — Windows workstations at the **Norris site**; both upgraded to Win11 on 2026-05-04 (Syncro #32244). Both were inherited from the **previous MSP** with **Sophos Endpoint Protection** (managed via the previous MSP's Sophos Central — no ACG access). Sophos removal is in progress (see Patterns and Active Work). Both enrolled in **GuruRMM** during the 2026-05 removal work; ScreenConnect + GuruRMM agents registered for Safe Mode (`SafeBoot\Network`). +- **LS-1, LS-2** — Windows workstations at the **Norris site**; both upgraded to Win11 on 2026-05-04 (Syncro #32244). Both were inherited from the **previous MSP** with **Sophos Endpoint Protection** (managed via the previous MSP's Sophos Central — no ACG access). **Sophos has been fully removed from both machines as of 2026-06-02** (Syncro #32347; see Patterns for full procedure). Both enrolled in **GuruRMM** during the 2026-05 removal work; ScreenConnect + GuruRMM agents registered for Safe Mode (`SafeBoot\Network`). + - **LS-1 GuruRMM agent:** `6b9617fa-5c77-40e1-8b64-a1545e730895` + - **LS-2 GuruRMM agent:** `97fe5582-aa3d-4132-94a6-f4c8582bca31` + - **Windows Defender:** active and real-time protection enabled on both as of 2026-06-02. ### Unraid Server @@ -92,26 +95,51 @@ Electrical contractor in Tucson, AZ. ACG-managed client. Distinctive in the flee ## Patterns & Known Issues -- **Inherited Sophos with no Central access — kernel-driver tamper-protection removal (execution started 2026-06-02).** LS-1 and LS-2 came from the previous MSP running Sophos Endpoint Protection managed via the previous MSP's Sophos Central account — ACG has **no Central access**, so no remote uninstall and no way to disable tamper protection from the management plane. Tamper protection is enforced by the **`SophosED.sys` kernel boot driver** (`Start=0`, loads before `smss.exe`), which defeats every user-mode removal: `SophosZap` (blocked by TP), `SophosUninstall.exe` (only removes user-mode parts), `PendingFileRenameOperations` delete (driver loads too early), `sc config` (kernel callback), and ACL reset (kernel-level). **Resolution path is offline via WinRE/PE:** delete `D:\Windows\System32\drivers\SophosED.sys`, load the offline SYSTEM hive and set the `Sophos Endpoint Defense` service `Start=4`, reboot, then `SophosZap.exe --confirm` (TP check now passes). Full step list in the 2026-05-29 session log. **Reusable for any inherited-MSP Sophos/CrowdStrike/SentinelOne removal where tamper protection is enforced and the management console is inaccessible.** (Related: GuruRMM SPEC-015 safeboot-network-registration aims to automate exactly this remote-Safe-Mode removal flow.) +- **Inherited Sophos with no Central access — kernel-driver tamper-protection removal (procedure proven and COMPLETE on LS-1 and LS-2, 2026-06-02).** LS-1 and LS-2 came from the previous MSP running Sophos Endpoint Protection managed via the previous MSP's Sophos Central account — ACG has **no Central access**, so no remote uninstall and no way to disable tamper protection from the management plane. The procedure is now proven end-to-end and reusable. Key findings from the full execution: + + - **SophosZap's gate is a registry flag, not just the driver.** SophosZap checks `HKLM\SYSTEM\CurrentControlSet\services\Sophos Endpoint Defense\TamperProtection\Config\SEDEnabled` — if this is `1`, SophosZap exits with "does not run with tamper protection on" even when the kernel driver is renamed/disabled. The driver disable alone is not sufficient; `SEDEnabled=0` must be set. + + - **Two Sophos boot drivers — treat them differently:** + - **`SophosED.sys`** = "Sophos Endpoint Defense" (the TAMPER driver). `Start=0` by default (Boot-start). Safe to rename/remove. Correct procedure: set service `Start=4` in the offline hive AND clear `SEDEnabled=0`. With `SEDEnabled=0`, SophosZap passes the tamper check and removes it cleanly. + - **`SophosEL.sys`** = "Sophos ELAM" (Early Launch Anti-Malware). `Start=0`, **`ErrorControl=3` (CRITICAL)**. **NEVER rename or delete this file manually.** If `SophosEL.sys` is missing on boot, Windows drops to Automatic Repair: `SrtTrail.txt` root cause: "Boot critical file C:\WINDOWS\system32\DRIVERS\SophosEL.sys is corrupt." Recovery requires booting back to PE and restoring the file. SophosZap removes the ELAM driver and its service itself, the boot-safe way, after tamper protection is neutralized. + + - **Offline hive editing: always read the active ControlSet first.** `CurrentControlSet` does not exist in an offline hive. Read `HKLM\OFFSYS\Select\Current` to determine which numbered set is active (e.g., `0x1` = `ControlSet001`) before editing service entries. Editing the wrong ControlSet leaves the machine unchanged. + + - **Correct offline procedure (PE):** + 1. `reg load HKLM\OFFSYS X:\Windows\System32\config\SYSTEM` + 2. `reg query HKLM\OFFSYS\Select /v Current` — note the active set number + 3. Under `HKLM\OFFSYS\ControlSet00N\Services\Sophos Endpoint Defense`: set `Start=4`; under `...\TamperProtection\Config`: set `SEDEnabled=0` + 4. `reg unload HKLM\OFFSYS` + 5. Reboot to normal Windows. Do NOT rename or delete `SophosEL.sys`. + 6. Verify Defender is active. Run `SophosZap.exe --confirm` via RMM or locally. Reboot as prompted. + 7. Run `SophosZap.exe --confirm` a second time. Confirm: services/drivers/folders NONE, Defender RTP True. + + - **PE helper script:** `clients/lonestar-electrical/scripts/Remove-Sophos-Offline-PE.ps1` (hardened with top-level try/catch and guaranteed `Read-Host` pause). + - **Reusable for any inherited-MSP Sophos/CrowdStrike/SentinelOne removal where tamper protection is enforced and the management console is inaccessible.** (Related: GuruRMM SPEC-015 safeboot-network-registration aims to automate exactly this remote-Safe-Mode removal flow.) + - **Sophos shell extensions + Datto Cloud Continuity startup conflict (LS-2).** Presented as unresponsive desktop mouse clicks (until Ctrl+Alt+Del) and dead Start-menu right-click. Root cause: Sophos shell extensions competing with the Datto Cloud Continuity `/pop` startup entry during logon. Removing the Datto startup registry entry addressed the logon contention. + - **ManageEngine + Google Workspace dual-EMM trap (resolved 2026-03-24).** A personal phone repeatedly prompted for MDM enrollment when the user added their Lonestar Google account. Root cause was **two independent triggers**: (1) ManageEngine MDM self-enrollment was enabled for all directory groups, AND (2) ManageEngine was configured as a **third-party EMM provider inside Google Workspace** (Devices > Mobile & endpoints > Settings > Third-party integrations). The Google integration enforces enrollment on any device that adds a Lonestar account — independent of ManageEngine's own self-enrollment setting. **Fix required both:** disable ManageEngine self-enrollment (Enrollment > Self Enrollment > Disable) AND remove ManageEngine as the third-party EMM in the GWS Admin Console. Disabling only one leaves the prompt in place. Company tablets enrolled directly via QR code are unaffected by either change. + - **Google Workspace, not M365.** Reach for GWS Admin Console + the ACG-MSP-Access service account for identity work. The M365 remediation-tool app suite does not apply to this client. + - **Field/mobile-first.** Most tickets are phone/tablet/field-device oriented (iPhone field setup, tablet PDF editing). Expect mobile, not desktop, as the primary support surface — the LS-1/LS-2 desktop work is the exception, not the norm. + - **Recurring `bzfirmware` checksum boot error = failing USB flash drive.** Replace the stick (Unraid USB Creator + copy old `config/` + re-register license to new GUID). Do NOT just replace the file — if the error recurs after a file-level fix, the stick itself is failing. Reusable for any Unraid box. --- ## Active Work -No open Syncro tickets as of 2026-06-01. +No open Syncro tickets as of 2026-06-02. -- **Sophos removal on LS-1 / LS-2 (ACTIVELY EXECUTING — LS-1 in progress, LS-2 not yet started).** Offline PE removal procedure is underway on LS-1: BitLocker confirmed OFF (verified from normal Windows before booting PE), `SophosZap.exe` staged in Downloads for post-reboot cleanup. LS-1 is awaiting a drive-letter check from PE (`dir C:\Windows & dir D:\Windows & dir E:\Windows`) before executing the `del /f \Windows\System32\drivers\SophosED.sys` + offline-hive `Start=4` disable sequence. LS-2 not yet started. Full offline command set in `clients/lonestar-electrical/session-logs/2026-05-29-sophos-removal.md`. Coord handoff: msg `689cfb7c` (2026-06-01). - - **Pending:** Verify or create Syncro ticket "Sophos Endpoint Removal - LS-1 and LS-2" before logging time (prepaid block, live-check `GET /customers/33809612`). -- **Unraid server USB replacement done (2026-06-02); PENDING:** - - Create Syncro ticket documenting the USB failure, replacement (Unraid 7.1.4 via USB Creator), config copy, and license re-registration. +- **Sophos removal on LS-1 / LS-2 — COMPLETE (2026-06-02).** Both machines are fully clean: no Sophos services, drivers, folders, or Add/Remove entries; Windows Defender real-time protection active on both. Billed and closed on Syncro #32347 (2.0h in-shop, prepaid). See Patterns for the full reusable procedure including the critical SophosEL ELAM boot-driver lesson. + +- **Unraid server USB replacement — COMPLETE (2026-06-02).** New stick running Unraid 7.1.4, config/ preserved, license re-registered. Documented and billed on Syncro #32372 (1.5h in-shop, prepaid, Closed). **Still open:** + - Vault the Lonestar Unraid root password and document the server (hostname, IP, Unraid 7.1.4, license type) in the wiki. - Capture and fold in the results of Mike's server health check (array start state, disk assignments, parity validity, registration status). - Verify array integrity: confirm all disks landed in correct slots from the copied `super.dat`; ensure no unwanted parity rebuild was triggered. - - Vault the Lonestar Unraid root password and document the server in the wiki (hostname, IP, Unraid 7.1.4, license type). + - Retire the old failing USB stick once the new stick is confirmed stable. --- @@ -129,12 +157,15 @@ No open Syncro tickets as of 2026-06-01. | 2026-05-28/29 | Sophos removal on LS-1/LS-2 begun: enrolled in GuruRMM, removed Datto startup conflict (LS-2), registered Safe Mode agents, removed user-mode Sophos; blocked by `SophosED.sys` kernel driver — WinRE offline removal staged (Ventoy USB), completion pending | | 2026-06-01 | Recovered the (previously unlogged) Sophos removal context, reconstructed it into a session log, and handed the WinRE completion procedure to Howard via coordinator (msg `689cfb7c`) | | 2026-06-02 | Unraid server USB flash drive failed (recurring bzfirmware checksum error); migrated to new stick (Unraid 7.1.4 via USB Creator), copied old config/, re-registered license to new GUID | -| 2026-06-02 | Began offline (PE) execution of Sophos removal on LS-1 — BitLocker confirmed off, SophosZap staged; SophosED.sys delete + offline-hive disable pending drive-letter check | +| 2026-06-02 | LS-1 Sophos offline-PE prep: BitLocker confirmed off, SophosZap staged, drive-letter check run; SED service Start=4 + SEDEnabled=0 set offline | +| 2026-06-02 | Sophos removal COMPLETED on LS-1 and LS-2 — offline tamper-disable (SED Start=4 + SEDEnabled=0) + SophosZap two-pass via GuruRMM; LS-2 hit Automatic Repair after boot-critical SophosEL.sys was renamed (recovered by restoring the file from PE, then relying on already-correct offline edits + SophosZap to remove it safely); Windows Defender active on both | +| 2026-06-02 | Syncro #32347 (Sophos removal, 2.0h in-shop) and #32372 (Unraid USB replacement, 1.5h in-shop) created, billed, and closed against prepaid block — 17.0 -> 13.5 hrs remaining | --- ## Compilation Notes +- Refreshed 2026-06-02 ~17:45 PT (recompile by Howard-Home/claude-main) to absorb the "17:39 PT — Sophos removal COMPLETE" update section of the 2026-06-02 session log: marked Sophos removal COMPLETE on both LS-1/LS-2 in Active Work and Infrastructure; updated hours remaining to 13.5 (Syncro #32347 2.0h + #32372 1.5h billed/closed); expanded Patterns with the proven full procedure including the critical two-driver distinction (SophosEL ELAM boot-critical — never rename/delete; SophosED tamper driver — disable via Start=4+SEDEnabled=0); added LS-1/LS-2 GuruRMM agent IDs; added two new History Highlights rows (PE+SophosZap completion, billing). - Refreshed 2026-06-02 22:10 PT (recompile by HOWARD-HOME/claude-main) to absorb the 22:10 PT update section of the 2026-06-02 session log: updated Active Work Sophos bullet to reflect execution-in-progress on LS-1 (BitLocker confirmed off, SophosZap staged, awaiting drive-letter check before PE delete); updated Patterns wording from "in progress 2026-05-28/29" to "execution started 2026-06-02"; added History Highlights row for the LS-1 PE execution start. - Refreshed 2026-06-02 (recompile by HOWARD-HOME/claude-main) to absorb the 2026-06-02 session log: added Unraid server infrastructure subsection, new `bzfirmware` checksum pattern, history row, and pending Active Work items. - Refreshed 2026-06-01 (full recompile) to incorporate the 2026-05-28/29 Sophos removal work, which had previously been lost — it was never written to a session log and survived only in a gitignored temp draft (`.claude/tmp/ollama_prompt.txt`) and coord message `8a5cb25c`. A proper session log was reconstructed at `clients/lonestar-electrical/session-logs/2026-05-29-sophos-removal.md` before this compile. diff --git a/wiki/index.md b/wiki/index.md index f76c28e..0e06ac9 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -41,7 +41,7 @@ Run `/wiki-lint` to check for stale entries and broken backlinks. | [Western Tire](clients/western-tire.md) | Tire retail (jackfurriers.com brand); Mike Furrier owner (Syncro ID 391491); email migrated from websvr to IX 2026-04-22; 30 mailboxes; SSL cert expires 2026-05-30 | 2026-05-24 | | [Kittle (general contractor)](clients/kittle.md) | General contractor Tucson AZ; Syncro 32460233; HPE MicroServer Gen11 WS2025 EVAL at 10.0.0.5; no backups, no firewall; DKIM/DMARC missing; 3 plaintext creds in Syncro notes; GuruRMM onboarding 2026-05-08 | 2026-05-24 | | [Khalsa (two-site)](clients/khalsa.md) | Two-site client (Camden + River); onboarding not completed; domain khalsa.local, DC TROUT at 10.11.12.254; Mac domain-join runbook documented; template docs otherwise empty | 2026-05-24 | -| [Lone Star Electrical Systems](clients/lonestar-electrical.md) | Electrical contractor Tucson AZ; Syncro 33809612, prepaid block 17.0 hrs; Google Workspace (not M365); ManageEngine MDM (Zoho); Unraid server (7.1.4, USB migrated 2026-06-02); LS-1/LS-2 inherited-Sophos kernel-driver removal in progress; field/mobile-first | 2026-06-02 | +| [Lone Star Electrical Systems](clients/lonestar-electrical.md) | Electrical contractor Tucson AZ; Syncro 33809612, prepaid block 13.5 hrs; Google Workspace (not M365); ManageEngine MDM (Zoho); Unraid server (7.1.4, USB migrated 2026-06-02); LS-1/LS-2 Sophos removal COMPLETE (2026-06-02); Defender active on both; field/mobile-first | 2026-06-02 | | [Anaise](clients/anaise.md) | Single workstation client; contact David (anaisedavid.office@gmail.com); DESKTOP-O8GF4SD; creds in vault at clients/anaise/desktop-o8gf4sd.sops.yaml; onboarding incomplete; M365 enrollment unconfirmed | 2026-05-24 | | [ACG Website (azcomputerguru.com)](clients/azcomputerguru.com.md) | Public website redesign (Astro); score 33/40; placeholder testimonials + no-backend form are pre-launch blockers; OKLCH token design system; see internal-infrastructure.md for ACG servers | 2026-05-24 | | [Quantum WMS](clients/quantumwms.md) | WMS company; quantumwms.com tenant (ddf3d2c9); GoDaddy decoupling + M365 migration; 2x Business Premium + Exchange Online Plan 1; deadline 2026-06-03; Tenant Admin consented 2026-05-26 | 2026-05-26 |