diff --git a/.claude/skills/coord/SKILL.md b/.claude/skills/coord/SKILL.md index da56d16..36c4add 100644 --- a/.claude/skills/coord/SKILL.md +++ b/.claude/skills/coord/SKILL.md @@ -35,6 +35,7 @@ py "$CLAUDETOOLS_ROOT/.claude/skills/coord/scripts/coord.py" ... | `lock claim "" [--ttl HOURS]` | Claim a work lock (default ttl 2h). | | `lock release ` | Release a lock. | | `lock list [--project KEY]` | List active locks. | +| `component set [--version V] [--notes TXT]` | Update a component's deploy state (auto-fills `updated_by`). E.g. `component set gururmm dashboard deployed --version 0.2.39`. | ## Conventions it handles for you diff --git a/.claude/skills/coord/scripts/coord.py b/.claude/skills/coord/scripts/coord.py index 22ea233..d607480 100644 --- a/.claude/skills/coord/scripts/coord.py +++ b/.claude/skills/coord/scripts/coord.py @@ -192,6 +192,17 @@ def c_lock_list(a): print(f" {str(l.get('id',''))[:8]} {l.get('project_key')}:{l.get('resource')} by {l.get('session_id')} exp {l.get('expires_at')}") +def c_comp_set(a): + payload = {"state": a.state, "updated_by": SESSION} + if a.version: + payload["version"] = a.version + if a.notes: + payload["notes"] = a.notes + st, r = call("PUT", f"/components/{a.project}/{a.component}", payload) + die(st, r, ok=(200, 201)) + print(f"[coord] component {a.project}/{a.component} -> state={a.state}" + (f" v{a.version}" if a.version else "")) + + def main(): p = argparse.ArgumentParser(prog="coord.py", description="ClaudeTools coordination API helper") sub = p.add_subparsers(dest="cmd", required=True) @@ -212,6 +223,10 @@ def main(): tl.add_argument("--status", default="pending"); tl.set_defaults(fn=c_todo_list) td = t.add_parser("done"); td.add_argument("id"); td.set_defaults(fn=c_todo_done) + cp = sub.add_parser("component").add_subparsers(dest="sub", required=True) + cs = cp.add_parser("set"); cs.add_argument("project"); cs.add_argument("component"); cs.add_argument("state") + cs.add_argument("--version"); cs.add_argument("--notes"); cs.set_defaults(fn=c_comp_set) + lk = sub.add_parser("lock").add_subparsers(dest="sub", required=True) lc = lk.add_parser("claim"); lc.add_argument("project"); lc.add_argument("resource") lc.add_argument("desc"); lc.add_argument("--ttl", type=int, default=2); lc.set_defaults(fn=c_lock_claim) diff --git a/.claude/skills/grok/SKILL.md b/.claude/skills/grok/SKILL.md index 6d421bf..27ec3eb 100644 --- a/.claude/skills/grok/SKILL.md +++ b/.claude/skills/grok/SKILL.md @@ -30,8 +30,9 @@ bash "$CLAUDETOOLS_ROOT/.claude/skills/grok/scripts/ask-grok.sh" ... | Mode | Usage | What it does | |------|-------|--------------| -| `text` | `ask-grok.sh text ""` | One-shot text answer (independent model). Prints to stdout. | -| `verify` | `ask-grok.sh verify ""` | Adversarial second opinion — Grok tries to REFUTE, returns CONFIRMED/REFUTED + reason. | +| `text` | `ask-grok.sh text ""` or `text --prompt-file ` | One-shot text answer (independent model). `--prompt-file` for long content (review/summarize a doc). | +| `verify` | `ask-grok.sh verify ""` or `verify --prompt-file ` | Adversarial second opinion — Grok tries to REFUTE/find gaps, returns a verdict + reasons. | +| `review` | `ask-grok.sh review [""]` | Grok reads the file at `` itself (its `read_file` tool, run in the repo) and reviews it — no embedding, handles large files, can pull in referenced files. | | `image` | `ask-grok.sh image "" [out.png]` | `image_gen` (Imagine) → copies the artifact to `out` (default `grok-image.png`). | | `video` | `ask-grok.sh video "" [out.mp4]` | `image_to_video` on an input image → copies to `out`. ~60-90s. | | `xsearch` | `ask-grok.sh xsearch ""` | Live `web_search` + X/Twitter tools; returns text with citations. | diff --git a/.claude/skills/grok/scripts/ask-grok.sh b/.claude/skills/grok/scripts/ask-grok.sh index 782660d..e0d534f 100644 --- a/.claude/skills/grok/scripts/ask-grok.sh +++ b/.claude/skills/grok/scripts/ask-grok.sh @@ -67,13 +67,15 @@ MODE="${1:-}"; shift 2>/dev/null || true TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT WORK="$TMP/work"; mkdir -p "$WORK" PF="$TMP/prompt.txt"; OUT="$TMP/out.json" +RUN_CWD="$WORK" # grok's working dir; the 'review' mode overrides to the repo so read_file can reach repo files +REPO_ROOT="${CLAUDETOOLS_ROOT:-$(cd "$SCRIPT_DIR/../../../.." 2>/dev/null && pwd)}" # run grok headless. $1=timeout secs; rest=extra flags. Reads $PF -> $OUT. # Never fails the script on grok's exit code (Cancelled is expected; we read artifacts). run_grok() { local to="$1"; shift timeout "$to" "$GROK" --prompt-file "$PF" --output-format json \ - --permission-mode dontAsk --no-subagents --no-plan --cwd "$WORK" "$@" \ + --permission-mode dontAsk --no-subagents --no-plan --cwd "$RUN_CWD" "$@" \ >"$OUT" 2>"$TMP/err.txt" || true } @@ -92,16 +94,24 @@ find_artifact() { case "$MODE" in text|verify) - [ -z "${1:-}" ] && { echo "usage: $SELF $MODE \"\"" >&2; exit 2; } + # content from --prompt-file (good for long docs) or the positional arg + SRC="" + if [ "${1:-}" = "--prompt-file" ]; then + [ -f "${2:-}" ] || { echo "[$SELF] prompt file not found: ${2:-}" >&2; exit 2; } + SRC="$(cat "$2")" + else + SRC="${1:-}" + fi + [ -z "$SRC" ] && { echo "usage: $SELF $MODE \"\" | $SELF $MODE --prompt-file " >&2; exit 2; } # Prompt-level steering keeps it text-only and (for verify) adversarial. # (The --disallowed-tools/--rules flags tripped the CLI; --check adds a slow # multi-turn self-check loop — both avoided in favor of prompt steering.) if [ "$MODE" = "verify" ]; then - printf 'Adversarially evaluate the following claim/finding: try hard to find any reason it is WRONG. Then give a one-line verdict (CONFIRMED or REFUTED) plus a one-sentence justification. Answer in text only; do not use tools. Claim/finding: %s' "$1" > "$PF" + printf 'Adversarially evaluate the following claim/finding/document: try hard to find any reason it is WRONG, incomplete, or overstated. Then give a verdict plus specific justification. Answer in text only; do not use tools. Content:\n%s' "$SRC" > "$PF" else - printf 'Answer directly in text; do not use tools. %s' "$1" > "$PF" + printf 'Answer directly in text; do not use tools.\n%s' "$SRC" > "$PF" fi - run_grok 120 --disable-web-search --max-turns 2 + run_grok 180 --disable-web-search --max-turns 3 txt="$(jfield text)" if [ -n "$txt" ]; then printf '%s\n' "$txt"; else echo "[$SELF] no text (stopReason=$(jfield stopReason)); raw: $OUT" >&2; exit 1; fi @@ -136,6 +146,19 @@ case "$MODE" in if [ -n "$txt" ]; then printf '%s\n' "$txt"; else echo "[$SELF] no result (stopReason=$(jfield stopReason))" >&2; exit 1; fi ;; + review|file) + [ -z "${1:-}" ] && { echo "usage: $SELF review [instructions]" >&2; exit 2; } + target="$1" + instr="${2:-Give an independent, critical review of this file: accuracy, gaps/omissions, and concrete improvements. Be specific.}" + # Grok reads the file itself (no embedding) -- run it in the repo so read_file resolves repo-relative paths. + [ -f "$target" ] || [ -f "$REPO_ROOT/$target" ] || { echo "[$SELF] file not found: $target" >&2; exit 2; } + RUN_CWD="$REPO_ROOT" + printf 'Use your read_file tool to read the file at this path (relative to your current directory), then do the task and stop. You may also read closely-related files it references if that helps. Do not modify anything.\nPath: %s\n\nTask: %s' "$target" "$instr" > "$PF" + run_grok 240 --max-turns 12 + txt="$(jfield text)" + if [ -n "$txt" ]; then printf '%s\n' "$txt"; else + echo "[$SELF] no result (session=$(jfield sessionId), stopReason=$(jfield stopReason))" >&2; exit 1; fi + ;; raw) "$GROK" "$@" ;; diff --git a/clients/glaztech/reports/2026-06-03-website-security-assessment.md b/clients/glaztech/reports/2026-06-03-website-security-assessment.md index 39baf27..5daf5a5 100644 --- a/clients/glaztech/reports/2026-06-03-website-security-assessment.md +++ b/clients/glaztech/reports/2026-06-03-website-security-assessment.md @@ -1,18 +1,53 @@ # Glaz-Tech Industries — Website Security Assessment **Classification:** CONFIDENTIAL — Security -**Date:** 2026-06-03 +**Date:** 2026-06-03 (updated 2026-06-04 — Appendix A; deep host + SQL-infrastructure recon folded into **C0-Extended**, the findings table, and the roadmap) **Assessor:** Arizona Computer Guru (Mike Swanson) -**Target:** Glaztech customer/e-commerce web application — server `WWW` (192.168.8.72, public 65.113.52.88), site `glaztech_new` at `D:\web\glaztech_4`, SQL backend `192.168.8.62,3436` -**Method:** Authorized read-only assessment via GuruRMM (config/registry inspection, source-code review of the on-server VB.NET source, and read-only/aggregate DB inspection). **No cardholder data and no passwords were retrieved** — sensitive columns were classified by aggregate only. +**Target:** Glaztech customer/e-commerce web application — server `WWW` (192.168.8.72, public 65.113.52.88), site `glaztech_new` at `D:\web\glaztech_4`, SQL backend `192.168.8.62,3436` (`GTI-INV-SQL`) +**Method:** Authorized read-only assessment via GuruRMM (config/registry inspection, source-code review of the on-server VB.NET source, read-only/aggregate DB inspection, and — 2026-06-04 — read-only host telemetry and SQL **metadata** inspection: IIS/app-pool config, services, scheduled tasks, file ACLs, listening ports, firewall state, certificate metadata, plus `sys.configurations` / `sys.servers` / `msdb` job + backup metadata and `sys.databases` encryption state). **No cardholder data was retrieved** and **no secret values were intentionally retrieved**; sensitive columns were classified by aggregate only. *(One domain-administrator password was returned incidentally inside a SQL Agent job-step preview; it was redacted from all assessment artifacts and is not reproduced here — see Assumptions and C0-Extended.)* > Companion report: `2026-06-03-pci-cardholder-data-finding.md` (cardholder-data storage detail). --- +## Executive Summary (for Steve Eastman / Tom) + +This is written plainly for the business owners. The detail and evidence are in the sections that follow. + +**What is at stake.** Your **internal billing system** (the staff-operated GTIware/PSA program that auto-charges open invoices) is still writing customers' full credit-card numbers — and the security code on the back of the card (CVV) — into your databases in plain, readable text, and it is adding more of them every day. The **public website itself does not store cards**, but it is the dangerous **access path** to those same databases: its shared administrative database login plus a query-injection flaw reach the exact tables where the stored cards live. The website is **also in PCI scope** in its own right because its pay pages **receive and forward** customers' card numbers to the processor. Storing CVV and unencrypted card numbers like this is a direct violation of the credit-card industry's security rules (PCI DSS), so you are **out of PCI compliance right now**, not at some future date. + +The bigger problem is how the website is wired to the database. The website logs into the shared database server with an account that has **full administrative control of that entire server** — not just the website's own data. That same server holds **every office's stored credit cards, roughly 9,000 customer and employee passwords (also in plain text), years of business archives, and payroll data.** Because of a flaw in how the website builds its database queries (SQL injection), an attacker who simply **guesses one customer login** — and the logins are easy to guess and have no lockout — is **one step away from reaching that administrative account.** From there they could: + +- **Steal everything:** all offices' card numbers and security codes, all ~9,000 passwords, all customer/financial records, and the multi-year archives. +- **Destroy everything:** delete or encrypt the databases for every office and demand a ransom. The backups sit on the same reachable server, so recovery could be impossible. +- **Commit fraud:** alter invoices, balances, and payment records, or run charges on stored cards. + +A breach of this kind would very likely trigger **legally required breach notifications to your customers, card-brand fines, and loss of your ability to accept cards** — on top of the direct theft and any ransom. + +**New, and more urgent (verified 2026-06-04).** A deeper inspection of the server confirmed the danger goes beyond "someone could reach the database." The same easy-to-guess website login leads, through flaws already on the box, **straight toward control of your whole Windows network** — because (a) the database server is configured so a successful break-in can run operating-system commands immediately, and (b) your **main Windows administrator password is sitting in plain text inside the database's own scheduled jobs**, where that same break-in can simply read it. In short, one guessed customer password could lead not just to stolen cards but toward **takeover of the entire company network.** Full technical detail is in **C0-Extended**; these findings move several items into a "do first, this week" containment list below. + +**The key point about "we switched to Payrilla/Paya."** (This same point is reinforced in "Current State Verified" and finding **C0** below — the repetition is deliberate, because it is the single most common misunderstanding about this risk.) Changing credit-card processors does **not** fix any of this on its own. We verified on 2026-06-03 that the website is **still** running on the old processor (CyberSource/PNC) and that the internal billing system is **still** writing plaintext cards to your databases every day. Switching processors only protects this website **after** it is actually re-pointed to the new processor's secure **tokenized/hosted payment system** — *a setup where the card number goes straight to the processor and is never stored on Glaztech's servers* — **and** the stored plaintext cards are purged. Until both of those happen, the exposure described here is fully live and growing. + +**Do first — emergency containment (this week, before the moves below):** +- **Rotate the Windows administrator password** and remove the plain-text copy of it stored inside the database's scheduled jobs (updating every place that uses it in the same change window). +- **Turn off the database's ability to run operating-system commands** (`xp_cmdshell`), and **fix the file permissions** so the website's config file — which holds the database and payment-processor passwords — is no longer readable by everyone on the box. +- **Turn the server's firewall back on** (it is currently off for two of three network profiles) and **replace the end-of-life RealVNC** (a ~2009 version, Steve's tool) with a current, supported remote-access option. + +**Then this week (top moves):** +1. **Stop the website from logging into the database as an administrator.** Give it a new, restricted login that can only see its own data — and that explicitly **cannot** reach the stored cards, the passwords, or the other offices' databases. (This must be sequenced carefully — see the roadmap — so we don't break the live site.) +2. **Stop writing the CVV security code today** — a small change to the internal billing system's card-save path so no new CVVs are recorded — and **purge the CVVs already stored** (CVV must never be retained at all). The CVV purge is safe to do immediately once you confirm the auto-charge engine does not re-submit CVV when it runs a saved card; if it does, fix that path first. (Full card-*number* purge comes later — after the tokenized payment path is live — so auto-billing isn't broken.) +3. **Turn on failed-login logging and basic lockout immediately** so guessing attacks can be seen and slowed, and **rotate the administrator password that is sitting in plain text in the website's config file.** Important: before rotating, identify **everything** that uses that login (the internal billing engine, jobs, linked servers) and update them in the same change window — rotating it blindly will break live internal systems. Best done once the new restricted website login above is proven working. +4. **Start the real migration** to the new processor's tokenized/hosted payment system so card numbers never land on Glaztech servers — then, once that is live, **purge the plaintext card numbers** already stored. **Get development tools off the live payment server, replace the end-of-life RealVNC,** and lock down who and what can reach the database server over the network. + +The rest of this report is the technical detail, evidence, and a careful step-by-step remediation order so the fixes can be made **without taking the live website down.** + +--- + ## Overall Risk: CRITICAL -The site stores cardholder data (PAN + CVV) and **all user passwords in plaintext**, contains SQL injection and reflected XSS, and runs on a server that doubles as a developer workstation with extensive remote-access and end-of-life software. Multiple findings are independently sufficient to cause a reportable breach. +The site stores cardholder data (PAN + CVV) and **all user passwords in plaintext**, contains SQL injection and reflected XSS, and runs on a server that doubles as a developer workstation and carries end-of-life software (e.g. RealVNC 4.x). Multiple findings are independently sufficient to cause a reportable breach. + +**PCI scope note.** The website does not just sit near the cardholder data — the quick-pay flows **process and transmit** it: the pages **receive the PAN and CVV via POST and forward them through the CyberSource SDK** to the processor. Receiving/forwarding card data places the internet-facing server squarely **inside the PCI cardholder-data environment (CDE) today**, independent of whether the website itself writes cards to disk (it does not — see "Where the Cards Are Stored"). Read any "no card-handling" phrasing in this report as "no card **storage** code on the website," **not** "out of PCI scope." | # | Finding | Severity | |---|---|---| @@ -21,19 +56,24 @@ The site stores cardholder data (PAN + CVV) and **all user passwords in plaintex | C2 | All user passwords stored in plaintext; passwords emailed in cleartext | Critical | | C3 | SQL injection via fake `quo()` escaper (incl. payment pages) | Critical | | C4 | Reflected XSS in `gt_errorpage.aspx` | High | +| C5 | **`xp_cmdshell` already enabled + domain-admin password stored cleartext in `msdb` jobs** — a `tom` SQLi yields immediate OS command execution on the DB server *and* domain-admin takeover (see C0-Extended) | Critical | | H1 | Production payment server is also a dev workstation (VS, SDKs, build tools) | High | -| H2 | Remote-access sprawl incl. end-of-life RealVNC 4.2.8 + stale ScreenConnect v6 | High | +| H2 | End-of-life RealVNC 4.2.8 (≈2009) listening on the CDE host (owner-operated) | High | | H3 | `debug="true"` + `customErrors=Off` + exceptions echoed to users | High | | H4 | Server accepts TLS 1.0/1.1 on the listener | High | | H5 | No cookie Secure/HttpOnly hardening, no MFA, no lockout, session-fixation risk | High | | H6 | Single shared SQL login with full card-column read; creds in `Web.config` | High | +| H7 | No anti-forgery (CSRF) tokens on payment / ACH / quick-pay flows | High | +| H8 | Missing security response headers (no CSP / HSTS / X-Frame-Options / etc.) | High (low-risk server-config quick win) | +| H9 | No at-rest encryption (TDE) on any of 47 DBs; CHD `.bak` backups copied to an SMB share | High | +| H10 | Host firewall off (Private/Public profiles); RDP/SMB/VNC/WinRM/MSMQ listening; **7 linked servers** form a cross-subnet lateral mesh reachable from `tom` | High | | M1 | Outdated/unused third-party components; SHA1 machineKey; source on prod | Medium | --- ## Current State Verified — 2026-06-03 (exposure is ACTIVE and ONGOING) -Re-checked the live system after a report that card processing had moved to a different provider ("Payrilla"): +Re-checked the live system after a report that card processing had moved to a different provider ("Payrilla"). *(This processor-switch point is also summarized in the Executive Summary and revisited in C0 — repeated deliberately because it is the most common misunderstanding about this risk.)* - **The website is still on CyberSource/PNC.** There is **no Payrilla integration** anywhere in the site code or config; the live payment pages remain `online-payment-pnc.aspx` / `quick-pay-pnc.aspx` → CyberSource (`api.cybersource.com`). - **Card processing is live via PNC today** — `web_payment_header` shows `CC-WebPayment-PNC` approvals/errors on 2026-06-03 (e.g. 14:47 Approved, 15:07 Error) plus E-Check. - **Plaintext cards are still being written to the local databases every day** — `cc_file` last write: Tucson **2026-06-03 14:15** (8 writes/60d), Phoenix **2026-06-03 10:19** (14 writes/60d). This is ongoing accumulation, not historical data. @@ -60,18 +100,60 @@ Because a SQL injection runs **as the connecting login**, the website's SQLi fla **Steal everything:** dump **all offices' stored cards (full PAN + CVV)** from `cc_file` + the entire `cof_payments_header` history (tens of thousands of card numbers); dump **~9,000+ plaintext passwords** (`web_security`); dump every customer/invoice/order/balance and the multi-year **archive** DBs; read TimeForce/payroll (`qqest`) and anything else on the instance. **Destroy everything:** `DROP DATABASE`/`DROP TABLE`/`DELETE`/`TRUNCATE` across all 46 databases — wipe live production for **every office** and the archives; or encrypt/drop and ransom (backups are also reachable, so recovery may be impossible). **Manipulate / commit fraud:** alter invoices, balances, prices, payment records; change account access levels (grant themselves admin); insert fraudulent transactions; falsify or delete audit history. -**Take over the server + network:** enable `xp_cmdshell` to run **OS commands on the SQL server (192.168.8.62)** as the SQL service account — install malware/ransomware, create OS accounts, exfiltrate files, **disable backups and AV**; harvest credentials and **pivot into the wider GTI/Glaztech network** (linked servers, shares, the service account's domain rights); plant persistent SQL backdoors and erase SQL audit trails. +**Take over the server + network:** use the **already-enabled `xp_cmdshell`** (verified `value_in_use=1` — no enable step needed) to run **OS commands on the SQL server (192.168.8.62)** — install malware/ransomware, create OS accounts, exfiltrate files, **disable backups and AV**; recover the **`glaztech\administrator` domain-admin password, which is stored in cleartext in `msdb` SQL Agent job steps** (readable by `tom` directly, even without `xp_cmdshell`); and **pivot across the 7 linked servers** into accounting (`mas_gti`), payroll (`qqest`), and backup hosts on two subnets — i.e. **domain-admin takeover.** Plant persistent SQL backdoors and erase SQL audit trails. **This escalation chain is verified and detailed in C0-Extended.** **Bottom line: one guessed customer password is one SQL-injection away from full theft, destruction, or ransom of the entire GTIware database server — and a foothold into the rest of the network.** ### Required remediation (do this FIRST) -1. **Stop the website connecting as `sysadmin` immediately.** Give the website a dedicated **least-privilege** SQL login scoped to only the specific tables/views/procs it needs, with **no access to the GTIware databases** (`cc_file`, `web_security`, other offices, archives). Revoke `sysadmin`/`securityadmin`/`dbcreator`. +1. **Stop the website connecting as `sysadmin`.** Give the website a dedicated **least-privilege** SQL login scoped to only the specific tables/views/procs it needs, with **no access to the GTIware databases** (`cc_file`, `web_security`, other offices, archives). Then revoke `sysadmin`/`securityadmin`/`dbcreator`. **Do this in the safe order** in the "Sequencing the least-privilege DB migration" callout under the Remediation Roadmap — `tom` may be used by GTIware/other processes, so demoting it in place can break the live site. 2. **Separate the website's data from GTIware** — the internet-facing app must not share a SQL instance with the internal PSA's cardholder data. Either move the website to its own database/instance, **or** strictly partition permissions so the public web login **cannot see or reach** the GTIware databases. -3. **Stop storing cardholder data on-prem at all** (best fix): either **call the processor's API directly** (processor-hosted fields / vault — no card data lands on Glaztech systems) or, at minimum, **tokenize the entire process** (store a processor token, never the PAN). **CVV must never be stored — period (PCI Req 3.2).** Any cardholder data that must be at rest has to be **encrypted**. +3. **Stop storing cardholder data on-prem at all** (best fix): either **call the processor's API directly** (processor-hosted fields / vault — no card data lands on Glaztech systems) or, at minimum, **tokenize the entire process** (store a processor token, never the PAN). **CVV must never be stored — period (PCI Req 3.2).** Any cardholder data that must be at rest has to be **encrypted**. *(Note: simply switching processors — the "we moved to Payrilla/Paya" point raised in the Executive Summary and Current State sections — does not accomplish this on its own; the stored-card exposure persists until the tokenized path is live and the stored PANs are purged. The repetition of this point across those sections is deliberate.)* 4. Then: fix the SQL injection (parameterize, remove `quo()`), add login lockout/rate-limiting, hash passwords. --- +## C0-Extended — Verified blast radius and privilege-escalation chain (deep recon, 2026-06-04) + +A follow-up read-only recon of the `WWW` host and the SQL instance (host telemetry + SQL **metadata** only — no CHD, no secret values) confirmed C0's "foothold into the rest of the network" warning is not hypothetical. The public website is the front door to a **multi-server, sysadmin-and-domain-admin blast radius**, and the path from a web compromise to full domain takeover needs **no additional vulnerability** beyond the SQLi already documented in C3. + +### The card engine runs in-process on the public server, on the `sysadmin` login +- The IIS worker for the public site (`w3wp.exe`, PID 13752) runs as `IIS APPPOOL\glaztech_new` — a low-privilege *local* identity (good locally). **But** the GTIware card-on-file engine DLLs **`gt_auto_process_2020.dll` and `glaztech_utilities_2020.dll` are present in the live `D:\web\glaztech_4\Bin\`** (plus duplicate copies in `Bin\Old_bin\` and `Old_code\Bin\`). +- Their `.dll.config` files carry **no connection strings of their own**, so the in-process engine uses the website's `Web.config` `tom` (sysadmin) connections. **A web-tier compromise therefore lands directly inside the card-on-file write/charge engine, with `sysadmin` DB rights** — the "website vs. GTIware" distinction collapses for any in-process or host-level attacker. The CDE is effectively the public host itself. + +### From SQLi to OS command execution — already armed +- `sys.configurations`: **`xp_cmdshell = 1` (enabled, `value_in_use = 1`).** There is no "enable it first" step — a `tom` SQLi can `EXEC xp_cmdshell` for **immediate OS command execution on the SQL server** (`192.168.8.62` / `GTI-INV-SQL`). (`clr enabled`, `Ole Automation Procedures`, `Ad Hoc Distributed Queries`, and `remote admin connections` are all `0` — `xp_cmdshell` is the live one.) +- The **SQL Server Agent service runs as `Administrator@glaztech.com`** (a domain-admin-class account). Any Agent job step — which sysadmin `tom` can create — executes as that account. + +### Domain-admin password sitting in cleartext in `msdb` +- Multiple backup-copy SQL Agent job steps run `EXEC xp_cmdshell 'net use \\192.168.8.52\sql_backup\... /user:glaztech\administrator /persistent:yes'`. **The `glaztech\administrator` domain-admin password is stored in cleartext in `msdb.dbo.sysjobsteps`,** readable by any sysadmin/`tom` connection — i.e. **harvestable straight from the web-app SQLi, without even invoking `xp_cmdshell`.** (The password value was redacted from all assessment artifacts and is not reproduced; it must be rotated — see the Emergency-containment bucket in the roadmap.) + +### Lateral reach — a cross-subnet linked-server mesh +- The instance defines **7 linked servers, all with remote-login enabled and most with data access enabled**, spanning two subnets: `192.168.0.54,55181`; `192.168.0.55,55181` (the `mas_gti` Sage/accounting system); `192.168.8.212,3436`; `192.168.8.52,3436`; `192.168.8.62,3430`; `GLAZ\TIMEFORCE` (payroll `qqest`); and `GTI-INV-SQL\GTISQL`. From the single internet-reachable `tom` credential an attacker pivots across the whole SQL estate — **including accounting and payroll** — on two subnets. +- The real automated card-charging runs **server-side** via CmdExec Agent jobs invoking `d:\sql_jobs\bin\gt_console_apps.exe` (`cp` = courtesy/card payments, `is` = invoice statements, `oa` = order acks, `lo` = load PDFs) as the domain-admin Agent account — a second GTIware execution host beyond the in-process DLLs. + +### Over-privileged logins; no encryption at rest +- **Sysadmin logins:** `tom` (the website's app login), the built-in **`sa` is enabled**, `GTI-INV-SQL\Administrator`, plus SQL service virtual accounts. Most databases run at **compatibility level 90 (SQL Server 2005-era).** +- **No encryption at rest:** **all 47 databases report `is_encrypted = False`** (no TDE), including every CHD-bearing `glaz_prod*`. Production `.bak` backups are written to `d:\sql_backup\prod\` **and copied to the `\\192.168.8.52\sql_backup\` SMB share** — unencrypted cardholder data on a reachable file share (this confirms and locates the "backups contain plaintext PAN/CVV" liability noted under C1 / "Where the Cards Are Stored"). + +### Secrets on the web host, readable by everyone +- `Web.config` `appSettings` hold **two payment processors' credentials in cleartext** — CyberSource (`cybs.Password`, `cybs.merchantID.password`, plus on-disk signing keys at `cybs.keysDirectory` / `cybs.keyFilename`) **and PNC** (`pnc_key`, `pnc_pin`, `pnc_token`, transaction/surcharge URLs). +- The web-root ACL grants **`Everyone:(R)`** on `D:\web\glaztech_4`, its `bin`, **and `Web.config`** — so any local principal (or any of the remote-access agents below) can read the `tom` SQL password and both processors' secrets. (Not world-*writable*, so no direct web-shell drop via ACL.) + +### Remote access — clarified +- The remote-management agents on this host — **both ScreenConnect clients (`support.azcomputer.guru` and the `kgc7jt` ScreenConnect Cloud instance), Datto RMM (CentraStage), Datto EDR (Infocyte), Syncro, Splashtop, and GuruRMM** — are **ACG's own sanctioned management stack**, not a third-party intrusion. *(An initial pass flagged the `kgc7jt` ScreenConnect as "foreign/unrecognized"; on confirmation it is an ACG cloud instance — withdrawn, no finding.)* +- The one genuine remote-access finding is **end-of-life RealVNC 4.x (`WinVNC4`), listening on 5900/5800** — **the owner's (Steve's) tool**, a ~2009 build with known auth-bypass-class issues, running on the CDE host. Update/replace it with a current supported version (coordinate with Steve) — see **H2**. + +### Host network posture (H10) +- **Host firewall: Domain profile On, but Private and Public profiles Off.** Listening services include RDP (3389), SMB (445), NetBIOS (139), VNC (5800/5900), ScreenConnect (6783), WinRM (5985/47001), MSMQ (1801/210x), and Dell OMSA (1311) — on a flat network these are lateral-movement surface. +- **TLS 1.0 is explicitly enabled** in SChannel (`Enabled=1`, Server and Client), not merely left at OS default (sharpens H4). +- **New internet-facing endpoints** under the same site: **`/webhooks`** (a Samsara webhook receiver, custom `samsara_webhook_receiver.dll`, physical path `...\incoming\samsara`) and **`/webhooks1`** (a `webhooks_test` app) — unreviewed input surfaces not in the original scope. +- **Additional hosts surfaced:** a second `\\192.168.0.147\web\glaztech_4` (referenced by a cleanup job), backup servers `192.168.8.52` / `192.168.8.212`, the MAS system `192.168.0.55`, and linked `192.168.0.54`. + +### The chain, end to end +Public site → SQLi as sysadmin `tom` (C3/C0) → **(a)** read all CHD + ~9,000 plaintext passwords (C1/C2); **(b)** `xp_cmdshell` → OS RCE on the DB server; **(c)** read the cleartext `glaztech\administrator` password from `msdb`; **(d)** pivot across 7 linked servers (accounting, payroll, backups) on two subnets; **(e)** reach **domain admin.** In parallel, the in-process GTIware engine and the `Everyone:(R)` secrets give a host-level attacker the same outcome without SQLi. **This is why the containment items now lead the remediation roadmap.** + +--- + ## Critical Findings ### C1 — Plaintext PAN and stored CVV (see companion PCI report) @@ -81,7 +163,7 @@ Because a SQL injection runs **as the connecting login**, the website's SQLi fla - **Customer portal:** auth stored proc `get_web_accesslevel` compares `web_security.web_password = @passwd` **with no hashing**. `web_security` holds **~9,000+ plaintext passwords** (corp 6,017 + tuc 3,012 + other offices), 0 hash-like values, lengths 3–19. - **Employees:** `emp/employee-login.aspx` "forgot password" verifies last name + email, then **emails the user their existing plaintext password** (`"The password to your employee profile is: " + pword`) — only possible with reversible/plaintext storage. - **Impact:** any DB read (or the existing SQLi) exposes every customer/employee credential in the clear; password reuse means broad downstream compromise. Weak "lastname + email" knowledge check gates the password email. -- **Fix:** store only salted password hashes (PBKDF2/bcrypt/Argon2); never email passwords — implement a reset-token flow; force a global password reset after remediation. +- **Fix:** store only salted password hashes (PBKDF2/bcrypt/Argon2); never email passwords — implement a time-limited reset-token flow. For the ~9,000-account cutover, prefer a lower-disruption rollout (hash on the fly via "force change on next successful login" + reset tokens) over a single forced global reset — see the roadmap. ### C3 — SQL injection via non-escaping `quo()` helper ```vb @@ -107,6 +189,8 @@ Chaining the findings into the realistic worst case, with difficulty ratings. **Step 2 — Normal UI (masked display).** Payment pages *display* cards masked to last-4 (`xxxx-xxxx-xxxx-1234`), so a point-and-click attacker sees last-4 + expiry + cardholder/billing data and can transact on saved cards. Note: the read proc `get_cc_data` is `SELECT * FROM cc_file WHERE acct_no=@acctno` — it returns the **full PAN and CVV to the application server**; only the *display* is masked, and the `@acctno` parameter makes it an IDOR-shaped full-card read. Any endpoint returning that proc's output unmasked (or the SQLi below) yields full numbers. +> **TO VERIFY (high priority) — IDOR on `get_cc_data(@acctno)`.** We have **not** confirmed how the `@acctno` passed to `get_cc_data` is bound to the authenticated session. **Recommend confirming** whether the application strictly derives `@acctno` from the logged-in user's own account (so a customer can only retrieve **their own** cards), versus accepting an `acct_no` supplied by the client — in which case **any logged-in user could request an arbitrary account number** and pull that account's full PAN/CVV to the app server (a classic Insecure Direct Object Reference). This is an open verification task; if the binding is not server-side and session-enforced, it is a second full-PAN read path independent of the SQLi and should be treated as Critical. + **Step 3 — SQL injection (FULL exposure).** The post-login payment pages (`quick-pay`, `ach`, `quick-pay-pnc`) build SQL with the non-escaping `quo()` helper and require only a valid session. A logged-in attacker can UNION-inject `SELECT cc_number, cc_code FROM cc_file` and exfiltrate **every stored full card number AND CVV** for the office — directly, because the data is **plaintext** (no encryption/key to defeat). UI masking is irrelevant at this layer. | Goal | Difficulty | @@ -121,7 +205,7 @@ Chaining the findings into the realistic worst case, with difficulty ratings. ## Why the Cards Are Stored, and Where They Flow -**Business purpose — card-on-file invoice auto-pay.** Cards are stored (with an `activate` flag on `cc_file`) so the business can automatically charge customers' open invoices. The proc `i_get_cc_on_file_invoices` joins `invoice` × `cc_file` for active cards with an outstanding, delivered balance; `gt_auto_process_2020.dll` / `glaztech_utilities_2020.dll` are the engine that reads the stored card and bills it (currently via CyberSource). **This is GTIware (the internal PSA), staff-operated — NOT the website.** The website has **zero** card-handling code (no `cc_file`/`save_cc_data`/`gt_auto_process` references in its `.aspx`/`.vb`); its quick-pay pages even disclaim saving cards. Saved-card rows are stamped with **staff** usernames (Victoria, Bryce, Diana) and notes like "run card when requested." The large `cof_payments_header` history (e.g. 14,496 rows in Phoenix) is years of these GTIware charges. +**Business purpose — card-on-file invoice auto-pay.** Cards are stored (with an `activate` flag on `cc_file`) so the business can automatically charge customers' open invoices. The proc `i_get_cc_on_file_invoices` joins `invoice` × `cc_file` for active cards with an outstanding, delivered balance; `gt_auto_process_2020.dll` / `glaztech_utilities_2020.dll` are the engine that reads the stored card and bills it (currently via CyberSource). **This is GTIware (the internal PSA), staff-operated — NOT the website's storage code.** *Important nuance verified 2026-06-04 (see C0-Extended): those engine DLLs are physically deployed in the website's own `D:\web\glaztech_4\Bin\` and load **in-process into the public IIS worker**, and they carry no DB credentials of their own — so they run on the website's `tom` (sysadmin) connection. A second copy of the billing logic, `d:\sql_jobs\bin\gt_console_apps.exe`, runs server-side from SQL Agent jobs. So while the **storage feature** is GTIware's, the **execution surface co-resides with the internet-facing site** — which is why a web compromise reaches the card engine directly.* The website has **no card-storage code** (no `cc_file`/`save_cc_data`/`gt_auto_process` references in its `.aspx`/`.vb`), and its quick-pay pages even disclaim saving cards. **It does, however, still receive and forward card data:** the quick-pay flows accept the PAN/CVV via POST and pass them through the CyberSource SDK to the processor, which is what keeps the website inside the PCI CDE (see the PCI scope note under Overall Risk). "No card-handling" here means **no storage**, not out-of-scope. Saved-card rows are stamped with **staff** usernames (Victoria, Bryce, Diana) and notes like "run card when requested." The large `cof_payments_header` history (e.g. 14,496 rows in Phoenix) is years of these GTIware charges. **Where the full PAN is used.** Only five DB objects reference the full `cc_number`: `save_cc_data`/`save_cc_data1`/`save_cc_data2` (writes) and the `is_cc_active`/`is_cc_on_file` functions. However, **`get_cc_data` is `SELECT *`**, so it also returns the full PAN + CVV whenever a saved card is read for charging — the full number crosses to the app server on every card-on-file charge; the UI only masks the *display*. @@ -136,8 +220,8 @@ Chaining the findings into the realistic worst case, with difficulty ratings. ### H1 — Production payment server is also a developer workstation Installed on the live server: **Visual Studio Community 2015 and 2022, .NET 8 SDKs, MSBuild/Build Tools, TFS office integration, IIS 10 Express, Notepad++, WinRAR 7.22, OpenSSL 3.5.0**. Full application **source code is on the box** (128 `.vb` + 125 `.aspx.vb`, not precompiled). This massively expands attack surface and blast radius on the host that processes cardholder data. **Fix:** move development off the production host; deploy precompiled; remove SDKs/IDEs/dev tools. -### H2 — Remote-access sprawl, including end-of-life software -Present: **RealVNC Enterprise E4.2.8** (≈2009 — critically outdated, known auth-bypass-class issues), **ScreenConnect client v6.0.11622 (2018, stale)** alongside a current ScreenConnect, **Splashtop**, **Datto RMM + Datto EDR**, **Syncro**, plus GuruRMM. 6+ remote-management agents = large unmonitored access surface. **Fix:** remove RealVNC and the stale ScreenConnect immediately; rationalize to a single sanctioned remote-access tool; inventory who controls each. +### H2 — End-of-life RealVNC on the cardholder-data host (owner-operated) +**RealVNC 4.x (`WinVNC4`) is installed and listening on 5900/5800** — a ≈2009 build, critically outdated with known auth-bypass-class issues, running on the server that processes cardholder data. This is **Steve's own remote-access tool**, not ACG's. *(For clarity: the other remote-management agents on the box — both ScreenConnect clients including the `kgc7jt` cloud instance, Datto RMM/EDR, Syncro, Splashtop, GuruRMM — are **ACG's sanctioned management stack** and are not findings; an earlier draft mis-flagged the `kgc7jt` ScreenConnect as third-party and that has been withdrawn.)* **Fix:** update or replace RealVNC 4.x with a current supported version (or retire it), coordinated with Steve; ensure whatever remains is patched and access-controlled. ### H3 — Debug/error information disclosure `Web.config`: **``** in production and **``** present; login/employee code echo `ex.Message` to the page or via `errmsg`. Leaks stack traces, SQL errors, internal paths. **Fix:** `debug="false"`, `customErrors="On"` with a generic page, stop surfacing exception text to users. @@ -155,6 +239,21 @@ SChannel: **TLS 1.0 Server `Enabled=1`** (and TLS 1.1 at OS default = enabled); ### H6 — Database access model Web app connects with a **single shared SQL login (`tom`)** that has full read on card and password columns (no column-level control); **connection strings with credentials are in `Web.config`** on the web server (15+ per-office DBs). **Fix:** least-privilege per-function accounts, remove blanket card/password read, protect/secret-manage connection strings, enable TDE at rest. +### H7 — No anti-forgery (CSRF) protection on payment / ACH / quick-pay flows +Authentication is custom **Session-variable** based (see H5), and we observed **no ASP.NET anti-forgery / CSRF tokens** on the state-changing flows — the payment, ACH, and quick-pay POST handlers act on the session alone, with no per-request token tying the request to the user's own page. The consequence: a logged-in customer (or a staff member with a session) who is lured to a malicious page could have a **payment, ACH submission, or account change submitted on their behalf cross-site**, without their intent. This is a distinct **financial / business-logic** risk from the SQLi (C3) and XSS (C4) — those are injection flaws; this is forged-but-valid requests. **Fix:** add per-request anti-forgery tokens (e.g. ASP.NET `AntiForgeryToken` or an equivalent synchronizer-token pattern) to all state-changing POSTs, validate them server-side, and reject requests without a matching token. *(Note: the absence of anti-forgery tokens is the verified observation; a working cross-site exploit was not constructed — see Assumptions.)* + +### H8 — Missing security response headers +*Severity: **High** — but a low-risk, server-config-only quick win (no application code change), which is why it is sequenced early in the roadmap ahead of higher-effort code fixes.* + +The application does not emit the standard hardening response headers: +- **No Content-Security-Policy (CSP)** — a CSP would materially reduce the impact of the reflected XSS in **C4** (it constrains what injected script can load/run) and is the single highest-value header to add here. +- **No HSTS (`Strict-Transport-Security`)** beyond the recently-added HTTP→HTTPS redirect (see H5) — without HSTS the browser is not told to refuse plain HTTP on future visits. +- **No `X-Frame-Options` / frame-ancestors** — leaves the site open to clickjacking/framing. +- **No `X-Content-Type-Options: nosniff`** — allows MIME-sniffing of responses. +- **No `Referrer-Policy`** — leaks full referrer URLs cross-site. + +**Fix:** add these headers at the IIS/site level (CSP, HSTS, X-Frame-Options/`frame-ancestors`, X-Content-Type-Options, Referrer-Policy). CSP first, as a partial mitigation for C4 while the XSS itself is fixed. + --- ## Medium / Component Hygiene @@ -169,28 +268,114 @@ Web app connects with a **single shared SQL login (`tom`)** that has full read o ## What is Acceptable (balanced view) - **OS patching is current-ish:** Windows Server 2019, build 17763.8755, patched through **May 2026** (supported to 2029) — the OS itself is *not* the weak point. - **Most data access is parameterized** (948 parameterized calls) — the SQLi exposure is a bounded set of concatenated queries, not pervasive. -- **The Sage 100 ERP DB** (`mas_gti`) stores **no** cardholder data — its native CC module is **disabled** (`CreditCardEnable = N`, `AR_CustomerCreditCard` = 0 rows; tokenization columns exist in the schema but are unused). The plaintext exposure is **entirely GTIware's card-on-file feature** (the internal PSA, staff-operated, storing into databases the website shares) — **not** Sage, and **not** the website's payment pages (which store nothing). The website's role is the *access path* (C0), not the storer. +- **The Sage 100 ERP DB** (`mas_gti`) stores **no** cardholder data — its native CC module is **disabled** (`CreditCardEnable = N`, `AR_CustomerCreditCard` = 0 rows). The plaintext exposure is **entirely GTIware's card-on-file feature** — **not** Sage, and **not** the website's payment pages (which store nothing, though they remain in PCI scope as a card-data conduit, see Overall Risk). The website's role is the *access path* (C0) and a card-data *conduit*, not the storer. *(Detail in "Why the Cards Are Stored" below.)* - TLS 1.2 to CyberSource now works (payment outage fixed 2026-06-03). --- +## PCI DSS Control Mapping + +Key findings mapped to the PCI DSS requirements they implicate. Where an exact sub-requirement number is not certain, the requirement **family** is cited; treat this as indicative for the QSA/SAQ conversation, not a formal gap assessment. + +| Finding | PCI DSS Requirement(s) | Why | +|---|---|---| +| C1 — plaintext PAN + stored CVV | **3.2** (no storage of sensitive auth data / CVV after authorization) and **3.4** (render PAN unreadable at rest) | CVV must never be retained; stored PAN must be encrypted/tokenized/truncated/hashed | +| C2 — plaintext passwords; emailed cleartext | **8.2.x** (strong cryptography for authentication credentials / one-way hashing of passwords) | App credentials must be hashed, never reversible or emailed | +| C3 — SQL injection (`quo()`) | **6.5.1** (injection flaws) | Address injection in secure-coding practices | +| C4 — reflected XSS | **6.5.7** (cross-site scripting) | Output encoding / XSS mitigation required | +| C0 / H6 — website connects as `sysadmin`; shared instance; blanket card/password read | **1.2 / 1.3** (segmentation / restrict CDE connectivity) and **7.x** (least privilege / need-to-know) | Flat reach into the CDE and an over-privileged DB login violate segmentation and least-privilege | +| H4 — TLS 1.0/1.1 on the listener | **4.1** (strong cryptography/TLS for cardholder data in transit) | Deprecated TLS on the card-accepting endpoint | +| H5 / Appendix A — no failed-login logging, no audit of auth events | **10.2 / 10.3** (audit logging of access and required log detail) | Auth attempts must be logged with the required fields (user, timestamp, source) | +| H7 — no anti-forgery tokens | **6.5.9** (CSRF) | State-changing requests must be protected against cross-site request forgery | +| H8 — missing CSP/HSTS/etc. | **6.5.x** (secure-coding controls for the public-facing app) | Hardening response headers are part of secure-coding controls for the public-facing app | +| H1 / M1 — dev tooling, source, outdated components on prod | **6.x** (secure systems/components) and **2.x** (minimize/secure system components) | Reduce attack surface; keep components current; don't run dev tooling on the CDE host | +| C5 — `xp_cmdshell` enabled; cleartext domain-admin in `msdb` | **2.2 / 7.x / 8.2.x** (secure configuration / least privilege / protect credentials) | OS-command surface enabled by default and a reusable admin secret stored in the clear | +| H2 — end-of-life RealVNC on the CDE host | **6.2 / 2.x** (patch/replace EOL components; minimize/secure system components) | EOL remote-access software with known vulns on the card-data host | +| H9 — no TDE; CHD backups on a share | **3.4** (render PAN unreadable at rest); **9.x** (protect backup media) | Stored PAN and its backups are unencrypted | +| H10 — flat network; firewall off; linked-server mesh | **1.2 / 1.3** (segmentation / restrict CDE connectivity) | Lateral reach across subnets from the CDE | + +> The website **processes and transmits** cardholder data (see Overall Risk / PCI scope note), so it is in PCI scope today. The merchant's SAQ type and the controls above should be re-confirmed with the acquirer/QSA after remediation. + +--- + +## Assumptions, Scope & Limitations + +So the findings are read with the right confidence level: +- **Read-only / passive methods only.** The assessment used config/registry inspection, on-server source review, and read-only/aggregate database inspection via GuruRMM. **No dynamic scanning** and **no exploit verification** were performed beyond the noted **live cross-database reads** (e.g. reading other offices' `cc_file` row counts to confirm C0's cross-DB reach). The SQLi, XSS, CSRF, and IDOR findings are assessed from code/config evidence; working exploits were **not** built. +- **GTIware source and the auto-process DLLs were not decompiled.** `gt_auto_process_2020.dll` / `glaztech_utilities_2020.dll` were not reverse-engineered; their **deployment** was verified, however (2026-06-04): both are present in the live web `Bin\`, load in-process into the public IIS worker, and have no connection strings in their `.dll.config` — so they run on the website's `tom` connection (see C0-Extended). Internal GTIware application source beyond what is on the web box was not reviewed. +- **Database backups were located but not opened.** Backup **destinations** are now known (`d:\sql_backup\prod\` and the `\\192.168.8.52\sql_backup\` SMB share) and at-rest encryption is confirmed absent (no TDE on any of 47 DBs), so backups demonstrably contain plaintext PAN/CVV; the retention schedule, share ACLs, and disposal process were not separately inspected. +- **No cardholder data and no passwords were intentionally retrieved.** Sensitive columns were classified by aggregate only (counts, lengths, hash-likeness). **One exception, handled:** a `glaztech\administrator` domain-admin password was returned *incidentally* inside a SQL Agent job-step command preview (`msdb` metadata); it was immediately redacted from all assessment artifacts, is not reproduced anywhere in this report, and is flagged for rotation (see C0-Extended / Emergency containment). +- **RMM/EDR agent security not deeply tested.** The remote-access/management agents (H2) were inventoried for presence and version, not security-tested for their own configuration or exposure. + +--- + ## Prioritized Remediation Roadmap -**Now (days):** -1. Purge stored CVV (`cc_file.cc_code`); stop writing it. -2. `debug="false"`, `customErrors="On"`; HTML-encode `gt_errorpage` output; stop echoing exceptions. -3. Remove RealVNC 4.2.8 and the stale ScreenConnect v6 client. -4. Disable TLS 1.0/1.1 on the listener. +**Order-of-operations matters here.** Several of these fixes can take the live site down if done in the wrong sequence — most notably the database-privilege change (C0/H6). The buckets below are ordered for safety, and the high-risk steps carry explicit sequencing and rollback notes. Read the "Sequencing the least-privilege DB migration" callout before touching login `tom`. -**Short term (weeks):** -5. Convert passwords to salted hashes; replace the email-the-password flow with reset tokens; force a global reset. -6. Parameterize the concatenated SQL (payment pages first); delete `quo()`. -7. Secure+HttpOnly cookies; session regeneration; login throttling/lockout. -8. Move card-on-file to the CyberSource token vault; purge/encrypt historical PAN columns. +> ### Sequencing the least-privilege DB migration (the riskiest step — read first) +> **This callout is the single authoritative sequence for the C0/H6 database-privilege fix.** Where "Now" item 2 and "Short-term" item 9 reference this change, they defer to these steps rather than restating them — follow the order here. +> +> The single most important fix (C0) is also the easiest to break the live site with, because **we have not confirmed that login `tom` is used only by the website.** GTIware (the internal card-on-file engine), internal jobs/configs, and linked servers may authenticate to the same instance as `tom`. Do **not** simply demote or re-password `tom` in place. Correct sequence: +> 1. **Inventory two things at once:** (a) the exact tables/views/stored procs the website's queries actually touch (the 948 parameterized calls + the 59 concatenated statements give the object list), and (b) **every consumer of the `tom` login** — the GTIware card-on-file engine, internal jobs/scheduled tasks/configs, and any linked servers — since `tom`'s password must later be rotated and every consumer updated in the same change window. +> 2. **Create a NEW least-privilege login** for the website with grants on **only** the website's objects, plus **explicit `DENY`** on `cc_file`, `web_security`, every **other office's** databases, the `*_archive` databases, and the system DBs (`master`, `msdb`, etc.). +> 3. **Stage the `Web.config` swap** to the new login with **error monitoring** in place (watch the app and SQL error logs) so any missed object permission surfaces immediately and can be granted or rolled back. Confirm the new login is proven in production. +> 4. **Only THEN** revoke `sysadmin` / `securityadmin` / `dbcreator` from `tom` — and only after confirming nothing else depends on those rights. +> 5. **Rotate the `tom` password** so the plaintext `sysadmin` credential is no longer sitting in `Web.config`. Do this **only after** the new website login is proven (steps 2–3) **and** all `tom` consumers from step 1b are identified and updated to the new password **in the same change window**. Rotating `tom` before its consumers are inventoried and updated will break live internal systems. +> +> **Getting this wrong breaks the live website.** Treat steps 2–5 as a change with a validation + rollback plan (revert the `Web.config` connection string), not a one-shot edit. -**Structural (project):** -9. Separate development from the production host; deploy precompiled; remove dev tooling and source from prod. -10. Least-privilege DB accounts, secret management for connection strings, TDE; re-scope the merchant PCI SAQ after remediation. +**Emergency containment (THIS WEEK, before everything below) — closes the trivially-exploitable internet-to-domain-admin path while it is open (see C0-Extended). These are mostly server-config/account actions, not application code:** +- **E1. Remove `Everyone:(R)`** from `D:\web\glaztech_4`, its `bin`, and `Web.config` (scope to the app-pool identity + required admins) so local principals can no longer read the `tom` SQL password and the CyberSource/PNC secrets. +- **E2. Rotate `glaztech\administrator`** and remove every embedded cleartext use of it (the `xp_cmdshell 'net use …'` backup-copy Agent job steps), updating all of them in the same change window; pause those jobs until they use a dedicated low-privilege account. +- **E3. Disable `xp_cmdshell`** on `GTI-INV-SQL` (`EXEC sp_configure 'show advanced options',1; RECONFIGURE; EXEC sp_configure 'xp_cmdshell',0; RECONFIGURE;`) to remove the direct RCE step; verify on the linked instances too. +- **E4. De-privilege the SQL Agent service account** off domain-admin to the minimum needed to run `gt_console_apps.exe`; **disable the built-in `sa`** login (or confirm a strong, unique password and restricted use). +- **E5. Turn the Private/Public host-firewall profiles back On**, **replace/disable the end-of-life RealVNC** (5900/5800 listener — Steve's tool, coordinate with him), and restrict WinRM/MSMQ/Dell-OMSA to specific management IPs. + +**Now (days) — start immediately; lower-risk items, EXCEPT item 2 which must follow the sequencing callout:** +1. **Stop CVV storage, then purge stored CVV** (`cc_file.cc_code`): + - **Immediate, safe now — STOP WRITING new CVV.** A code change to the GTIware `save_cc_data*` path so `cc_code` is no longer populated going forward. *(This is a GTIware/internal-billing-system change, tracked as a separate workstream — see Structural item 17.)* + - **Immediate, after a quick confirmation — PURGE existing `cc_file.cc_code` values.** CVV must never be stored (PCI Req 3.2), and card-on-file recharges normally do **not** re-submit CVV at charge time, so this should be doable right away. **Caveat:** first confirm the auto-charge engine (`gt_auto_process`) does not re-submit CVV when it runs a saved card — if it does, defer the value-purge until that path is fixed. (Full **PAN** purge is **not** here — it is strictly post-tokenization; see Structural item 16.) +2. **Stand up the new least-privilege website login and revoke `tom`'s `sysadmin` rights — strictly per the "Sequencing the least-privilege DB migration" callout above (do not improvise the order).** This is the **riskiest** step in this bucket: getting it wrong breaks the live website. The `tom` password rotation that removes the plaintext `sysadmin` credential from `Web.config` is part of that sequence (callout step 5) and must wait until the new login is proven **and** every `tom` consumer has been inventoried and updated in the same change window. +3. **Turn on failed-login logging** (timestamp, username, source IP) **and basic throttling/lockout immediately** — this closes the detection blind spot (H5 / Appendix A) and slows guessing while the deeper fixes land. +4. **Interim defense-in-depth:** consider a **WAF or IIS request-filtering rule** in front of the app as a stop-gap against SQLi/XSS **while the code is being fixed** — not a substitute for the code fixes, but valuable coverage during the window. +5. `debug="false"`, `customErrors="On"`; HTML-encode `gt_errorpage` output; stop echoing exceptions (H3/C4). +6. Add the **security response headers** (CSP first — partial mitigation for C4 — plus HSTS, X-Frame-Options, X-Content-Type-Options, Referrer-Policy) (H8). *(Server-config only, no code change — quick win; this is why H8 lands in the "Now" bucket ahead of the code-level CSRF fix H7, despite H7 being scored higher.)* +7. Remove **RealVNC 4.2.8** and the stale **ScreenConnect v6** client (H2). +8. Disable **TLS 1.0/1.1** on the listener (H4) — **with a rollback plan** (re-enable if a legacy client breaks); confirm no legacy client dependency first. + +**Short term (weeks) — code and credential fixes (validate + rollback each; see testing gates):** +9. **Complete the least-privilege DB migration** per the "Sequencing the least-privilege DB migration" callout — finish callout steps 4–5 (revoke `sysadmin`/`securityadmin`/`dbcreator` from `tom`, then rotate `tom`'s password with all consumers updated in the same change window) once the new login is proven in production. *(Refer to the callout for the authoritative step order; not restated here.)* +10. **Parameterize the concatenated SQL** (payment pages first); delete `quo()` (C3). +11. **Convert passwords to salted hashes** (PBKDF2/bcrypt/Argon2) and replace the email-the-password flow with **time-limited reset tokens**; **never email passwords** (C2). *Rollout nuance:* a forced **global reset for ~9,000 customers + staff is operationally heavy** — prefer a lower-disruption path: hash on the fly via **"force change on next successful login"** plus time-limited reset tokens, rather than invalidating everyone at once. +12. **Add per-request anti-forgery (CSRF) tokens** to the payment/ACH/quick-pay POSTs and validate them server-side (H7). +13. Secure+HttpOnly cookies; **session regeneration on login** (H5). +14. **Verify and, if needed, fix the `get_cc_data` IDOR** (the TO-VERIFY item) — bind `@acctno` to the session server-side so a user can only read their own cards. +15. **Encrypt the `Web.config` connection strings** / move to proper secret management (H6) — so the DB credential is not plaintext on disk even on the new login. + +**Structural (project) — the durable fixes:** +16. **Move card-on-file to a tokenized / processor-hosted model.** Prefer the processor's **hosted fields / direct-API vault so the PAN never touches Glaztech servers** over "collect the card ourselves and store a token after." Then **purge / encrypt the historical PAN columns** and decommission CyberSource (ties to the Current-State migration note). + - **Prerequisite — the GTIware token migration (item 17) MUST be complete before the historical PAN columns are purged.** The auto-charge engine still reads the full PAN to bill saved cards; purging PAN while it still depends on the full number would break auto-billing and risk data loss. Until the GTIware side lands, expect **new PANs to keep accumulating** — the purge is a one-time action taken only after writes have stopped and tokens have replaced the stored numbers. +17. **GTIware-side work (PARALLEL, REQUIRED — not the website team's job):** stop the GTIware engine writing `cc_file` / `cof_payments_header`, and migrate `save_cc_data*` and `gt_auto_process_2020.dll` to tokens. This is a separate workstream from the website fixes and must be tracked and resourced as such; the website remediation does **not** cover it. + - **Dependency — this must complete before the historical PAN purge in item 16.** The token migration is the gate: only once `gt_auto_process` bills by token (no longer reading full PAN) can the stored PAN columns be safely purged. New plaintext PANs may keep accumulating until this lands. +18. **Encrypt or securely destroy the CHD-laden database backups** per the retention policy as part of the data purge — every existing backup of the affected DBs contains plaintext PAN + CVV and is a perpetual liability until handled. +19. **Network segmentation:** restrict the SQL listener (`192.168.8.62,3436`) via host firewall / ACL to **only the web server IP(s)** and required internal hosts; the network currently appears **flat**. This is part of the C0/H6 segmentation fix. +20. Separate development from the production host; deploy **precompiled**; remove dev tooling and source from prod (H1). +21. Enable **TDE** at rest; re-scope / re-confirm the merchant **PCI SAQ** with the acquirer/QSA after remediation. (The website now clearly hosts in-process card-on-file logic and is a multi-backend credential hub — the SAQ type must reflect that.) +22. **Decouple the GTIware card engine from the public host (C0-Extended).** Run `gt_auto_process_2020` / `glaztech_utilities_2020` off the internet-facing IIS process — on a non-CDE host, with its own dedicated **low-privilege** DB login (no `sysadmin`, explicit `DENY` on other offices, `msdb`, and the linked servers). You cannot safely swap only the website's connection string while the engine shares the same process and config. +23. **Audit and minimize the 7 linked servers.** Remove any not strictly required; restrict `is_remote_login_enabled` / data access and apply least privilege on the remote ends. Extend the segmentation in item 19 to the backup shares (`192.168.8.52` / `.212`), MAS (`192.168.0.55`), and TimeForce so the CDE is not flatly trusted across subnets. +24. **Move the CyberSource + PNC gateway secrets out of `Web.config` `appSettings`** into proper secret management (or encrypted config / DPAPI) and **rotate** them as part of the change. +25. **Lock down `/webhooks` and `/webhooks1` (Samsara).** Confirm authentication/authorization, restrict source IPs where possible, verify they allow no unauthenticated DB or card operations, and remove the `webhooks_test` app from production. +26. **Recon the second host `\\192.168.0.147\web\glaztech_4`** (surfaced by a cleanup job) — same code/secrets? also public-facing? same GTIware DLLs? — and bring it into scope. + +### Testing & rollback gates (applies to all of the above) +- **There may be NO staging environment.** Finding **H1** (dev tooling and source on the live box) strongly implies development happens on production. **Assume there is no separate test environment** until confirmed, and plan accordingly. +- **Require a validation + rollback plan** for each higher-risk change: the **DB-login change** (revert the `Web.config` connection string), the **`quo()` removal / parameterization** (per-page validation that queries still return correct results), the **error-path changes** (confirm `customErrors`/encoding don't break legitimate error flows), and the **TLS 1.0/1.1 disablement** (re-enable if a legacy client breaks). +- **Do NOT make in-place hotfixes on prod.** Stand up at least a minimal validation environment (or a maintenance-window + immediate-rollback procedure) before applying code or DB-permission changes. In-place edits to a live payment site with no rollback are how these fixes cause an outage. + +### Open follow-up tasks +- **`cc_file` "Invalid object name" anomaly (corp DB).** The Current-State check found `cc_file` returning "Invalid object name" in the `corp` DB though it existed there earlier the same day (per-office DBs unaffected). **Action:** confirm whether the object was dropped/renamed, moved, or is a permissions/visibility artifact — and whether any process is mid-migration. Track to closure; it changes the purge inventory if `corp` no longer holds that table. +- **IDOR verification on `get_cc_data(@acctno)`** (see Attack Path / item 14) — close this out as a concrete task. --- @@ -227,4 +412,4 @@ The review found no attacker, but it confirmed the **detection gap** noted in H5 --- -**Status:** Assessment complete 2026-06-03; intrusion/brute-force log review added 2026-06-04 (Appendix A). **No changes were made to the application, database, or data** (read-only throughout). Findings to be reviewed with the client (Steve Eastman / Tom) as priority security and PCI remediation. This report contains no card numbers or passwords. +**Status:** Assessment complete 2026-06-03; intrusion/brute-force log review added 2026-06-04 (Appendix A); **deep host + SQL-infrastructure recon added 2026-06-04 (C0-Extended; findings C5/H9/H10; Emergency-containment roadmap bucket).** **No changes were made to the application, database, or data** (read-only throughout). Findings to be reviewed with the client (Steve Eastman / Tom) as priority security and PCI remediation. This report contains no card numbers or passwords; one incidentally-returned domain-admin password was redacted and is not reproduced. diff --git a/clients/glaztech/session-logs/2026-06-04-session.md b/clients/glaztech/session-logs/2026-06-04-session.md index 0a93a21..01b68b7 100644 --- a/clients/glaztech/session-logs/2026-06-04-session.md +++ b/clients/glaztech/session-logs/2026-06-04-session.md @@ -63,3 +63,46 @@ Folded the findings into the security record: added **Appendix A — Intrusion / - Wiki: `wiki/clients/glaztech.md` - Ticket: Syncro **#32378** (Glaztech website security — Waiting on Customer) - RMM API: `http://172.16.3.30:3001` | agent id `455a1bc7-1c29-42bc-b597-fa1e64f08eec` + +--- + +## Update: 09:38 PT — Deep host + SQL-infrastructure recon (Grok gap-analysis loop) + +### Session Summary +Ran a collaborative gap-analysis loop (Claude + Grok CLI, 4 Grok turns on session `019e9351-…`) plus 3 read-only GuruRMM pulls on the `WWW` host to find what the website security assessment had not yet checked. Grok identified overlooked surfaces; Claude pulled the requested read-only host telemetry + SQL **metadata**; results were fed back to Grok each turn. Findings were folded into the assessment report. All read-only; no CHD and no secret values intentionally retrieved. + +### Key Findings (new vs. prior report) +- **GTIware card engine is co-resident in the public IIS worker on the `tom` sysadmin connection.** `gt_auto_process_2020.dll` / `glaztech_utilities_2020.dll` live in `D:\web\glaztech_4\Bin\`; their `.dll.config` carry no own connection strings → they use the website's `tom` connections. `w3wp` PID 13752 = `IIS APPPOOL\glaztech_new`. +- **`xp_cmdshell = 1` (enabled) on `GTI-INV-SQL`** → instant OS RCE from a `tom` SQLi (`clr`/`Ole`/`Ad Hoc`/`remote admin` all 0). +- **SQL Agent service runs as `Administrator@glaztech.com`** (domain-admin class). +- **Cleartext `glaztech\administrator` domain-admin password embedded in `msdb` SQL Agent backup-copy job steps** (`xp_cmdshell 'net use … /user:glaztech\administrator '`) — readable by any sysadmin/`tom` connection. **Value redacted from all artifacts (sed-scrubbed from the recon output); flagged for rotation.** +- **7 linked servers** (remote-login + data-access) across `192.168.0.x` and `192.168.8.x` (`mas_gti`, `qqest`/TimeForce, backup hosts, `GTI-INV-SQL\GTISQL`). +- **`sa` login enabled; all 47 DBs `is_encrypted=False` (no TDE);** CHD `.bak` backups copied to `\\192.168.8.52\sql_backup\`. +- **`Web.config` holds two processors' secrets in cleartext** (CyberSource `cybs.*` + on-disk signing keys; PNC `pnc_key/pin/token`); **`Everyone:(R)` ACL** on web root + `bin` + `Web.config`. +- Host firewall Domain=On, **Private/Public=Off**; **TLS 1.0 explicitly `Enabled=1`**; new endpoints `/webhooks` (Samsara receiver) + `/webhooks1`. +- Server-side billing engine `d:\sql_jobs\bin\gt_console_apps.exe` (`cp`/`is`/`oa`/`lo`) run by the Agent (domain-admin). +- Extra hosts surfaced: `\\192.168.0.147\web\glaztech_4`, backups `8.52`/`8.212`, MAS `0.55`, linked `0.54`. + +### Corrections / retractions +- An initial pass mis-flagged the `kgc7jt` ScreenConnect Cloud instance as "foreign/unrecognized third-party access" (draft finding **C6**). Per Mike: ScreenConnect (both), Datto RMM/EDR, Syncro, Splashtop are **ACG's sanctioned management stack**. **C6 fully retracted.** RealVNC is **Steve's** tool, kept as **H2** strictly for its end-of-life age (RealVNC 4.x ≈2009, listening 5900/5800). +- `machineKey` is AutoGenerate (not hardcoded); the app-pool identity is least-privilege locally — the privilege issue is the SQL `tom` login, not the Windows identity. + +### Key Decisions +- Folded findings into the existing report as a new **C0-Extended** section + findings **C5/H9/H10** + an **Emergency-containment** roadmap bucket (E1–E5) ahead of all other work, rather than a separate report. +- Domain-admin password returned incidentally in a job-step preview was scrubbed from the on-disk recon artifact and never written to the report (no-secrets constraint). + +### Configuration Changes +- Edited `clients/glaztech/reports/2026-06-03-website-security-assessment.md` (347 → 415 lines): method/date, Exec summary, findings table (+C5/H9/H10, −C6), C0 attack chain, new **C0-Extended** section, H2 rewrite, "Why the Cards Are Stored" nuance, Assumptions, PCI mapping, Emergency-containment bucket + structural items 22–26, status footer. + +### Commands & Outputs +- GuruRMM read-only PowerShell to `WWW` (agent `455a1bc7…`): **recon1** (IIS/app-pool/tasks/services/GTIware hunt/ACLs/net/firewall/TLS/certs), **recon2** (w3wp identity, GTIware `.dll.config` safe-parse, SQL metadata via the app's `tom` connection — `sys.configurations`/`dm_server_services`/`sys.servers`/`msdb` jobs+backups/principals/`sys.databases`), **recon3** (ScreenConnect probe). +- Grok CLI session `019e9351-ed1c-7bc3-b171-b4cf4b53745d` (4 turns) for gap analysis + corrected remediation sequencing. + +### Pending / Incomplete Tasks +- Coord todos filed: **6d15fc88** (rotate `glaztech\administrator` + purge `msdb` cleartext use; disable `xp_cmdshell`/`sa`), **aebaf751** (least-privilege `tom` migration). +- Recon the second host `\\192.168.0.147\web\glaztech_4`; lock down `/webhooks`. +- Carried: `corp` DB `cc_file` anomaly; IDOR on `get_cc_data`; Payrilla scope. + +### Reference Information +- Grok session `019e9351-ed1c-7bc3-b171-b4cf4b53745d`; SQL host `GTI-INV-SQL` `192.168.8.62,3436` (instance `GTISQL`). +- Coord todos `6d15fc88-db4f-4a35-a76a-a5a6a9f50795`, `aebaf751-d778-423f-a84b-314fbb294f30`. diff --git a/clients/peaceful-spirit/session-logs/2026-06-04-session.md b/clients/peaceful-spirit/session-logs/2026-06-04-session.md new file mode 100644 index 0000000..498223b --- /dev/null +++ b/clients/peaceful-spirit/session-logs/2026-06-04-session.md @@ -0,0 +1,77 @@ +# Peaceful Spirit — Session Log 2026-06-04 + +## User +- **User:** Mike Swanson (mike) +- **Machine:** GURU-5070 +- **Role:** admin + +## Session Summary + +Investigated a report that Bridgette's home machine (BridgettePSHomeComputer) was throwing VPN errors. The investigation established that this was not a Bridgette-specific or account-specific problem but a site-wide VPN outage: all Peaceful Spirit L2TP/IPsec clients were failing at the IPsec negotiation layer (Windows RAS error 789), which occurs before user authentication. MaraHomeNew failed identically while connecting as a different user (pst-admin), confirming the fault was below the per-user layer. + +Diagnosis proceeded by elimination via GuruRMM remote commands. The RRAS endpoint on PST-SERVER was confirmed fully healthy: 30-day uptime (no overnight reboot), services running (RemoteAccess, IKEEXT, PolicyAgent, RasMan), IKE listening on UDP 500/4500, firewall allow-rules present, PSK matching the vault, and the public/egress IP unchanged at 98.190.129.150 — exactly what clients dial. The client-side NAT-T registry key was correct (value 2). IPsec auditing was temporarily enabled on the server and a live dial triggered from Bridgette; the server logged zero IKE/IPsec security events, proving the clients' negotiation packets were not reaching the server at all. + +Root cause was isolated to the edge gateway. The site router (UDR Ultra, hostname UCG-PST-CC, 192.168.0.10) was reached read-only via an SSH key jump through PST-SERVER (the device's WAN SSH on :22 had also gone unreachable). Its live iptables ruleset contained no DNAT/port-forward for the VPN, and `last reboot` showed the device rebooted 2026-06-04 03:59 — the outage cutoff. After the reboot it came back without the UDP 500/4500 to 192.168.0.2 port-forward, so all inbound IPsec was silently dropped at the edge. The UniFi config's legacy Mongo collections (portforward/network/firewallrule/routing) all read 0, indicating this UniFi OS build (5.1.15) stores these in a migrated schema, so the controller UI was the authoritative place to confirm/restore the rule. + +Mike re-added the Port Forward (UDP 500 + 4500 to 192.168.0.2) in the UniFi controller. Verification confirmed the DNAT rules were live in the UDR ruleset, and a live dial test showed Bridgette fully connected (assigned VPN IP 192.168.0.242, RAS event 20224 link established, no 789). Mara's IPsec link also established (no 789); her test returned 691 only because the test rasdial ran as SYSTEM, the documented wrong-principal artifact for this site, not a real fault. The outage was resolved end-to-end. Resolution and root cause were posted to #dev-alerts. + +## Key Decisions + +- Treated the report as a whole-site diagnosis rather than a single-machine fix once MaraHomeNew showed the identical 789, since error 789 is a pre-authentication IPsec failure. +- Used IPsec auditing + a live dial as the decisive test to distinguish "packets not arriving (edge)" from "PSK/policy mismatch (server)" — zero IKE events conclusively pointed upstream of the server. +- Reached the UDR read-only via the PST-SERVER LAN jump (SSH key pushed to the server temporarily, used, then deleted) because the UDR's WAN SSH was unreachable and RMM was the only live channel to the LAN. +- Did not edit the UDR firewall over SSH; the UniFi controller re-pushes config, so iptables edits would not persist. The fix was directed to the controller UI. +- Left the controller change to Mike (per his choice) rather than driving the UniFi browser, then verified via RMM. + +## Problems Encountered + +- Two GuruRMM agents matched "BridgettePSHomeComputer"; the wiki UUID (074141d7…) was stale/offline. The machine re-enrolled — live agent is 01160fc8… (v0.6.49). Resolved by always resolving hostname to UUID live. +- Remote command sent to the UDR over PowerShell→ssh→bash lost its quoting and the remote shell choked on parentheses/pipes. Resolved by base64-encoding the remote script and running `echo | base64 -d | sh`. +- A transient "/mingw64/bin/curl: Permission denied" during one dispatch. Resolved by writing the JSON payload to a file and using `--data-binary @file`. +- Mara's verification dial returned 691; identified as the known SYSTEM-rasdial wrong-principal artifact, not a real failure (IPsec link established successfully). + +## Configuration Changes + +- **UniFi controller (UDR Ultra, 192.168.0.10):** Re-added Port Forward UDP 500 + 4500 → 192.168.0.2 (done by Mike in UI). This is the fix. +- **PST-SERVER:** IPsec auditing (Main/Extended/Quick Mode) temporarily enabled for the live-dial test, then restored to "No Auditing". No persistent change. +- No repo code changes. Session log + wiki article only. + +## Credentials & Secrets + +- No new credentials created. Existing vault entries referenced: + - `clients/peaceful-spirit/server.sops.yaml` → `credentials.ucg` (UDR SSH key `~/.ssh/pst-cc-ucg`, ssh_password, vpn_psk). + - `clients/peaceful-spirit/vpn.sops.yaml` → L2TP PSK `z5zkNBds2V9eIkdey09Zm6Khil3DAZs8` (matches server). +- **Vault drift noted (not yet fixed):** `vpn.sops.yaml` lists pst-admin password `24Hearts$`, but the wiki records a reset to `SpiritWalk26!` on 2026-05-22. Needs reconciliation. + +## Infrastructure & Servers + +- **PST-SERVER** — 192.168.0.2, Windows Server 2016 Essentials, PEACEFULSPIRIT.local DC, RRAS L2TP/IPsec endpoint. Public IP 98.190.129.150. RMM agent 87293069-33b6-45e8-a68f-6811216cdb96 (v0.6.52). +- **UCG-PST-CC** — UDR Ultra, LAN 192.168.0.10 (default gateway), WAN 98.190.129.150. UniFi OS 5.1.15, kernel 5.4.213-ui-ipq5322 (aarch64). Rebooted 2026-06-04 03:59. +- **BridgettePSHomeComputer** — RMM agent 01160fc8-4c2e-4e47-a591-e4e0f9ba5ea7 (v0.6.49). Connects as PEACEFULSPIRIT\BridgetteSH (SSO) via logon task "Connect Peaceful Spirit VPN". Got VPN IP 192.168.0.242 after fix. +- **MaraHomeNew** — RMM agent e9645594-6d7c-4c97-8cb4-920cb5d06c8e (v0.6.52). Connects as pst-admin via AllUserConnection cmdkey path. +- VPN: L2TP/IPsec, MSCHAPv2 + PSK, pool 192.168.0.240+, DNS 192.168.0.2. + +## Commands & Outputs + +- Client error: `error code returned on failure is 789` (RasClient event 20227) on every dial; one earlier termination 832. +- Server during live dial w/ auditing on: `NO IKE/IPsec security events in last 5 min -> negotiation packets not reaching server`. +- UDR `last reboot`: `reboot ... Thu Jun 4 03:59 still running` (prior boot May 22). +- UDR post-fix NAT: `-A UBIOS_PREROUTING_USER_HOOK -p udp ... --dport 500 ... -j DNAT --to-destination 192.168.0.2:500` and `...4500 ... -j DNAT --to-destination 192.168.0.2:4500`. +- Bridgette post-fix: `ConnectionStatus: Connected`, `IPAddress: 192.168.0.242`, event 20224 "link ... established by user PEACEFULSPIRIT\BridgetteSH". +- Mara post-fix: event 20224 link established; `rasdial` as SYSTEM → 691 (expected wrong-principal artifact). + +## Pending / Incomplete Tasks + +- **Update original Syncro ticket** with resolution + 1hr warranty labor (in progress this session). +- **Reboot-persistence test:** confirm the re-added port-forward survives a deliberate UDR reboot — a port-forward vanishing on reboot is abnormal (possible firmware bug or uncommitted rule). +- **DDNS for client profiles:** clients hardcode 98.190.129.150; a DDNS hostname would future-proof against a Cox WAN-IP change. +- **Vault reconcile:** fix pst-admin password drift in `vpn.sops.yaml` (24Hearts$ vs SpiritWalk26!). +- **Wiki:** record re-enrolled BridgettePSHomeComputer agent UUID and the "UDR reboot drops VPN port-forward" known issue. +- Optional real-world confirmation of Mara's auto-connect when a user is at the machine. + +## Reference Information + +- Syncro customer: Peaceful Spirit Massage, ID 278525. +- GuruRMM API: http://172.16.3.30:3001. +- UDR read-only access path: SSH key `~/.ssh/pst-cc-ucg` via PST-SERVER LAN jump to root@192.168.0.10. +- #dev-alerts messages: outage root cause 1512129381093474324; resolution 1512133532221444348. diff --git a/wiki/clients/peaceful-spirit.md b/wiki/clients/peaceful-spirit.md index db689f5..802d388 100644 --- a/wiki/clients/peaceful-spirit.md +++ b/wiki/clients/peaceful-spirit.md @@ -2,7 +2,7 @@ type: client name: peaceful-spirit display_name: Peaceful Spirit Therapeutic Massage -last_compiled: 2026-06-02 +last_compiled: 2026-06-04 compiled_by: GURU-5070/claude-main sources: - clients/peaceful-spirit/session-logs/2026-05-10-recovered-setup-radius-authentication-for-vpn-access.md @@ -10,6 +10,7 @@ sources: - clients/peaceful-spirit/session-logs/2026-05-11-session.md - clients/peaceful-spirit/session-logs/2026-05-22-session.md - clients/peaceful-spirit/session-logs/2026-05-27-session.md + - clients/peaceful-spirit/session-logs/2026-06-04-session.md - clients/peaceful-spirit/server.sops.yaml (vault) - clients/peaceful-spirit/vpn.sops.yaml (vault) backlinks: @@ -18,7 +19,7 @@ backlinks: # Peaceful Spirit Therapeutic Massage -Massage therapy practice with at least two sites: Country Club (primary, all work performed here) and a Northwest (NW) site. On-premises Windows Server 2016 Essentials domain environment. Domain-joined workstations for Mara (owner/operator) and other staff. L2TP/IPsec VPN fully deployed to all known machines as of 2026-05-27. +Massage therapy practice with at least two sites: Country Club (primary, all work performed here) and a Northwest (NW) site. On-premises Windows Server 2016 Essentials domain environment. Domain-joined workstations for Mara (owner/operator) and other staff. L2TP/IPsec VPN fully deployed to all known machines as of 2026-05-27. Site-wide VPN outage occurred 2026-06-04 due to UDR Ultra reboot dropping VPN port-forward — resolved same day by re-adding UDP 500/4500 -> 192.168.0.2 in UniFi controller. --- @@ -40,8 +41,8 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor | Host | IP | Role | OS | Notes | |---|---|---|---|---| -| PST-SERVER | 192.168.0.2 | DC, DNS, RRAS (L2TP/IPsec VPN), NPS, Enterprise Root CA (AD CS) | Windows Server 2016 Essentials (build 14393) | GuruRMM agent ID: `6b6106a7-8515-4b6b-857d-0dc6ede53f35`. Win32-OpenSSH installed 2026-05-11 (`C:\Program Files\OpenSSH\OpenSSH-Win64\`). Machine cert: `DB71981ABE4CBA1DE96FEEEAF178F6259663B543` (CN=PST-SERVER.PEACEFULSPIRIT.local, valid 5/9/2027). | -| UCG-PST-CC | 192.168.0.10 (LAN) / 98.190.129.150 (WAN) | UniFi Cloud Gateway — perimeter router + DNAT for VPN | UniFi OS | SSH: `root@192.168.0.10` via key `~/.ssh/pst-cc-ucg` (password-auth is keyboard-interactive; password: vault). WAN SSH (98.190.129.150:22) is NOT accessible remotely — timed out from all tested sources. UCG VPN (strongSwan/xl2tpd) abandoned 2026-05-22 in favor of RRAS on PST-SERVER. DNAT persistence: `/data/on_boot.d/10-vpn-portforward.sh`. | +| PST-SERVER | 192.168.0.2 | DC, DNS, RRAS (L2TP/IPsec VPN), NPS, Enterprise Root CA (AD CS) | Windows Server 2016 Essentials (build 14393) | GuruRMM agent ID: `87293069-33b6-45e8-a68f-6811216cdb96` (v0.6.52, confirmed 2026-06-04; prior ID `6b6106a7-8515-4b6b-857d-0dc6ede53f35` is retired/re-enrolled). Win32-OpenSSH installed 2026-05-11 (`C:\Program Files\OpenSSH\OpenSSH-Win64\`). Machine cert: `DB71981ABE4CBA1DE96FEEEAF178F6259663B543` (CN=PST-SERVER.PEACEFULSPIRIT.local, valid 5/9/2027). 30-day uptime confirmed 2026-06-04 (no reboot during VPN outage). | +| UCG-PST-CC | 192.168.0.10 (LAN) / 98.190.129.150 (WAN) | UniFi Cloud Gateway Ultra (UDR Ultra) — perimeter router + DNAT for VPN | UniFi OS 5.1.15, kernel 5.4.213-ui-ipq5322 (aarch64) | SSH: `root@192.168.0.10` via key `~/.ssh/pst-cc-ucg` (password-auth is keyboard-interactive; password: vault). WAN SSH (98.190.129.150:22) is NOT accessible remotely — timed out from all tested sources; LAN SSH reachable via PST-SERVER jump. UCG VPN (strongSwan/xl2tpd) abandoned 2026-05-22 in favor of RRAS on PST-SERVER. DNAT persistence: `/data/on_boot.d/10-vpn-portforward.sh`. NOTE: Rebooted 2026-06-04 03:59 and dropped the VPN port-forward (see Known Issues). Port-forward re-added in UniFi controller by Mike 2026-06-04. UniFi OS 5.1.15 stores port-forward rules in a migrated schema — legacy Mongo collections read 0; use the controller UI as authoritative. | **Note:** An NW (Northwest) site exists with a separate UCG that previously had an OpenVPN server at 64.139.88.249:1194 (TCP). No further NW site details are documented. @@ -75,10 +76,10 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor | Machine | Role | GuruRMM Agent ID | Notes | |---|---|---|---| -| MaraHomeNew | Mara's home desktop | `c778b6a3-c646-4454-a065-8c8bdcb1578e` | Domain-joined. VPN working (confirmed via rasdial 2026-05-11). Machine cert installed (D067E07B, CN=MaraHomeNew.PEACEFULSPIRIT.local, valid to 5/10/2027). Connects as pst-admin. | +| MaraHomeNew | Mara's home desktop | `e9645594-6d7c-4c97-8cb4-920cb5d06c8e` (v0.6.52, confirmed 2026-06-04; prior ID `c778b6a3-c646-4454-a065-8c8bdcb1578e` retired) | Domain-joined. VPN working (confirmed via rasdial 2026-05-11; IPsec link established 2026-06-04 post-fix). Machine cert installed (D067E07B, CN=MaraHomeNew.PEACEFULSPIRIT.local, valid to 5/10/2027). Connects as pst-admin. | | Maras-HP-Laptop | Mara's HP laptop | `13cb3629-5043-4bd6-b977-6968eeccf804` | Domain-joined. VPN deployed 2026-05-22 (PSK set on-site by Mike). OneDrive per-machine deployed 2026-05-11. pst-admin profile wiped and rebuilt 2026-05-11. Connects as pst-admin. | | PST-SURFACE | Surface device | `4a993b61-59b3-42f4-bdb5-d4362941f7d6` | Domain-joined. VPN deployed 2026-05-22 (PSK set on-site by Mike). Connects as pst-admin. | -| BridgettePSHomeComputer | Bridgette's home PC | `074141d7-bd96-49ff-8f64-edf31159c00b` | Domain-joined. VPN deployed remotely 2026-05-27 via GuruRMM `user_session`. Connects as BridgetteSH (SSO). Logon scheduled task `Connect Peaceful Spirit VPN` auto-connects ~20s after sign-in. NAT-T key was missing — set and rebooted 2026-05-27. | +| BridgettePSHomeComputer | Bridgette's home PC | `01160fc8-4c2e-4e47-a591-e4e0f9ba5ea7` (v0.6.49, re-enrolled 2026-06-04; old UUID `074141d7-bd96-49ff-8f64-edf31159c00b` is dead/offline) | Domain-joined. VPN deployed remotely 2026-05-27 via GuruRMM `user_session`. Connects as BridgetteSH (SSO). Logon scheduled task `Connect Peaceful Spirit VPN` auto-connects ~20s after sign-in. NAT-T key was missing — set and rebooted 2026-05-27. Got VPN IP 192.168.0.242 after 2026-06-04 port-forward fix (event 20224 link established). | --- @@ -93,11 +94,11 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor | Host | Agent ID | Enrolled | Last Known Status | |---|---|---|---| -| PST-SERVER | `6b6106a7-8515-4b6b-857d-0dc6ede53f35` | 2026-05-10 23:19 UTC | Active (2026-05-11 01:29 UTC) | -| MaraHomeNew | `c778b6a3-c646-4454-a065-8c8bdcb1578e` | [unverified date] | — | +| PST-SERVER | `87293069-33b6-45e8-a68f-6811216cdb96` (v0.6.52) | [re-enrolled; prior `6b6106a7...` retired] | Active — confirmed 2026-06-04 | +| MaraHomeNew | `e9645594-6d7c-4c97-8cb4-920cb5d06c8e` (v0.6.52) | [re-enrolled; prior `c778b6a3...` retired] | Active — confirmed 2026-06-04 | | Maras-HP-Laptop | `13cb3629-5043-4bd6-b977-6968eeccf804` | [unverified date] | — | | PST-SURFACE | `4a993b61-59b3-42f4-bdb5-d4362941f7d6` | [unverified date] | — | -| BridgettePSHomeComputer | `074141d7-bd96-49ff-8f64-edf31159c00b` | 2026-05-27 | Confirmed active 2026-05-27 | +| BridgettePSHomeComputer | `01160fc8-4c2e-4e47-a591-e4e0f9ba5ea7` (v0.6.49) | Re-enrolled 2026-06-04 (old `074141d7...` dead/offline) | Active — confirmed 2026-06-04 | --- @@ -108,7 +109,7 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor - **GuruRMM (external):** https://rmm.azcomputerguru.com - **Vault paths:** - `clients/peaceful-spirit/server.sops.yaml` — PST-SERVER credentials (sysadmin) and UCG details (root, keyboard-interactive); raw secrets live in the vault entry, not here. Created during the 2026-05-10 recovered session. - - `clients/peaceful-spirit/vpn.sops.yaml` — VPN PSK, pst-admin credentials, network details. Note: pst-admin password updated to SpiritWalk26! on 2026-05-22 — vault entry needs updating. + - `clients/peaceful-spirit/vpn.sops.yaml` — VPN PSK (z5zkNBds2V9eIkdey09Zm6Khil3DAZs8, confirmed 2026-06-04 matches server), pst-admin credentials, network details. [WARNING] VAULT DRIFT: vault lists pst-admin password as `24Hearts$` but wiki records a reset to `SpiritWalk26!` on 2026-05-22 — needs reconciliation (verify with Mara, update whichever is stale). --- @@ -118,7 +119,8 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor - **NRPT instead of VPN DNS suffix push.** `Add-VpnConnectionTriggerDnsConfiguration` fails for AllUserConnection profiles. Use `Add-DnsClientNrptRule -Namespace ".peacefulspirit.local" -NameServers "192.168.0.2"` instead. - **cmdkey as SYSTEM for pre-login credential persistence.** Machine credential store entries (cmdkey in SYSTEM context) are available at the Windows login screen; per-user cmdkey entries are not. - **Stale hosts file.** During 2026-05-22 on-site, MaraHomeNew (and likely other machines) had a stale hosts entry mapping PST-SERVER to 72.194.62.5 (Mara's router's bogus DNS response). This caused name resolution failures even with VPN up. A GuruRMM cleanup script was deployed; verify no residual entries if name resolution issues recur. The hosts-file path encoding bug (`driverstc` artifact) means the cleanup script may not have fully run on all machines. -- **UCG iptables DNAT required — UniFi Traffic Rules are firewall-allow only, NOT DNAT.** Port-forward rules must be placed via CLI in `/data/on_boot.d/10-vpn-portforward.sh` for persistence across reboots. +- **UDR Ultra reboot can silently drop the VPN port-forward (site-wide outage risk).** Confirmed 2026-06-04: the UDR Ultra (UCG-PST-CC) rebooted at 03:59 and came back without the UDP 500/4500 -> 192.168.0.2 port-forward, taking the entire site VPN offline with error 789 (IKE packets silently dropped at the edge). The `/data/on_boot.d/10-vpn-portforward.sh` persistence script was present but the UniFi OS 5.1.15 controller schema migration appears to have superseded it. **After any site-wide error 789, check the UDR port-forward in the UniFi controller FIRST** — IPsec auditing on the server (zero IKE events) is the confirmatory test. Long-term open items: (1) verify the re-added rule persists across a deliberate reboot (possible firmware bug or uncommitted rule), (2) add a DDNS hostname so the hardcoded 98.190.129.150 in client profiles is not a single point of failure for a Cox WAN-IP change. +- **UCG iptables DNAT required — UniFi Traffic Rules are firewall-allow only, NOT DNAT.** Port-forward rules must be managed via the UniFi controller UI; `/data/on_boot.d/10-vpn-portforward.sh` is a legacy CLI fallback and may not persist reliably on UniFi OS 5.1.15+ (see above). Always verify iptables live after a reboot. - **UCG SSH unreachable from office WAN.** All remote UCG administration must go through GuruRMM (for PST-SERVER) or the UniFi cloud portal (for UCG itself). LAN SSH (192.168.0.10) requires keyboard-interactive auth — password auth via plink fails; use paramiko with kb_handler or interactive terminal. - **GuruRMM PowerShell invocation quirk.** Running `command_type: powershell` fails on PST machines with "-OutputEncoding is not recognized." Use `command_type: cmd` and call `powershell.exe` explicitly within the script body. - **Machine cert template (PEACEFULSPIRIT-PST-SERVER-CA / Machine template).** `msPKI-Certificate-Name-Flag` was changed from `0x18000000` to `0x1` (ENROLLEE_SUPPLIES_SUBJECT) on 2026-05-11. This is a domain-wide template change. New machine certs will use the CSR Subject/SAN rather than the submitting machine's AD DNS identity. RRAS UserAuthProtocolAccepted now includes Certificate (added 2026-05-11). @@ -132,16 +134,19 @@ Massage therapy practice with at least two sites: Country Club (primary, all wor ## Active Work -As of 2026-05-27 session end: +As of 2026-06-04 session end: -- **VPN rollout: COMPLETE.** All four machines (MaraHomeNew, Maras-HP-Laptop, PST-SURFACE, BridgettePSHomeComputer) have working L2TP/IPsec VPN. -- **Vault update needed:** pst-admin password was reset to SpiritWalk26! on 2026-05-22; vault entry `clients/peaceful-spirit/vpn.sops.yaml` needs updating. (2026-05-27 session confirmed no SOPS entry existed for PSK/pst-admin at that time — secrets only in session logs.) +- **VPN rollout: COMPLETE.** All four machines (MaraHomeNew, Maras-HP-Laptop, PST-SURFACE, BridgettePSHomeComputer) have working L2TP/IPsec VPN. 2026-06-04 site-wide outage (UDR reboot) confirmed resolved: Bridgette connected (192.168.0.242), Mara IPsec established. +- **[OPEN] UDR port-forward reboot-persistence test:** Confirm the re-added UDP 500/4500 -> 192.168.0.2 rule survives a deliberate UDR reboot. The prior rule vanished on the 2026-06-04 03:59 reboot — may be a firmware bug or an uncommitted rule. If it doesn't persist, re-examine `/data/on_boot.d/10-vpn-portforward.sh` or escalate to UniFi. +- **[OPEN] DDNS for VPN endpoint:** Client profiles hardcode 98.190.129.150 (Cox WAN). A DDNS hostname would prevent a site-wide VPN breakage on a Cox IP change. Deferred — low urgency but document path. +- **[OPEN] Vault drift — pst-admin password:** `vpn.sops.yaml` lists `24Hearts$`; wiki records reset to `SpiritWalk26!` on 2026-05-22. Reconcile: verify current password with Mara, update vault to match. +- **[OPEN] Syncro ticket #32271 update:** Resolution + 1hr warranty labor for 2026-06-04 outage (per session log, in progress at session end). - **Parity decision deferred:** Mara's 3 machines connect as shared `pst-admin`; BridgetteSH connects as her own domain account via SSO. Consider aligning all to per-user auth (cleaner audit trail) or aligning Bridgette to `pst-admin`. - **Pre-login VPN verification:** Confirmed working on MaraHomeNew via rasdial. Maras-HP-Laptop and PST-SURFACE need verification at the Windows login screen specifically. - **Hosts file cleanup verification:** The GuruRMM cleanup script had a path encoding bug (`driverstc` instead of `drivers\etc`) — DNS was flushed but hosts entries may not have been removed on all machines. Verify if name resolution issues recur. - **PST-SERVER temp file cleanup:** `C:\ProgramData\`: gen_certs.ps1, fix_acl.ps1, acl_result.txt, verify_acl.ps1, acl_verify.txt, and all *.inf, *.req, *.cer, *.pfx files. Also remove temporary firewall rules TEMP-CertEnroll-RPC (TCP 135) and TEMP-CertEnroll-DCOM (TCP 49152-65535). - **Machine cert VPN path (IKEv2) — deferred.** Machine certs were generated for MaraHomeNew (D067E07B), Maras-HP-Laptop (4CADDE8F, CA RequestId 66), and PST-SURFACE (197FF22A, CA RequestId 67) and PFXs (password: PstVpn2026!) were created. This IKEv2 machine-cert approach was superseded by the L2TP/RRAS decision on 2026-05-22. The certs and PFXs remain on PST-SERVER and DESKTOP-0O8A1RL — determine if IKEv2 path should be completed, abandoned, or the certs revoked. -- **Auto-connect task on BridgettePSHomeComputer:** Validated via `Start-ScheduledTask`; not yet observed through an actual sign-in cycle. +- **Auto-connect task on BridgettePSHomeComputer:** Validated via `Start-ScheduledTask`; Bridgette fully connected 2026-06-04 (logon-task path confirmed end-to-end during outage resolution). --- @@ -158,6 +163,7 @@ As of 2026-05-27 session end: | 2026-05-22 | L2TP/IPsec VPN successfully deployed to MaraHomeNew, Maras-HP-Laptop, PST-SURFACE during on-site visit at Mara's house. UCG-hosted strongSwan/xl2tpd abandoned; RRAS on PST-SERVER became the VPN endpoint. UCG DNAT rules created for UDP 500/4500/ESP. Stale hosts file entries removed. pst-admin and mara passwords reset to SpiritWalk26!. BridgettePSHomeComputer offline — VPN pending. | | 2026-05-27 | BridgettePSHomeComputer VPN deployed fully remotely via GuruRMM `user_session` context (no on-site visit). L2TP PSK set remotely. BridgetteSH added to WseRemoteAccessUsers and granted msNPAllowDialin. Logon scheduled task created for auto-connect. VPN rollout complete across all four machines. | | 2026-06-01 | Crashed 2026-05-10 session transcript (9700a3c6) recovered by the auto-reconstructor. Primary-source log saved as `clients/peaceful-spirit/session-logs/2026-05-10-recovered-setup-radius-authentication-for-vpn-access.md`, cross-linked with the manual `2026-05-10-session.md`. Covers UCG SSH key generation, paramiko tunneling, RADIUS/NPS extraction, and vault `server.sops.yaml` creation. | +| 2026-06-04 | Site-wide VPN outage: UDR Ultra (UCG-PST-CC) rebooted at 03:59 and returned without UDP 500/4500 -> 192.168.0.2 port-forward. All clients failed RAS error 789 (IPsec pre-auth, zero IKE packets reaching server). RRAS/PST-SERVER confirmed healthy (30-day uptime, services up, PSK correct). Root cause isolated to missing DNAT rule via IPsec auditing (zero IKE events on live dial). Mike re-added port-forward in UniFi controller. Bridgette connected (192.168.0.242, event 20224); Mara IPsec established. BridgettePSHomeComputer re-enrolled in GuruRMM (new UUID 01160fc8, old 074141d7 dead). PST-SERVER agent UUID confirmed 87293069; MaraHomeNew agent UUID confirmed e9645594. | --- diff --git a/wiki/index.md b/wiki/index.md index a6e116f..51a2994 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -1,7 +1,7 @@ # Wiki Index Last updated: 2026-06-04 -Compiled by: GURU-BEAST-ROG/claude-main +Compiled by: GURU-5070/claude-main This wiki is LLM-maintained. Do not edit articles manually — run `/wiki-compile` to update. Run `/wiki-lint` to check for stale entries and broken backlinks. @@ -29,7 +29,7 @@ Run `/wiki-lint` to check for stale entries and broken backlinks. | [Grabb & Durando Law Office](clients/grabb-durando.md) | Personal injury law firm; GND-SERVER GuruRMM enrolled; AI demand review app scoped ($4K–$7K); website migration pending; plaintext DB password in README needs vaulting | 2026-05-24 | | [Pavon](clients/pavon.md) | Former/archive client; GeoVision NVR surveillance; OwnCloud at 172.16.3.22 backed by Uranus; cron stacking fixed; Nextcloud migration deferred 3–6 months | 2026-05-24 | | [Rednour Law Offices](clients/rednour.md) | Law firm; M365 rednourlaw.com (tenant 4a4ca18a) fully onboarded 2026-05-31; all 5 ComputerGuru SPs consented; no MDE license; 3 workstations GuruRMM enrolled (FRONTDESKRECEPT/LEGALASST/REDNOURCARRIEVI); Carla Skinner renamed from Emma; prior MSP agents (ScreenConnect/Splashtop/Datto) still present; shared-drive access for Nick Pafford deferred | 2026-06-02 | -| [Peaceful Spirit Therapeutic Massage](clients/peaceful-spirit.md) | Massage therapy practice; PST-SERVER (192.168.0.2) + 5 GuruRMM agents; L2TP/IPsec RRAS VPN complete across all machines; Syncro 278525 (Peaceful Spirit Massage) | 2026-06-02 | +| [Peaceful Spirit Therapeutic Massage](clients/peaceful-spirit.md) | Massage therapy practice; PST-SERVER (192.168.0.2) + 5 GuruRMM agents; L2TP/IPsec RRAS VPN complete; 2026-06-04 site-wide outage resolved (UDR Ultra reboot dropped VPN port-forward, re-added in controller); BridgettePSHomeComputer re-enrolled (new UUID 01160fc8); vault drift open (pst-admin password); Syncro 278525 (Peaceful Spirit Massage) | 2026-06-04 | | [Sombra Residential LLC](clients/sombra-residential.md) | Property management; Server2013 (actually WS2012 EOL, unpatched) + DESKTOP-UQRN4K3 GuruRMM enrolled; Transwiz migration artifacts cause Office credential prompts | 2026-05-24 | | [Stamback Septic](clients/stamback-septic.md) | Septic services; prepaid block ~3.5 hrs remaining; DESKTOP-BTR2AM3 + StambackLaptopNew GuruRMM enrolled; OneDrive identity wipe pattern documented | 2026-05-24 | | [BG Builders LLC](clients/bg-builders.md) | Construction; M365 bgbuildersllc.com (CIPP: sonorangreenllc.com); terminated employee (Lesley Roth) — account disabled, litigation hold, device wipes pending; no Intune | 2026-05-24 | @@ -92,7 +92,7 @@ Run `/wiki-lint` to check for stale entries and broken backlinks. | Glaz-Tech Industries | M365, ~200 users, 9 locations; WWW (192.168.8.72 / 65.113.52.88, IIS 10 / .NET 4.8, GuruRMM agent 455a1bc7); SQL backend (192.168.8.62,3436); CyberSource REST (PNC merchant) | GuruRMM (WWW agent enrolled) | | Grabb & Durando Law Office | GND-SERVER (WS2019, GuruRMM enrolled) | GuruRMM; AI demand review app (scoped) | | Pavon | OwnCloud VM (172.16.3.22), Uranus /Archive storage | — | -| Peaceful Spirit | PST-SERVER (192.168.0.2, GuruRMM enrolled), UCG (98.190.129.150) | GuruRMM | +| Peaceful Spirit | PST-SERVER (192.168.0.2, agent 87293069), UCG-PST-CC UDR Ultra (192.168.0.10 / 98.190.129.150), 4 workstations | GuruRMM | | Sombra Residential LLC | Server2013 (WS2012 EOL) + DESKTOP-UQRN4K3, GuruRMM enrolled | GuruRMM | | Stamback Septic | DESKTOP-BTR2AM3 + StambackLaptopNew, GuruRMM enrolled | GuruRMM | | BG Builders LLC | M365 bgbuildersllc.com; no on-prem infra documented | — |