sync: auto-sync from GURU-5070 at 2026-06-15 09:41:53
Author: Mike Swanson Machine: GURU-5070 Timestamp: 2026-06-15 09:41:53
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>(function(){try{var s=localStorage.getItem("acg-theme");var m=s||(window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light");document.documentElement.setAttribute("data-theme",m);document.documentElement.setAttribute("data-skin",(localStorage.getItem("acg-skin")||"ledger"));document.documentElement.classList.add("js");}catch(e){}})();</script>
|
||||
</head>
|
||||
<body>
|
||||
@@ -17,7 +18,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span><span class="brand__name">Arizona Computer Guru</span><br /><span class="brand__since">Concierge IT · Tucson · since 2001</span></span>
|
||||
</a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 104" role="img" aria-label="Arizona Computer Guru">
|
||||
<g transform="rotate(20 50 54)" fill="none" stroke="#F2922E" stroke-width="11" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2" />
|
||||
<line x1="50" y1="14" x2="50" y2="50" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 348 B |
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>(function(){try{var s=localStorage.getItem("acg-theme");var m=s||(window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light");document.documentElement.setAttribute("data-theme",m);document.documentElement.setAttribute("data-skin",(localStorage.getItem("acg-skin")||"ledger"));document.documentElement.classList.add("js");}catch(e){}})();</script>
|
||||
</head>
|
||||
<body>
|
||||
@@ -17,7 +18,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span><span class="brand__name">Arizona Computer Guru</span><br /><span class="brand__since">Concierge IT · Tucson · since 2001</span></span>
|
||||
</a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>(function(){try{var s=localStorage.getItem("acg-theme");var m=s||(window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light");document.documentElement.setAttribute("data-theme",m);document.documentElement.setAttribute("data-skin",(localStorage.getItem("acg-skin")||"ledger"));document.documentElement.classList.add("js");}catch(e){}})();</script>
|
||||
<script type="application/ld+json">
|
||||
{"@context":"https://schema.org","@type":"LocalBusiness","name":"Arizona Computer Guru","image":"assets/images/contact.png","address":{"@type":"PostalAddress","streetAddress":"7437 E. 22nd St","addressLocality":"Tucson","addressRegion":"AZ","postalCode":"85710","addressCountry":"US"},"telephone":"+1-520-304-8300","email":"info@azcomputerguru.com","url":"https://azcomputerguru.com","openingHours":"Mo-Fr 09:00-17:00"}
|
||||
@@ -20,7 +21,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span><span class="brand__name">Arizona Computer Guru</span><br /><span class="brand__since">Concierge IT · Tucson · since 2001</span></span>
|
||||
</a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
|
||||
@@ -297,11 +297,11 @@ h3 { font-size: 1.5rem; font-weight: 600; }
|
||||
.site-header .wrap { display: flex; align-items: center; gap: 1.5rem;
|
||||
min-height: calc(var(--base) * 3); }
|
||||
.brand { display: flex; align-items: baseline; gap: 0.6rem; text-decoration: none; color: var(--ink); }
|
||||
.brand__mark { /* CSS-drawn monogram, not an image icon */
|
||||
font-family: var(--f-display); font-weight: 700; font-size: 1.05rem;
|
||||
letter-spacing: 0.05em; color: var(--on-accent); background: var(--accent);
|
||||
padding: 0.15rem 0.5rem; border-radius: 2px; align-self: center;
|
||||
.brand__mark { /* official ACG "G" askew-power-symbol mark; tints to the active skin accent */
|
||||
display: inline-flex; align-items: center; align-self: center;
|
||||
color: var(--accent); line-height: 0;
|
||||
}
|
||||
.brand__mark svg { width: 34px; height: 35px; }
|
||||
.brand__name { font-family: var(--f-display); font-weight: 600; font-size: 1.35rem;
|
||||
letter-spacing: 0.005em; }
|
||||
.brand__since { font-family: var(--f-mono); font-size: 0.66rem; letter-spacing: 0.18em;
|
||||
|
||||
BIN
projects/acg-website-showcase/multipage/design/_markcheck.png
Normal file
BIN
projects/acg-website-showcase/multipage/design/_markcheck.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
BIN
projects/acg-website-showcase/multipage/design/lg-bold.png
Normal file
BIN
projects/acg-website-showcase/multipage/design/lg-bold.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
BIN
projects/acg-website-showcase/multipage/design/lg-paper.png
Normal file
BIN
projects/acg-website-showcase/multipage/design/lg-paper.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
BIN
projects/acg-website-showcase/multipage/design/lg-verdigris.png
Normal file
BIN
projects/acg-website-showcase/multipage/design/lg-verdigris.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>
|
||||
(function () { try {
|
||||
var s = localStorage.getItem("acg-theme");
|
||||
@@ -27,7 +28,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span>
|
||||
<span class="brand__name">Arizona Computer Guru</span><br />
|
||||
<span class="brand__since">Concierge IT · Tucson · since 2001</span>
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>(function(){try{var s=localStorage.getItem("acg-theme");var m=s||(window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light");document.documentElement.setAttribute("data-theme",m);document.documentElement.setAttribute("data-skin",(localStorage.getItem("acg-skin")||"ledger"));document.documentElement.classList.add("js");}catch(e){}})();</script>
|
||||
</head>
|
||||
<body>
|
||||
@@ -17,7 +18,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span><span class="brand__name">Arizona Computer Guru</span><br /><span class="brand__since">Concierge IT · Tucson · since 2001</span></span>
|
||||
</a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Anton&family=Hanken+Grotesk:wght@400;500;700&family=JetBrains+Mono:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/radical.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
</head>
|
||||
<body>
|
||||
<a href="#main" class="sr-only">Skip to content</a>
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Barlow+Condensed:wght@500;600;700&family=Lexend:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;600;700&family=Fraunces:wght@400;500;600;700&family=Source+Sans+3:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600;700&family=Anton&family=Hanken+Grotesk:wght@400;500;700&display=swap" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="css/styles.css" />
|
||||
<link rel="icon" href="assets/logo/acg-mark.svg" type="image/svg+xml" />
|
||||
<script>(function(){try{var s=localStorage.getItem("acg-theme");var m=s||(window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light");document.documentElement.setAttribute("data-theme",m);document.documentElement.setAttribute("data-skin",(localStorage.getItem("acg-skin")||"ledger"));document.documentElement.classList.add("js");}catch(e){}})();</script>
|
||||
</head>
|
||||
<body>
|
||||
@@ -17,7 +18,7 @@
|
||||
<header class="site-header">
|
||||
<div class="wrap">
|
||||
<a class="brand" href="index.html" aria-label="Arizona Computer Guru home">
|
||||
<span class="brand__mark" aria-hidden="true">ACG</span>
|
||||
<span class="brand__mark" aria-hidden="true"><svg viewBox="0 0 100 104" fill="none" stroke="currentColor" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"><g transform="rotate(20 50 54)"><path d="M61.2 26.2 A30 30 0 1 1 38.8 26.2"/><line x1="50" y1="14" x2="50" y2="50"/></g></svg></span>
|
||||
<span><span class="brand__name">Arizona Computer Guru</span><br /><span class="brand__since">Concierge IT · Tucson · since 2001</span></span>
|
||||
</a>
|
||||
<nav class="nav" aria-label="Primary">
|
||||
|
||||
@@ -213,3 +213,53 @@ still computes $952/mo; `node --check js/app.js` clean.
|
||||
State: multipage 4-skin switcher (Paper / Midnight / Verdigris / Bold) at :4328, all in
|
||||
light+dark. Bold is the dialed-back-then-premium-softened radical. Open: Mike to confirm
|
||||
Bold is the keeper, or pick among the four skins for the real azcomputerguru.com redesign.
|
||||
|
||||
## Update: 09:41 PT (2026-06-15) — logo mark, errorlog skill fixes, Graphifyy eval
|
||||
|
||||
Three workstreams after the Bold-skin softening.
|
||||
|
||||
**1. Official ACG logo (partial).** Mike supplied the official wordmark ("Arizona ComputerGuru",
|
||||
the "G" = an askew power symbol) and noted the short logo is just that stylized G. Hand-built the
|
||||
short mark as a faithful SVG (`multipage/assets/logo/acg-mark.svg`, askew power symbol), wired it
|
||||
as the header brand mark (tints to each skin's accent via currentColor) + favicon across all 6
|
||||
multipage pages + radical.html. NOT done: the full wordmark — it's an exact brand asset with
|
||||
custom type; can't save Mike's pasted raster from here and AI-recreation would be off-brand, so
|
||||
the real file needs to be dropped into `multipage/assets/logo/` (color + a white/knockout for dark
|
||||
skins). Open design choice for Mike: official fixed wordmark sitewide vs. keep the per-skin
|
||||
adaptive lockup.
|
||||
|
||||
**2. errorlog review -> skill hardening.** 4 entries: #2 (py-on-Linux) + #3 ($CLAUDETOOLS_ROOT)
|
||||
already resolved on this machine; #4 (sync.sh submodule abort) fixed upstream by the GURU-BEAST-ROG
|
||||
coord broadcast (resolve_submodule_collisions; lands on next pull here). #1 (mailbox/FABB) was the
|
||||
open one: app `fabb3421` is DELETED (AADSTS700016), and it held the only Mail.Send/ReadWrite/
|
||||
Contacts scopes -> `/mailbox` + M365 contacts blocked, and the 3 legacy "old app only" tenants
|
||||
(Valleywide/Dataforth/Cascades) now have NO working remediation app. Hardened the docs:
|
||||
`gotchas.md` (DELETED + blast radius + URGENT migration), `mailbox.md` (BLOCKED/AADSTS700016
|
||||
banner), `errorlog.md`. **Mike's decision:** Mail.Send belongs in the SUITE (real use = IR
|
||||
victim-notification during box takeovers) -> add Mail.Send to the **Exchange Operator** tier;
|
||||
implementation NOT executed (production multi-tenant app change, needs go + admin-consent). Also
|
||||
surfaced 2 stale for-mike.md items: Intune Manager needs signInAudience->AzureADMultipleOrgs PATCH
|
||||
(unblocks Cascades MDM); Mike's per-user Syncro key.
|
||||
|
||||
**3. Graphifyy vs GrepAI evaluation (`projects/graphifyy-eval/`).** Question: adopt Graphifyy
|
||||
(getzep-style code+doc knowledge graph, `safishamsi/graphify`, PyPI `graphifyy`) for day-to-day?
|
||||
Built a 3-arm protocol (A=GrepAI, B=Graphifyy, C=control) with a 10-query test set + rubric.
|
||||
- Arm A (GrepAI): 18/20, ~3k ctx tokens/query, already indexed. 2 misses: phrasing-sensitivity
|
||||
(C1) and stale-duplicate retrieval (D2 pulled superseded kittle-design).
|
||||
- Arm B (Graphifyy): code extraction = AST (instant/free/local, but redundant with GrepAI).
|
||||
Doc/non-code = generative LLM per chunk. Local ollama (apples-to-apples, since GrepAI uses
|
||||
ollama embeddings) was IMPRACTICAL: qwen3:8b DNF 20 files/10min ("too small for JSON"),
|
||||
codestral:22b got 1 of 2 chunks on 3 files/6min. Claude backend (vault gururmm anthropic key,
|
||||
~$0.20 real spend) WORKED: 3 docs in 120s, but graphify reported ~$0.068/doc (recurring on an
|
||||
active repo), and the decisive finding: `graphify query` returns the concept/relationship MAP,
|
||||
NOT the content values (the prices were absent) -> you still Read the file for facts. GrepAI
|
||||
returns content directly.
|
||||
- **Verdict: keep GrepAI, do not adopt Graphifyy.** Cheap part duplicates GrepAI; unique part
|
||||
(doc/relationship graph) is impractical local / costs recurring $ cloud / hands a map not facts.
|
||||
Revisit only with GPU+fast local generative model, or a recurring architecture-relationship
|
||||
need worth a metered ingest budget. Full writeup: `projects/graphifyy-eval/FINDINGS.md`.
|
||||
- Cleanup done: GrepAI re-enabled in settings.local.json; graphifyy + openai uninstalled
|
||||
(anthropic kept); scratch graphs deleted; PROTOCOL/results/FINDINGS retained.
|
||||
|
||||
Note: untracked discord-dm skill/command/scripts + 2 feedback memories present in the tree are
|
||||
fleet additions (not this session's work); auto-sync sweeps them.
|
||||
|
||||
81
projects/graphifyy-eval/FINDINGS.md
Normal file
81
projects/graphifyy-eval/FINDINGS.md
Normal file
@@ -0,0 +1,81 @@
|
||||
# Graphifyy vs GrepAI — findings (GURU-5070, 2026-06-15)
|
||||
|
||||
## TL;DR
|
||||
On this machine, **Graphifyy does not clear the bar for day-to-day adoption.** Its code-graph
|
||||
is fast/free but largely redundant with GrepAI (already wired in); its one real differentiator
|
||||
(a graph over docs/PDFs) requires LLM semantic extraction that is **impractically slow on the
|
||||
apples-to-apples local-ollama config**, and making it usable would require a cloud LLM backend
|
||||
= ongoing API cost, which negates the local/free premise. Recommend **do not adopt**; keep GrepAI.
|
||||
|
||||
## Arm A — GrepAI (baseline) [complete]
|
||||
10/10 answered, **18/20** rubric. Medians: ~3,025 ctx tokens/query, 3 calls, ~55s. Already
|
||||
indexed (no build step). Two notable retrieval misses: C1 (phrasing surfaced the old timeout
|
||||
reaper, not the comms-durability fix) and D2 (returned the SUPERSEDED kittle-design copy, missed
|
||||
the canonical June BEC report) — i.e. it trips on indexed stale duplicates. Cost: ~514k agent
|
||||
tokens for the 10-query arm.
|
||||
|
||||
## Arm B — Graphifyy (local ollama) [BLOCKED at indexing]
|
||||
Setup done: `pip install graphifyy` (v0.8.39) + `openai` dep; GrepAI disabled per-machine
|
||||
(backed up). Backend = local ollama (apples-to-apples: GrepAI also uses ollama).
|
||||
|
||||
Architecture confirmed:
|
||||
- **Code extraction = AST (tree-sitter), no LLM** — instant, free, local. Strong.
|
||||
- **Query/path/explain = local BFS over graph.json with a token budget** — cheap at query time.
|
||||
- **Doc/PDF/image extraction = generative LLM (JSON) per chunk via the chosen backend** — heavy.
|
||||
|
||||
Indexing measurements (the blocker):
|
||||
- `msp-pricing` (2 code + 18 docs + 2 PDFs), `qwen3:8b`, --mode deep: **did NOT finish in 600s**;
|
||||
graphify warned the 8B model is "too small for JSON instruction following" + VRAM/truncation.
|
||||
- `msp-pricing/docs` (3 markdown files), `codestral:22b`, --token-budget 4096: got through
|
||||
**chunk 1 of 2 in 360s** and did not finish. ~minutes per chunk.
|
||||
- AST-only code extraction ran instantly in both runs.
|
||||
|
||||
### Why local ollama is the wrong workload for Graphifyy (key insight)
|
||||
"Both use ollama" is true but the workloads differ fundamentally:
|
||||
- **GrepAI/ollama = embeddings** (nomic-embed-text): one fast forward pass per chunk. Cheap.
|
||||
- **Graphifyy/ollama = generative structured (JSON) extraction** per chunk: slow, needs a
|
||||
large instruction-following model; small models fail JSON, large models are minutes/chunk.
|
||||
|
||||
So the doc-graph that is Graphifyy's only edge over GrepAI is gated behind an indexing cost that
|
||||
is impractical locally on this hardware, and a cloud backend (gemini/claude/openai) would add
|
||||
real per-ingest API cost + break the local/free + ollama-parity premise.
|
||||
|
||||
## Arm B addendum — Claude backend (cloud, breaks ollama-parity) [tested]
|
||||
To see if a capable cloud backend makes the doc-graph viable: re-ran the SAME 3 `msp-pricing/docs`
|
||||
files with `--backend claude` (key from vault `projects/gururmm/anthropic-api.sops.yaml`,
|
||||
`anthropic` pip dep added).
|
||||
- **Build: succeeded in 120s** (vs local-ollama DNF). 41 nodes, 68 edges, 9 communities.
|
||||
- **Cost reported by graphify: 7,813 in / 12,008 out tokens, ~$0.20 for 3 small docs** (~$0.068/doc).
|
||||
Extrapolated: the doc slice (wiki + msp-pricing + kittle + dataforth + gc docs) is ~hundreds of
|
||||
files = ~$10-$30 initial; the full repo's docs (wiki + hundreds of client session-logs/reports)
|
||||
= ~$50-$200+; plus steady re-ingest as docs change (SHA256 cache skips unchanged, so steady-state
|
||||
= changed/new docs only — a constant trickle in an active repo). Code stays free (AST).
|
||||
- **Query quality (the decisive finding):** `graphify query` is a local, free, 1s BFS over
|
||||
graph.json. For "GPS pricing tiers and prices" it returned the **concept/relationship MAP**
|
||||
(all tier + plan NODES, cross-doc concept links like "GPS Support Plans (Cross-Document
|
||||
Concept)") in ~1,573 tokens — but **NOT the actual prices** ($19/$26/$39 absent; nodes carry
|
||||
label + src file, not leaf values). To get the facts you still Read the source file. GrepAI
|
||||
(Arm A D1) returned the file chunk WITH the prices and answered outright.
|
||||
|
||||
### What this means
|
||||
- **Fact/content retrieval (the common day-to-day query):** GrepAI is more direct (returns
|
||||
content). Graphifyy returns a map -> you still open the file. Extra hop.
|
||||
- **Structural/relationship retrieval (architecture, impact, cross-doc concept links):**
|
||||
Graphifyy's genuine edge, and the cross-document concept synthesis is nice — but it's the rarer
|
||||
query, and overlaps GrepAI's RPG for code.
|
||||
|
||||
## Cost/benefit verdict (Mike's day-to-day)
|
||||
- Day-to-day is mostly MSP ops + bursts of dev. The retrieval that helps is code (dev bursts) +
|
||||
docs/knowledge (client history). GrepAI already serves code well (18/20) and is zero-setup.
|
||||
- Graphifyy's code side ≈ redundant with GrepAI. Its doc side (the differentiator) can't be
|
||||
cheaply/locally indexed here. Net marginal benefit is low; the standing index/maintenance cost
|
||||
(and the second-system overhead) is real.
|
||||
- **Recommendation: do not adopt fleet-wide; do not replace GrepAI.** Revisit only if (a) a fast
|
||||
local generative model + GPU make doc-graph indexing cheap, or (b) the doc/PDF knowledge-graph
|
||||
becomes a must-have and a metered cloud-backend ingest budget is acceptable.
|
||||
|
||||
## Reversal / cleanup
|
||||
- Re-enable GrepAI: restore `enabledMcpjsonServers` (backup at
|
||||
`projects/graphifyy-eval/settings.local.json.bak`) — needs session restart.
|
||||
- Remove Graphifyy: `py -m pip uninstall -y graphifyy` (and `openai` if unwanted). Delete
|
||||
`projects/graphifyy-eval/out/` scratch graphs.
|
||||
106
projects/graphifyy-eval/PROTOCOL.md
Normal file
106
projects/graphifyy-eval/PROTOCOL.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# Graphifyy vs GrepAI — evaluation protocol (GURU-5070)
|
||||
|
||||
Goal: real, comparable data on whether **Graphifyy** beats the incumbent **GrepAI** for
|
||||
Mike's day-to-day in ClaudeTools, enough to make an adopt / skip / adopt-narrowly call.
|
||||
Decision hinges on token efficiency + retrieval quality, weighed against maintenance cost.
|
||||
|
||||
## Tools under test
|
||||
- **GrepAI** — `D:\claudetools\grepai.exe mcp-serve`, exposed as `mcp__grepai__*` (semantic
|
||||
search + RPG graph: explore / trace_callers / trace_callees / trace_graph). Repo-wide index
|
||||
already built (`.grepai/`). Enabled per-machine via `enabledMcpjsonServers:["grepai"]` in
|
||||
`.claude/settings.local.json`.
|
||||
- **Graphifyy** — `pip install graphifyy && graphify install`. Local graph (NetworkX +
|
||||
tree-sitter + Leiden). CLI/skill: `graphify <path> [--mode deep|--update]`,
|
||||
`graphify query "q"`, `graphify path "A" "B"`, `graphify explain "C"`. Docs/PDF/images
|
||||
ingested via Claude API (token cost); code parsed locally.
|
||||
|
||||
## Arms (run in separate sessions; MCP toggles need a restart)
|
||||
- **A — GrepAI** (baseline / "before"): grepai ON, Graphifyy not used. Run FIRST, this session.
|
||||
- **B — Graphifyy** ("after"): Graphifyy ON, grepai DISABLED (removed from
|
||||
`enabledMcpjsonServers`). New session.
|
||||
- **C — Control** (optional): both off; only `grep`/`glob`/`Read`. Shows whether either graph
|
||||
tool beats plain search.
|
||||
|
||||
Same model for all arms. Each query answered in a FRESH sub-agent constrained to that arm's
|
||||
tools, to avoid cross-arm contamination. Scoring done against the rubric, blind to arm where
|
||||
feasible.
|
||||
|
||||
## Fixed test corpus (both tools index the SAME slice)
|
||||
To keep it fair and bounded (not the whole repo + node_modules):
|
||||
- Code: `projects/msp-tools/guru-rmm/` (Rust server + agent + React dashboard)
|
||||
- Docs: `wiki/`, `projects/msp-pricing/`, `clients/kittle/`, `clients/dataforth/`
|
||||
- PDF: `projects/msp-pricing/marketing/The Arizona Business Owner's Guide to Choosing an MSP - Arizona Computer Guru.pdf`
|
||||
|
||||
Note asymmetry: GrepAI's existing index is repo-wide (slight recall edge, more noise);
|
||||
Graphifyy indexes exactly this slice. All test queries are answerable from the slice.
|
||||
|
||||
## Metrics (per query x arm)
|
||||
| Metric | How captured |
|
||||
|---|---|
|
||||
| `ctx_tokens` | chars of retrieved context the agent consumed / 4 (consistent approx) |
|
||||
| `tool_calls` | number of retrieval round-trips to reach the answer |
|
||||
| `latency_s` | wall-clock for the query |
|
||||
| `score` | 0 = wrong/missing, 1 = partial, 2 = complete & correct (vs rubric) |
|
||||
|
||||
One-time / maintenance (measured once per tool):
|
||||
- `index_build_s` — full index of the test corpus (code-only, then code+docs)
|
||||
- `reindex_s` — incremental update after touching ONE file
|
||||
- `ingest_api_tokens` — Graphifyy's Claude-API tokens to ingest docs/PDF/images
|
||||
(GrepAI: note its embedding model/cost; LLM-ingestion ≈ 0)
|
||||
|
||||
## Test set (10 queries; code-heavy + docs-heavy, since docs is Graphifyy's claimed edge)
|
||||
Each has a rubric = key facts a correct answer MUST contain.
|
||||
|
||||
CODE
|
||||
- C1: "In GuruRMM, how does the server avoid false-failing commands that were delivered but not
|
||||
acked? Name the mechanism + migrations." Rubric: agent CommandAck on receipt + dedup; reaper
|
||||
RE-DELIVERS un-acked instead of false-failing; migrations 058 acked_at / 059 delivery_attempts.
|
||||
- C2: "Trace where un-acked command re-delivery is handled in the RMM server and what calls it."
|
||||
Rubric: the reaper fn + its caller path. (grepai trace_callers vs graphify path)
|
||||
- C3: "Where is GuruRMM agent self-update with rollback implemented and what guards it?"
|
||||
Rubric: agent `updater/mod.rs` + watchdog.
|
||||
- C4: "What does GuruConnect SPEC-018 propose?" Rubric: session broker / capture worker as SYSTEM.
|
||||
|
||||
DOCS / KNOWLEDGE (Graphifyy's claimed strength)
|
||||
- D1: "GPS pricing structure (tiers + prices)?" Rubric: Basic $19 / Pro $26 / Advanced $39 per
|
||||
endpoint; support plans Essential/Standard/Premium/Priority.
|
||||
- D2: "Summarize the Kittle BEC/ACH-fraud incident and root cause." Rubric: Ken+marco+Accounting
|
||||
compromised; fraudulent bank-change to City of Tucson + Marana ($130K+ prevented); IC3 filed;
|
||||
root cause = April credential theft + incomplete remediation (password never reset, ~2mo).
|
||||
- D3: "Which ACG clients had M365 breach/credential incidents in 2026 and each root cause?"
|
||||
Rubric (relationship query): Kittle (BEC), Dataforth (2026-03-27 phishing -> MFA), mvaninc
|
||||
(unauthorized sign-in OKC). Partial credit per client.
|
||||
- D4: "List the 7 red flags of a bad MSP from the Buyers Guide." Rubric: the 7 from
|
||||
MSP-Buyers-Guide-Content.md (unlimited-support, high-pressure sales, offshore-only, no
|
||||
proactive monitoring, long lock-ins, one-size packages, no local presence). PDF/doc ingestion.
|
||||
- D5: "Canonical Kittle article path + what it superseded?" Rubric: clients/kittle.md canonical;
|
||||
kittle-design.md superseded 2026-06-09.
|
||||
|
||||
MIXED (code + docs)
|
||||
- M1: "How do new GuruRMM builds get promoted from beta to stable?" Rubric: builds tag beta;
|
||||
promote via POST /api/updates/rollouts/:version/promote; build-server.sh auto-deploys.
|
||||
|
||||
## Procedure
|
||||
1. (Arm A, now) For each query, spawn a sub-agent: tools = grepai + Read only; instruct it to
|
||||
use ONLY grepai for retrieval, answer, and report (answer, total retrieved chars, # grepai
|
||||
calls, elapsed). Log to results.csv with arm=A.
|
||||
2. Score each answer 0/1/2 vs rubric.
|
||||
3. Disable GrepAI (below), install + index Graphifyy, measure one-time costs.
|
||||
4. (Arm B, new session) Same queries, sub-agent tools = Bash(graphify) + Read; use ONLY
|
||||
graphify for retrieval. Log arm=B. Score.
|
||||
5. (Arm C, optional) grep/glob/Read only. Log arm=C.
|
||||
6. Analyze: per-metric medians by arm; weight ctx_tokens + score (the day-to-day levers);
|
||||
factor in index/maintenance cost and the doc-vs-code split.
|
||||
|
||||
## Reversible environment changes (per-machine only)
|
||||
Disable GrepAI (edit `.claude/settings.local.json`, remove "grepai" from
|
||||
`enabledMcpjsonServers`; restart session). Re-enable = add it back. **Do NOT edit `.mcp.json`**
|
||||
(shared/fleet). Install Graphifyy: `py -m pip install graphifyy && graphify install`. Uninstall
|
||||
= `py -m pip uninstall graphifyy` + remove its skill. Snapshot of `settings.local.json` kept at
|
||||
`projects/graphifyy-eval/settings.local.json.bak` before any edit.
|
||||
|
||||
## Open setup unknowns to resolve at install
|
||||
- Which API key/env var Graphifyy uses for doc/PDF/image ingestion (README didn't say; it bills
|
||||
as "a Claude Code skill"). Confirm before indexing docs so ingest cost is attributable.
|
||||
- Whether `graphify query` itself spends LLM tokens to answer (vs returning raw graph context) —
|
||||
affects per-query cost comparison; measure.
|
||||
21
projects/graphifyy-eval/results.csv
Normal file
21
projects/graphifyy-eval/results.csv
Normal file
@@ -0,0 +1,21 @@
|
||||
query,arm,ctx_tokens,grepai_or_graphify_calls,files_read,latency_s,score,notes
|
||||
C1,A,8500,6,0,75,1,"described OLD timeout reaper (mig 014/043) + interrupt-on-reconnect; MISSED the comms-durability CommandAck/dedup + re-deliver + migrations 058/059 (the actual fix). query-phrasing steered to wrong mechanism"
|
||||
C2,A,6000,3,2,76,2,"nailed it: fail_timed_out_commands (db/commands.rs:337) w/ acked_at/delivery_attempts gating + re-deliver via get_pending_commands (ws/mod.rs); caller=main.rs tokio task. (1 stray grep for a line number)"
|
||||
C3,A,3100,2,1,63,2,"updater/mod.rs AgentUpdater full flow + rollback watchdog + guards. complete"
|
||||
C4,A,1800,1,1,37,2,"SPEC-018 SYSTEM service host + session broker, capture workers as SYSTEM. complete"
|
||||
D1,A,2750,1,0,23,2,"GPS Basic $19/Pro $26/Adv $39 + 4 support plans + equip pack. complete, 1 call"
|
||||
D2,A,2412,1,0,33,1,"retrieved the SUPERSEDED clients/kittle-design/ April breach-check (Alexis/Ken inbox rules); MISSED the canonical June BEC/ACH-fraud event ($130K to City of Tucson/Marana prevented, IC3 filed). stale-duplicate retrieval"
|
||||
D3,A,12250,4,0,74,2,"open-ended relationship query; comprehensive + well-sourced (Valley Wide confirmed; Cascades/Bardach/Barbara blocked; Kittle unconfirmed). NOTE: my rubric was inaccurate - weak gold query"
|
||||
D4,A,2185,1,0,39,2,"all 7 red flags correct from MSP-Buyers-Guide-Content.md. (1 stray grep for titles)"
|
||||
D5,A,2950,3,0,56,2,"wiki/clients/kittle.md canonical, superseded kittle-design.md 2026-06-09. correct. (1 stray grep)"
|
||||
M1,A,7625,3,0,54,2,"beta-first + POST /api/updates/rollouts/:version/promote + .channel sidecars + dashboard promote. complete"
|
||||
C1,B,,,,,,
|
||||
C2,B,,,,,,
|
||||
C3,B,,,,,,
|
||||
C4,B,,,,,,
|
||||
D1,B,,,,,,
|
||||
D2,B,,,,,,
|
||||
D3,B,,,,,,
|
||||
D4,B,,,,,,
|
||||
D5,B,,,,,,
|
||||
M1,B,,,,,,
|
||||
|
Reference in New Issue
Block a user