sync: auto-sync from GURU-5070 at 2026-07-01 15:49:56

Author: Mike Swanson
Machine: GURU-5070
Timestamp: 2026-07-01 15:49:56
This commit is contained in:
2026-07-01 15:50:48 -07:00
parent 1775571abb
commit 2937b00ebf
15 changed files with 1217 additions and 67 deletions

View File

@@ -0,0 +1,347 @@
#!/usr/bin/env python
"""errorlog-dream: lint the fleet error log (errorlog.md).
Read-only by default. Parses the canonical entry format
YYYY-MM-DD | MACHINE | skill/context | [type] message [ctx: k=v ...] (xN)
and reports the patterns the log exists to surface: which contexts generate
the most failures, which documented rules keep getting violated (repeat ref=
citations), which identical failures recur across days (noise clusters that
need a skill-side fix, not more logging), resolved entries, machine-name
drift, and entries old enough to archive.
The single mutating mode, --apply-archive, moves entries older than --days
(default 60) into errorlog-archive/YYYY-MM.md. Everything judgment-shaped
stays in the PROPOSED section for the operator, mirroring memory-dream.
"""
import argparse
import io
import json
import os
import re
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone
MARKER = "<!-- Append entries below this line -->"
ENTRY_RE = re.compile(
r"^(\d{4}-\d{2}-\d{2}) \| ([^|]+?) \| ([^|]+?) \| (.*)$"
)
TYPE_RE = re.compile(r"^\[(correction|friction|[a-z-]+)\]\s+")
CTX_RE = re.compile(r"\[ctx: ([^\]]*)\]")
REF_RE = re.compile(r"ref=([A-Za-z0-9_./#-]+)")
COUNT_RE = re.compile(r" \(x(\d+)\)\s*$")
RESOLVED_RE = re.compile(r"\[RESOLVED[^\]]*\]", re.IGNORECASE)
def find_root():
env = os.environ.get("CLAUDETOOLS_ROOT")
if env and os.path.isdir(env):
return env
here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
idf = os.path.join(here, ".claude", "identity.json")
if os.path.isfile(idf):
try:
with io.open(idf, encoding="utf-8") as fh:
root = json.load(fh).get("claudetools_root")
if root and os.path.isdir(root):
return root
except Exception:
pass
return here
class Entry(object):
__slots__ = ("date", "machine", "skill", "msg", "etype", "ctx", "refs",
"count", "resolved", "lines", "raw_first")
def __init__(self, date, machine, skill, msg, lines):
self.date = date
self.machine = machine.strip()
self.skill = skill.strip()
self.lines = lines # verbatim block lines (for archiving)
self.raw_first = lines[0]
m = COUNT_RE.search(msg)
self.count = int(m.group(1)) if m else 1
msg = COUNT_RE.sub("", msg)
t = TYPE_RE.match(msg)
self.etype = t.group(1) if t else "exec"
cm = CTX_RE.search(msg)
self.ctx = cm.group(1) if cm else ""
self.refs = REF_RE.findall(msg)
self.resolved = bool(RESOLVED_RE.search(" ".join(lines)))
self.msg = msg
@property
def context_group(self):
return self.skill.split("/", 1)[0].strip()
def norm_msg(self):
"""Message with volatile tokens (ids, numbers, hex, paths' digits)
collapsed, for grouping recurring failures across days."""
m = TYPE_RE.sub("", self.msg)
m = CTX_RE.sub("", m)
m = re.sub(r"[0-9a-fA-F]{8}-[0-9a-fA-F-]{27,}", "<uuid>", m)
m = re.sub(r"\b[0-9a-fA-F]{12,}\b", "<hex>", m)
m = re.sub(r"\d+", "<n>", m)
return re.sub(r"\s+", " ", m).strip().lower()
def parse_log(path):
"""Return (header_lines, entries, unparsed_blocks, trailing_map).
Blocks are runs of consecutive non-blank lines after the marker. A block
whose first line matches ENTRY_RE is an Entry (continuation lines belong
to it -- the pre-helper era wrote multi-line entries by hand); anything
else is an unparsed block, reported but never touched.
"""
with io.open(path, encoding="utf-8") as fh:
lines = fh.read().splitlines()
header, rest, seen_marker = [], [], False
for ln in lines:
(rest if seen_marker else header).append(ln)
if not seen_marker and ln.strip() == MARKER:
seen_marker = True
if not seen_marker:
rest, header = header, []
blocks, cur = [], []
for ln in rest:
if ln.strip():
cur.append(ln)
elif cur:
blocks.append(cur)
cur = []
if cur:
blocks.append(cur)
entries, unparsed = [], []
for b in blocks:
m = ENTRY_RE.match(b[0])
if m:
entries.append(Entry(m.group(1), m.group(2), m.group(3), m.group(4), b))
else:
unparsed.append(b)
return header, entries, unparsed
def analyze(entries, unparsed, root, archive_days, today):
r = {}
weight = lambda es: sum(e.count for e in es)
r["total"] = len(entries)
r["weighted"] = weight(entries)
r["unparsed"] = len(unparsed)
if entries:
r["span"] = (min(e.date for e in entries), max(e.date for e in entries))
by_type = defaultdict(int)
for e in entries:
by_type[e.etype] += 1
r["by_type"] = dict(by_type)
ctxs = defaultdict(list)
for e in entries:
ctxs[e.context_group].append(e)
r["top_contexts"] = sorted(
((c, weight(es), len(es)) for c, es in ctxs.items()),
key=lambda t: -t[1])[:15]
# repeat ref= citations: >=2 means a documented rule/memory is not sticking
refs = defaultdict(list)
for e in entries:
for ref in e.refs:
refs[ref].append(e)
mem_dir = os.path.join(root, ".claude", "memory")
rep = []
for ref, es in sorted(refs.items(), key=lambda kv: -len(kv[1])):
if len(es) < 2:
continue
base = ref.split("#", 1)[0].split("/")[-1]
cand = base if base.endswith(".md") else base + ".md"
exists = os.path.isfile(os.path.join(mem_dir, cand))
rep.append((ref, len(es), exists, sorted({e.date for e in es})[-3:]))
r["repeat_refs"] = rep
# noise clusters: same machine+skill+normalized message on >=3 distinct
# days (the helper's (xN) dedup already collapses same-day repeats)
clusters = defaultdict(list)
for e in entries:
clusters[(e.machine, e.skill, e.norm_msg())].append(e)
noise = []
for (mach, skill, norm), es in clusters.items():
days = sorted({e.date for e in es})
if len(days) >= 3 or weight(es) >= 5:
noise.append((mach, skill, norm[:110], weight(es), len(days)))
r["noise"] = sorted(noise, key=lambda t: -t[3])[:15]
r["resolved"] = [e for e in entries if e.resolved]
machines = defaultdict(set)
for e in entries:
machines[e.machine.lower()].add(e.machine)
r["machine_drift"] = {k: sorted(v) for k, v in machines.items() if len(v) > 1}
r["by_machine"] = sorted(
((m, weight(es)) for m, es in
((m, [e for e in entries if e.machine.lower() == m]) for m in machines)),
key=lambda t: -t[1])
cutoff = (today - timedelta(days=archive_days)).strftime("%Y-%m-%d")
r["cutoff"] = cutoff
r["archive"] = [e for e in entries if e.date < cutoff]
return r
def render(r, archive_days):
L = []
add = L.append
add("# errorlog-dream report")
add("")
add("## SUMMARY")
span = r.get("span")
add("- entries: %d parsed (%d weighted with (xN) counters), %d unparsed legacy block(s)"
% (r["total"], r["weighted"], r["unparsed"]))
if span:
add("- span: %s .. %s" % span)
add("- by type: " + ", ".join("%s=%d" % kv for kv in sorted(r["by_type"].items())))
add("- by machine: " + ", ".join("%s=%d" % kv for kv in r["by_machine"]))
add("")
add("## TOP CONTEXTS (weighted)")
for c, w, n in r["top_contexts"]:
add("- %-22s %4d (%d entries)" % (c, w, n))
add("")
add("## REPEAT REFS -- documented rules that are NOT sticking")
if r["repeat_refs"]:
for ref, n, exists, dates in r["repeat_refs"]:
add("- ref=%s cited %dx (last: %s) -- memory file %s"
% (ref, n, ", ".join(dates), "exists" if exists else "NOT FOUND"))
else:
add("- none")
add("")
add("## NOISE CLUSTERS -- identical failures recurring across days")
if r["noise"]:
for mach, skill, norm, w, days in r["noise"]:
add("- %s | %s | %dx over %d day(s): %s" % (mach, skill, w, days, norm))
else:
add("- none")
add("")
add("## RESOLVED entries (archive candidates regardless of age)")
for e in r["resolved"]:
add("- %s | %s | %s" % (e.date, e.machine, e.skill))
if not r["resolved"]:
add("- none")
add("")
add("## MACHINE-NAME DRIFT")
if r["machine_drift"]:
for k, variants in sorted(r["machine_drift"].items()):
add("- %s spelled %s -- normalize identity.json .machine on the odd one out"
% (k, " / ".join(variants)))
else:
add("- none")
add("")
add("## ARCHIVE CANDIDATES (older than %d days, cutoff %s)" % (archive_days, r["cutoff"]))
add("- %d entr%s -- run --apply-archive to move them to errorlog-archive/YYYY-MM.md"
% (len(r["archive"]), "y" if len(r["archive"]) == 1 else "ies"))
add("")
add("## PROPOSED (needs human approval)")
for ref, n, exists, dates in r["repeat_refs"]:
add("- [STRENGTHEN?] ref=%s keeps repeating (%dx)%s -- the prose rule failed; "
"add a mechanical guard (hook/wrapper/preflight) or rewrite the memory"
% (ref, n, "" if exists else " (and the cited memory file is MISSING)"))
for mach, skill, norm, w, days in r["noise"]:
add("- [SUPPRESS?] %s/%s fails identically %dx over %d days -- fix the skill "
"(backoff, expected-condition filter, or health-gate), don't keep logging it"
% (mach, skill, w, days))
for e in r["resolved"]:
add("- [ARCHIVE?] resolved entry %s | %s | %s can move to the archive now"
% (e.date, e.machine, e.skill))
if not (r["repeat_refs"] or r["noise"] or r["resolved"]):
add("- nothing to propose")
add("")
return "\n".join(L)
def apply_archive(log_path, root, header, entries, unparsed, cutoff_entries):
"""Move cutoff_entries' blocks into errorlog-archive/YYYY-MM.md (append,
newest-first order preserved as-is) and rewrite errorlog.md without them.
Unparsed blocks are never moved."""
arch_dir = os.path.join(root, "errorlog-archive")
if not os.path.isdir(arch_dir):
os.makedirs(arch_dir)
by_month = defaultdict(list)
for e in cutoff_entries:
by_month[e.date[:7]].append(e)
for month, es in sorted(by_month.items()):
p = os.path.join(arch_dir, "%s.md" % month)
new = not os.path.isfile(p)
with io.open(p, "a", encoding="utf-8", newline="\n") as fh:
if new:
fh.write("# Error Log archive -- %s\n\nMoved out of errorlog.md by "
"errorlog-dream --apply-archive.\n" % month)
for e in es:
fh.write("\n" + "\n".join(e.lines) + "\n")
print("[OK] archived %d entr%s -> errorlog-archive/%s.md"
% (len(es), "y" if len(es) == 1 else "ies", month))
keep_ids = {id(e) for e in entries} - {id(e) for e in cutoff_entries}
out = list(header)
for e in entries:
if id(e) in keep_ids:
out.append("")
out.extend(e.lines)
for b in unparsed:
out.append("")
out.extend(b)
out.append("")
with io.open(log_path, "w", encoding="utf-8", newline="\n") as fh:
fh.write("\n".join(out))
print("[OK] errorlog.md rewritten: %d entries kept, %d archived, %d unparsed block(s) untouched"
% (len(keep_ids), len(cutoff_entries), len(unparsed)))
def main(argv=None):
ap = argparse.ArgumentParser(description="lint errorlog.md")
ap.add_argument("--days", type=int, default=60,
help="archive-candidate age threshold (default 60)")
ap.add_argument("--apply-archive", action="store_true",
help="move entries older than --days to errorlog-archive/")
ap.add_argument("--no-file", action="store_true",
help="print report to stdout only")
ap.add_argument("--report-file", default=None)
ap.add_argument("--log", default=None, help="path to errorlog.md (for tests)")
ap.add_argument("--root", default=None, help="repo root override (for tests)")
args = ap.parse_args(argv)
root = args.root or find_root()
log_path = args.log or os.path.join(root, "errorlog.md")
if not os.path.isfile(log_path):
print("[ERROR] %s not found" % log_path, file=sys.stderr)
return 2
header, entries, unparsed = parse_log(log_path)
today = datetime.now(timezone.utc)
r = analyze(entries, unparsed, root, args.days, today)
report = render(r, args.days)
print(report)
if not args.no_file:
rp = args.report_file
if not rp:
rdir = os.path.join(root, "errorlog-archive", "_reports")
if not os.path.isdir(rdir):
os.makedirs(rdir)
rp = os.path.join(rdir, today.strftime("%Y-%m-%d-%H%M") + "-dream.md")
with io.open(rp, "w", encoding="utf-8", newline="\n") as fh:
fh.write(report + "\n")
print("[OK] report written: %s" % os.path.relpath(rp, root))
if args.apply_archive:
if r["archive"]:
apply_archive(log_path, root, header, entries, unparsed, r["archive"])
else:
print("[OK] nothing old enough to archive (cutoff %s)" % r["cutoff"])
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,136 @@
#!/usr/bin/env python
"""selftest for errorlog-dream: run the analyzer against a synthetic errorlog
in a temp dir and assert each detector fires, then assert --apply-archive
moves exactly the old entries and leaves the marker, recent entries, and
unparsed legacy blocks intact."""
import io
import os
import shutil
import sys
import tempfile
from datetime import datetime, timedelta, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import errorlog_dream as ed
def build_fixture(root, today):
d = lambda n: (today - timedelta(days=n)).strftime("%Y-%m-%d")
old1 = d(95)
old2 = d(65)
recent = d(3)
lines = [
"# Error Log",
"",
ed.MARKER,
"",
# noise cluster: same machine+skill+shape on 3 distinct days
"%s | BOX-A | widget/api | HTTP 500 on id 12345 [ctx: cmd=list]" % d(1),
"",
"%s | BOX-A | widget/api | HTTP 500 on id 99881 [ctx: cmd=list] (x3)" % d(2),
"",
"%s | BOX-A | widget/api | HTTP 500 on id 40404 [ctx: cmd=list]" % recent,
"",
# repeat ref (2x) citing an existing memory
"%s | BOX-B | bash/env | [friction] used /tmp again [ctx: ref=fix_tmp_rule]" % d(4),
"",
"%s | BOX-B | bash/env | [friction] used /tmp AGAIN again [ctx: ref=fix_tmp_rule]" % recent,
"",
# machine-name case drift
"%s | box-b | coord | HTTP 0 talking to coord" % d(5),
"",
# resolved entry
"%s | BOX-A | sync/submodules | checkout aborted. [RESOLVED %s] fixed in sync.sh" % (d(6), d(5)),
"",
# correction
"%s | BOX-A | client/foo | [correction] assumed X; correct is Y" % d(7),
"",
# old entries -> archive candidates (two months)
"%s | BOX-A | oldskill | ancient failure one" % old1,
"",
"%s | BOX-B | oldskill | ancient failure two" % old2,
"",
# legacy multi-line unparsed block (no pipes on first line)
"Some legacy hand-written note",
"spanning two lines with no format.",
"",
]
log = os.path.join(root, "errorlog.md")
with io.open(log, "w", encoding="utf-8", newline="\n") as fh:
fh.write("\n".join(lines))
mem = os.path.join(root, ".claude", "memory")
os.makedirs(mem)
with io.open(os.path.join(mem, "fix_tmp_rule.md"), "w", encoding="utf-8") as fh:
fh.write("---\nname: fix_tmp_rule\n---\nrule body\n")
return log
def main():
tmp = tempfile.mkdtemp(prefix="eldream-test-")
failures = []
ok = lambda cond, name: failures.append(name) if not cond else None
try:
today = datetime.now(timezone.utc)
log = build_fixture(tmp, today)
header, entries, unparsed = ed.parse_log(log)
ok(len(entries) == 10, "parse: 10 entries (got %d)" % len(entries))
ok(len(unparsed) == 1, "parse: 1 unparsed block")
ok(any(e.count == 3 for e in entries), "parse: (x3) counter read")
r = ed.analyze(entries, unparsed, tmp, 60, today)
ok(r["weighted"] == 12, "weighted count incl x3 (got %d)" % r["weighted"])
ok(r["by_type"].get("friction") == 2 and r["by_type"].get("correction") == 1,
"type tally")
ok(any(ref == "fix_tmp_rule" and n == 2 and exists
for ref, n, exists, _ in r["repeat_refs"]),
"repeat-ref detector (existing memory)")
ok(any(skill == "widget/api" and w == 5 for _, skill, _, w, _ in r["noise"]),
"noise-cluster detector")
ok(len(r["resolved"]) == 1, "resolved detector")
ok("box-b" in r["machine_drift"], "machine-drift detector")
ok(len(r["archive"]) == 2, "archive candidates (got %d)" % len(r["archive"]))
report = ed.render(r, 60)
for token in ("STRENGTHEN?", "SUPPRESS?", "ARCHIVE?", "MACHINE-NAME DRIFT"):
ok(token in report, "report contains %s" % token)
# --apply-archive: moves the 2 old entries, keeps everything else
rc = ed.main(["--log", log, "--root", tmp, "--no-file", "--apply-archive"])
ok(rc == 0, "apply-archive exit 0")
h2, e2, u2 = ed.parse_log(log)
ok(len(e2) == 8, "post-archive: 8 entries kept (got %d)" % len(e2))
ok(len(u2) == 1, "post-archive: unparsed block untouched")
with io.open(log, encoding="utf-8") as fh:
body = fh.read()
ok(ed.MARKER in body, "post-archive: marker intact")
ok("ancient failure" not in body, "post-archive: old entries removed")
arch = os.path.join(tmp, "errorlog-archive")
months = sorted(f for f in os.listdir(arch) if f.endswith(".md"))
ok(len(months) == 2, "archive split by month (got %s)" % months)
joined = ""
for f in months:
with io.open(os.path.join(arch, f), encoding="utf-8") as fh:
joined += fh.read()
ok("ancient failure one" in joined and "ancient failure two" in joined,
"archived content present")
# idempotent: second run archives nothing
rc = ed.main(["--log", log, "--root", tmp, "--no-file", "--apply-archive"])
_, e3, _ = ed.parse_log(log)
ok(rc == 0 and len(e3) == 8, "second apply-archive is a no-op")
finally:
shutil.rmtree(tmp, ignore_errors=True)
if failures:
print("[ERROR] selftest FAILED:")
for f in failures:
print(" - " + f)
return 1
print("[OK] errorlog-dream selftest: all assertions passed")
return 0
if __name__ == "__main__":
raise SystemExit(main())