azcomputerguru/claudetools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: auto-sync from DESKTOP-0O8A1RL at 2026-05-12 07:04:17

Browse Source 
Author: Mike Swanson
Machine: DESKTOP-0O8A1RL
Timestamp: 2026-05-12 07:04:17
This commit is contained in:
Mike Swanson
2026-05-12 07:04:18 -07:00
parent b1a588d0db
commit e75ddfbc53
3 changed files with 196 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
projects/dataforth-dos/database/notify.js
View File

@@ -2,7 +2,8 @@
const os = require('os');
const fs = require('fs');
const path = require('path');
const https = require('https');
const qs = require('querystring');
const CREDS_PATH = 'C:\\ProgramData\\dataforth-uploader\\credentials.json';
const TO = ['mike@azcomputerguru.com']; // add jlehman@dataforth.com once confirmed working
@@ -12,46 +13,76 @@ const PREFIX = '[NOTIFY]';
const SEP_EQ = '='.repeat(60);
const SEP_DAS = '-'.repeat(60);
function loadSmtpCreds() {
function loadGraphCreds() {
try {
const raw = fs.readFileSync(CREDS_PATH, 'utf8');
const c = JSON.parse(raw);
if (!c.SMTP_USER || !c.SMTP_PASS) {
process.stderr.write(`${PREFIX} SMTP_USER/SMTP_PASS not present in credentials.json — skipping email\n`);
if (!c.GRAPH_TENANT_ID || !c.GRAPH_CLIENT_ID || !c.GRAPH_CLIENT_SECRET) {
process.stderr.write(`${PREFIX} GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — skipping email\n`);
return null;
}
return { user: c.SMTP_USER, pass: c.SMTP_PASS };
return { tenantId: c.GRAPH_TENANT_ID, clientId: c.GRAPH_CLIENT_ID, clientSecret: c.GRAPH_CLIENT_SECRET };
} catch (e) {
process.stderr.write(`${PREFIX} Could not load credentials.json: ${e.message} — skipping email\n`);
return null;
}
}
function buildTransport(creds) {
const nodemailer = require('nodemailer');
return nodemailer.createTransport({
host: 'smtp.office365.com',
port: 587,
requireTLS: true,
auth: {
user: creds.user,
pass: creds.pass,
},
function httpsPost(hostname, path, headers, body) {
return new Promise((resolve, reject) => {
const data = Buffer.from(body);
const req = https.request({ hostname, path, method: 'POST', headers: { ...headers, 'Content-Length': data.length } }, (res) => {
const chunks = [];
res.on('data', (c) => chunks.push(c));
res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }));
});
req.on('error', reject);
req.write(data);
req.end();
});
}
async function getToken(creds) {
const body = qs.stringify({
grant_type: 'client_credentials',
client_id: creds.clientId,
client_secret: creds.clientSecret,
scope: 'https://graph.microsoft.com/.default',
});
const res = await httpsPost(
'login.microsoftonline.com',
`/${creds.tenantId}/oauth2/v2.0/token`,
{ 'Content-Type': 'application/x-www-form-urlencoded' },
body
);
const parsed = JSON.parse(res.body);
if (!parsed.access_token) throw new Error(`Token error: ${parsed.error} — ${parsed.error_description}`);
return parsed.access_token;
}
async function sendMail(subject, text) {
const creds = loadSmtpCreds();
const creds = loadGraphCreds();
if (!creds) return;
try {
const transport = buildTransport(creds);
await transport.sendMail({
from: FROM,
to: TO,
const token = await getToken(creds);
const payload = JSON.stringify({
message: {
subject,
text,
body: { contentType: 'Text', content: text },
toRecipients: TO.map((a) => ({ emailAddress: { address: a } })),
from: { emailAddress: { address: FROM } },
},
});
const res = await httpsPost(
'graph.microsoft.com',
`/v1.0/users/${FROM}/sendMail`,
{ 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
payload
);
if (res.status !== 202) {
process.stderr.write(`${PREFIX} sendMail HTTP ${res.status}: ${res.body.slice(0, 200)}\n`);
}
} catch (e) {
process.stderr.write(`${PREFIX} sendMail failed: ${e.message}\n`);
}
@@ -66,9 +97,7 @@ function buildAlertText(subject, context) {
lines.push(`Subject: [TestDataDB] ALERT: ${subject}`);
lines.push(SEP_DAS);
if (context.stage !== undefined) {
lines.push(`Stage: ${context.stage}`);
}
if (context.stage !== undefined) lines.push(`Stage: ${context.stage}`);
if (context.error !== undefined) {
lines.push('');
@@ -77,9 +106,7 @@ function buildAlertText(subject, context) {
if (Array.isArray(context.details) && context.details.length > 0) {
lines.push('');
for (const line of context.details) {
lines.push(` ${line}`);
}
for (const line of context.details) lines.push(` ${line}`);
}
if (context.stats !== undefined) {
@@ -104,7 +131,6 @@ function buildAlertText(subject, context) {
function alert(subject, context) {
context = context || {};
// Always log to stderr so the service log captures it regardless of SMTP
try {
const preview = [
`${PREFIX} ${SEP_EQ}`,
@@ -117,18 +143,14 @@ function alert(subject, context) {
if (context.stats) preview.push(`${PREFIX} Stats: ${JSON.stringify(context.stats)}`);
preview.push(`${PREFIX} ${SEP_EQ}`);
for (const line of preview) process.stderr.write(line + '\n');
} catch (_) {
// stderr write failure — nothing to do
}
} catch (_) {}
const text = buildAlertText(subject, context);
// fire-and-forget; caller never awaits notify.alert
sendMail(`[TestDataDB] ALERT: ${subject}`, text).catch(() => {});
}
/**
* Send a daily pipeline summary email.
* Called from run-pipeline.ps1 via a separate Node invocation.
*
* @param {object} stats
* @param {number} stats.received
@@ -160,7 +182,6 @@ async function summary(stats) {
await sendMail(subject, lines.join('\n'));
}
// Allow run-pipeline.ps1 to invoke `node notify.js summary <json>`
if (require.main === module) {
const cmd = process.argv[2];
if (cmd === 'summary') {

59
projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1
View File

@@ -7,7 +7,7 @@ $nodeExe = 'C:\Program Files\nodejs\node.exe'
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
$stamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
$log = Join-Path $logDir "pipeline-$stamp.log"
$smtpCred = $null
$graphCreds = $null
function Log([string]$m) {
$line = "[$(Get-Date -Format o)] $m"
@@ -16,17 +16,37 @@ function Log([string]$m) {
}
function SendEmail([string]$Subject, [string]$Body) {
if ($null -eq $smtpCred) { return }
if ($null -eq $graphCreds) { return }
try {
Send-MailMessage `
-From 'sysadmin@dataforth.com' `
-To @('mike@azcomputerguru.com') `
-Subject $Subject `
-Body $Body `
-SmtpServer 'smtp.office365.com' `
-Port 587 `
-UseSsl `
-Credential $smtpCred
# Acquire Graph token
$tokenBody = @{
grant_type = 'client_credentials'
client_id = $graphCreds.clientId
client_secret = $graphCreds.clientSecret
scope = 'https://graph.microsoft.com/.default'
}
$tokenResp = Invoke-RestMethod `
-Method Post `
-Uri "https://login.microsoftonline.com/$($graphCreds.tenantId)/oauth2/v2.0/token" `
-Body $tokenBody
$token = $tokenResp.access_token
# Send mail via Graph
$mailPayload = @{
message = @{
subject = $Subject
body = @{ contentType = 'Text'; content = $Body }
toRecipients = @(@{ emailAddress = @{ address = 'mike@azcomputerguru.com' } })
from = @{ emailAddress = @{ address = 'sysadmin@dataforth.com' } }
}
} | ConvertTo-Json -Depth 10
Invoke-RestMethod `
-Method Post `
-Uri 'https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail' `
-Headers @{ Authorization = "Bearer $token"; 'Content-Type' = 'application/json' } `
-Body $mailPayload | Out-Null
Log "Email sent: $Subject"
} catch {
Log "WARNING: Email send failed: $_"
@@ -44,12 +64,15 @@ try {
$env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
$env:CF_SCOPE = $creds.CF_SCOPE
if ($creds.SMTP_USER -and $creds.SMTP_PASS) {
$secPass = ConvertTo-SecureString $creds.SMTP_PASS -AsPlainText -Force
$smtpCred = New-Object System.Management.Automation.PSCredential($creds.SMTP_USER, $secPass)
Log "SMTP credentials loaded for $($creds.SMTP_USER)"
if ($creds.GRAPH_TENANT_ID -and $creds.GRAPH_CLIENT_ID -and $creds.GRAPH_CLIENT_SECRET) {
$graphCreds = [PSCustomObject]@{
tenantId = $creds.GRAPH_TENANT_ID
clientId = $creds.GRAPH_CLIENT_ID
clientSecret = $creds.GRAPH_CLIENT_SECRET
}
Log "Graph credentials loaded (app $($creds.GRAPH_CLIENT_ID))"
} else {
Log "WARNING: SMTP_USER/SMTP_PASS not in credentials.json — email notifications disabled"
Log "WARNING: GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — email notifications disabled"
}
# [1] DFWDS process
@@ -77,7 +100,7 @@ try {
# [4] Daily summary email
Log '[4] sending daily summary email'
if ($smtpCred) {
if ($graphCreds) {
try {
# Parse totals from most recent upload log; default to zeros if not found
$totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 }
@@ -109,7 +132,7 @@ Log: $($latestUploadLog.FullName)
Log "WARNING: Summary email failed: $_"
}
} else {
Log ' SMTP not configured — skipping summary email'
Log ' Graph credentials not configured — skipping summary email'
}
Log '=== pipeline end (OK) ==='

95
projects/dataforth-dos/session-logs/2026-05-12-session.md
View File

@@ -146,3 +146,98 @@ pg: True, express: True, nodemailer: True, pdfkit: True, better-sqlite3: False (
- **AJ contact email:** dataforthgit@ (forwards to AJ)
- **Session logs:** `projects/dataforth-dos/session-logs/`
- **Process docs:** `projects/dataforth-dos/TEST-DATASHEET-PROCESS.md`
---
## Update: 07:02 PT — Graph API email implementation
### Session Summary
This update resolved the SMTP AUTH blocker by switching the entire email path from SMTP/nodemailer to Microsoft Graph API using the existing Claude-Code-M365 Entra app. The SMTP blocker (`535 5.7.139 SmtpClientAuthentication is disabled`) was already known from the earlier session; this update pivoted to Graph API as the user indicated tenant admin access was available via remediation credentials.
The first step was testing whether the Claude-Code-M365 app already had `Mail.Send` permission. A Python test script (`df_graph_mailtest.py`) confirmed it did not — the `POST /v1.0/users/sysadmin@dataforth.com/sendMail` call returned HTTP 403. The app had 6 existing appRoleAssignments (Directory, Group, User read/write permissions) but no mail permissions. Attempting to grant `Mail.Send` via the app's own client_credentials token failed (403 — app lacks `AppRoleAssignment.ReadWrite.All`). Attempting with a ROPC token using the Claude-Code-M365 app as the OAuth client also failed, because the app has only application permissions (no delegated), so ROPC yields only the user's basic Graph access.
The solution was ROPC with the Azure PowerShell public client (`1950a258-227b-4e31-a9cf-717495945fc2`) — a well-known public client that supports broad delegated scopes without requiring a pre-consented app. Using `sysadmin@dataforth.com` credentials, a token was acquired with `AppRoleAssignment.ReadWrite.All` and `Application.ReadWrite.All`. The `POST /servicePrincipals/{sp_id}/appRoleAssignments` call succeeded (HTTP 201). After 30 seconds of propagation, the mail send test returned HTTP 202 and the test email arrived (confirmed by user: "test received").
With Graph API confirmed working, `notify.js` was rewritten to use the built-in `https` module instead of nodemailer — no external dependency. The file reads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from `credentials.json`, acquires a `client_credentials` token, and calls `POST /v1.0/users/sysadmin@dataforth.com/sendMail`. `run-pipeline.ps1` was updated to replace `Send-MailMessage` (SMTP) with `Invoke-RestMethod` to the Graph token and sendMail endpoints, using the same Graph creds. `credentials.json` on AD2 was updated: SMTP_USER/SMTP_PASS removed, GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET added. All three paths tested successfully: direct Python API call (HTTP 202), PowerShell SendEmail function (confirmed working), and Node `notify.alert()` + `notify.summary()` (both completed, no errors).
### Key Decisions
- **Azure PowerShell public client for permission grant** — Claude-Code-M365 has no delegated permissions, so ROPC with it yields minimal access. The Azure PowerShell app (`1950a258-227b-4e31-a9cf-717495945fc2`) is a well-known public client that supports `AppRoleAssignment.ReadWrite.All` as a delegated scope, allowing grant operations when the user (sysadmin) has admin role. No secret needed.
- **Removed nodemailer from notify.js** — Graph API uses only the built-in `https` module. This eliminates the npm dependency entirely. The `require('nodemailer')` was the only reason nodemailer was installed; it's now unused but remains in node_modules (harmless, can be removed with `npm uninstall nodemailer` if desired).
- **Removed SMTP_USER/SMTP_PASS from credentials.json** — SMTP auth is disabled for this tenant and will likely remain so (Microsoft security default). Keeping dead creds adds noise and confusion. Replaced with Graph creds in the same file.
- **Browser automation skipped** — Attempted to use claude-in-chrome to grant consent via Entra portal but the extension's screenshot/click/JS injection was blocked on the MS login page. Pivoted to pure API approach (ROPC + public client) which proved cleaner and faster.
- **notify.js CLI summary path not removed** — The `node notify.js summary <json>` CLI path still exists in the code but is not called from run-pipeline.ps1 (PS5 double-quote stripping makes it unusable from PS). run-pipeline.ps1 now uses its own PowerShell-native SendEmail function. The CLI path could be useful if called from Node-to-Node in future; no reason to remove it.
### Problems Encountered
- **Python urllib `InvalidURL` for OData filter with spaces** — `$filter=appId eq '...'` contains spaces; Python 3.14 is strict about control characters in URLs. Fixed: wrap path in `urllib.parse.quote(path, safe='/?=&$\'()')` in the `graph()` helper.
- **App-only token can't grant its own permissions** — `POST /servicePrincipals/{id}/appRoleAssignments` with the app's own client_credentials token returned 403 (needs `AppRoleAssignment.ReadWrite.All` which the app doesn't have). Fixed: used Azure PowerShell public client + sysadmin ROPC to get a delegated admin token.
- **ROPC with Claude-Code-M365 client insufficient** — ROPC using our app as the OAuth client gives only the app's pre-consented delegated scopes (none). Fixed: switched to Azure PowerShell public client ID.
- **Permission propagation delay** — Immediately after granting Mail.Send, the sendMail call still returned 403 and the new assignment wasn't visible in `appRoleAssignments`. Resolved after ~30 second sleep.
- **Browser extension blocked on MS login page** — `computer.screenshot`, `computer.left_click`, `computer.key`, and `javascript_tool` all failed with "Cannot access a chrome-extension:// URL of different extension" when the tab was on login.microsoftonline.com. `read_page` and `form_input` worked but were insufficient for login. Did not block the work — switched to pure API approach.
- **PS5 double-quote stripping (reprise)** — Test of `node notify.js summary $stats` from PowerShell failed with "Expected property name or '}' in JSON at position 1". This is the same PS5 bug documented in the earlier session. Confirmed: this CLI path cannot be called from run-pipeline.ps1. Summary email in PS remains entirely PowerShell-native via Invoke-RestMethod.
- **Here-string terminator error in SFTP-transferred PS scripts** — PS scripts containing `@"..."@` here-strings failed with "The string is missing the terminator" when transferred via SFTP. Root cause: unclear (encoding or line ending). Fixed: rewrote test scripts using regular string concatenation with `` `n `` escapes.
### Configuration Changes
**Repo (D:\claudetools):**
- `projects/dataforth-dos/database/notify.js` — rewritten. Removed nodemailer/SMTP. Now uses built-in `https` module + Graph API client_credentials. Loads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from credentials.json. Function signatures (`alert`, `summary`) unchanged.
- `projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1` — updated. Replaced `$smtpCred`/`Send-MailMessage` with `$graphCreds`/`Invoke-RestMethod` Graph API flow. Token acquisition + sendMail call inline in `SendEmail` function.
**On AD2 (192.168.0.6):**
- `C:\ProgramData\dataforth-uploader\credentials.json` — removed `SMTP_USER`, `SMTP_PASS`; added `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET`
- `C:\Shares\testdatadb\database\notify.js` — deployed (Graph API version)
- `C:\ProgramData\dataforth-uploader\run-pipeline.ps1` — deployed (Graph API version)
**Entra (Dataforth tenant 7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584):**
- Claude-Code-M365 SP (`11b0aa55-314f-49bd-b8b3-baafaf49b44c`) — `Mail.Send` application permission granted. Assignment ID: `VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc`. Resource: Microsoft Graph SP `bea71f59-7956-41ed-8f99-45f31c4a6342`.
### Credentials & Secrets
- **Dataforth M365 tenant:** `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` — vault: `clients/dataforth/m365.sops.yaml`
- **Claude-Code-M365 app:** app-id=`7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29`, secret=`tXo8Q~ZNG9zoBpbK9HwJTkzx.YEigZ9AynoSrca3`, expires 2027-12-22 — vault: `clients/dataforth/m365.sops.yaml → credentials.entra-app`
- **sysadmin@dataforth.com:** password `Paper123!@#` — vault: `clients/dataforth/m365.sops.yaml → credentials.password`
- **Azure PowerShell public client (used for permission grant):** `1950a258-227b-4e31-a9cf-717495945fc2` — no secret, public client, well-known Microsoft app ID
### Commands & Outputs
```
# Mail.Send permission grant (Python)
POST /v1.0/servicePrincipals/11b0aa55-314f-49bd-b8b3-baafaf49b44c/appRoleAssignments
{principalId: "11b0aa55...", resourceId: "bea71f59...", appRoleId: "b633e1c5-b582-4048-a93e-9f11b44c7e96"}
→ HTTP 201, Assignment ID: VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc
# Mail send test (after propagation)
POST /v1.0/users/sysadmin@dataforth.com/sendMail
→ HTTP 202 — [OK] Email sent (or queued)
User confirmed: "test received"
# run-pipeline.ps1 SendEmail test on AD2 (PowerShell)
Graph creds loaded: app=7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29
Token acquired (len=2180)
[OK] Email sent
# notify.js alert/summary test on AD2 (Node)
[NOTIFY] ALERT: [TestDataDB] Graph API test from notify.alert()
[NOTIFY] Host: AD2 | Time: 2026-05-12T14:02:26.690Z
[NOTIFY] Sending daily summary (OK) for 2026-05-12
[TEST] summary() completed
```
### Pending / Incomplete Tasks
1. **Add John Lehman to TO list** — `jlehman@dataforth.com` must be added to `TO` array in `notify.js` (line 8) and to `toRecipients` in `run-pipeline.ps1 SendEmail`. Deploy both to AD2. Do this after Mike confirms the next real pipeline run email arrives correctly.
2. **Clean diagnostic scripts on AD2** — `C:\Shares\testdatadb\database\_*.js` (~20 files from 2026-04-15 session). Safe to delete.
3. **Clean vault entry** — `ad2.sops.yaml` stale backslash in password field.
4. **Investigate 2026-04-22 undocumented session** — import.js, notify.js, upload-to-api.js modified that date with no log.
5. **Optional: uninstall nodemailer** — `npm uninstall nodemailer` in `C:\Shares\testdatadb`. Not urgent — unused but harmless.
### Reference Information
- **Mail.Send appRole ID (Graph):** `b633e1c5-b582-4048-a93e-9f11b44c7e96`
- **Claude-Code-M365 SP ID:** `11b0aa55-314f-49bd-b8b3-baafaf49b44c`
- **Microsoft Graph SP ID (Dataforth tenant):** `bea71f59-7956-41ed-8f99-45f31c4a6342`
- **Azure PowerShell public client:** `1950a258-227b-4e31-a9cf-717495945fc2`
- **Graph sendMail endpoint:** `POST https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail`
- **Token endpoint:** `POST https://login.microsoftonline.com/7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584/oauth2/v2.0/token`