sync: auto-sync from DESKTOP-0O8A1RL at 2026-05-12 07:04:17

Author: Mike Swanson
Machine: DESKTOP-0O8A1RL
Timestamp: 2026-05-12 07:04:17

projects/dataforth-dos/database/notify.js

@@ -1,8 +1,9 @@
 'use strict';
 
-const os   = require('os');
+const os    = require('os');
-const fs   = require('fs');
+const fs    = require('fs');
-const path = require('path');
+const https = require('https');
+const qs    = require('querystring');
 
 const CREDS_PATH = 'C:\\ProgramData\\dataforth-uploader\\credentials.json';
 const TO         = ['mike@azcomputerguru.com'];  // add jlehman@dataforth.com once confirmed working
@@ -12,46 +13,76 @@ const PREFIX     = '[NOTIFY]';
 const SEP_EQ     = '='.repeat(60);
 const SEP_DAS    = '-'.repeat(60);
 
-function loadSmtpCreds() {
+function loadGraphCreds() {
     try {
         const raw = fs.readFileSync(CREDS_PATH, 'utf8');
         const c   = JSON.parse(raw);
-        if (!c.SMTP_USER || !c.SMTP_PASS) {
+        if (!c.GRAPH_TENANT_ID || !c.GRAPH_CLIENT_ID || !c.GRAPH_CLIENT_SECRET) {
-            process.stderr.write(`${PREFIX} SMTP_USER/SMTP_PASS not present in credentials.json — skipping email\n`);
+            process.stderr.write(`${PREFIX} GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — skipping email\n`);
             return null;
         }
-        return { user: c.SMTP_USER, pass: c.SMTP_PASS };
+        return { tenantId: c.GRAPH_TENANT_ID, clientId: c.GRAPH_CLIENT_ID, clientSecret: c.GRAPH_CLIENT_SECRET };
     } catch (e) {
         process.stderr.write(`${PREFIX} Could not load credentials.json: ${e.message} — skipping email\n`);
         return null;
     }
 }
 
-function buildTransport(creds) {
+function httpsPost(hostname, path, headers, body) {
-    const nodemailer = require('nodemailer');
+    return new Promise((resolve, reject) => {
-    return nodemailer.createTransport({
+        const data = Buffer.from(body);
-        host:       'smtp.office365.com',
+        const req  = https.request({ hostname, path, method: 'POST', headers: { ...headers, 'Content-Length': data.length } }, (res) => {
-        port:       587,
+            const chunks = [];
-        requireTLS: true,
+            res.on('data', (c) => chunks.push(c));
-        auth: {
+            res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }));
-            user: creds.user,
+        });
-            pass: creds.pass,
+        req.on('error', reject);
-        },
+        req.write(data);
+        req.end();
     });
 }
 
+async function getToken(creds) {
+    const body = qs.stringify({
+        grant_type:    'client_credentials',
+        client_id:     creds.clientId,
+        client_secret: creds.clientSecret,
+        scope:         'https://graph.microsoft.com/.default',
+    });
+    const res = await httpsPost(
+        'login.microsoftonline.com',
+        `/${creds.tenantId}/oauth2/v2.0/token`,
+        { 'Content-Type': 'application/x-www-form-urlencoded' },
+        body
+    );
+    const parsed = JSON.parse(res.body);
+    if (!parsed.access_token) throw new Error(`Token error: ${parsed.error} — ${parsed.error_description}`);
+    return parsed.access_token;
+}
+
 async function sendMail(subject, text) {
-    const creds = loadSmtpCreds();
+    const creds = loadGraphCreds();
     if (!creds) return;
 
     try {
-        const transport = buildTransport(creds);
+        const token = await getToken(creds);
-        await transport.sendMail({
+        const payload = JSON.stringify({
-            from:    FROM,
+            message: {
-            to:      TO,
+                subject,
-            subject,
+                body:         { contentType: 'Text', content: text },
-            text,
+                toRecipients: TO.map((a) => ({ emailAddress: { address: a } })),
+                from:         { emailAddress: { address: FROM } },
+            },
         });
+        const res = await httpsPost(
+            'graph.microsoft.com',
+            `/v1.0/users/${FROM}/sendMail`,
+            { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
+            payload
+        );
+        if (res.status !== 202) {
+            process.stderr.write(`${PREFIX} sendMail HTTP ${res.status}: ${res.body.slice(0, 200)}\n`);
+        }
     } catch (e) {
         process.stderr.write(`${PREFIX} sendMail failed: ${e.message}\n`);
     }
@@ -66,9 +97,7 @@ function buildAlertText(subject, context) {
     lines.push(`Subject: [TestDataDB] ALERT: ${subject}`);
     lines.push(SEP_DAS);
 
-    if (context.stage !== undefined) {
+    if (context.stage !== undefined) lines.push(`Stage:   ${context.stage}`);
-        lines.push(`Stage:   ${context.stage}`);
-    }
 
     if (context.error !== undefined) {
         lines.push('');
@@ -77,9 +106,7 @@ function buildAlertText(subject, context) {
 
     if (Array.isArray(context.details) && context.details.length > 0) {
         lines.push('');
-        for (const line of context.details) {
+        for (const line of context.details) lines.push(`  ${line}`);
-            lines.push(`  ${line}`);
-        }
     }
 
     if (context.stats !== undefined) {
@@ -104,7 +131,6 @@ function buildAlertText(subject, context) {
 function alert(subject, context) {
     context = context || {};
 
-    // Always log to stderr so the service log captures it regardless of SMTP
     try {
         const preview = [
             `${PREFIX} ${SEP_EQ}`,
@@ -117,18 +143,14 @@ function alert(subject, context) {
         if (context.stats)   preview.push(`${PREFIX} Stats: ${JSON.stringify(context.stats)}`);
         preview.push(`${PREFIX} ${SEP_EQ}`);
         for (const line of preview) process.stderr.write(line + '\n');
-    } catch (_) {
+    } catch (_) {}
-        // stderr write failure — nothing to do
-    }
 
     const text = buildAlertText(subject, context);
-    // fire-and-forget; caller never awaits notify.alert
     sendMail(`[TestDataDB] ALERT: ${subject}`, text).catch(() => {});
 }
 
 /**
  * Send a daily pipeline summary email.
- * Called from run-pipeline.ps1 via a separate Node invocation.
  *
  * @param {object} stats
  * @param {number} stats.received
@@ -139,8 +161,8 @@ function alert(subject, context) {
  * @returns {Promise<void>}
  */
 async function summary(stats) {
-    const date   = new Date().toISOString().slice(0, 10);
+    const date    = new Date().toISOString().slice(0, 10);
-    const status = (stats.errors > 0) ? 'FAIL' : 'OK';
+    const status  = (stats.errors > 0) ? 'FAIL' : 'OK';
     const subject = `[TestDataDB] Daily pipeline ${status} — ${date}`;
 
     const lines = [
@@ -160,7 +182,6 @@ async function summary(stats) {
     await sendMail(subject, lines.join('\n'));
 }
 
-// Allow run-pipeline.ps1 to invoke `node notify.js summary <json>`
 if (require.main === module) {
     const cmd = process.argv[2];
     if (cmd === 'summary') {

projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1

@@ -7,7 +7,7 @@ $nodeExe  = 'C:\Program Files\nodejs\node.exe'
 New-Item -ItemType Directory -Force -Path $logDir | Out-Null
 $stamp    = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
 $log      = Join-Path $logDir "pipeline-$stamp.log"
-$smtpCred = $null
+$graphCreds = $null
 
 function Log([string]$m) {
     $line = "[$(Get-Date -Format o)] $m"
@@ -16,17 +16,37 @@ function Log([string]$m) {
 }
 
 function SendEmail([string]$Subject, [string]$Body) {
-    if ($null -eq $smtpCred) { return }
+    if ($null -eq $graphCreds) { return }
     try {
-        Send-MailMessage `
+        # Acquire Graph token
-            -From       'sysadmin@dataforth.com' `
+        $tokenBody = @{
-            -To         @('mike@azcomputerguru.com') `
+            grant_type    = 'client_credentials'
-            -Subject    $Subject `
+            client_id     = $graphCreds.clientId
-            -Body       $Body `
+            client_secret = $graphCreds.clientSecret
-            -SmtpServer 'smtp.office365.com' `
+            scope         = 'https://graph.microsoft.com/.default'
-            -Port       587 `
+        }
-            -UseSsl `
+        $tokenResp = Invoke-RestMethod `
-            -Credential $smtpCred
+            -Method Post `
+            -Uri    "https://login.microsoftonline.com/$($graphCreds.tenantId)/oauth2/v2.0/token" `
+            -Body   $tokenBody
+        $token = $tokenResp.access_token
+
+        # Send mail via Graph
+        $mailPayload = @{
+            message = @{
+                subject      = $Subject
+                body         = @{ contentType = 'Text'; content = $Body }
+                toRecipients = @(@{ emailAddress = @{ address = 'mike@azcomputerguru.com' } })
+                from         = @{ emailAddress = @{ address = 'sysadmin@dataforth.com' } }
+            }
+        } | ConvertTo-Json -Depth 10
+
+        Invoke-RestMethod `
+            -Method  Post `
+            -Uri     'https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail' `
+            -Headers @{ Authorization = "Bearer $token"; 'Content-Type' = 'application/json' } `
+            -Body    $mailPayload | Out-Null
+
         Log "Email sent: $Subject"
     } catch {
         Log "WARNING: Email send failed: $_"
@@ -44,12 +64,15 @@ try {
     $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET
     $env:CF_SCOPE         = $creds.CF_SCOPE
 
-    if ($creds.SMTP_USER -and $creds.SMTP_PASS) {
+    if ($creds.GRAPH_TENANT_ID -and $creds.GRAPH_CLIENT_ID -and $creds.GRAPH_CLIENT_SECRET) {
-        $secPass  = ConvertTo-SecureString $creds.SMTP_PASS -AsPlainText -Force
+        $graphCreds = [PSCustomObject]@{
-        $smtpCred = New-Object System.Management.Automation.PSCredential($creds.SMTP_USER, $secPass)
+            tenantId     = $creds.GRAPH_TENANT_ID
-        Log "SMTP credentials loaded for $($creds.SMTP_USER)"
+            clientId     = $creds.GRAPH_CLIENT_ID
+            clientSecret = $creds.GRAPH_CLIENT_SECRET
+        }
+        Log "Graph credentials loaded (app $($creds.GRAPH_CLIENT_ID))"
     } else {
-        Log "WARNING: SMTP_USER/SMTP_PASS not in credentials.json — email notifications disabled"
+        Log "WARNING: GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — email notifications disabled"
     }
 
     # [1] DFWDS process
@@ -77,7 +100,7 @@ try {
 
     # [4] Daily summary email
     Log '[4] sending daily summary email'
-    if ($smtpCred) {
+    if ($graphCreds) {
         try {
             # Parse totals from most recent upload log; default to zeros if not found
             $totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 }
@@ -109,7 +132,7 @@ Log: $($latestUploadLog.FullName)
             Log "WARNING: Summary email failed: $_"
         }
     } else {
-        Log '    SMTP not configured — skipping summary email'
+        Log '    Graph credentials not configured — skipping summary email'
     }
 
     Log '=== pipeline end (OK) ==='

projects/dataforth-dos/session-logs/2026-05-12-session.md

@@ -146,3 +146,98 @@ pg: True, express: True, nodemailer: True, pdfkit: True, better-sqlite3: False (
 - **AJ contact email:** dataforthgit@ (forwards to AJ)
 - **Session logs:** `projects/dataforth-dos/session-logs/`
 - **Process docs:** `projects/dataforth-dos/TEST-DATASHEET-PROCESS.md`
+
+---
+
+## Update: 07:02 PT — Graph API email implementation
+
+### Session Summary
+
+This update resolved the SMTP AUTH blocker by switching the entire email path from SMTP/nodemailer to Microsoft Graph API using the existing Claude-Code-M365 Entra app. The SMTP blocker (`535 5.7.139 SmtpClientAuthentication is disabled`) was already known from the earlier session; this update pivoted to Graph API as the user indicated tenant admin access was available via remediation credentials.
+
+The first step was testing whether the Claude-Code-M365 app already had `Mail.Send` permission. A Python test script (`df_graph_mailtest.py`) confirmed it did not — the `POST /v1.0/users/sysadmin@dataforth.com/sendMail` call returned HTTP 403. The app had 6 existing appRoleAssignments (Directory, Group, User read/write permissions) but no mail permissions. Attempting to grant `Mail.Send` via the app's own client_credentials token failed (403 — app lacks `AppRoleAssignment.ReadWrite.All`). Attempting with a ROPC token using the Claude-Code-M365 app as the OAuth client also failed, because the app has only application permissions (no delegated), so ROPC yields only the user's basic Graph access.
+
+The solution was ROPC with the Azure PowerShell public client (`1950a258-227b-4e31-a9cf-717495945fc2`) — a well-known public client that supports broad delegated scopes without requiring a pre-consented app. Using `sysadmin@dataforth.com` credentials, a token was acquired with `AppRoleAssignment.ReadWrite.All` and `Application.ReadWrite.All`. The `POST /servicePrincipals/{sp_id}/appRoleAssignments` call succeeded (HTTP 201). After 30 seconds of propagation, the mail send test returned HTTP 202 and the test email arrived (confirmed by user: "test received").
+
+With Graph API confirmed working, `notify.js` was rewritten to use the built-in `https` module instead of nodemailer — no external dependency. The file reads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from `credentials.json`, acquires a `client_credentials` token, and calls `POST /v1.0/users/sysadmin@dataforth.com/sendMail`. `run-pipeline.ps1` was updated to replace `Send-MailMessage` (SMTP) with `Invoke-RestMethod` to the Graph token and sendMail endpoints, using the same Graph creds. `credentials.json` on AD2 was updated: SMTP_USER/SMTP_PASS removed, GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET added. All three paths tested successfully: direct Python API call (HTTP 202), PowerShell SendEmail function (confirmed working), and Node `notify.alert()` + `notify.summary()` (both completed, no errors).
+
+### Key Decisions
+
+- **Azure PowerShell public client for permission grant** — Claude-Code-M365 has no delegated permissions, so ROPC with it yields minimal access. The Azure PowerShell app (`1950a258-227b-4e31-a9cf-717495945fc2`) is a well-known public client that supports `AppRoleAssignment.ReadWrite.All` as a delegated scope, allowing grant operations when the user (sysadmin) has admin role. No secret needed.
+- **Removed nodemailer from notify.js** — Graph API uses only the built-in `https` module. This eliminates the npm dependency entirely. The `require('nodemailer')` was the only reason nodemailer was installed; it's now unused but remains in node_modules (harmless, can be removed with `npm uninstall nodemailer` if desired).
+- **Removed SMTP_USER/SMTP_PASS from credentials.json** — SMTP auth is disabled for this tenant and will likely remain so (Microsoft security default). Keeping dead creds adds noise and confusion. Replaced with Graph creds in the same file.
+- **Browser automation skipped** — Attempted to use claude-in-chrome to grant consent via Entra portal but the extension's screenshot/click/JS injection was blocked on the MS login page. Pivoted to pure API approach (ROPC + public client) which proved cleaner and faster.
+- **notify.js CLI summary path not removed** — The `node notify.js summary <json>` CLI path still exists in the code but is not called from run-pipeline.ps1 (PS5 double-quote stripping makes it unusable from PS). run-pipeline.ps1 now uses its own PowerShell-native SendEmail function. The CLI path could be useful if called from Node-to-Node in future; no reason to remove it.
+
+### Problems Encountered
+
+- **Python urllib `InvalidURL` for OData filter with spaces** — `$filter=appId eq '...'` contains spaces; Python 3.14 is strict about control characters in URLs. Fixed: wrap path in `urllib.parse.quote(path, safe='/?=&$\'()')` in the `graph()` helper.
+- **App-only token can't grant its own permissions** — `POST /servicePrincipals/{id}/appRoleAssignments` with the app's own client_credentials token returned 403 (needs `AppRoleAssignment.ReadWrite.All` which the app doesn't have). Fixed: used Azure PowerShell public client + sysadmin ROPC to get a delegated admin token.
+- **ROPC with Claude-Code-M365 client insufficient** — ROPC using our app as the OAuth client gives only the app's pre-consented delegated scopes (none). Fixed: switched to Azure PowerShell public client ID.
+- **Permission propagation delay** — Immediately after granting Mail.Send, the sendMail call still returned 403 and the new assignment wasn't visible in `appRoleAssignments`. Resolved after ~30 second sleep.
+- **Browser extension blocked on MS login page** — `computer.screenshot`, `computer.left_click`, `computer.key`, and `javascript_tool` all failed with "Cannot access a chrome-extension:// URL of different extension" when the tab was on login.microsoftonline.com. `read_page` and `form_input` worked but were insufficient for login. Did not block the work — switched to pure API approach.
+- **PS5 double-quote stripping (reprise)** — Test of `node notify.js summary $stats` from PowerShell failed with "Expected property name or '}' in JSON at position 1". This is the same PS5 bug documented in the earlier session. Confirmed: this CLI path cannot be called from run-pipeline.ps1. Summary email in PS remains entirely PowerShell-native via Invoke-RestMethod.
+- **Here-string terminator error in SFTP-transferred PS scripts** — PS scripts containing `@"..."@` here-strings failed with "The string is missing the terminator" when transferred via SFTP. Root cause: unclear (encoding or line ending). Fixed: rewrote test scripts using regular string concatenation with `` `n `` escapes.
+
+### Configuration Changes
+
+**Repo (D:\claudetools):**
+- `projects/dataforth-dos/database/notify.js` — rewritten. Removed nodemailer/SMTP. Now uses built-in `https` module + Graph API client_credentials. Loads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from credentials.json. Function signatures (`alert`, `summary`) unchanged.
+- `projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1` — updated. Replaced `$smtpCred`/`Send-MailMessage` with `$graphCreds`/`Invoke-RestMethod` Graph API flow. Token acquisition + sendMail call inline in `SendEmail` function.
+
+**On AD2 (192.168.0.6):**
+- `C:\ProgramData\dataforth-uploader\credentials.json` — removed `SMTP_USER`, `SMTP_PASS`; added `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET`
+- `C:\Shares\testdatadb\database\notify.js` — deployed (Graph API version)
+- `C:\ProgramData\dataforth-uploader\run-pipeline.ps1` — deployed (Graph API version)
+
+**Entra (Dataforth tenant 7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584):**
+- Claude-Code-M365 SP (`11b0aa55-314f-49bd-b8b3-baafaf49b44c`) — `Mail.Send` application permission granted. Assignment ID: `VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc`. Resource: Microsoft Graph SP `bea71f59-7956-41ed-8f99-45f31c4a6342`.
+
+### Credentials & Secrets
+
+- **Dataforth M365 tenant:** `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` — vault: `clients/dataforth/m365.sops.yaml`
+- **Claude-Code-M365 app:** app-id=`7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29`, secret=`tXo8Q~ZNG9zoBpbK9HwJTkzx.YEigZ9AynoSrca3`, expires 2027-12-22 — vault: `clients/dataforth/m365.sops.yaml → credentials.entra-app`
+- **sysadmin@dataforth.com:** password `Paper123!@#` — vault: `clients/dataforth/m365.sops.yaml → credentials.password`
+- **Azure PowerShell public client (used for permission grant):** `1950a258-227b-4e31-a9cf-717495945fc2` — no secret, public client, well-known Microsoft app ID
+
+### Commands & Outputs
+
+```
+# Mail.Send permission grant (Python)
+POST /v1.0/servicePrincipals/11b0aa55-314f-49bd-b8b3-baafaf49b44c/appRoleAssignments
+{principalId: "11b0aa55...", resourceId: "bea71f59...", appRoleId: "b633e1c5-b582-4048-a93e-9f11b44c7e96"}
+→ HTTP 201, Assignment ID: VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc
+
+# Mail send test (after propagation)
+POST /v1.0/users/sysadmin@dataforth.com/sendMail
+→ HTTP 202 — [OK] Email sent (or queued)
+User confirmed: "test received"
+
+# run-pipeline.ps1 SendEmail test on AD2 (PowerShell)
+Graph creds loaded: app=7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29
+Token acquired (len=2180)
+[OK] Email sent
+
+# notify.js alert/summary test on AD2 (Node)
+[NOTIFY] ALERT: [TestDataDB] Graph API test from notify.alert()
+[NOTIFY] Host: AD2 | Time: 2026-05-12T14:02:26.690Z
+[NOTIFY] Sending daily summary (OK) for 2026-05-12
+[TEST] summary() completed
+```
+
+### Pending / Incomplete Tasks
+
+1. **Add John Lehman to TO list** — `jlehman@dataforth.com` must be added to `TO` array in `notify.js` (line 8) and to `toRecipients` in `run-pipeline.ps1 SendEmail`. Deploy both to AD2. Do this after Mike confirms the next real pipeline run email arrives correctly.
+2. **Clean diagnostic scripts on AD2** — `C:\Shares\testdatadb\database\_*.js` (~20 files from 2026-04-15 session). Safe to delete.
+3. **Clean vault entry** — `ad2.sops.yaml` stale backslash in password field.
+4. **Investigate 2026-04-22 undocumented session** — import.js, notify.js, upload-to-api.js modified that date with no log.
+5. **Optional: uninstall nodemailer** — `npm uninstall nodemailer` in `C:\Shares\testdatadb`. Not urgent — unused but harmless.
+
+### Reference Information
+
+- **Mail.Send appRole ID (Graph):** `b633e1c5-b582-4048-a93e-9f11b44c7e96`
+- **Claude-Code-M365 SP ID:** `11b0aa55-314f-49bd-b8b3-baafaf49b44c`
+- **Microsoft Graph SP ID (Dataforth tenant):** `bea71f59-7956-41ed-8f99-45f31c4a6342`
+- **Azure PowerShell public client:** `1950a258-227b-4e31-a9cf-717495945fc2`
+- **Graph sendMail endpoint:** `POST https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail`
+- **Token endpoint:** `POST https://login.microsoftonline.com/7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584/oauth2/v2.0/token`