diff --git a/projects/dataforth-dos/database/notify.js b/projects/dataforth-dos/database/notify.js index ca584cb..090f4e7 100644 --- a/projects/dataforth-dos/database/notify.js +++ b/projects/dataforth-dos/database/notify.js @@ -1,8 +1,9 @@ 'use strict'; -const os = require('os'); -const fs = require('fs'); -const path = require('path'); +const os = require('os'); +const fs = require('fs'); +const https = require('https'); +const qs = require('querystring'); const CREDS_PATH = 'C:\\ProgramData\\dataforth-uploader\\credentials.json'; const TO = ['mike@azcomputerguru.com']; // add jlehman@dataforth.com once confirmed working @@ -12,46 +13,76 @@ const PREFIX = '[NOTIFY]'; const SEP_EQ = '='.repeat(60); const SEP_DAS = '-'.repeat(60); -function loadSmtpCreds() { +function loadGraphCreds() { try { const raw = fs.readFileSync(CREDS_PATH, 'utf8'); const c = JSON.parse(raw); - if (!c.SMTP_USER || !c.SMTP_PASS) { - process.stderr.write(`${PREFIX} SMTP_USER/SMTP_PASS not present in credentials.json — skipping email\n`); + if (!c.GRAPH_TENANT_ID || !c.GRAPH_CLIENT_ID || !c.GRAPH_CLIENT_SECRET) { + process.stderr.write(`${PREFIX} GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — skipping email\n`); return null; } - return { user: c.SMTP_USER, pass: c.SMTP_PASS }; + return { tenantId: c.GRAPH_TENANT_ID, clientId: c.GRAPH_CLIENT_ID, clientSecret: c.GRAPH_CLIENT_SECRET }; } catch (e) { process.stderr.write(`${PREFIX} Could not load credentials.json: ${e.message} — skipping email\n`); return null; } } -function buildTransport(creds) { - const nodemailer = require('nodemailer'); - return nodemailer.createTransport({ - host: 'smtp.office365.com', - port: 587, - requireTLS: true, - auth: { - user: creds.user, - pass: creds.pass, - }, +function httpsPost(hostname, path, headers, body) { + return new Promise((resolve, reject) => { + const data = Buffer.from(body); + const req = https.request({ hostname, path, method: 'POST', headers: { ...headers, 'Content-Length': data.length } }, (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') })); + }); + req.on('error', reject); + req.write(data); + req.end(); }); } +async function getToken(creds) { + const body = qs.stringify({ + grant_type: 'client_credentials', + client_id: creds.clientId, + client_secret: creds.clientSecret, + scope: 'https://graph.microsoft.com/.default', + }); + const res = await httpsPost( + 'login.microsoftonline.com', + `/${creds.tenantId}/oauth2/v2.0/token`, + { 'Content-Type': 'application/x-www-form-urlencoded' }, + body + ); + const parsed = JSON.parse(res.body); + if (!parsed.access_token) throw new Error(`Token error: ${parsed.error} — ${parsed.error_description}`); + return parsed.access_token; +} + async function sendMail(subject, text) { - const creds = loadSmtpCreds(); + const creds = loadGraphCreds(); if (!creds) return; try { - const transport = buildTransport(creds); - await transport.sendMail({ - from: FROM, - to: TO, - subject, - text, + const token = await getToken(creds); + const payload = JSON.stringify({ + message: { + subject, + body: { contentType: 'Text', content: text }, + toRecipients: TO.map((a) => ({ emailAddress: { address: a } })), + from: { emailAddress: { address: FROM } }, + }, }); + const res = await httpsPost( + 'graph.microsoft.com', + `/v1.0/users/${FROM}/sendMail`, + { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, + payload + ); + if (res.status !== 202) { + process.stderr.write(`${PREFIX} sendMail HTTP ${res.status}: ${res.body.slice(0, 200)}\n`); + } } catch (e) { process.stderr.write(`${PREFIX} sendMail failed: ${e.message}\n`); } @@ -66,9 +97,7 @@ function buildAlertText(subject, context) { lines.push(`Subject: [TestDataDB] ALERT: ${subject}`); lines.push(SEP_DAS); - if (context.stage !== undefined) { - lines.push(`Stage: ${context.stage}`); - } + if (context.stage !== undefined) lines.push(`Stage: ${context.stage}`); if (context.error !== undefined) { lines.push(''); @@ -77,9 +106,7 @@ function buildAlertText(subject, context) { if (Array.isArray(context.details) && context.details.length > 0) { lines.push(''); - for (const line of context.details) { - lines.push(` ${line}`); - } + for (const line of context.details) lines.push(` ${line}`); } if (context.stats !== undefined) { @@ -104,7 +131,6 @@ function buildAlertText(subject, context) { function alert(subject, context) { context = context || {}; - // Always log to stderr so the service log captures it regardless of SMTP try { const preview = [ `${PREFIX} ${SEP_EQ}`, @@ -117,18 +143,14 @@ function alert(subject, context) { if (context.stats) preview.push(`${PREFIX} Stats: ${JSON.stringify(context.stats)}`); preview.push(`${PREFIX} ${SEP_EQ}`); for (const line of preview) process.stderr.write(line + '\n'); - } catch (_) { - // stderr write failure — nothing to do - } + } catch (_) {} const text = buildAlertText(subject, context); - // fire-and-forget; caller never awaits notify.alert sendMail(`[TestDataDB] ALERT: ${subject}`, text).catch(() => {}); } /** * Send a daily pipeline summary email. - * Called from run-pipeline.ps1 via a separate Node invocation. * * @param {object} stats * @param {number} stats.received @@ -139,8 +161,8 @@ function alert(subject, context) { * @returns {Promise} */ async function summary(stats) { - const date = new Date().toISOString().slice(0, 10); - const status = (stats.errors > 0) ? 'FAIL' : 'OK'; + const date = new Date().toISOString().slice(0, 10); + const status = (stats.errors > 0) ? 'FAIL' : 'OK'; const subject = `[TestDataDB] Daily pipeline ${status} — ${date}`; const lines = [ @@ -160,7 +182,6 @@ async function summary(stats) { await sendMail(subject, lines.join('\n')); } -// Allow run-pipeline.ps1 to invoke `node notify.js summary ` if (require.main === module) { const cmd = process.argv[2]; if (cmd === 'summary') { diff --git a/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1 b/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1 index 7648d4b..93f310a 100644 --- a/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1 +++ b/projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1 @@ -7,7 +7,7 @@ $nodeExe = 'C:\Program Files\nodejs\node.exe' New-Item -ItemType Directory -Force -Path $logDir | Out-Null $stamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss' $log = Join-Path $logDir "pipeline-$stamp.log" -$smtpCred = $null +$graphCreds = $null function Log([string]$m) { $line = "[$(Get-Date -Format o)] $m" @@ -16,17 +16,37 @@ function Log([string]$m) { } function SendEmail([string]$Subject, [string]$Body) { - if ($null -eq $smtpCred) { return } + if ($null -eq $graphCreds) { return } try { - Send-MailMessage ` - -From 'sysadmin@dataforth.com' ` - -To @('mike@azcomputerguru.com') ` - -Subject $Subject ` - -Body $Body ` - -SmtpServer 'smtp.office365.com' ` - -Port 587 ` - -UseSsl ` - -Credential $smtpCred + # Acquire Graph token + $tokenBody = @{ + grant_type = 'client_credentials' + client_id = $graphCreds.clientId + client_secret = $graphCreds.clientSecret + scope = 'https://graph.microsoft.com/.default' + } + $tokenResp = Invoke-RestMethod ` + -Method Post ` + -Uri "https://login.microsoftonline.com/$($graphCreds.tenantId)/oauth2/v2.0/token" ` + -Body $tokenBody + $token = $tokenResp.access_token + + # Send mail via Graph + $mailPayload = @{ + message = @{ + subject = $Subject + body = @{ contentType = 'Text'; content = $Body } + toRecipients = @(@{ emailAddress = @{ address = 'mike@azcomputerguru.com' } }) + from = @{ emailAddress = @{ address = 'sysadmin@dataforth.com' } } + } + } | ConvertTo-Json -Depth 10 + + Invoke-RestMethod ` + -Method Post ` + -Uri 'https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail' ` + -Headers @{ Authorization = "Bearer $token"; 'Content-Type' = 'application/json' } ` + -Body $mailPayload | Out-Null + Log "Email sent: $Subject" } catch { Log "WARNING: Email send failed: $_" @@ -44,12 +64,15 @@ try { $env:CF_CLIENT_SECRET = $creds.CF_CLIENT_SECRET $env:CF_SCOPE = $creds.CF_SCOPE - if ($creds.SMTP_USER -and $creds.SMTP_PASS) { - $secPass = ConvertTo-SecureString $creds.SMTP_PASS -AsPlainText -Force - $smtpCred = New-Object System.Management.Automation.PSCredential($creds.SMTP_USER, $secPass) - Log "SMTP credentials loaded for $($creds.SMTP_USER)" + if ($creds.GRAPH_TENANT_ID -and $creds.GRAPH_CLIENT_ID -and $creds.GRAPH_CLIENT_SECRET) { + $graphCreds = [PSCustomObject]@{ + tenantId = $creds.GRAPH_TENANT_ID + clientId = $creds.GRAPH_CLIENT_ID + clientSecret = $creds.GRAPH_CLIENT_SECRET + } + Log "Graph credentials loaded (app $($creds.GRAPH_CLIENT_ID))" } else { - Log "WARNING: SMTP_USER/SMTP_PASS not in credentials.json — email notifications disabled" + Log "WARNING: GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET not in credentials.json — email notifications disabled" } # [1] DFWDS process @@ -77,7 +100,7 @@ try { # [4] Daily summary email Log '[4] sending daily summary email' - if ($smtpCred) { + if ($graphCreds) { try { # Parse totals from most recent upload log; default to zeros if not found $totals = [PSCustomObject]@{ received=0; created=0; updated=0; unchanged=0; errors=0 } @@ -109,7 +132,7 @@ Log: $($latestUploadLog.FullName) Log "WARNING: Summary email failed: $_" } } else { - Log ' SMTP not configured — skipping summary email' + Log ' Graph credentials not configured — skipping summary email' } Log '=== pipeline end (OK) ===' diff --git a/projects/dataforth-dos/session-logs/2026-05-12-session.md b/projects/dataforth-dos/session-logs/2026-05-12-session.md index c6e68f8..bde2e50 100644 --- a/projects/dataforth-dos/session-logs/2026-05-12-session.md +++ b/projects/dataforth-dos/session-logs/2026-05-12-session.md @@ -146,3 +146,98 @@ pg: True, express: True, nodemailer: True, pdfkit: True, better-sqlite3: False ( - **AJ contact email:** dataforthgit@ (forwards to AJ) - **Session logs:** `projects/dataforth-dos/session-logs/` - **Process docs:** `projects/dataforth-dos/TEST-DATASHEET-PROCESS.md` + +--- + +## Update: 07:02 PT — Graph API email implementation + +### Session Summary + +This update resolved the SMTP AUTH blocker by switching the entire email path from SMTP/nodemailer to Microsoft Graph API using the existing Claude-Code-M365 Entra app. The SMTP blocker (`535 5.7.139 SmtpClientAuthentication is disabled`) was already known from the earlier session; this update pivoted to Graph API as the user indicated tenant admin access was available via remediation credentials. + +The first step was testing whether the Claude-Code-M365 app already had `Mail.Send` permission. A Python test script (`df_graph_mailtest.py`) confirmed it did not — the `POST /v1.0/users/sysadmin@dataforth.com/sendMail` call returned HTTP 403. The app had 6 existing appRoleAssignments (Directory, Group, User read/write permissions) but no mail permissions. Attempting to grant `Mail.Send` via the app's own client_credentials token failed (403 — app lacks `AppRoleAssignment.ReadWrite.All`). Attempting with a ROPC token using the Claude-Code-M365 app as the OAuth client also failed, because the app has only application permissions (no delegated), so ROPC yields only the user's basic Graph access. + +The solution was ROPC with the Azure PowerShell public client (`1950a258-227b-4e31-a9cf-717495945fc2`) — a well-known public client that supports broad delegated scopes without requiring a pre-consented app. Using `sysadmin@dataforth.com` credentials, a token was acquired with `AppRoleAssignment.ReadWrite.All` and `Application.ReadWrite.All`. The `POST /servicePrincipals/{sp_id}/appRoleAssignments` call succeeded (HTTP 201). After 30 seconds of propagation, the mail send test returned HTTP 202 and the test email arrived (confirmed by user: "test received"). + +With Graph API confirmed working, `notify.js` was rewritten to use the built-in `https` module instead of nodemailer — no external dependency. The file reads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from `credentials.json`, acquires a `client_credentials` token, and calls `POST /v1.0/users/sysadmin@dataforth.com/sendMail`. `run-pipeline.ps1` was updated to replace `Send-MailMessage` (SMTP) with `Invoke-RestMethod` to the Graph token and sendMail endpoints, using the same Graph creds. `credentials.json` on AD2 was updated: SMTP_USER/SMTP_PASS removed, GRAPH_TENANT_ID/CLIENT_ID/CLIENT_SECRET added. All three paths tested successfully: direct Python API call (HTTP 202), PowerShell SendEmail function (confirmed working), and Node `notify.alert()` + `notify.summary()` (both completed, no errors). + +### Key Decisions + +- **Azure PowerShell public client for permission grant** — Claude-Code-M365 has no delegated permissions, so ROPC with it yields minimal access. The Azure PowerShell app (`1950a258-227b-4e31-a9cf-717495945fc2`) is a well-known public client that supports `AppRoleAssignment.ReadWrite.All` as a delegated scope, allowing grant operations when the user (sysadmin) has admin role. No secret needed. +- **Removed nodemailer from notify.js** — Graph API uses only the built-in `https` module. This eliminates the npm dependency entirely. The `require('nodemailer')` was the only reason nodemailer was installed; it's now unused but remains in node_modules (harmless, can be removed with `npm uninstall nodemailer` if desired). +- **Removed SMTP_USER/SMTP_PASS from credentials.json** — SMTP auth is disabled for this tenant and will likely remain so (Microsoft security default). Keeping dead creds adds noise and confusion. Replaced with Graph creds in the same file. +- **Browser automation skipped** — Attempted to use claude-in-chrome to grant consent via Entra portal but the extension's screenshot/click/JS injection was blocked on the MS login page. Pivoted to pure API approach (ROPC + public client) which proved cleaner and faster. +- **notify.js CLI summary path not removed** — The `node notify.js summary ` CLI path still exists in the code but is not called from run-pipeline.ps1 (PS5 double-quote stripping makes it unusable from PS). run-pipeline.ps1 now uses its own PowerShell-native SendEmail function. The CLI path could be useful if called from Node-to-Node in future; no reason to remove it. + +### Problems Encountered + +- **Python urllib `InvalidURL` for OData filter with spaces** — `$filter=appId eq '...'` contains spaces; Python 3.14 is strict about control characters in URLs. Fixed: wrap path in `urllib.parse.quote(path, safe='/?=&$\'()')` in the `graph()` helper. +- **App-only token can't grant its own permissions** — `POST /servicePrincipals/{id}/appRoleAssignments` with the app's own client_credentials token returned 403 (needs `AppRoleAssignment.ReadWrite.All` which the app doesn't have). Fixed: used Azure PowerShell public client + sysadmin ROPC to get a delegated admin token. +- **ROPC with Claude-Code-M365 client insufficient** — ROPC using our app as the OAuth client gives only the app's pre-consented delegated scopes (none). Fixed: switched to Azure PowerShell public client ID. +- **Permission propagation delay** — Immediately after granting Mail.Send, the sendMail call still returned 403 and the new assignment wasn't visible in `appRoleAssignments`. Resolved after ~30 second sleep. +- **Browser extension blocked on MS login page** — `computer.screenshot`, `computer.left_click`, `computer.key`, and `javascript_tool` all failed with "Cannot access a chrome-extension:// URL of different extension" when the tab was on login.microsoftonline.com. `read_page` and `form_input` worked but were insufficient for login. Did not block the work — switched to pure API approach. +- **PS5 double-quote stripping (reprise)** — Test of `node notify.js summary $stats` from PowerShell failed with "Expected property name or '}' in JSON at position 1". This is the same PS5 bug documented in the earlier session. Confirmed: this CLI path cannot be called from run-pipeline.ps1. Summary email in PS remains entirely PowerShell-native via Invoke-RestMethod. +- **Here-string terminator error in SFTP-transferred PS scripts** — PS scripts containing `@"..."@` here-strings failed with "The string is missing the terminator" when transferred via SFTP. Root cause: unclear (encoding or line ending). Fixed: rewrote test scripts using regular string concatenation with `` `n `` escapes. + +### Configuration Changes + +**Repo (D:\claudetools):** +- `projects/dataforth-dos/database/notify.js` — rewritten. Removed nodemailer/SMTP. Now uses built-in `https` module + Graph API client_credentials. Loads `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` from credentials.json. Function signatures (`alert`, `summary`) unchanged. +- `projects/dataforth-dos/datasheet-pipeline/run-pipeline.ps1` — updated. Replaced `$smtpCred`/`Send-MailMessage` with `$graphCreds`/`Invoke-RestMethod` Graph API flow. Token acquisition + sendMail call inline in `SendEmail` function. + +**On AD2 (192.168.0.6):** +- `C:\ProgramData\dataforth-uploader\credentials.json` — removed `SMTP_USER`, `SMTP_PASS`; added `GRAPH_TENANT_ID`, `GRAPH_CLIENT_ID`, `GRAPH_CLIENT_SECRET` +- `C:\Shares\testdatadb\database\notify.js` — deployed (Graph API version) +- `C:\ProgramData\dataforth-uploader\run-pipeline.ps1` — deployed (Graph API version) + +**Entra (Dataforth tenant 7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584):** +- Claude-Code-M365 SP (`11b0aa55-314f-49bd-b8b3-baafaf49b44c`) — `Mail.Send` application permission granted. Assignment ID: `VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc`. Resource: Microsoft Graph SP `bea71f59-7956-41ed-8f99-45f31c4a6342`. + +### Credentials & Secrets + +- **Dataforth M365 tenant:** `7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584` — vault: `clients/dataforth/m365.sops.yaml` +- **Claude-Code-M365 app:** app-id=`7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29`, secret=`tXo8Q~ZNG9zoBpbK9HwJTkzx.YEigZ9AynoSrca3`, expires 2027-12-22 — vault: `clients/dataforth/m365.sops.yaml → credentials.entra-app` +- **sysadmin@dataforth.com:** password `Paper123!@#` — vault: `clients/dataforth/m365.sops.yaml → credentials.password` +- **Azure PowerShell public client (used for permission grant):** `1950a258-227b-4e31-a9cf-717495945fc2` — no secret, public client, well-known Microsoft app ID + +### Commands & Outputs + +``` +# Mail.Send permission grant (Python) +POST /v1.0/servicePrincipals/11b0aa55-314f-49bd-b8b3-baafaf49b44c/appRoleAssignments +{principalId: "11b0aa55...", resourceId: "bea71f59...", appRoleId: "b633e1c5-b582-4048-a93e-9f11b44c7e96"} +→ HTTP 201, Assignment ID: VaqwEU8xvUm4s7qvr0m0TE9AF5g2N8VBuPvGm30gKxc + +# Mail send test (after propagation) +POST /v1.0/users/sysadmin@dataforth.com/sendMail +→ HTTP 202 — [OK] Email sent (or queued) +User confirmed: "test received" + +# run-pipeline.ps1 SendEmail test on AD2 (PowerShell) +Graph creds loaded: app=7a8c0b2e-57fb-4d79-9b5a-4b88d21b1f29 +Token acquired (len=2180) +[OK] Email sent + +# notify.js alert/summary test on AD2 (Node) +[NOTIFY] ALERT: [TestDataDB] Graph API test from notify.alert() +[NOTIFY] Host: AD2 | Time: 2026-05-12T14:02:26.690Z +[NOTIFY] Sending daily summary (OK) for 2026-05-12 +[TEST] summary() completed +``` + +### Pending / Incomplete Tasks + +1. **Add John Lehman to TO list** — `jlehman@dataforth.com` must be added to `TO` array in `notify.js` (line 8) and to `toRecipients` in `run-pipeline.ps1 SendEmail`. Deploy both to AD2. Do this after Mike confirms the next real pipeline run email arrives correctly. +2. **Clean diagnostic scripts on AD2** — `C:\Shares\testdatadb\database\_*.js` (~20 files from 2026-04-15 session). Safe to delete. +3. **Clean vault entry** — `ad2.sops.yaml` stale backslash in password field. +4. **Investigate 2026-04-22 undocumented session** — import.js, notify.js, upload-to-api.js modified that date with no log. +5. **Optional: uninstall nodemailer** — `npm uninstall nodemailer` in `C:\Shares\testdatadb`. Not urgent — unused but harmless. + +### Reference Information + +- **Mail.Send appRole ID (Graph):** `b633e1c5-b582-4048-a93e-9f11b44c7e96` +- **Claude-Code-M365 SP ID:** `11b0aa55-314f-49bd-b8b3-baafaf49b44c` +- **Microsoft Graph SP ID (Dataforth tenant):** `bea71f59-7956-41ed-8f99-45f31c4a6342` +- **Azure PowerShell public client:** `1950a258-227b-4e31-a9cf-717495945fc2` +- **Graph sendMail endpoint:** `POST https://graph.microsoft.com/v1.0/users/sysadmin@dataforth.com/sendMail` +- **Token endpoint:** `POST https://login.microsoftonline.com/7dfa3ce8-c496-4b51-ab8d-bd3dcd78b584/oauth2/v2.0/token`