fix: dataforth API upload — unregistered model skip list, batch-500 fallback, FAIL filter
- UNREGISTERED_MODELS set: 9 model numbers not in Hoffman API catalog; skipped silently instead of generating errors - batch-500 fallback: when a bulk batch returns HTTP 500, retry each record individually so good records get stamped and only truly-bad records count as errors - FAIL-parameter filter: records with any FAIL on a parameter line are excluded from the push before the batch is assembled - notify.js integration: wired in existing notification module Files added: - projects/dataforth-dos/database/upload-to-api.js - projects/dataforth-dos/database/notify.js Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
71
projects/dataforth-dos/database/notify.js
Normal file
71
projects/dataforth-dos/database/notify.js
Normal file
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Fire-and-forget alert notification module.
|
||||
* Email sending is not yet implemented (Entra app pending).
|
||||
* Logs a formatted preview of what would be emailed to stderr.
|
||||
* Never throws — callers do not need to guard this.
|
||||
*/
|
||||
|
||||
const os = require('os');
|
||||
|
||||
const TO = 'mike@azcomputerguru.com';
|
||||
const FROM = 'sysadmin@dataforth.com';
|
||||
const HOST = os.hostname();
|
||||
const PREFIX = '[NOTIFY]';
|
||||
const SEP_EQ = '='.repeat(60);
|
||||
const SEP_DAS = '-'.repeat(60);
|
||||
|
||||
/**
|
||||
* Log a formatted email preview to stderr.
|
||||
*
|
||||
* @param {string} subject - Email subject line (prefixed with [TestDataDB] automatically)
|
||||
* @param {object} [context={}]
|
||||
* @param {string} [context.stage] - Pipeline stage where the alert fired
|
||||
* @param {string} [context.error] - Error message or description
|
||||
* @param {string[]} [context.details] - Additional detail lines
|
||||
* @param {object} [context.stats] - Arbitrary stats object, serialized as JSON
|
||||
*/
|
||||
function alert(subject, context) {
|
||||
context = context || {};
|
||||
const lines = [];
|
||||
|
||||
lines.push(`${PREFIX} ${SEP_EQ}`);
|
||||
lines.push(`${PREFIX} TO: ${TO}`);
|
||||
lines.push(`${PREFIX} FROM: ${FROM} (pending Entra app)`);
|
||||
lines.push(`${PREFIX} SUBJECT: [TestDataDB] ${subject}`);
|
||||
lines.push(`${PREFIX} ${SEP_DAS}`);
|
||||
lines.push(`${PREFIX} Host: ${HOST}`);
|
||||
lines.push(`${PREFIX} Time: ${new Date().toISOString()}`);
|
||||
|
||||
if (context.stage !== undefined) {
|
||||
lines.push(`${PREFIX} Stage: ${context.stage}`);
|
||||
}
|
||||
|
||||
if (context.error !== undefined) {
|
||||
lines.push(`${PREFIX}`);
|
||||
lines.push(`${PREFIX} ${context.error}`);
|
||||
}
|
||||
|
||||
if (Array.isArray(context.details) && context.details.length > 0) {
|
||||
lines.push(`${PREFIX}`);
|
||||
for (const line of context.details) {
|
||||
lines.push(`${PREFIX} ${line}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (context.stats !== undefined) {
|
||||
lines.push(`${PREFIX} Stats: ${JSON.stringify(context.stats)}`);
|
||||
}
|
||||
|
||||
lines.push(`${PREFIX} ${SEP_EQ}`);
|
||||
lines.push(`${PREFIX} (email not sent — Entra app pending)`);
|
||||
|
||||
try {
|
||||
for (const line of lines) {
|
||||
process.stderr.write(line + '\n');
|
||||
}
|
||||
} catch (_) {
|
||||
// stderr write failure — nothing we can do
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { alert };
|
||||
262
projects/dataforth-dos/database/upload-to-api.js
Normal file
262
projects/dataforth-dos/database/upload-to-api.js
Normal file
@@ -0,0 +1,262 @@
|
||||
/**
|
||||
* Post-import uploader — pushes just-imported records to Dataforth's Hoffman
|
||||
* API. Called from import.js after insertBatch, and from the /api/upload
|
||||
* endpoint for individual/bulk UI pushes.
|
||||
*
|
||||
* Datasheet content is rendered in memory from the DB row via
|
||||
* render-datasheet.renderContent — no For_Web filesystem dependency.
|
||||
*
|
||||
* Credentials come from C:\ProgramData\dataforth-uploader\credentials.json
|
||||
* (ACL'd to SYSTEM + Administrators + svc_testdatadb).
|
||||
*
|
||||
* The API is idempotent — already-present records return Unchanged.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const https = require('https');
|
||||
const { URL } = require('url');
|
||||
const db = require('./db');
|
||||
const { renderContent } = require('./render-datasheet');
|
||||
const notify = require('./notify');
|
||||
|
||||
const CREDS_PATH = 'C:\\ProgramData\\dataforth-uploader\\credentials.json';
|
||||
const BATCH = 100;
|
||||
const TOKEN_LEEWAY_MS = 60 * 1000;
|
||||
const HTTP_TIMEOUT_MS = 120 * 1000;
|
||||
|
||||
const RECORD_COLUMNS = 'id, log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file';
|
||||
|
||||
// Models not yet registered in Hoffman's API — skip silently instead of generating errors.
|
||||
// When Hoffman adds these, remove them from this list and they will push on the next run.
|
||||
const UNREGISTERED_MODELS = new Set([
|
||||
'7B21', '7B30-02D', '7B32-02D', '7B39-02',
|
||||
'7B47K-03', '7B47K-1254', '7B47K-1574', '7B47K-1650',
|
||||
'SCM5B36-04',
|
||||
]);
|
||||
|
||||
let _creds = null;
|
||||
function loadCreds() {
|
||||
if (_creds) return _creds;
|
||||
if (!fs.existsSync(CREDS_PATH)) {
|
||||
throw new Error(`creds file not found: ${CREDS_PATH}`);
|
||||
}
|
||||
_creds = JSON.parse(fs.readFileSync(CREDS_PATH, 'utf8'));
|
||||
for (const k of ['CF_TOKEN_URL','CF_API_BASE','CF_CLIENT_ID','CF_CLIENT_SECRET','CF_SCOPE']) {
|
||||
if (!_creds[k]) throw new Error(`${CREDS_PATH} missing field ${k}`);
|
||||
}
|
||||
return _creds;
|
||||
}
|
||||
|
||||
let _tok = { value: null, expiresAt: 0 };
|
||||
|
||||
function httpPost(uri, body, headers) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const u = new URL(uri);
|
||||
const req = https.request({
|
||||
hostname: u.hostname, port: u.port || 443, path: u.pathname + u.search,
|
||||
method: 'POST',
|
||||
headers: Object.assign({}, headers, {'Content-Length': Buffer.byteLength(body)}),
|
||||
timeout: HTTP_TIMEOUT_MS,
|
||||
}, res => {
|
||||
let data = '';
|
||||
res.on('data', c => data += c);
|
||||
res.on('end', () => {
|
||||
try { resolve({status: res.statusCode, body: JSON.parse(data)}); }
|
||||
catch (e) { resolve({status: res.statusCode, body: {_raw: data}}); }
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => req.destroy(new Error('http timeout')));
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function getToken(force = false) {
|
||||
const c = loadCreds();
|
||||
if (!force && _tok.value && Date.now() < _tok.expiresAt - TOKEN_LEEWAY_MS) {
|
||||
return _tok.value;
|
||||
}
|
||||
const form = Object.entries({
|
||||
grant_type: 'client_credentials',
|
||||
client_id: c.CF_CLIENT_ID,
|
||||
client_secret: c.CF_CLIENT_SECRET,
|
||||
scope: c.CF_SCOPE,
|
||||
}).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&');
|
||||
const r = await httpPost(c.CF_TOKEN_URL, form, {'Content-Type': 'application/x-www-form-urlencoded'});
|
||||
if (r.status !== 200 || !r.body.access_token) {
|
||||
throw new Error(`token fetch failed: ${r.status} ${JSON.stringify(r.body).slice(0,200)}`);
|
||||
}
|
||||
_tok.value = r.body.access_token;
|
||||
_tok.expiresAt = Date.now() + (r.body.expires_in || 3600) * 1000;
|
||||
return _tok.value;
|
||||
}
|
||||
|
||||
async function bulkPost(items) {
|
||||
const c = loadCreds();
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
const tok = await getToken(attempt > 0);
|
||||
try {
|
||||
const r = await httpPost(
|
||||
`${c.CF_API_BASE}/api/v1/TestReportDataFiles/bulk`,
|
||||
JSON.stringify({Items: items}),
|
||||
{'Authorization': `Bearer ${tok}`, 'Content-Type': 'application/json'},
|
||||
);
|
||||
if (r.status === 401 && attempt === 0) continue;
|
||||
return r;
|
||||
} catch (e) {
|
||||
if (attempt === 0) { await new Promise(r => setTimeout(r, 5000)); continue; }
|
||||
return {status: 0, body: {_error: e.message}};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function stampConfirmed(items, errors) {
|
||||
const badSns = new Set();
|
||||
for (const e of (errors || [])) {
|
||||
const matches = String(e).match(/\b\d+-\d+[A-Z]?\b/gi) || [];
|
||||
for (const m of matches) badSns.add(m);
|
||||
}
|
||||
const confirmedSns = items.map(it => it.SerialNumber).filter(sn => !badSns.has(sn));
|
||||
if (confirmedSns.length === 0) return;
|
||||
try {
|
||||
const placeholders = confirmedSns.map((_, j) => `$${j + 1}`).join(',');
|
||||
await db.execute(
|
||||
`UPDATE test_records SET api_uploaded_at = NOW() WHERE serial_number IN (${placeholders})`,
|
||||
confirmedSns,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(`[API-UPLOAD] stamp failed: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadRecords(records, result) {
|
||||
const t0 = Date.now();
|
||||
for (let i = 0; i < records.length; i += BATCH) {
|
||||
const chunk = records.slice(i, i + BATCH);
|
||||
const items = [];
|
||||
for (const r of chunk) {
|
||||
let content;
|
||||
try {
|
||||
content = renderContent(r);
|
||||
} catch (e) {
|
||||
console.error(`[API-UPLOAD] render fail ${r.serial_number}: ${e.message}`);
|
||||
result.errors++;
|
||||
continue;
|
||||
}
|
||||
if (!content) { result.skipped++; continue; }
|
||||
// Records with any FAIL parameter are not published — unit either
|
||||
// failed and was never re-tested to a clean pass, or has an
|
||||
// out-of-spec measurement that was accepted internally but should
|
||||
// not appear on the public datasheet site.
|
||||
if (/\bFAIL\s*$/m.test(content)) { result.skipped++; continue; }
|
||||
// Model not yet registered in Hoffman's API — skip silently.
|
||||
if (UNREGISTERED_MODELS.has(r.model_number)) { result.skipped++; continue; }
|
||||
items.push({ SerialNumber: r.serial_number, Content: content });
|
||||
}
|
||||
if (items.length === 0) continue;
|
||||
|
||||
let resp;
|
||||
try { resp = await bulkPost(items); }
|
||||
catch (e) {
|
||||
console.error(`[API-UPLOAD] batch threw: ${e.message}`);
|
||||
result.errors += items.length;
|
||||
continue;
|
||||
}
|
||||
if (resp.status === 500 && items.length > 1) {
|
||||
// One bad record poisons the batch — retry each record individually so
|
||||
// good records get stamped and only the truly-bad ones count as errors.
|
||||
for (const item of items) {
|
||||
let r2;
|
||||
try { r2 = await bulkPost([item]); }
|
||||
catch (e2) { result.errors++; continue; }
|
||||
if (r2.status !== 200) {
|
||||
result.errors++;
|
||||
} else {
|
||||
result.created += r2.body.Created || 0;
|
||||
result.updated += r2.body.Updated || 0;
|
||||
result.unchanged += r2.body.Unchanged || 0;
|
||||
result.errors += (r2.body.Errors || []).length;
|
||||
await stampConfirmed([item], r2.body.Errors);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (resp.status !== 200) {
|
||||
console.error(`[API-UPLOAD] HTTP ${resp.status}: ${JSON.stringify(resp.body).slice(0,200)}`);
|
||||
result.errors += items.length;
|
||||
continue;
|
||||
}
|
||||
result.created += resp.body.Created || 0;
|
||||
result.updated += resp.body.Updated || 0;
|
||||
result.unchanged += resp.body.Unchanged || 0;
|
||||
result.errors += (resp.body.Errors || []).length;
|
||||
|
||||
await stampConfirmed(items, resp.body.Errors);
|
||||
}
|
||||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
console.log(`[API-UPLOAD] done in ${elapsed}s: created=${result.created} updated=${result.updated} unchanged=${result.unchanged} errors=${result.errors} skipped=${result.skipped}`);
|
||||
|
||||
if (result.errors > 0) {
|
||||
notify.alert('API push partial failure', {
|
||||
stage: 'Hoffman API bulk push',
|
||||
details: [`${result.errors} record(s) failed to push to Dataforth API`],
|
||||
stats: result,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-import upload. Non-throwing — upload failure must not wedge the import flow.
|
||||
* @param {string[]} filePaths - source_file values from the just-completed import
|
||||
*/
|
||||
async function uploadNewRecords(filePaths) {
|
||||
const result = {created: 0, updated: 0, unchanged: 0, errors: 0, skipped: 0};
|
||||
try {
|
||||
if (!filePaths || filePaths.length === 0) return result;
|
||||
if (!fs.existsSync(CREDS_PATH)) {
|
||||
console.log(`[API-UPLOAD] credentials not configured (${CREDS_PATH}); skipping`);
|
||||
return result;
|
||||
}
|
||||
const placeholders = filePaths.map((_, i) => `$${i + 1}`).join(',');
|
||||
const records = await db.query(
|
||||
`SELECT ${RECORD_COLUMNS} FROM test_records WHERE overall_result = 'PASS' AND source_file IN (${placeholders})`,
|
||||
filePaths,
|
||||
);
|
||||
if (records.length === 0) {
|
||||
console.log('[API-UPLOAD] no records eligible for upload');
|
||||
return result;
|
||||
}
|
||||
console.log(`[API-UPLOAD] ${records.length} records to upload`);
|
||||
await uploadRecords(records, result);
|
||||
} catch (e) {
|
||||
console.error(`[API-UPLOAD] fatal: ${e.message}`);
|
||||
notify.alert('API push fatal error', {
|
||||
stage: 'uploadNewRecords',
|
||||
error: e.message,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload records identified by serial numbers. Called by /api/upload for
|
||||
* per-record and bulk UI pushes. Throws on hard failures so the endpoint
|
||||
* can return 500.
|
||||
*/
|
||||
async function uploadBySerialNumbers(sns) {
|
||||
const result = {created: 0, updated: 0, unchanged: 0, errors: 0, skipped: 0};
|
||||
if (!sns || sns.length === 0) return result;
|
||||
if (!fs.existsSync(CREDS_PATH)) {
|
||||
throw new Error(`credentials not configured (${CREDS_PATH})`);
|
||||
}
|
||||
const placeholders = sns.map((_, i) => `$${i + 1}`).join(',');
|
||||
const records = await db.query(
|
||||
`SELECT ${RECORD_COLUMNS} FROM test_records WHERE serial_number IN (${placeholders}) AND overall_result = 'PASS'`,
|
||||
sns,
|
||||
);
|
||||
if (records.length === 0) return result;
|
||||
await uploadRecords(records, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
module.exports = { uploadNewRecords, uploadBySerialNumbers };
|
||||
Reference in New Issue
Block a user