dataforth(datasheet): converge 8B/5B/SCM onto the DSCA template machinery (gated; WIP)
Generalized the shared renderer (datasheet-exact.js) so the per-model template path is no longer DSCA-only — 8B/SCM5B now route through the same templates/slotMap/fround path as DSCA. SCM7B stays on its separate parse7BRawData/legacy path (excluded). Changes (all behind the per-model `validated` gate; non-templated families byte-unchanged, and DSCA verified non-regressed at 54/56 + staged-92 still render): - Load 8b5bscm-templates.json (MINED_8B5BSCM); for family 8B/SCM5B, a mined template takes the template path (else falls through to the legacy DATA_LINES path). - Final-Test header is now the verbatim captured `ftHeader` (8B/5B use two schemes — legacy "Measured Value" vs DSCA "Measured Value*"); enrich-8b5bscm.js captured it for all 136 from the Hoffman originals. - Accuracy data formatter chosen by input type detected from accHeader: frequency(Hz)/ AC(AAC/VAC) -> formatAccuracyLineDSCA3345 (now takes inputType, not model prefix); standard voltage/current/temp -> legacy formatAccuracyLine. - parseRawData step-skip generalized to any templated model (skipStepIfStatus). - Footer 240VAC/Hi-Pot family blocks skipped for templated models (rows carry them). - Templated SCM5B prepends "SCM" to the printed model name. - slotmap-from-hoffman.js parameterized (DSCA_TPL) — derived 80 slotMaps for 8b5bscm. Validation vs live Hoffman originals (validate-mined.js): 37/136 pass and are marked validated -> 13 real 8B/SCM5B render live via the template path; 24 SCM7B already match via the unchanged legacy path. Everything else stays null (gated, safe) — nothing published. Remaining long tail (well-characterized, next session): ~66 render-null need slotMaps the Hoffman-oracle derivation couldn't get (vintage / parse); ~33 multi-nuance fails: Final-Test sign handling on some 8B/5B measured values, accuracy-precision fround in the legacy std formatter, "Packing Check List" vs "Check List" variants, and other per-model footer/format quirks. Each failing model stacks several, so single fixes don't flip them. Tools: enrich-8b5bscm.js, validate-mined.js (generalized validator), slotmap-from-hoffman.js. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -30,6 +30,17 @@ try {
|
||||
DSCA3345_TEMPLATES = {};
|
||||
}
|
||||
|
||||
// 8B / SCM5B per-model templates recovered from the Hoffman API (same data gap as
|
||||
// DSCA33/45). Same superset schema plus a verbatim `ftHeader` (8B/5B use two Final-Test
|
||||
// header schemes: legacy "Measured Value" vs DSCA-style "Measured Value*"). Per-model
|
||||
// `validated` gate. SCM7B is NOT templated here (separate parse path). Loaded once.
|
||||
let MINED_8B5BSCM = {};
|
||||
try {
|
||||
MINED_8B5BSCM = require('../8b5bscm-templates.json');
|
||||
} catch (e) {
|
||||
MINED_8B5BSCM = {};
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// DATA LINES: parameter names and units per family
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -180,7 +191,7 @@ function getSensorNum(sentype) {
|
||||
// Parse raw_data from DB record
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function parseRawData(rawData, family) {
|
||||
function parseRawData(rawData, family, skipStepIfStatus) {
|
||||
if (!rawData) return null;
|
||||
|
||||
const lines = rawData.split('\n').map(l => l.trim()).filter(l => l.length > 0);
|
||||
@@ -213,13 +224,15 @@ function parseRawData(rawData, family) {
|
||||
}
|
||||
|
||||
// Next line: step response / placeholders.
|
||||
// SCM5B/8B: "0","0",value DSCT: just value. Many DSCA models OMIT this bare
|
||||
// line and go straight to the STATUS groups; consuming a STATUS group here
|
||||
// drops a Final-Test row (the "lines drop" defect). For DSCA, skip consuming
|
||||
// when the line is actually a STATUS group (starts with PASS/FAIL).
|
||||
// SCM5B/8B: "0","0",value DSCT: just value. Many DSCA models — and the templated
|
||||
// 8B/5B subtypes (8B45/8B49/5B39/SCM5B33...) — OMIT this bare line and go straight to
|
||||
// the STATUS groups; consuming a STATUS group here drops a Final-Test row (the "lines
|
||||
// drop" defect). Skip consuming when the line is actually a STATUS group (PASS/FAIL)
|
||||
// for DSCA or any templated model (skipStepIfStatus); non-templated families keep the
|
||||
// original behavior byte-for-byte.
|
||||
if (lineIdx < lines.length) {
|
||||
const looksLikeStatus = /^"?(PASS|FAIL)/i.test(lines[lineIdx].trim());
|
||||
if (!(family === 'DSCA' && looksLikeStatus)) {
|
||||
if (!((family === 'DSCA' || skipStepIfStatus) && looksLikeStatus)) {
|
||||
const parts = parseCSVLine(lines[lineIdx++]);
|
||||
const lastVal = parts[parts.length - 1];
|
||||
result.stepResponse = parseFloat(lastVal) || 0;
|
||||
@@ -532,9 +545,9 @@ function formatAccuracyLine(point, sensorNum, maxIn) {
|
||||
* to 3 decimals.
|
||||
* - DSCA45 (frequency input) prints stim as an UNSIGNED integer Hz; calc/meas/error SIGNED.
|
||||
*/
|
||||
function formatAccuracyLineDSCA3345(point, model, accOut) {
|
||||
function formatAccuracyLineDSCA3345(point, inputType, accOut) {
|
||||
const scale = /mA/.test(accOut || '') ? 1000 : 1;
|
||||
const isDSCA45 = /^DSCA45/i.test((model || '').trim());
|
||||
const isDSCA45 = inputType === 'freq'; // frequency-input (Hz stim); else AC-RMS
|
||||
// values were computed in QB single precision; recover the single before formatting
|
||||
// so last-digit rounding at the .5 boundary matches the original (Math.fround).
|
||||
const num = (val, decimals, signed) => ((signed && val >= 0) ? '+' : '') + Math.fround(val).toFixed(decimals);
|
||||
@@ -598,24 +611,29 @@ function generateExactDatasheet(record, specs) {
|
||||
// specs + accuracy label). Source is the staged-original set (dsca-templates) or
|
||||
// the Hoffman-mined set (dsca33-45-templates) for the families whose specs were
|
||||
// lost. No template -> do not guess; skip this cert.
|
||||
const dscaKey = (record.model_number || '').trim();
|
||||
// Hoffman-mined templates take PRECEDENCE: DSCA33/45 were also captured by the
|
||||
// STAGE 1 staged extractor (sometimes with accOut "?" and no accHeader), and that
|
||||
// stale entry must not shadow the authoritative Hoffman-mined one.
|
||||
const dscaTpl = (family === 'DSCA')
|
||||
? (DSCA3345_TEMPLATES[dscaKey] || DSCA_TEMPLATES[dscaKey] || null)
|
||||
: null;
|
||||
if (family === 'DSCA' && !dscaTpl) return null;
|
||||
// Hoffman-mined DSCA33/45 render only once the model is byte-validated against its
|
||||
// Hoffman original — otherwise stay null so an unverified render can't overwrite a
|
||||
// live original. The validation harness sets DSCA_VALIDATE_MODE to render
|
||||
// unvalidated models for the byte-compare; the live service never sets it.
|
||||
if (family === 'DSCA' && DSCA3345_TEMPLATES[dscaKey] && !DSCA3345_TEMPLATES[dscaKey].validated
|
||||
&& !process.env.DSCA_VALIDATE_MODE) return null;
|
||||
const modelKey = (record.model_number || '').trim();
|
||||
// Per-model template driving the Final-Test (+ accuracy) layout. DSCA uses the staged
|
||||
// set or the Hoffman-mined DSCA33/45 set (mined takes PRECEDENCE over a stale staged
|
||||
// entry); 8B/SCM5B use the Hoffman-mined 8b5bscm set. SCM7B is excluded (separate
|
||||
// parse path). `minedTpl` = the Hoffman-mined entry (accHeader/ftHeader + a per-model
|
||||
// `validated` gate).
|
||||
let tpl = null, minedTpl = null;
|
||||
if (family === 'DSCA') {
|
||||
minedTpl = DSCA3345_TEMPLATES[modelKey] || null;
|
||||
tpl = minedTpl || DSCA_TEMPLATES[modelKey] || null;
|
||||
if (!tpl) return null; // no DSCA template -> do not guess the layout
|
||||
} else if (family === '8B' || family === 'SCM5B') {
|
||||
minedTpl = MINED_8B5BSCM[modelKey] || null;
|
||||
tpl = minedTpl; // null -> falls through to the legacy DATA_LINES path
|
||||
}
|
||||
// Hoffman-mined models render only once byte-validated against their original (else
|
||||
// null) so an unverified render can't overwrite a live original. The validation
|
||||
// harness sets DSCA_VALIDATE_MODE to open the gate for its compare.
|
||||
if (minedTpl && minedTpl.accHeader && !minedTpl.validated && !process.env.DSCA_VALIDATE_MODE) return null;
|
||||
|
||||
const parsed = (family === 'SCM7B')
|
||||
? parse7BRawData(record.raw_data)
|
||||
: parseRawData(record.raw_data, family);
|
||||
: parseRawData(record.raw_data, family, !!tpl);
|
||||
if (!parsed) return null;
|
||||
if (family !== 'SCM7B' && parsed.accuracy.length < 5) return null;
|
||||
|
||||
@@ -634,10 +652,14 @@ function generateExactDatasheet(record, specs) {
|
||||
: record.test_date || '';
|
||||
|
||||
let modelName = specs ? specs.MODNAME : record.model_number;
|
||||
// 7B header prepends "SCM" to the model name
|
||||
// 7B header prepends "SCM" to the model name; templated SCM5B do the same (the spec
|
||||
// MODNAME is stored as "5B49-03" but the original prints "SCM5B49-03").
|
||||
if (family === 'SCM7B' && !modelName.toUpperCase().startsWith('SCM')) {
|
||||
modelName = 'SCM' + modelName;
|
||||
}
|
||||
if (tpl && family === 'SCM5B' && !modelName.toUpperCase().startsWith('SCM')) {
|
||||
modelName = 'SCM' + modelName;
|
||||
}
|
||||
|
||||
const lines = [];
|
||||
const TAB5 = ' '; // 4 spaces = TAB(5) in QB (0-indexed)
|
||||
@@ -667,12 +689,12 @@ function generateExactDatasheet(record, specs) {
|
||||
} else {
|
||||
lines.push(' ACCURACY TEST');
|
||||
lines.push('');
|
||||
if (dscaTpl && Array.isArray(dscaTpl.accHeader) && dscaTpl.accHeader.length >= 2) {
|
||||
if (tpl && Array.isArray(tpl.accHeader) && tpl.accHeader.length >= 2) {
|
||||
// DSCA33/45 (Hoffman-mined): the accuracy header carries model-specific tokens
|
||||
// the sensor-type logic can't synthesize (Vin (mVAC), Iin (AAC), Frequency (Hz),
|
||||
// Output (VDC)/(mADC)). Emit the verbatim 2-line header from the original.
|
||||
lines.push(dscaTpl.accHeader[0]);
|
||||
lines.push(dscaTpl.accHeader[1]);
|
||||
lines.push(tpl.accHeader[0]);
|
||||
lines.push(tpl.accHeader[1]);
|
||||
lines.push(TAB5 + '-'.repeat(10) + ' ' + '-'.repeat(11) + ' ' + '-'.repeat(11) + ' ' + '-'.repeat(10) + ' ' + '-'.repeat(8));
|
||||
} else {
|
||||
lines.push(' Calculated Measured');
|
||||
@@ -688,15 +710,22 @@ function generateExactDatasheet(record, specs) {
|
||||
}
|
||||
// DSCA labels its accuracy output column "Output (V)"/"Output (mA)" (from the
|
||||
// template) with '-' rule separators; 5B/8B/etc. use "Vout (V)" with '='.
|
||||
const accOut = (family === 'DSCA' && dscaTpl) ? dscaTpl.accOut : 'Vout (V)';
|
||||
const accOut = (family === 'DSCA' && tpl) ? tpl.accOut : 'Vout (V)';
|
||||
const accSep = (family === 'DSCA') ? '-' : '=';
|
||||
lines.push(' ' + inputHeader + ' ' + accOut + ' ' + accOut + '* Error (%) Status');
|
||||
lines.push(TAB5 + accSep.repeat(10) + ' ' + accSep.repeat(10) + ' ' + accSep.repeat(10) + ' ' + accSep.repeat(9) + ' ' + accSep.repeat(8));
|
||||
}
|
||||
|
||||
// For a mined template, the accuracy DATA columns follow the input type: frequency
|
||||
// (Hz) and AC (AAC/VAC) use the DSCA33/45 conventions (unsigned/integer stim, mA
|
||||
// scaling); standard voltage/current/temperature inputs use the legacy signed
|
||||
// formatter (sensor-type + maxIn from specs, which 8B/5B have).
|
||||
const accHdrStr = (tpl && Array.isArray(tpl.accHeader)) ? tpl.accHeader.join(' ') : '';
|
||||
const accInput = /\(Hz\)/.test(accHdrStr) ? 'freq'
|
||||
: (/AAC|VAC|mVAC/.test(accHdrStr) ? 'ac' : 'std');
|
||||
for (const point of parsed.accuracy) {
|
||||
if (dscaTpl && Array.isArray(dscaTpl.accHeader)) {
|
||||
lines.push(formatAccuracyLineDSCA3345(point, record.model_number, dscaTpl.accOut));
|
||||
if (tpl && Array.isArray(tpl.accHeader) && (accInput === 'freq' || accInput === 'ac')) {
|
||||
lines.push(formatAccuracyLineDSCA3345(point, accInput, tpl.accOut));
|
||||
continue;
|
||||
}
|
||||
lines.push(formatAccuracyLine(point, sensorNum, maxIn));
|
||||
@@ -708,12 +737,12 @@ function generateExactDatasheet(record, specs) {
|
||||
// QB column positions (1-indexed): TAB(31), TAB(47), TAB(60-speclen), TAB(61), TAB(71)
|
||||
lines.push(' FINAL TEST RESULTS');
|
||||
lines.push('');
|
||||
if (family === 'DSCA') {
|
||||
// DSCA Final-Test renders from the per-model staged template: the rows give
|
||||
// the parameter names + specs (and accuracy label) directly; the value-bearing
|
||||
// raw_data STATUS groups map positionally onto the spec-bearing rows. Rows with
|
||||
// an empty spec (240VAC Withstand / Hi-Pot) carry no measured value and render
|
||||
// as PASS. Header/column scheme matches the staged originals.
|
||||
if (tpl) {
|
||||
// Template-driven Final-Test (DSCA, and Hoffman-mined 8B/SCM5B): the rows give
|
||||
// the parameter names + specs directly; the value-bearing raw_data STATUS groups
|
||||
// map positionally onto the spec-bearing rows (via slotMap when counts differ).
|
||||
// Spec-less pass/fail rows (240VAC Withstand / Hi-Pot) render PASS; spec-less
|
||||
// section sub-heads (Zero-Crossing/TTL) render with no status.
|
||||
|
||||
// Value-bearing measurements in source order (drop "PASS"/"" padding entries).
|
||||
const measurements = [];
|
||||
@@ -721,37 +750,42 @@ function generateExactDatasheet(record, specs) {
|
||||
const m = formatMeasuredExact(s);
|
||||
if (m) measurements.push(m);
|
||||
}
|
||||
const specRowCount = dscaTpl.rows.filter(r => (r.spec || '').trim()).length;
|
||||
const specRowCount = tpl.rows.filter(r => (r.spec || '').trim()).length;
|
||||
// The simple positional zip is sound only when there is exactly one measured
|
||||
// value per spec-bearing row. When counts differ, this subtype measures slots
|
||||
// the template omits (e.g. an extra load pair); use the per-model slotMap
|
||||
// (absolute statusEntries index per spec-bearing row, derived from the staged
|
||||
// originals) to pull the right value. With no usable slotMap, skip rather than
|
||||
// misalign ("do not guess").
|
||||
// value per spec-bearing row. When counts differ, the subtype measures slots the
|
||||
// template omits; use the per-model slotMap (absolute statusEntries index per
|
||||
// spec-bearing row). With no usable slotMap, skip rather than misalign.
|
||||
const useSlot = (measurements.length !== specRowCount)
|
||||
&& Array.isArray(dscaTpl.slotMap) && dscaTpl.slotMap.length === specRowCount;
|
||||
&& Array.isArray(tpl.slotMap) && tpl.slotMap.length === specRowCount;
|
||||
if (measurements.length !== specRowCount && !useSlot) return null;
|
||||
|
||||
let h1 = setCol('', 12, 'Parameter');
|
||||
h1 = setCol(h1, 31, 'Measured Value*');
|
||||
h1 = setCol(h1, 51, 'Specification');
|
||||
h1 = setCol(h1, 69, 'Status');
|
||||
lines.push(h1);
|
||||
// Final-Test header: use the verbatim captured header when present (8B/5B use
|
||||
// two schemes — legacy "Measured Value" vs DSCA-style "Measured Value*"); else
|
||||
// the DSCA-style header. The separator is a rule line (cosmetic, canonicalized).
|
||||
if (tpl.ftHeader) {
|
||||
lines.push(tpl.ftHeader);
|
||||
} else {
|
||||
let h1 = setCol('', 12, 'Parameter');
|
||||
h1 = setCol(h1, 31, 'Measured Value*');
|
||||
h1 = setCol(h1, 51, 'Specification');
|
||||
h1 = setCol(h1, 69, 'Status');
|
||||
lines.push(h1);
|
||||
}
|
||||
let h2 = setCol('', 4, '='.repeat(25));
|
||||
h2 = setCol(h2, 31, '='.repeat(15));
|
||||
h2 = setCol(h2, 48, '='.repeat(19));
|
||||
h2 = setCol(h2, 69, '='.repeat(6));
|
||||
lines.push(h2);
|
||||
|
||||
const is3345 = Array.isArray(dscaTpl.accHeader); // Hoffman-mined DSCA33/45
|
||||
const is3345 = Array.isArray(tpl.accHeader); // Hoffman-mined (DSCA33/45 + 8B/5B)
|
||||
let mi = 0, si = 0;
|
||||
for (const row of dscaTpl.rows) {
|
||||
for (const row of tpl.rows) {
|
||||
const spec = (row.spec || '').trim();
|
||||
let line = setCol('', 4, row.name);
|
||||
if (spec) {
|
||||
const su = splitSpecUnit(spec);
|
||||
const m = useSlot
|
||||
? formatMeasuredExact(parsed.statusEntries[dscaTpl.slotMap[si++]])
|
||||
? formatMeasuredExact(parsed.statusEntries[tpl.slotMap[si++]])
|
||||
: measurements[mi++];
|
||||
if (m) {
|
||||
// measured value right-justified ending at col 38, unit at col 40.
|
||||
@@ -780,9 +814,9 @@ function generateExactDatasheet(record, specs) {
|
||||
// Footer load note ("Standard output load for test is ... ohms.") — printed
|
||||
// before the underline, only by the models whose staged original had it
|
||||
// (captured per-model in STAGE 1; not all current-output models print it).
|
||||
if (dscaTpl.loadNote) {
|
||||
if (tpl.loadNote) {
|
||||
lines.push('');
|
||||
lines.push(TAB5 + dscaTpl.loadNote);
|
||||
lines.push(TAB5 + tpl.loadNote);
|
||||
}
|
||||
} else {
|
||||
// QB: TAB(12); "Parameter"; TAB(30); "Measured Value"; TAB(51); "Specification "; TAB(70); "Status"
|
||||
@@ -837,8 +871,11 @@ function generateExactDatasheet(record, specs) {
|
||||
} // end non-DSCA Final Test Results
|
||||
|
||||
// ---- Footer ----
|
||||
// 240 VAC / Hi-Pot (conditional by family/model)
|
||||
if (family === 'SCM5B') {
|
||||
// 240 VAC / Hi-Pot (conditional by family/model). Templated models already render
|
||||
// these as spec-less rows from the template, so skip the family-specific footer block.
|
||||
if (tpl) {
|
||||
// no footer Withstand/Hi-Pot block — handled in the template rows
|
||||
} else if (family === 'SCM5B') {
|
||||
const mn = (modelName || '').trim();
|
||||
if (!mn.startsWith('SCM5BPT') && !mn.startsWith('SCM5B-1369')) {
|
||||
lines.push(TAB5 + '240 VAC Withstand' + ''.padEnd(49) + 'PASS');
|
||||
|
||||
30
projects/dataforth-dos/tools/enrich-8b5bscm.js
Normal file
30
projects/dataforth-dos/tools/enrich-8b5bscm.js
Normal file
@@ -0,0 +1,30 @@
|
||||
// Enrich 8b5bscm-templates.json: capture each model's verbatim Final-Test HEADER line
|
||||
// (the "Parameter ... Measured Value[*] ... Specification ... Status" line) from its
|
||||
// _srcSerial Hoffman original, since 8B/5B use two different header schemes (legacy
|
||||
// "Measured Value" vs DSCA-style "Measured Value*"). Stored as `ftHeader`. --apply writes.
|
||||
const fs = require('fs'), https = require('https');
|
||||
const TPL = './8b5bscm-templates.json';
|
||||
const c = JSON.parse(fs.readFileSync('C:\\ProgramData\\dataforth-uploader\\credentials.json', 'utf8'));
|
||||
const APPLY = process.argv.includes('--apply');
|
||||
function req(m, uri, h, b) { return new Promise((res, rej) => { const u = new URL(uri); const r = https.request({ hostname: u.hostname, port: 443, path: u.pathname, method: m, headers: h, timeout: 30000 }, x => { let d = ''; x.on('data', c => d += c); x.on('end', () => { try { res(JSON.parse(d)); } catch { res({ _raw: d }); } }); }); r.on('error', rej); if (b) r.write(b); r.end(); }); }
|
||||
(async () => {
|
||||
const tpl = JSON.parse(fs.readFileSync(TPL, 'utf8'));
|
||||
const form = Object.entries({ grant_type: 'client_credentials', client_id: c.CF_CLIENT_ID, client_secret: c.CF_CLIENT_SECRET, scope: c.CF_SCOPE }).map(([k, v]) => encodeURIComponent(k) + '=' + encodeURIComponent(v)).join('&');
|
||||
const t = (await req('POST', c.CF_TOKEN_URL, { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(form) }, form)).access_token;
|
||||
let done = 0; const schemes = {};
|
||||
for (const m of Object.keys(tpl)) {
|
||||
const sn = tpl[m]._srcSerial; if (!sn) continue;
|
||||
const g = await req('GET', c.CF_API_BASE + '/api/v1/TestReportDataFiles/' + encodeURIComponent(sn), { Authorization: 'Bearer ' + t });
|
||||
if (!g.Content) continue;
|
||||
const L = g.Content.replace(/\r/g, '').split('\n');
|
||||
const fi = L.findIndex(l => /FINAL TEST RESULTS/.test(l)); if (fi < 0) continue;
|
||||
const hi = L.findIndex((l, i) => i > fi && /Parameter\s+Measured/.test(l)); if (hi < 0) continue;
|
||||
tpl[m].ftHeader = L[hi];
|
||||
const star = /Measured Value\*/.test(L[hi]) ? 'star' : 'plain';
|
||||
schemes[star] = (schemes[star] || 0) + 1;
|
||||
done++;
|
||||
}
|
||||
console.log('captured ftHeader for ' + done + ' models; header schemes: ' + JSON.stringify(schemes));
|
||||
if (APPLY) { fs.writeFileSync(TPL, JSON.stringify(tpl)); console.log('[APPLY] wrote ' + TPL); }
|
||||
else console.log('(dry run — pass --apply)');
|
||||
})().catch(e => { console.error('ERR', e.message); process.exit(1); });
|
||||
@@ -6,7 +6,7 @@ process.env.DSCA_VALIDATE_MODE = '1';
|
||||
const fs = require('fs'), https = require('https');
|
||||
const db = require('./database/db');
|
||||
const dse = require('./templates/datasheet-exact');
|
||||
const TPL = './dsca33-45-templates.json';
|
||||
const TPL = process.env.DSCA_TPL || './dsca33-45-templates.json';
|
||||
const c = JSON.parse(fs.readFileSync('C:\\ProgramData\\dataforth-uploader\\credentials.json', 'utf8'));
|
||||
const APPLY = process.argv.includes('--apply');
|
||||
const only = process.argv.slice(2).filter(a => !a.startsWith('--'));
|
||||
|
||||
97
projects/dataforth-dos/tools/validate-mined.js
Normal file
97
projects/dataforth-dos/tools/validate-mined.js
Normal file
@@ -0,0 +1,97 @@
|
||||
// Fix 2 — validate DSCA33/45 Hoffman-mined renders against the live Hoffman originals.
|
||||
// For each model: render its _srcSerial (an already-uploaded unit) via the new render
|
||||
// path and content-normalized-compare it to GET /api/v1/TestReportDataFiles/{_srcSerial}.
|
||||
// --apply marks passing models `validated:true` in dsca33-45-templates.json (the render
|
||||
// gate). Read-only otherwise (no DB writes, no Hoffman writes).
|
||||
process.env.DSCA_VALIDATE_MODE = '1'; // open the render gate for the compare
|
||||
const fs = require('fs');
|
||||
const https = require('https');
|
||||
const db = require('./database/db');
|
||||
const { renderContent } = require('./database/render-datasheet');
|
||||
|
||||
const TPL_PATH = process.env.DSCA_TPL || './dsca33-45-templates.json';
|
||||
const CREDS_PATH = 'C:\\ProgramData\\dataforth-uploader\\credentials.json';
|
||||
const APPLY = process.argv.includes('--apply');
|
||||
const only = process.argv.slice(2).filter(a => !a.startsWith('--'));
|
||||
|
||||
function creds() { return JSON.parse(fs.readFileSync(CREDS_PATH, 'utf8')); }
|
||||
function httpReq(method, uri, headers, body) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const u = new URL(uri);
|
||||
const req = https.request({ hostname: u.hostname, port: u.port || 443, path: u.pathname + u.search, method, headers, timeout: 30000 }, res => {
|
||||
let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve({ status: res.statusCode, body: JSON.parse(d) }); } catch { resolve({ status: res.statusCode, body: { _raw: d } }); } });
|
||||
});
|
||||
req.on('error', reject); req.on('timeout', () => req.destroy(new Error('timeout')));
|
||||
if (body) req.write(body); req.end();
|
||||
});
|
||||
}
|
||||
async function getToken() {
|
||||
const c = creds();
|
||||
const form = Object.entries({ grant_type: 'client_credentials', client_id: c.CF_CLIENT_ID, client_secret: c.CF_CLIENT_SECRET, scope: c.CF_SCOPE })
|
||||
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&');
|
||||
const r = await httpReq('POST', c.CF_TOKEN_URL, { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(form) }, form);
|
||||
if (r.status !== 200 || !r.body.access_token) throw new Error('token fail ' + r.status);
|
||||
return r.body.access_token;
|
||||
}
|
||||
async function fetchOriginal(token, serial) {
|
||||
const c = creds();
|
||||
const r = await httpReq('GET', `${c.CF_API_BASE}/api/v1/TestReportDataFiles/${encodeURIComponent(serial)}`, { Authorization: 'Bearer ' + token });
|
||||
if (r.status !== 200) return null;
|
||||
return r.body && r.body.Content ? r.body.Content : null;
|
||||
}
|
||||
// content-normalize: collapse whitespace per line; DROP rule lines (pure separators —
|
||||
// runs of = ~ _ -) and blank lines. Rule lines carry no content and their
|
||||
// presence/position is the deferred cosmetic gap (e.g. the leading === letterhead line
|
||||
// the originals have and our renders omit), so removing them isolates real content.
|
||||
function norm(s) {
|
||||
return s.replace(/\r/g, '').split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(t => t.length > 0 && !(/^[=~_\- ]+$/.test(t) && /[=~_\-]/.test(t)))
|
||||
.map(t => t.replace(/\s+/g, ' '));
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const tpl = JSON.parse(fs.readFileSync(TPL_PATH, 'utf8'));
|
||||
const models = (only.length ? only : Object.keys(tpl)).filter(m => tpl[m]);
|
||||
const token = await getToken();
|
||||
const pass = [], fail = [], noOracle = [], noRec = [];
|
||||
for (const m of models) {
|
||||
const sn = tpl[m]._srcSerial;
|
||||
if (!sn) { noOracle.push(m); continue; }
|
||||
const original = await fetchOriginal(token, sn);
|
||||
if (!original) { noOracle.push(m + '(no Hoffman ' + sn + ')'); continue; }
|
||||
const rec = await db.queryOne('SELECT * FROM test_records WHERE serial_number=$1 AND model_number=$2 LIMIT 1', [sn, m]);
|
||||
if (!rec) { noRec.push(m + '(' + sn + ')'); continue; }
|
||||
let rendered; try { rendered = renderContent(rec); } catch (e) { rendered = null; }
|
||||
if (!rendered) { fail.push({ m, sn, reason: 'render null' }); continue; }
|
||||
const a = norm(rendered), b = norm(original);
|
||||
let diff = -1; const mx = Math.max(a.length, b.length);
|
||||
for (let i = 0; i < mx; i++) { if (a[i] !== b[i]) { diff = i; break; } }
|
||||
if (diff === -1) pass.push(m);
|
||||
else fail.push({ m, sn, line: diff, render: a[diff], golden: b[diff] });
|
||||
}
|
||||
console.log('\n=== DSCA33/45 Hoffman validation ===');
|
||||
console.log('PASS (' + pass.length + '): ' + pass.join(', '));
|
||||
console.log('\nFAIL (' + fail.length + '):');
|
||||
for (const f of fail) {
|
||||
if (f.reason) { console.log(' ' + f.m + ' (' + f.sn + '): ' + f.reason); continue; }
|
||||
console.log(' ' + f.m + ' (' + f.sn + ') first diff L' + f.line);
|
||||
console.log(' render: ' + JSON.stringify(f.render));
|
||||
console.log(' golden: ' + JSON.stringify(f.golden));
|
||||
}
|
||||
if (noOracle.length) console.log('\nNO ORACLE: ' + noOracle.join(', '));
|
||||
if (noRec.length) console.log('NO DB REC: ' + noRec.join(', '));
|
||||
|
||||
if (APPLY) {
|
||||
const passSet = new Set(pass);
|
||||
for (const m of Object.keys(tpl)) {
|
||||
if (passSet.has(m)) tpl[m].validated = true;
|
||||
else if (only.length === 0) delete tpl[m].validated; // full run: clear stale
|
||||
}
|
||||
fs.writeFileSync(TPL_PATH, JSON.stringify(tpl));
|
||||
console.log('\n[APPLY] marked validated on ' + pass.length + ' models in ' + TPL_PATH);
|
||||
} else {
|
||||
console.log('\n(dry run — pass --apply to mark validated)');
|
||||
}
|
||||
await db.close();
|
||||
})().catch(e => { console.error('ERR', e.message, e.stack); process.exit(1); });
|
||||
Reference in New Issue
Block a user