dataforth(datasheet): Fix 2 STAGE 3 — DSCA render validator + first full report
validate-dsca-stage3.js: read-only harness that, for every staged DSCA original
we have ground truth for (2806 across 126 models), looks up the DB record,
renders it through the live path, and content-compares. GATE = the FINAL TEST
RESULTS section (rule lines canonicalized, whitespace collapsed — so the deferred
column-spacing cosmetic doesn't register); accuracy-section diffs reported
separately as informational.
First run verdict (report attached):
- 68 models FINAL-TEST CONTENT-CLEAN (0 mismatch over compared certs).
- 2123/2316 certs match the Final-Test content exactly (91.7%).
- 26 models show measured-value last-digit diffs only — structure (names, specs,
row alignment, statuses) is correct. Two root causes, neither structural:
* rounding-mode: JS double toFixed vs QB single-precision half-up
(e.g. raw 9.9995 code3 -> "9.999" here, "10.000" in golden). Fixable but
float-precision-sensitive; risks regressing currently-clean values.
* data vintage: staged .TXT is a different test run than the DB latest-wins
record (Fix 3) — e.g. Supply Current 19.6 vs 20.3, 0.7 apart. Not a render
bug; can't be reconciled against an older staged sheet.
- ~32 models render null (count-guard): DSCA33-*, DSCA45-*, DSCA49-* families
whose raw_data carries load points the template omits -> need per-subtype slot
mapping (the canonical-slot approach) before they can render.
Still NOT published: service not restarted, nothing re-pushed to Hoffman.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
171
projects/dataforth-dos/tools/validate-dsca-stage3.js
Normal file
171
projects/dataforth-dos/tools/validate-dsca-stage3.js
Normal file
@@ -0,0 +1,171 @@
|
||||
// Fix 2 STAGE 3 — per-subtype byte-validation of DSCA renders vs staged originals.
|
||||
// For every staged DSCA .TXT we have ground truth for, look up the DB record,
|
||||
// render it through the live render path, and content-normalize-compare the two.
|
||||
// Grouped by model (layout). Whitespace is collapsed per line (column spacing is
|
||||
// the deferred cosmetic gap) so the compare tests CONTENT — names, values, specs,
|
||||
// statuses — not pixel alignment. Read-only; no DB writes, no Hoffman push.
|
||||
//
|
||||
// Usage: node _validate_dsca_stage3.js [--limit-per-model N] [--report path]
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database/db');
|
||||
const { renderContent } = require('./database/render-datasheet');
|
||||
|
||||
const STAGE = 'C:/Shares/test/STAGE';
|
||||
const args = process.argv.slice(2);
|
||||
const LIMIT = (() => { const i = args.indexOf('--limit-per-model'); return i >= 0 ? parseInt(args[i + 1], 10) : Infinity; })();
|
||||
const REPORT = (() => { const i = args.indexOf('--report'); return i >= 0 ? args[i + 1] : 'C:/Shares/testdatadb/_dsca-stage3-report.txt'; })();
|
||||
|
||||
function walk(d, out) {
|
||||
let it = [];
|
||||
try { it = fs.readdirSync(d, { withFileTypes: true }); } catch { return out; }
|
||||
for (const e of it) {
|
||||
const p = path.join(d, e.name);
|
||||
if (e.isDirectory()) walk(p, out);
|
||||
else if (/\.txt$/i.test(e.name)) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Content-normalize a single line: trim + collapse whitespace. Rule lines (runs
|
||||
// of = - ~ _ separated by spaces) canonicalize to <RULE> so the deferred cosmetic
|
||||
// dash/equal-count differences don't register as content diffs.
|
||||
function normLine(l) {
|
||||
const t = l.trim();
|
||||
if (t.length && /^[=~_\- ]+$/.test(t) && /[=~_\-]/.test(t)) return '<RULE>';
|
||||
return t.replace(/\s+/g, ' ');
|
||||
}
|
||||
function norm(s) {
|
||||
return s.replace(/\r/g, '').split('\n').map(normLine).filter(l => l.length > 0);
|
||||
}
|
||||
|
||||
// Extract the FINAL TEST RESULTS section: from its header to just before the
|
||||
// footer underline (___). This is the Fix 2 deliverable; compared content-strict.
|
||||
function finalTestLines(s) {
|
||||
const L = s.replace(/\r/g, '').split('\n');
|
||||
const fi = L.findIndex(l => /FINAL TEST RESULTS/.test(l));
|
||||
if (fi < 0) return [];
|
||||
const out = [];
|
||||
for (let i = fi; i < L.length; i++) {
|
||||
const t = L[i].trim();
|
||||
if (i > fi && /^_{5,}$/.test(t)) break; // footer underline
|
||||
if (/It is hereby certified/.test(t)) break;
|
||||
out.push(normLine(L[i]));
|
||||
}
|
||||
return out.filter(l => l.length > 0);
|
||||
}
|
||||
|
||||
// Accuracy section lines (informational — spacing + any calc rounding live here).
|
||||
function accuracyLines(s) {
|
||||
const L = s.replace(/\r/g, '').split('\n');
|
||||
const ai = L.findIndex(l => /ACCURACY TEST/.test(l));
|
||||
if (ai < 0) return [];
|
||||
const fi = L.findIndex(l => /FINAL TEST RESULTS/.test(l));
|
||||
const end = fi < 0 ? L.length : fi;
|
||||
return L.slice(ai, end).map(normLine).filter(l => l.length > 0);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const files = walk(STAGE, []);
|
||||
// index staged DSCA files: model + SN + text
|
||||
const staged = [];
|
||||
for (const f of files) {
|
||||
let t; try { t = fs.readFileSync(f, 'utf8'); } catch { continue; }
|
||||
const model = (t.match(/^\s*Model:\s*(\S+)/m) || [])[1] || '';
|
||||
if (!/^DSCA/i.test(model)) continue;
|
||||
const sn = (t.match(/^\s*SN:\s*(\S+)/m) || [])[1] || '';
|
||||
if (!sn) continue;
|
||||
staged.push({ f, model: model.trim(), sn: sn.trim(), text: t });
|
||||
}
|
||||
console.log(`staged DSCA originals: ${staged.length}`);
|
||||
|
||||
const byModel = {}; // model -> tallies
|
||||
function rec(model) {
|
||||
if (!byModel[model]) byModel[model] = { compared: 0, ftMatch: 0, ftMismatch: 0, accMismatch: 0, noRecord: 0, notRendered: 0, samples: [] };
|
||||
return byModel[model];
|
||||
}
|
||||
function firstDiff(a, b) {
|
||||
const max = Math.max(a.length, b.length);
|
||||
for (let i = 0; i < max; i++) if (a[i] !== b[i]) return i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const seenPerModel = {};
|
||||
for (const s of staged) {
|
||||
seenPerModel[s.model] = (seenPerModel[s.model] || 0) + 1;
|
||||
if (seenPerModel[s.model] > LIMIT) continue;
|
||||
const r = rec(s.model);
|
||||
let row = await db.queryOne('SELECT * FROM test_records WHERE serial_number=$1 AND model_number=$2 LIMIT 1', [s.sn, s.model]);
|
||||
if (!row) row = await db.queryOne('SELECT * FROM test_records WHERE raw_serial_number=$1 AND model_number=$2 LIMIT 1', [s.sn, s.model]);
|
||||
if (!row) { r.noRecord++; continue; }
|
||||
let rendered;
|
||||
try { rendered = renderContent(row); } catch (e) { rendered = null; }
|
||||
if (!rendered) { r.notRendered++; continue; }
|
||||
r.compared++;
|
||||
// GATE: Final-Test section content must match exactly (rules canonicalized).
|
||||
const ftA = finalTestLines(rendered), ftB = finalTestLines(s.text);
|
||||
const fd = firstDiff(ftA, ftB);
|
||||
if (fd === -1) r.ftMatch++;
|
||||
else {
|
||||
r.ftMismatch++;
|
||||
if (r.samples.length < 3) r.samples.push({ sn: s.sn, line: fd, render: ftA[fd], golden: ftB[fd] });
|
||||
}
|
||||
// INFO: accuracy section (deferred cosmetic spacing + any calc rounding).
|
||||
const acA = accuracyLines(rendered), acB = accuracyLines(s.text);
|
||||
if (firstDiff(acA, acB) !== -1) r.accMismatch++;
|
||||
}
|
||||
|
||||
// report
|
||||
const models = Object.keys(byModel).sort();
|
||||
let totC = 0, totFM = 0, totFMM = 0, totAcc = 0, totNR = 0, totNRn = 0, cleanModels = 0;
|
||||
const ftDirty = [];
|
||||
for (const m of models) {
|
||||
const x = byModel[m];
|
||||
totC += x.compared; totFM += x.ftMatch; totFMM += x.ftMismatch; totAcc += x.accMismatch;
|
||||
totNR += x.noRecord; totNRn += x.notRendered;
|
||||
if (x.compared > 0 && x.ftMismatch === 0) cleanModels++;
|
||||
if (x.ftMismatch > 0) ftDirty.push(m);
|
||||
}
|
||||
const out = [];
|
||||
out.push('Fix 2 STAGE 3 — DSCA Final-Test render vs staged-original content validation');
|
||||
out.push('GATE = FINAL TEST RESULTS section, content-strict (rule lines canonicalized,');
|
||||
out.push('whitespace collapsed). Accuracy-section diffs reported separately (deferred');
|
||||
out.push('cosmetic spacing + any pre-existing calc rounding — NOT a Fix 2 gate).');
|
||||
out.push('Corpus: ' + staged.length + ' staged DSCA originals across ' + models.length + ' models.');
|
||||
out.push('='.repeat(78));
|
||||
out.push('');
|
||||
out.push('SUMMARY');
|
||||
out.push(' models with staged originals: ' + models.length);
|
||||
out.push(' models FINAL-TEST CLEAN (>=1 compared, 0 mismatch): ' + cleanModels);
|
||||
out.push(' models with FINAL-TEST mismatches: ' + ftDirty.length);
|
||||
out.push(' certs compared: ' + totC);
|
||||
out.push(' Final-Test match: ' + totFM);
|
||||
out.push(' Final-Test mismatch: ' + totFMM);
|
||||
out.push(' (certs with accuracy-section diffs: ' + totAcc + ' — informational)');
|
||||
out.push(' staged serials not in DB: ' + totNR);
|
||||
out.push(' in DB but not rendered (skipped/null): ' + totNRn);
|
||||
out.push('');
|
||||
out.push('MODELS WITH FINAL-TEST CONTENT MISMATCHES (investigate before re-push):');
|
||||
if (!ftDirty.length) out.push(' (none — Final-Test renders are content-clean for all compared models)');
|
||||
for (const m of ftDirty) {
|
||||
const x = byModel[m];
|
||||
out.push(' ' + m + ' compared=' + x.compared + ' ftMatch=' + x.ftMatch + ' ftMismatch=' + x.ftMismatch);
|
||||
for (const s of x.samples) {
|
||||
out.push(' [finaltest L' + s.line + '] SN ' + s.sn);
|
||||
out.push(' render: ' + JSON.stringify(s.render));
|
||||
out.push(' golden: ' + JSON.stringify(s.golden));
|
||||
}
|
||||
}
|
||||
out.push('');
|
||||
out.push('FINAL-TEST CLEAN MODELS (' + cleanModels + '):');
|
||||
out.push(' ' + models.filter(m => byModel[m].compared > 0 && byModel[m].ftMismatch === 0).join(', '));
|
||||
out.push('');
|
||||
out.push('MODELS WITH NO COMPARABLE CERT (no staged serial in DB, or all skipped/null):');
|
||||
out.push(' ' + models.filter(m => byModel[m].compared === 0).map(m => m + '(' + (byModel[m].notRendered ? 'null' : 'noDBrec') + ')').join(', '));
|
||||
|
||||
const text = out.join('\n');
|
||||
fs.writeFileSync(REPORT, text);
|
||||
console.log(text);
|
||||
console.log('\n[report written] ' + REPORT);
|
||||
await db.close();
|
||||
})().catch(e => { console.error('ERR', e.message, e.stack); process.exit(1); });
|
||||
Reference in New Issue
Block a user