Add SCMVAS/SCMHVAS datasheet pipeline extension (Dataforth)
Extends the Test Datasheet Pipeline on AD2:C:\Shares\testdatadb to
generate web-published datasheets for the SCMVAS-Mxxx (obsolete) and
SCMHVAS-Mxxxx (replacement) High Voltage Input Module product lines.
Both are tested either with the existing TESTHV3 software (production
VASLOG .DAT logs) or in Engineering with plain .txt output.
Key changes on AD2 (all deployed 2026-04-12 with dated backups):
- parsers/spec-reader.js: getSpecs() returns a `{_family:'SCMVAS',
_noSpecs:true}` sentinel for SCMVAS/SCMHVAS/VAS-M/HVAS-M model prefixes
so the export pipeline does not silently skip them for missing specs.
- templates/datasheet-exact.js: new Accuracy-only template branch
(generateSCMVASDatasheet + helpers) that mirrors the existing shipped
format byte-for-byte. Extraction regex covers both QuickBASIC STR$()
output formats: scientific-with-trailing-status-digit (98.4% of
records) and plain-decimal (1.6% of records above QB's threshold).
- parsers/vaslog-engtxt.js (new): parses the Engineering-Tested .txt
files in TS-3R\LOGS\VASLOG\VASLOG - Engineering Tested\. Filename SN
regex strips optional trailing 14-digit timestamp; in-file "SN:"
header is the authoritative source when the filename is malformed.
- database/import.js: LOG_TYPES grows a VASLOG_ENG entry with
subfolder + recursive flags. Pre-existing 7 log types keep their
implicit recursive=true behaviour (config.recursive !== false).
importFiles() routes VASLOG_ENG paths before the generic loop so a
VASLOG - Engineering Tested/*.txt path does not mis-dispatch to the
multiline parser.
- database/export-datasheets.js: VASLOG_ENG records are written
verbatim via fs.copyFileSync(source_file, For_Web/<SN>.TXT) for true
byte-level pass-through, with a graceful raw_data fallback when the
source file is no longer on disk.
Deploy outcome:
- 27,503 SCMVAS/SCMHVAS datasheets rendered (27,065 from scientific +
438 from plain-decimal PASS lines, post-patch rerun)
- 434 Engineering-Tested .txt files pass-through-copied to For_Web
- 0 errors across both batches
Repo layout added here:
- scmvas-hvas-research/: discovery artifacts (source .BAS, hvin.dat,
sample .DAT + .txt, binary-format notes, IMPLEMENTATION_PLAN.md)
- implementation/: staged final code + deploy helpers + local test
harness + per-step verification scripts
- backups/pre-deploy-20260412/: independent local snapshot of the 4
AD2 files replaced, pulled byte-for-byte before deploy
All helper scripts fetch the AD2 password at runtime from the SOPS
vault (clients/dataforth/ad2.sops.yaml). None of the committed files
contain the plaintext credential. Known vault-entry hygiene issue
(stale shell-escape backslash before the `!`) is documented in the
fetcher comments and stripped at read-time; flagged separately for
cleanup.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
"""Probe testdatadb API on port 3000 of AD2 via SSH tunnel hop."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print('=== root HTTP probe ===')
|
||||
out, err, rc = ps(c, r'try { $r = Invoke-WebRequest -Uri "http://localhost:3000/" -UseBasicParsing -TimeoutSec 10; Write-Host ("HTTP " + $r.StatusCode + " len=" + $r.Content.Length) } catch { Write-Host ("HTTP FAIL: " + $_.Exception.Message) }')
|
||||
print(out)
|
||||
|
||||
print('=== /api/search probe (hit live DB) ===')
|
||||
out, err, rc = ps(c, r'try { $r = Invoke-WebRequest -Uri "http://localhost:3000/api/search?limit=1" -UseBasicParsing -TimeoutSec 20; Write-Host ("HTTP " + $r.StatusCode + " len=" + $r.Content.Length); Write-Host ($r.Content.Substring(0, [math]::Min(300, $r.Content.Length))) } catch { Write-Host ("HTTP FAIL: " + $_.Exception.Message) }')
|
||||
print(out)
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Backfill SCMVAS/SCMHVAS datasheets to \\ad2\webshare\For_Web.
|
||||
|
||||
Deploys a one-off node script that:
|
||||
- Queries PASS records with NULL forweb_exported_at AND (SCMVAS/SCMHVAS/VAS-M/HVAS-M
|
||||
model OR log_type=VASLOG_ENG)
|
||||
- For VASLOG_ENG: copies source .txt verbatim to For_Web\<SN>.TXT (pass-through)
|
||||
- For VASLOG SCMVAS/SCMHVAS: runs generateExactDatasheet and writes
|
||||
- Updates forweb_exported_at per batch
|
||||
|
||||
Runs in --dry-run mode by default; pass --go to actually write. Also supports
|
||||
--limit N to cap.
|
||||
"""
|
||||
import argparse, base64, subprocess, sys, yaml, paramiko
|
||||
|
||||
HOST='192.168.0.6'; USER='sysadmin'
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database/db');
|
||||
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('./templates/datasheet-exact');
|
||||
|
||||
const OUTPUT_DIR = '\\\\ad2\\webshare\\For_Web';
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const dry = args.includes('--dry-run');
|
||||
const limitIdx = args.indexOf('--limit');
|
||||
const limit = limitIdx >= 0 ? parseInt(args[limitIdx + 1], 10) : 0;
|
||||
|
||||
console.log('[INFO] output: ' + OUTPUT_DIR);
|
||||
console.log('[INFO] dry-run: ' + dry);
|
||||
console.log('[INFO] limit: ' + (limit || 'none'));
|
||||
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
console.error('[FAIL] output dir not reachable: ' + OUTPUT_DIR);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('[INFO] loading specs...');
|
||||
const specMap = loadAllSpecs();
|
||||
|
||||
const where = [
|
||||
"overall_result = 'PASS'",
|
||||
"forweb_exported_at IS NULL",
|
||||
"((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%') OR log_type = 'VASLOG_ENG')"
|
||||
].join(' AND ');
|
||||
let sql = 'SELECT * FROM test_records WHERE ' + where + ' ORDER BY test_date DESC';
|
||||
if (limit > 0) sql += ' LIMIT ' + limit;
|
||||
|
||||
const rows = await db.query(sql);
|
||||
console.log('[INFO] ' + rows.length + ' records to process');
|
||||
|
||||
let exported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let passthrough = 0;
|
||||
let rendered = 0;
|
||||
const batchIds = [];
|
||||
const BATCH = 200;
|
||||
|
||||
async function flush() {
|
||||
if (batchIds.length === 0) return;
|
||||
if (dry) { batchIds.length = 0; return; }
|
||||
const now = new Date().toISOString();
|
||||
await db.transaction(async (tx) => {
|
||||
for (const id of batchIds) {
|
||||
await tx.execute('UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2', [now, id]);
|
||||
}
|
||||
});
|
||||
batchIds.length = 0;
|
||||
}
|
||||
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const record = rows[i];
|
||||
try {
|
||||
const outPath = path.join(OUTPUT_DIR, record.serial_number + '.TXT');
|
||||
|
||||
if (record.log_type === 'VASLOG_ENG') {
|
||||
if (record.source_file && fs.existsSync(record.source_file)) {
|
||||
if (!dry) fs.copyFileSync(record.source_file, outPath);
|
||||
passthrough++;
|
||||
} else {
|
||||
if (!dry) fs.writeFileSync(outPath, record.raw_data || '', 'utf8');
|
||||
passthrough++;
|
||||
}
|
||||
} else {
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) { skipped++; continue; }
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) { skipped++; continue; }
|
||||
if (!dry) fs.writeFileSync(outPath, txt, 'utf8');
|
||||
rendered++;
|
||||
}
|
||||
|
||||
batchIds.push(record.id);
|
||||
exported++;
|
||||
|
||||
if (batchIds.length >= BATCH) {
|
||||
await flush();
|
||||
process.stdout.write('[PROGRESS] ' + exported + '/' + rows.length + '\n');
|
||||
}
|
||||
} catch (e) {
|
||||
errors++;
|
||||
console.error('[ERR] ' + record.serial_number + ': ' + e.message);
|
||||
}
|
||||
}
|
||||
await flush();
|
||||
|
||||
console.log('');
|
||||
console.log('========================================');
|
||||
console.log('Backfill Complete' + (dry ? ' (DRY RUN)' : ''));
|
||||
console.log('========================================');
|
||||
console.log('Processed: ' + exported);
|
||||
console.log(' rendered: ' + rendered);
|
||||
console.log(' passthrough: ' + passthrough);
|
||||
console.log('Skipped: ' + skipped);
|
||||
console.log('Errors: ' + errors);
|
||||
await db.close();
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('[FATAL] ' + e.message); process.exit(1); });
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=7200):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--go', action='store_true', help='actually write (default is dry-run)')
|
||||
ap.add_argument('--limit', type=int, default=0, help='cap records processed')
|
||||
args = ap.parse_args()
|
||||
dry = not args.go
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_backfill_scmvas.js'
|
||||
with sftp.open(remote,'w') as fh:
|
||||
fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
print(f'[OK] deployed {remote}', flush=True)
|
||||
|
||||
flags = ['--dry-run'] if dry else []
|
||||
if args.limit > 0:
|
||||
flags += ['--limit', str(args.limit)]
|
||||
cmd = r'cd C:\Shares\testdatadb; & node ./_backfill_scmvas.js ' + ' '.join(flags)
|
||||
print(f'[RUN] {cmd}', flush=True)
|
||||
out, err, rc = ps(c, cmd)
|
||||
print(f'[rc={rc}]', flush=True)
|
||||
print(out, flush=True)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('--- STDERR ---', flush=True)
|
||||
print(err[:2000], flush=True)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Count the backlog for task #12 backfill + confirm X: access context."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const total = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL"
|
||||
);
|
||||
console.log('Total PASS backlog: ' + total.c);
|
||||
|
||||
const vaslog = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL AND log_type='VASLOG'"
|
||||
);
|
||||
console.log(' of which VASLOG (production .DAT): ' + vaslog.c);
|
||||
|
||||
const vaslog_eng = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL AND log_type='VASLOG_ENG'"
|
||||
);
|
||||
console.log(' of which VASLOG_ENG (Eng .txt): ' + vaslog_eng.c);
|
||||
|
||||
const scmvas = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%')"
|
||||
);
|
||||
console.log('SCMVAS/SCMHVAS/VAS-M/HVAS-M backlog: ' + scmvas.c);
|
||||
|
||||
const bymodel = await db.query(
|
||||
"SELECT model_number, log_type, COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%') " +
|
||||
"GROUP BY model_number, log_type ORDER BY c DESC"
|
||||
);
|
||||
console.log('By model:');
|
||||
for (const r of bymodel) console.log(' ' + r.model_number.padEnd(18) + ' ' + (r.log_type||'').padEnd(12) + ' ' + r.c);
|
||||
|
||||
await db.close();
|
||||
})().catch(e => { console.error('FAIL: ' + e.message); process.exit(1); });
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_backlog_probe.js'
|
||||
with sftp.open(remote,'w') as fh:
|
||||
fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_backlog_probe.js')
|
||||
print(out)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('STDERR:', err[:500])
|
||||
|
||||
# Check X: access path resolution from service account's perspective
|
||||
print('\n=== X: drive / UNC resolution ===')
|
||||
out, err, rc = ps(c, r'Get-PSDrive -Name X -ErrorAction SilentlyContinue | Format-Table Name,Root -AutoSize; Get-SmbMapping | Where-Object { $_.LocalPath -match "X:" } | Format-Table -AutoSize')
|
||||
print(out)
|
||||
|
||||
# Check testdatadb service account identity
|
||||
print('=== testdatadb service identity ===')
|
||||
out, err, rc = ps(c, r'Get-WmiObject -Class Win32_Service -Filter "Name=''testdatadb''" | Select Name,StartName,State,PathName | Format-List')
|
||||
print(out)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try:
|
||||
sftp.remove(remote)
|
||||
except Exception:
|
||||
pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Post-deploy health check for testdatadb on AD2.
|
||||
|
||||
Restart the Windows service, then curl the API and confirm it returns 200.
|
||||
Log any startup errors.
|
||||
"""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print('=== service list ===')
|
||||
out, err, rc = ps(c, 'Get-Service | Where-Object { $_.Name -match "testdata|testdb" } | Select Name,Status,DisplayName | Format-Table -AutoSize | Out-String')
|
||||
print(out)
|
||||
|
||||
print('=== node syntax-check the 5 deployed files ===')
|
||||
out, err, rc = ps(c, r'''
|
||||
$files = @(
|
||||
'C:\Shares\testdatadb\parsers\spec-reader.js',
|
||||
'C:\Shares\testdatadb\parsers\vaslog-engtxt.js',
|
||||
'C:\Shares\testdatadb\templates\datasheet-exact.js',
|
||||
'C:\Shares\testdatadb\database\import.js',
|
||||
'C:\Shares\testdatadb\database\export-datasheets.js'
|
||||
)
|
||||
foreach ($f in $files) {
|
||||
$r = & node --check $f 2>&1
|
||||
if ($LASTEXITCODE -eq 0) { Write-Host "[OK] $f" } else { Write-Host "[FAIL] $f : $r" }
|
||||
}
|
||||
''')
|
||||
print(out)
|
||||
if err: print('STDERR:', err[:500])
|
||||
|
||||
print('=== quick require-load test (no handlers invoked) ===')
|
||||
out, err, rc = ps(c, r'''
|
||||
$script = @'
|
||||
try {
|
||||
require("C:/Shares/testdatadb/parsers/spec-reader.js");
|
||||
console.log("[OK] spec-reader");
|
||||
require("C:/Shares/testdatadb/parsers/vaslog-engtxt.js");
|
||||
console.log("[OK] vaslog-engtxt");
|
||||
require("C:/Shares/testdatadb/templates/datasheet-exact.js");
|
||||
console.log("[OK] datasheet-exact");
|
||||
} catch (e) { console.log("[FAIL] " + e.message); process.exit(1); }
|
||||
'@
|
||||
$script | Out-File -FilePath $env:TEMP\loadtest.js -Encoding ascii
|
||||
& node $env:TEMP\loadtest.js
|
||||
''')
|
||||
print(out)
|
||||
if err: print('STDERR:', err[:500])
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,257 @@
|
||||
/**
|
||||
* Export Datasheets
|
||||
*
|
||||
* Generates TXT datasheets for unexported PASS records and writes them to X:\For_Web\.
|
||||
* Updates forweb_exported_at after successful export.
|
||||
*
|
||||
* Usage:
|
||||
* node export-datasheets.js Export all pending (batch mode)
|
||||
* node export-datasheets.js --limit 100 Export up to 100 records
|
||||
* node export-datasheets.js --file <paths> Export records matching specific source files
|
||||
* node export-datasheets.js --serial 178439-1 Export a specific serial number
|
||||
* node export-datasheets.js --dry-run Show what would be exported without writing
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./db');
|
||||
|
||||
const { loadAllSpecs, getSpecs } = require('../parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('../templates/datasheet-exact');
|
||||
|
||||
// Configuration
|
||||
const OUTPUT_DIR = 'X:\\For_Web';
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
async function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const limitIdx = args.indexOf('--limit');
|
||||
const limit = limitIdx >= 0 ? parseInt(args[limitIdx + 1]) : 0;
|
||||
const serialIdx = args.indexOf('--serial');
|
||||
const serial = serialIdx >= 0 ? args[serialIdx + 1] : null;
|
||||
const fileIdx = args.indexOf('--file');
|
||||
const files = fileIdx >= 0 ? args.slice(fileIdx + 1).filter(f => !f.startsWith('--')) : null;
|
||||
|
||||
console.log('========================================');
|
||||
console.log('Datasheet Export');
|
||||
console.log('========================================');
|
||||
console.log(`Output: ${OUTPUT_DIR}`);
|
||||
console.log(`Dry run: ${dryRun}`);
|
||||
if (limit) console.log(`Limit: ${limit}`);
|
||||
if (serial) console.log(`Serial: ${serial}`);
|
||||
console.log(`Start: ${new Date().toISOString()}`);
|
||||
|
||||
if (!dryRun && !fs.existsSync(OUTPUT_DIR)) {
|
||||
console.error(`ERROR: Output directory does not exist: ${OUTPUT_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('\nLoading model specs...');
|
||||
const specMap = loadAllSpecs();
|
||||
|
||||
// Build query
|
||||
const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
if (serial) {
|
||||
paramIdx++;
|
||||
conditions.push(`serial_number = $${paramIdx}`);
|
||||
params.push(serial);
|
||||
}
|
||||
|
||||
if (files && files.length > 0) {
|
||||
const placeholders = files.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
|
||||
conditions.push(`source_file IN (${placeholders})`);
|
||||
params.push(...files);
|
||||
}
|
||||
|
||||
let sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')} ORDER BY test_date DESC`;
|
||||
|
||||
if (limit) {
|
||||
paramIdx++;
|
||||
sql += ` LIMIT $${paramIdx}`;
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
const records = await db.query(sql, params);
|
||||
console.log(`\nFound ${records.length} records to export`);
|
||||
|
||||
if (records.length === 0) {
|
||||
console.log('Nothing to export.');
|
||||
await db.close();
|
||||
return { exported: 0, skipped: 0, errors: 0 };
|
||||
}
|
||||
|
||||
let exported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let noSpecs = 0;
|
||||
let pendingUpdates = [];
|
||||
|
||||
for (const record of records) {
|
||||
try {
|
||||
const filename = record.serial_number + '.TXT';
|
||||
const outputPath = path.join(OUTPUT_DIR, filename);
|
||||
|
||||
// VASLOG_ENG: verbatim byte-for-byte copy of the original file.
|
||||
// Using fs.copyFileSync avoids any utf-8 round-trip that would
|
||||
// corrupt non-ASCII bytes (CP1252 etc.) in customer datasheets.
|
||||
// Fall back to writing raw_data if the source file is gone.
|
||||
if (record.log_type === 'VASLOG_ENG') {
|
||||
if (dryRun) {
|
||||
console.log(` [DRY RUN] Would copy: ${record.source_file} -> ${filename}`);
|
||||
exported++;
|
||||
continue;
|
||||
}
|
||||
if (record.source_file && fs.existsSync(record.source_file)) {
|
||||
fs.copyFileSync(record.source_file, outputPath);
|
||||
} else {
|
||||
console.warn(`[WARN] source file missing, writing decoded raw_data for ${record.serial_number}`);
|
||||
if (!record.raw_data) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
fs.writeFileSync(outputPath, record.raw_data, 'utf8');
|
||||
}
|
||||
pendingUpdates.push(record.id);
|
||||
exported++;
|
||||
|
||||
if (pendingUpdates.length >= BATCH_SIZE) {
|
||||
await flushUpdates(pendingUpdates);
|
||||
pendingUpdates = [];
|
||||
process.stdout.write(`\r Exported: ${exported} / ${records.length}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Template-generated datasheet path.
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) {
|
||||
noSpecs++;
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
console.log(` [DRY RUN] Would write: ${filename}`);
|
||||
exported++;
|
||||
} else {
|
||||
fs.writeFileSync(outputPath, txt, 'utf8');
|
||||
pendingUpdates.push(record.id);
|
||||
exported++;
|
||||
|
||||
// Batch commit
|
||||
if (pendingUpdates.length >= BATCH_SIZE) {
|
||||
await flushUpdates(pendingUpdates);
|
||||
pendingUpdates = [];
|
||||
process.stdout.write(`\r Exported: ${exported} / ${records.length}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`\n ERROR exporting ${record.serial_number}: ${err.message}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining updates
|
||||
if (pendingUpdates.length > 0) {
|
||||
await flushUpdates(pendingUpdates);
|
||||
}
|
||||
|
||||
console.log(`\n\n========================================`);
|
||||
console.log(`Export Complete`);
|
||||
console.log(`========================================`);
|
||||
console.log(`Exported: ${exported}`);
|
||||
console.log(`Skipped: ${skipped} (${noSpecs} missing specs)`);
|
||||
console.log(`Errors: ${errors}`);
|
||||
console.log(`End: ${new Date().toISOString()}`);
|
||||
|
||||
await db.close();
|
||||
return { exported, skipped, errors };
|
||||
}
|
||||
|
||||
async function flushUpdates(ids) {
|
||||
const now = new Date().toISOString();
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const id of ids) {
|
||||
await txClient.execute(
|
||||
'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
|
||||
[now, id]
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Export function for use by import.js (no db argument -- uses shared pool)
|
||||
async function exportNewRecords(specMap, filePaths) {
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
console.log(`[EXPORT] Output directory not found: ${OUTPUT_DIR}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
if (filePaths && filePaths.length > 0) {
|
||||
const placeholders = filePaths.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
|
||||
conditions.push(`source_file IN (${placeholders})`);
|
||||
params.push(...filePaths);
|
||||
}
|
||||
|
||||
const sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')}`;
|
||||
const records = await db.query(sql, params);
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
let exported = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const record of records) {
|
||||
const filename = record.serial_number + '.TXT';
|
||||
const outputPath = path.join(OUTPUT_DIR, filename);
|
||||
|
||||
try {
|
||||
// VASLOG_ENG: verbatim copy, preserving original bytes.
|
||||
if (record.log_type === 'VASLOG_ENG') {
|
||||
if (record.source_file && fs.existsSync(record.source_file)) {
|
||||
fs.copyFileSync(record.source_file, outputPath);
|
||||
} else {
|
||||
console.warn(`[WARN] source file missing, writing decoded raw_data for ${record.serial_number}`);
|
||||
if (!record.raw_data) continue;
|
||||
fs.writeFileSync(outputPath, record.raw_data, 'utf8');
|
||||
}
|
||||
} else {
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) continue;
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) continue;
|
||||
fs.writeFileSync(outputPath, txt, 'utf8');
|
||||
}
|
||||
|
||||
await txClient.execute(
|
||||
'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
|
||||
[new Date().toISOString(), record.id]
|
||||
);
|
||||
exported++;
|
||||
} catch (err) {
|
||||
console.error(`[EXPORT] Error writing ${filename}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[EXPORT] Generated ${exported} datasheet(s)`);
|
||||
return exported;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = { exportNewRecords };
|
||||
@@ -0,0 +1,396 @@
|
||||
/**
|
||||
* Data Import Script
|
||||
* Imports test data from DAT and SHT files into PostgreSQL database
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./db');
|
||||
|
||||
const { parseMultilineFile, extractTestStation } = require('../parsers/multiline');
|
||||
const { parseCsvFile } = require('../parsers/csvline');
|
||||
const { parseShtFile } = require('../parsers/shtfile');
|
||||
const { parseVaslogEngTxt } = require('../parsers/vaslog-engtxt');
|
||||
|
||||
// Data source paths
|
||||
const TEST_PATH = 'C:/Shares/test';
|
||||
const RECOVERY_PATH = 'C:/Shares/Recovery-TEST';
|
||||
const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS');
|
||||
|
||||
// Log types and their parsers.
|
||||
// NOTE: `recursive` defaults to TRUE when absent (walk subfolders by default,
|
||||
// preserving pre-existing production behavior for DSCLOG/5BLOG/8BLOG/PWRLOG/
|
||||
// SCTLOG/7BLOG). Set it to FALSE explicitly on VASLOG so the .DAT walk does
|
||||
// NOT descend into the "VASLOG - Engineering Tested" subfolder (belt-and-
|
||||
// suspenders: the .DAT glob wouldn't match .txt, but be explicit anyway).
|
||||
// VASLOG_ENG also sets recursive:false -- the eng-tested dir is flat.
|
||||
const LOG_TYPES = {
|
||||
'DSCLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'5BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'8BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'PWRLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'SCTLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'VASLOG': { parser: 'multiline', ext: '.DAT', recursive: false },
|
||||
'7BLOG': { parser: 'csvline', ext: '.DAT' },
|
||||
// Engineering-tested SCMHVAS pre-rendered datasheets live under VASLOG/"VASLOG - Engineering Tested"/
|
||||
'VASLOG_ENG': { parser: 'vaslog-engtxt', ext: '.txt', dir: 'VASLOG/VASLOG - Engineering Tested', recursive: false }
|
||||
};
|
||||
|
||||
// Find all files of a specific type in a directory
|
||||
function findFiles(dir, pattern, recursive = true) {
|
||||
const results = [];
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const items = fs.readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item.name);
|
||||
|
||||
if (item.isDirectory() && recursive) {
|
||||
results.push(...findFiles(fullPath, pattern, recursive));
|
||||
} else if (item.isFile()) {
|
||||
if (pattern.test(item.name)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Ignore permission errors
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Parse records from a file (sync -- file I/O only)
|
||||
function parseFile(filePath, logType, parser) {
|
||||
const testStation = extractTestStation(filePath);
|
||||
|
||||
switch (parser) {
|
||||
case 'multiline':
|
||||
return parseMultilineFile(filePath, logType, testStation);
|
||||
case 'csvline':
|
||||
return parseCsvFile(filePath, testStation);
|
||||
case 'shtfile':
|
||||
return parseShtFile(filePath, testStation);
|
||||
case 'vaslog-engtxt':
|
||||
return parseVaslogEngTxt(filePath, testStation);
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Batch insert records into PostgreSQL
|
||||
async function insertBatch(txClient, records) {
|
||||
let imported = 0;
|
||||
for (const record of records) {
|
||||
try {
|
||||
const result = await txClient.execute(
|
||||
`INSERT INTO test_records
|
||||
(log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (log_type, model_number, serial_number, test_date, test_station)
|
||||
DO UPDATE SET raw_data = EXCLUDED.raw_data, overall_result = EXCLUDED.overall_result`,
|
||||
[
|
||||
record.log_type,
|
||||
record.model_number,
|
||||
record.serial_number,
|
||||
record.test_date,
|
||||
record.test_station,
|
||||
record.overall_result,
|
||||
record.raw_data,
|
||||
record.source_file
|
||||
]
|
||||
);
|
||||
if (result.rowCount > 0) imported++;
|
||||
} catch (err) {
|
||||
// Constraint error - skip
|
||||
}
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
|
||||
// Import records from a file
|
||||
async function importFile(txClient, filePath, logType, parser) {
|
||||
let records = [];
|
||||
|
||||
try {
|
||||
records = parseFile(filePath, logType, parser);
|
||||
const imported = await insertBatch(txClient, records);
|
||||
return { total: records.length, imported };
|
||||
} catch (err) {
|
||||
console.error(`Error importing ${filePath}: ${err.message}`);
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// Import from HISTLOGS (master consolidated logs)
|
||||
async function importHistlogs(txClient) {
|
||||
console.log('\n=== Importing from HISTLOGS ===');
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const subdir = config.dir || logType;
|
||||
const logDir = path.join(HISTLOGS_PATH, subdir);
|
||||
|
||||
if (!fs.existsSync(logDir)) {
|
||||
console.log(` ${logType}: directory not found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false);
|
||||
console.log(` ${logType}: found ${files.length} files`);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from test station logs
|
||||
async function importStationLogs(txClient, basePath, label) {
|
||||
console.log(`\n=== Importing from ${label} ===`);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
const stationPattern = /^TS-\d+[LR]?$/i;
|
||||
let stations = [];
|
||||
|
||||
try {
|
||||
const items = fs.readdirSync(basePath, { withFileTypes: true });
|
||||
stations = items
|
||||
.filter(i => i.isDirectory() && stationPattern.test(i.name))
|
||||
.map(i => i.name);
|
||||
} catch (err) {
|
||||
console.log(` Error reading ${basePath}: ${err.message}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(` Found stations: ${stations.join(', ')}`);
|
||||
|
||||
for (const station of stations) {
|
||||
const logsDir = path.join(basePath, station, 'LOGS');
|
||||
|
||||
if (!fs.existsSync(logsDir)) continue;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const subdir = config.dir || logType;
|
||||
const logDir = path.join(logsDir, subdir);
|
||||
|
||||
if (!fs.existsSync(logDir)) continue;
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also import SHT files
|
||||
const shtFiles = findFiles(basePath, /\.SHT$/i, true);
|
||||
console.log(` Found ${shtFiles.length} SHT files`);
|
||||
|
||||
for (const file of shtFiles) {
|
||||
const { total, imported } = await importFile(txClient, file, 'SHT', 'shtfile');
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from Recovery-TEST backups (newest first)
|
||||
async function importRecoveryBackups(txClient) {
|
||||
console.log('\n=== Importing from Recovery-TEST backups ===');
|
||||
|
||||
if (!fs.existsSync(RECOVERY_PATH)) {
|
||||
console.log(' Recovery-TEST directory not found');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true })
|
||||
.filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name))
|
||||
.map(i => i.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
|
||||
console.log(` Found backup dates: ${backups.join(', ')}`);
|
||||
|
||||
let totalImported = 0;
|
||||
|
||||
for (const backup of backups) {
|
||||
const backupPath = path.join(RECOVERY_PATH, backup);
|
||||
const imported = await importStationLogs(txClient, backupPath, `Recovery-TEST/${backup}`);
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Main import function
|
||||
async function runImport() {
|
||||
console.log('========================================');
|
||||
console.log('Test Data Import');
|
||||
console.log('========================================');
|
||||
console.log(`Start time: ${new Date().toISOString()}`);
|
||||
|
||||
let grandTotal = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
grandTotal += await importHistlogs(txClient);
|
||||
grandTotal += await importRecoveryBackups(txClient);
|
||||
grandTotal += await importStationLogs(txClient, TEST_PATH, 'test');
|
||||
});
|
||||
|
||||
const stats = await db.queryOne('SELECT COUNT(*) as count FROM test_records');
|
||||
|
||||
console.log('\n========================================');
|
||||
console.log('Import Complete');
|
||||
console.log('========================================');
|
||||
console.log(`Total records in database: ${stats.count}`);
|
||||
console.log(`End time: ${new Date().toISOString()}`);
|
||||
|
||||
await db.close();
|
||||
}
|
||||
|
||||
// Import a single file (for incremental imports from sync)
|
||||
async function importSingleFile(filePath) {
|
||||
console.log(`Importing: ${filePath}`);
|
||||
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
// VASLOG_ENG subpath must be checked before VASLOG (substring overlap).
|
||||
if (filePath.includes('VASLOG - Engineering Tested')) {
|
||||
logType = 'VASLOG_ENG';
|
||||
parser = LOG_TYPES['VASLOG_ENG'].parser;
|
||||
} else {
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (type === 'VASLOG_ENG') continue;
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Unknown log type for: ${filePath}`);
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
let result;
|
||||
await db.transaction(async (txClient) => {
|
||||
result = await importFile(txClient, filePath, logType, parser);
|
||||
});
|
||||
|
||||
console.log(` Imported ${result.imported} of ${result.total} records`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Import multiple files (for batch incremental imports)
|
||||
async function importFiles(filePaths) {
|
||||
console.log(`\n========================================`);
|
||||
console.log(`Incremental Import: ${filePaths.length} files`);
|
||||
console.log(`========================================`);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const filePath of filePaths) {
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
// VASLOG_ENG subpath must be checked before the generic loop --
|
||||
// otherwise `includes('VASLOG')` hits first and the eng .txt gets
|
||||
// dispatched to the multiline parser. Mirror importSingleFile().
|
||||
if (filePath.includes('VASLOG - Engineering Tested')) {
|
||||
logType = 'VASLOG_ENG';
|
||||
parser = LOG_TYPES['VASLOG_ENG'].parser;
|
||||
} else {
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (type === 'VASLOG_ENG') continue;
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Skipping unknown type: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const { total, imported } = await importFile(txClient, filePath, logType, parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
console.log(` ${path.basename(filePath)}: ${imported}/${total} records`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
|
||||
// Export datasheets for newly imported records
|
||||
if (totalImported > 0) {
|
||||
try {
|
||||
const { loadAllSpecs } = require('../parsers/spec-reader');
|
||||
const { exportNewRecords } = require('./export-datasheets');
|
||||
const specMap = loadAllSpecs();
|
||||
await exportNewRecords(specMap, filePaths);
|
||||
} catch (err) {
|
||||
console.error(`[EXPORT] Datasheet export failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { total: totalRecords, imported: totalImported };
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length > 0 && args[0] === '--file') {
|
||||
const files = args.slice(1);
|
||||
if (files.length === 0) {
|
||||
console.log('Usage: node import.js --file <file1> [file2] ...');
|
||||
process.exit(1);
|
||||
}
|
||||
importFiles(files).then(() => db.close()).catch(console.error);
|
||||
} else if (args.length > 0 && args[0] === '--help') {
|
||||
console.log('Usage:');
|
||||
console.log(' node import.js Full import from all sources');
|
||||
console.log(' node import.js --file <f> Import specific file(s)');
|
||||
process.exit(0);
|
||||
} else {
|
||||
runImport().catch(console.error);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { runImport, importSingleFile, importFiles };
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
Deploy staged pipeline changes to AD2:C:\\Shares\\testdatadb\\.
|
||||
|
||||
Backs up each existing target to <name>.bak-YYYYMMDD before overwriting.
|
||||
Fails if a target file does not exist on AD2 (excluding brand-new files
|
||||
declared in NEW_FILES below).
|
||||
|
||||
Usage:
|
||||
python deploy-to-ad2.py --dry-run
|
||||
python deploy-to-ad2.py
|
||||
|
||||
Credentials: fetched at runtime from the SOPS vault
|
||||
(clients/dataforth/ad2.sops.yaml -> credentials.password). No hardcoded
|
||||
password; no env-var / prompt fallback. Fails loud if the vault read fails.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import paramiko
|
||||
|
||||
HOST = '192.168.0.6'
|
||||
USER = 'sysadmin'
|
||||
|
||||
VAULT_SH = 'D:/vault/scripts/vault.sh'
|
||||
VAULT_ENTRY = 'clients/dataforth/ad2.sops.yaml'
|
||||
VAULT_FIELD = 'credentials.password'
|
||||
|
||||
|
||||
def get_ad2_password() -> str:
|
||||
"""Fetch the AD2 sysadmin password from the SOPS vault.
|
||||
|
||||
Fails loud (raises) on any error: missing vault, decryption failure,
|
||||
empty value. Do NOT fall back to env vars or prompts -- per CLAUDE.md
|
||||
deploy scripts must not hold credentials.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['bash', VAULT_SH, 'get-field', VAULT_ENTRY, VAULT_FIELD],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
raise RuntimeError(
|
||||
f'[FAIL] vault helper not runnable: {VAULT_SH} ({e})'
|
||||
) from e
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise RuntimeError(
|
||||
f'[FAIL] vault read timed out after 30s for {VAULT_ENTRY}'
|
||||
) from e
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or '').strip()
|
||||
raise RuntimeError(
|
||||
f'[FAIL] vault read failed (rc={result.returncode}) for '
|
||||
f'{VAULT_ENTRY}:{VAULT_FIELD}: {stderr}'
|
||||
)
|
||||
|
||||
pwd = (result.stdout or '').strip()
|
||||
if not pwd:
|
||||
raise RuntimeError(
|
||||
f'[FAIL] vault returned empty value for {VAULT_ENTRY}:{VAULT_FIELD}'
|
||||
)
|
||||
return pwd
|
||||
|
||||
REMOTE_ROOT = 'C:/Shares/testdatadb'
|
||||
LOCAL_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deployment file lists. Each list has different semantics:
|
||||
#
|
||||
# UPDATE_FILES -- file MUST already exist on AD2. Backup-then-overwrite.
|
||||
# Fails loud if the remote file is missing (that's a drift
|
||||
# signal -- something changed on the box we didn't expect).
|
||||
#
|
||||
# NEW_FILES -- file must NOT already exist on AD2. Creates it.
|
||||
# Fails loud if the remote file is already present (we would
|
||||
# otherwise silently clobber something we didn't back up).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Files that already exist on AD2 and will be backed up + overwritten.
|
||||
UPDATE_FILES = [
|
||||
('parsers/spec-reader.js', 'parsers/spec-reader.js'),
|
||||
('templates/datasheet-exact.js', 'templates/datasheet-exact.js'),
|
||||
('database/import.js', 'database/import.js'),
|
||||
('database/export-datasheets.js', 'database/export-datasheets.js'),
|
||||
]
|
||||
|
||||
# Files that do NOT yet exist on AD2 and must be created fresh.
|
||||
NEW_FILES = [
|
||||
('parsers/vaslog-engtxt.js', 'parsers/vaslog-engtxt.js'),
|
||||
]
|
||||
|
||||
|
||||
def connect() -> paramiko.SSHClient:
|
||||
pwd = get_ad2_password()
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(
|
||||
HOST, username=USER, password=pwd,
|
||||
timeout=15, look_for_keys=False, allow_agent=False, banner_timeout=30,
|
||||
)
|
||||
return c
|
||||
|
||||
|
||||
def remote_exists(sftp: paramiko.SFTPClient, path: str) -> bool:
|
||||
try:
|
||||
sftp.stat(path)
|
||||
return True
|
||||
except IOError:
|
||||
return False
|
||||
|
||||
|
||||
def to_remote(rel: str) -> str:
|
||||
return f'{REMOTE_ROOT}/{rel}'
|
||||
|
||||
|
||||
def backup_and_copy(sftp: paramiko.SFTPClient, ssh: paramiko.SSHClient,
|
||||
local_rel: str, remote_rel: str, dry_run: bool, stamp: str) -> None:
|
||||
local_path = os.path.join(LOCAL_ROOT, local_rel.replace('/', os.sep))
|
||||
remote_path = to_remote(remote_rel)
|
||||
backup_path = f'{remote_path}.bak-{stamp}'
|
||||
|
||||
if not os.path.isfile(local_path):
|
||||
raise FileNotFoundError(f'[FAIL] local file missing: {local_path}')
|
||||
|
||||
if not remote_exists(sftp, remote_path):
|
||||
raise FileNotFoundError(f'[FAIL] remote file missing on AD2: {remote_path}')
|
||||
|
||||
print(f'[INFO] {remote_rel}')
|
||||
if dry_run:
|
||||
print(f' would back up to: {backup_path}')
|
||||
print(f' would upload: {local_path} -> {remote_path}')
|
||||
return
|
||||
|
||||
# Backup via SFTP copy (read + re-upload). Paramiko has no server-side copy.
|
||||
with sftp.open(remote_path, 'rb') as src:
|
||||
data = src.read()
|
||||
with sftp.open(backup_path, 'wb') as dst:
|
||||
dst.write(data)
|
||||
print(f' backup: {backup_path} ({len(data)} bytes)')
|
||||
|
||||
sftp.put(local_path, remote_path)
|
||||
size = os.path.getsize(local_path)
|
||||
print(f' uploaded: {local_path} -> {remote_path} ({size} bytes)')
|
||||
|
||||
|
||||
def create_new(sftp: paramiko.SFTPClient, local_rel: str, remote_rel: str,
|
||||
dry_run: bool) -> None:
|
||||
"""Create a file that is expected to be NEW on AD2.
|
||||
|
||||
Fails loud if the remote file already exists -- NEW_FILES declares this
|
||||
is a brand-new file, so pre-existence is a drift signal. If a previous
|
||||
deploy partially ran, clean up manually or move the entry to
|
||||
UPDATE_FILES.
|
||||
"""
|
||||
local_path = os.path.join(LOCAL_ROOT, local_rel.replace('/', os.sep))
|
||||
remote_path = to_remote(remote_rel)
|
||||
|
||||
if not os.path.isfile(local_path):
|
||||
raise FileNotFoundError(f'[FAIL] local file missing: {local_path}')
|
||||
|
||||
print(f'[INFO] {remote_rel} (NEW)')
|
||||
|
||||
if remote_exists(sftp, remote_path):
|
||||
raise FileExistsError(
|
||||
f'[FAIL] remote target already exists but is declared NEW: {remote_path} '
|
||||
f'-- move to UPDATE_FILES or remove remote manually'
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
print(f' would create: {local_path} -> {remote_path}')
|
||||
return
|
||||
|
||||
sftp.put(local_path, remote_path)
|
||||
size = os.path.getsize(local_path)
|
||||
print(f' created: {remote_path} ({size} bytes)')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--dry-run', action='store_true', help='print actions without writing')
|
||||
args = ap.parse_args()
|
||||
|
||||
stamp = datetime.date.today().strftime('%Y%m%d')
|
||||
|
||||
print('=' * 72)
|
||||
print('Deploy staged pipeline changes to AD2')
|
||||
print('=' * 72)
|
||||
print(f'Host: {HOST}')
|
||||
print(f'Remote root: {REMOTE_ROOT}')
|
||||
print(f'Local root: {LOCAL_ROOT}')
|
||||
print(f'Dry run: {args.dry_run}')
|
||||
print(f'Backup tag: .bak-{stamp}')
|
||||
print('')
|
||||
|
||||
ssh = connect()
|
||||
try:
|
||||
sftp = ssh.open_sftp()
|
||||
try:
|
||||
for local_rel, remote_rel in UPDATE_FILES:
|
||||
backup_and_copy(sftp, ssh, local_rel, remote_rel, args.dry_run, stamp)
|
||||
|
||||
for local_rel, remote_rel in NEW_FILES:
|
||||
create_new(sftp, local_rel, remote_rel, args.dry_run)
|
||||
finally:
|
||||
sftp.close()
|
||||
finally:
|
||||
ssh.close()
|
||||
|
||||
print('')
|
||||
print('[OK] done' if not args.dry_run else '[OK] dry-run complete (no changes made)')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Exception as e:
|
||||
print(f'[FAIL] {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Post-backfill verification: counts + sample the 438 skipped records."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const before = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND ((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%') OR log_type='VASLOG_ENG')"
|
||||
);
|
||||
console.log('SCMVAS/SCMHVAS backlog remaining: ' + before.c);
|
||||
|
||||
const exported = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE forweb_exported_at IS NOT NULL " +
|
||||
"AND ((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') OR log_type='VASLOG_ENG')"
|
||||
);
|
||||
console.log('SCMVAS/SCMHVAS exported total: ' + exported.c);
|
||||
|
||||
// Sample of skipped model names
|
||||
console.log('');
|
||||
console.log('Skipped-record model breakdown:');
|
||||
const skipped = await db.query(
|
||||
"SELECT model_number, log_type, COUNT(*) c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND ((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%') OR log_type='VASLOG_ENG') " +
|
||||
"GROUP BY model_number, log_type ORDER BY c DESC LIMIT 30"
|
||||
);
|
||||
for (const r of skipped) console.log(' ' + r.model_number.padEnd(20) + ' ' + (r.log_type||'').padEnd(12) + ' ' + r.c);
|
||||
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_final_verify.js'
|
||||
with sftp.open(remote,'w') as fh: fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_final_verify.js')
|
||||
print(out)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('STDERR:', err[:400])
|
||||
|
||||
# Count For_Web files
|
||||
print('\n=== For_Web file count ===')
|
||||
out, err, rc = ps(c, r'(Get-ChildItem "\\ad2\webshare\For_Web" -File -Filter *.TXT | Measure-Object).Count')
|
||||
print('Total *.TXT in For_Web: ' + out.strip())
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Consolidated single-session script that completes tasks #10, #11, and stages #12.
|
||||
|
||||
Runs everything over ONE SSH session to avoid SSH rate-limiting.
|
||||
|
||||
Steps:
|
||||
1. Deploy inline generator script to AD2
|
||||
2. Generate datasheet for SN 179379-1, pull back for visual check (task #10)
|
||||
3. Run node import.js to ingest Engineering-Tested .txt files (task #11)
|
||||
4. Count VASLOG_ENG records now in DB
|
||||
5. Report backlog size for task #12 (full backfill) + stage scheduled-task cmd
|
||||
6. Clean up scratch files on AD2
|
||||
"""
|
||||
import base64, os, subprocess, yaml, paramiko, time
|
||||
|
||||
HOST = '192.168.0.6'
|
||||
USER = 'sysadmin'
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\live-export'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
GEN_ONE_JS = r'''
|
||||
const db = require('./database/db');
|
||||
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('./templates/datasheet-exact');
|
||||
|
||||
(async () => {
|
||||
const sn = process.argv[2];
|
||||
const rows = await db.query(
|
||||
"SELECT * FROM test_records WHERE serial_number = $1 AND model_number LIKE 'SCMHVAS%' ORDER BY test_date DESC LIMIT 1",
|
||||
[sn]
|
||||
);
|
||||
if (rows.length === 0) {
|
||||
console.error('[FAIL] no SCMHVAS record for ' + sn);
|
||||
process.exit(1);
|
||||
}
|
||||
const record = rows[0];
|
||||
console.log('[INFO] model=' + record.model_number +
|
||||
' log_type=' + record.log_type +
|
||||
' date=' + record.test_date +
|
||||
' status=' + record.overall_result);
|
||||
|
||||
const specMap = loadAllSpecs();
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
console.log('[INFO] specs stub keys: ' + (specs ? JSON.stringify(Object.keys(specs)) : 'null'));
|
||||
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) {
|
||||
console.error('[FAIL] formatter returned null');
|
||||
await db.close();
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[INFO] generated ' + txt.length + ' bytes');
|
||||
console.log('----- BEGIN DATASHEET -----');
|
||||
console.log(txt);
|
||||
console.log('----- END DATASHEET -----');
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
COUNT_JS = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const cnt = await db.queryOne("SELECT COUNT(*) as c FROM test_records WHERE log_type='VASLOG_ENG'");
|
||||
console.log('VASLOG_ENG count: ' + cnt.c);
|
||||
|
||||
const scmvas = await db.queryOne(
|
||||
"SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%')"
|
||||
);
|
||||
console.log('SCMVAS/SCMHVAS backlog (no forweb_exported_at): ' + scmvas.c);
|
||||
|
||||
const total = await db.queryOne(
|
||||
"SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL"
|
||||
);
|
||||
console.log('Total PASS backlog: ' + total.c);
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def get_pwd():
|
||||
r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '')
|
||||
|
||||
def ps(c, cmd, to=300):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
out = stdout.read().decode('utf-8', 'replace')
|
||||
err = stderr.read().decode('utf-8', 'replace')
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
return out, err, rc
|
||||
|
||||
def connect_with_retry():
|
||||
last = None
|
||||
for i in range(5):
|
||||
try:
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=get_pwd(),
|
||||
timeout=30, banner_timeout=45, auth_timeout=30,
|
||||
look_for_keys=False, allow_agent=False)
|
||||
return c
|
||||
except Exception as e:
|
||||
last = e
|
||||
print(f'[RETRY {i+1}/5] {type(e).__name__}: {e}')
|
||||
time.sleep(15 * (i + 1))
|
||||
raise last
|
||||
|
||||
def main():
|
||||
c = connect_with_retry()
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
print('\n=== STEP 1: deploy inline generator ===')
|
||||
remote_gen = 'C:/Shares/testdatadb/_gen_one.js'
|
||||
remote_count = 'C:/Shares/testdatadb/_count.js'
|
||||
with sftp.open(remote_gen, 'w') as fh:
|
||||
fh.write(GEN_ONE_JS)
|
||||
with sftp.open(remote_count, 'w') as fh:
|
||||
fh.write(COUNT_JS)
|
||||
print(f' deployed {remote_gen} and {remote_count}')
|
||||
|
||||
print('\n=== STEP 2: generate datasheet for 179379-1 ===')
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_gen_one.js 179379-1')
|
||||
print(f' rc={rc}')
|
||||
print(out)
|
||||
if err.strip(): print(f' STDERR: {err[:500]}')
|
||||
|
||||
# Save the generated datasheet to local for inspection
|
||||
if '----- BEGIN DATASHEET -----' in out:
|
||||
body = out.split('----- BEGIN DATASHEET -----', 1)[1]
|
||||
body = body.split('----- END DATASHEET -----', 1)[0]
|
||||
body = body.lstrip('\r\n')
|
||||
local_dst = os.path.join(LOCAL_OUT, '179379-1.TXT')
|
||||
with open(local_dst, 'w', encoding='utf-8', newline='') as fh:
|
||||
fh.write(body)
|
||||
print(f' saved locally: {local_dst}')
|
||||
|
||||
print('\n=== STEP 3: run full import to ingest Engineering-Tested .txt ===')
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node database/import.js', to=600)
|
||||
print(f' rc={rc}')
|
||||
# Only last ~40 lines to avoid log spam
|
||||
for line in out.splitlines()[-40:]:
|
||||
print(f' {line}')
|
||||
if err.strip(): print(f' STDERR (first 500): {err[:500]}')
|
||||
|
||||
print('\n=== STEP 4: count VASLOG_ENG records + backlog ===')
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_count.js')
|
||||
print(f' rc={rc}')
|
||||
print(out)
|
||||
if err.strip(): print(f' STDERR: {err[:300]}')
|
||||
|
||||
print('\n=== STEP 5: identify service account to stage backfill ===')
|
||||
out, err, rc = ps(c, r'Get-WmiObject -Class Win32_Service -Filter "Name=''testdatadb''" | Select-Object Name,StartName,State | Format-List | Out-String')
|
||||
print(out)
|
||||
|
||||
print('\n=== STEP 6: cleanup scratch files ===')
|
||||
try:
|
||||
sftp.remove(remote_gen); print(f' removed {remote_gen}')
|
||||
except Exception as e:
|
||||
print(f' [WARN] remove {remote_gen}: {e}')
|
||||
try:
|
||||
sftp.remove(remote_count); print(f' removed {remote_count}')
|
||||
except Exception as e:
|
||||
print(f' [WARN] remove {remote_count}: {e}')
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Inline generate one SCMHVAS datasheet on AD2 (no X: drive dependency)."""
|
||||
import base64, subprocess, yaml, paramiko, os
|
||||
|
||||
TEST_SN = '179379-1'
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\live-export'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const db = require('./database/db');
|
||||
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('./templates/datasheet-exact');
|
||||
|
||||
(async () => {
|
||||
const sn = process.argv[2];
|
||||
const rows = await db.query(
|
||||
"SELECT * FROM test_records WHERE serial_number = $1 ORDER BY test_date DESC LIMIT 1",
|
||||
[sn]
|
||||
);
|
||||
if (rows.length === 0) {
|
||||
console.error('[FAIL] no record for ' + sn);
|
||||
process.exit(1);
|
||||
}
|
||||
const record = rows[0];
|
||||
console.log('[INFO] record: model=' + record.model_number +
|
||||
' log_type=' + record.log_type +
|
||||
' date=' + record.test_date +
|
||||
' status=' + record.overall_result);
|
||||
|
||||
const specMap = loadAllSpecs();
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
console.log('[INFO] specs: ' + (specs ? JSON.stringify(Object.keys(specs)) : 'null'));
|
||||
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) {
|
||||
console.error('[FAIL] formatter returned null');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[INFO] generated ' + txt.length + ' bytes');
|
||||
console.log('----- BEGIN DATASHEET -----');
|
||||
console.log(txt);
|
||||
console.log('----- END DATASHEET -----');
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
# Script must live inside testdatadb/ so relative requires resolve.
|
||||
remote_js = 'C:/Shares/testdatadb/_gen_one.js'
|
||||
with sftp.open(remote_js, 'w') as fh:
|
||||
fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
|
||||
cmd = f'cd C:\\Shares\\testdatadb; & node ./_gen_one.js {TEST_SN}'
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=120)
|
||||
out = stdout.read().decode('utf-8','replace')
|
||||
err = stderr.read().decode('utf-8','replace')
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
print(f'[rc={rc}]')
|
||||
print(out)
|
||||
if err.strip():
|
||||
print('--- STDERR ---')
|
||||
print(err[:2000])
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Targeted import of the 434 VASLOG Engineering-Tested .txt files.
|
||||
|
||||
Runs node import.js --file <batch> to import directly, then counts VASLOG_ENG
|
||||
records in the DB. Avoids the slow full-import walk.
|
||||
"""
|
||||
import base64, os, subprocess, yaml, paramiko, sys
|
||||
|
||||
HOST = '192.168.0.6'
|
||||
USER = 'sysadmin'
|
||||
REMOTE_DIR = r'C:\Shares\test\TS-3R\LOGS\VASLOG\VASLOG - Engineering Tested'
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '')
|
||||
|
||||
def ps(c, cmd, to=600):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
out = stdout.read().decode('utf-8', 'replace')
|
||||
err = stderr.read().decode('utf-8', 'replace')
|
||||
rc = stdout.channel.recv_exit_status()
|
||||
return out, err, rc
|
||||
|
||||
def main():
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45,
|
||||
look_for_keys=False, allow_agent=False)
|
||||
sys.stdout.flush()
|
||||
try:
|
||||
print('[STEP 1] List Engineering-Tested .txt files on AD2', flush=True)
|
||||
out, err, rc = ps(c, f'Get-ChildItem -LiteralPath "{REMOTE_DIR}" -File -Filter *.txt | ForEach-Object {{ $_.FullName }}')
|
||||
files = [l.strip() for l in out.splitlines() if l.strip()]
|
||||
print(f' found {len(files)} .txt files', flush=True)
|
||||
|
||||
if not files:
|
||||
print(' [WARN] no files found', flush=True)
|
||||
return
|
||||
|
||||
print('[STEP 2] Build PowerShell command array and invoke import.js --file', flush=True)
|
||||
# Build a PS array literal to pass to node. We chunk to avoid CLI length limits.
|
||||
CHUNK = 50
|
||||
total_imported = 0
|
||||
total_parsed = 0
|
||||
for i in range(0, len(files), CHUNK):
|
||||
batch = files[i:i+CHUNK]
|
||||
# PowerShell @() array with paths quoted
|
||||
quoted = ','.join(f'"{p}"' for p in batch)
|
||||
script = (
|
||||
r'cd C:\Shares\testdatadb; ' +
|
||||
f'$files = @({quoted}); ' +
|
||||
r'& node database/import.js --file @files 2>&1'
|
||||
)
|
||||
out, err, rc = ps(c, script, to=300)
|
||||
lines = out.splitlines()
|
||||
# Print a summary tail of each chunk
|
||||
tail = [l for l in lines if 'records' in l.lower() or 'total' in l.lower() or 'error' in l.lower()]
|
||||
print(f' chunk {i//CHUNK + 1} ({len(batch)} files): rc={rc}', flush=True)
|
||||
for t in tail[-4:]:
|
||||
print(f' {t}', flush=True)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print(f' STDERR: {err[:400]}', flush=True)
|
||||
|
||||
print('[STEP 3] Count VASLOG_ENG in DB', flush=True)
|
||||
script = (
|
||||
r'cd C:\Shares\testdatadb; & node -e "'
|
||||
r"const db=require('./database/db');"
|
||||
r"(async()=>{const r=await db.queryOne(\"SELECT COUNT(*) c FROM test_records WHERE log_type='VASLOG_ENG'\");"
|
||||
r'console.log(\"VASLOG_ENG rows: \"+r.c);await db.close();})();"'
|
||||
)
|
||||
out, err, rc = ps(c, script, to=60)
|
||||
print(out, flush=True)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print(f' STDERR: {err[:400]}', flush=True)
|
||||
|
||||
print('[STEP 4] Cleanup scratch files on AD2', flush=True)
|
||||
sftp = c.open_sftp()
|
||||
for scratch in ['C:/Shares/testdatadb/_gen_one.js', 'C:/Shares/testdatadb/_count.js']:
|
||||
try:
|
||||
sftp.remove(scratch)
|
||||
print(f' removed {scratch}', flush=True)
|
||||
except Exception as e:
|
||||
print(f' [WARN] {scratch}: {e}', flush=True)
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Targeted Engineering-Tested .txt import — v2.
|
||||
|
||||
Drops a node script on AD2 that reads the directory itself and calls
|
||||
importFiles() with the full list. Avoids CLI-length limits and chunking.
|
||||
"""
|
||||
import base64, subprocess, yaml, paramiko, sys
|
||||
|
||||
HOST = '192.168.0.6'
|
||||
USER = 'sysadmin'
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database/db');
|
||||
const { importFiles } = require('./database/import');
|
||||
|
||||
const DIR = 'C:\\Shares\\test\\TS-3R\\LOGS\\VASLOG\\VASLOG - Engineering Tested';
|
||||
|
||||
(async () => {
|
||||
const entries = fs.readdirSync(DIR).filter(n => n.toLowerCase().endsWith('.txt'));
|
||||
const files = entries.map(n => path.join(DIR, n));
|
||||
console.log('[INFO] ' + files.length + ' .txt files queued for import');
|
||||
const result = await importFiles(files);
|
||||
console.log('[DONE] imported=' + result.imported + ' parsed=' + result.total);
|
||||
|
||||
const cnt = await db.queryOne("SELECT COUNT(*) c FROM test_records WHERE log_type='VASLOG_ENG'");
|
||||
console.log('[DB] VASLOG_ENG rows total: ' + cnt.c);
|
||||
|
||||
// Check forweb export status
|
||||
const forweb = await db.queryOne(
|
||||
"SELECT COUNT(*) c FROM test_records WHERE log_type='VASLOG_ENG' AND forweb_exported_at IS NOT NULL"
|
||||
);
|
||||
console.log('[DB] VASLOG_ENG already on X:\\For_Web: ' + forweb.c);
|
||||
|
||||
await db.close();
|
||||
})().catch(e => { console.error('[FAIL] ' + e.message); process.exit(1); });
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '')
|
||||
|
||||
def ps(c, cmd, to=1800):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8', 'replace'), stderr.read().decode('utf-8', 'replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
def main():
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45,
|
||||
look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote_js = 'C:/Shares/testdatadb/_import_engtxt.js'
|
||||
with sftp.open(remote_js, 'w') as fh:
|
||||
fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
print(f'[OK] deployed {remote_js}', flush=True)
|
||||
|
||||
print('[RUN] executing ./_import_engtxt.js (this may take a few minutes)', flush=True)
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_import_engtxt.js')
|
||||
print(f'[rc={rc}]', flush=True)
|
||||
print(out, flush=True)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print(f'--- STDERR ---\n{err[:2000]}', flush=True)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try:
|
||||
sftp.remove(remote_js)
|
||||
print(f'[OK] removed {remote_js}', flush=True)
|
||||
except Exception as e:
|
||||
print(f'[WARN] cleanup {remote_js}: {e}', flush=True)
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Sample a few skipped records to understand why they didn't render."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const db = require('./database/db');
|
||||
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('./templates/datasheet-exact');
|
||||
|
||||
(async () => {
|
||||
const rows = await db.query(
|
||||
"SELECT id, serial_number, model_number, raw_data FROM test_records " +
|
||||
"WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"ORDER BY test_date DESC LIMIT 5"
|
||||
);
|
||||
const specMap = loadAllSpecs();
|
||||
for (const r of rows) {
|
||||
console.log('====================');
|
||||
console.log('SN:' + r.serial_number + ' model:' + r.model_number);
|
||||
console.log('raw_data length: ' + (r.raw_data||'').length);
|
||||
console.log('first 200 chars: ' + JSON.stringify((r.raw_data||'').slice(0, 200)));
|
||||
const specs = getSpecs(specMap, r.model_number);
|
||||
console.log('specs: ' + (specs ? 'stub' : 'null'));
|
||||
try {
|
||||
const txt = generateExactDatasheet(r, specs);
|
||||
console.log('formatter output length: ' + (txt ? txt.length : 'null'));
|
||||
if (txt) console.log('snippet: ' + txt.slice(0, 200));
|
||||
} catch (e) {
|
||||
console.log('formatter threw: ' + e.message);
|
||||
}
|
||||
}
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_inspect.js'
|
||||
with sftp.open(remote,'w') as fh: fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_inspect.js')
|
||||
print(out)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('STDERR:', err[:500])
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Deep-dive on the 438 skipped records.
|
||||
|
||||
Looking for patterns: date range, test station, source file, model-family drift,
|
||||
prior ship status, accuracy magnitude.
|
||||
"""
|
||||
import base64, json, subprocess, yaml, paramiko
|
||||
|
||||
NODE_SCRIPT = r'''
|
||||
const db = require('./database/db');
|
||||
|
||||
(async () => {
|
||||
console.log('======================================================================');
|
||||
console.log('SKIPPED RECORDS INVESTIGATION');
|
||||
console.log('======================================================================');
|
||||
|
||||
const WHERE_SKIPPED = "overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"AND log_type='VASLOG'";
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [1] Date range of SKIPPED vs RENDERED ---');
|
||||
const dateRanges = await db.query(
|
||||
"SELECT CASE WHEN forweb_exported_at IS NULL THEN 'SKIPPED' ELSE 'RENDERED' END AS status, " +
|
||||
"MIN(test_date) mindate, MAX(test_date) maxdate, COUNT(*) cnt " +
|
||||
"FROM test_records WHERE overall_result='PASS' AND log_type='VASLOG' " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"GROUP BY CASE WHEN forweb_exported_at IS NULL THEN 'SKIPPED' ELSE 'RENDERED' END"
|
||||
);
|
||||
for (const r of dateRanges) console.log(' ' + r.status.padEnd(10) + ' ' + r.mindate + ' .. ' + r.maxdate + ' (' + r.cnt + ' records)');
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [2] Test station of SKIPPED ---');
|
||||
const stations = await db.query(
|
||||
"SELECT COALESCE(test_station,'(null)') ts, COUNT(*) cnt FROM test_records " +
|
||||
"WHERE " + WHERE_SKIPPED + " GROUP BY test_station ORDER BY cnt DESC"
|
||||
);
|
||||
for (const r of stations) console.log(' ' + r.ts.padEnd(10) + ' ' + r.cnt);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [3] Source file of SKIPPED (grouped) ---');
|
||||
const sources = await db.query(
|
||||
"SELECT source_file, COUNT(*) cnt FROM test_records " +
|
||||
"WHERE " + WHERE_SKIPPED + " GROUP BY source_file ORDER BY cnt DESC LIMIT 20"
|
||||
);
|
||||
for (const r of sources) console.log(' ' + r.cnt.toString().padEnd(6) + ' ' + r.source_file);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [4] Year distribution: SKIPPED ---');
|
||||
const skippedYears = await db.query(
|
||||
"SELECT strftime('%Y', test_date) yr, COUNT(*) cnt FROM test_records " +
|
||||
"WHERE " + WHERE_SKIPPED + " GROUP BY yr ORDER BY yr"
|
||||
);
|
||||
for (const r of skippedYears) console.log(' ' + r.yr + ' ' + r.cnt);
|
||||
|
||||
console.log('\n--- [5] Year distribution: RENDERED ---');
|
||||
const renderedYears = await db.query(
|
||||
"SELECT strftime('%Y', test_date) yr, COUNT(*) cnt FROM test_records " +
|
||||
"WHERE overall_result='PASS' AND log_type='VASLOG' AND forweb_exported_at IS NOT NULL " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"GROUP BY yr ORDER BY yr"
|
||||
);
|
||||
for (const r of renderedYears) console.log(' ' + r.yr + ' ' + r.cnt);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [6] Sample raw_data: SKIPPED vs same-model RENDERED ---');
|
||||
const pair = await db.query(
|
||||
"SELECT 'SKIPPED' AS tag, serial_number, model_number, test_date, test_station, source_file, raw_data " +
|
||||
"FROM test_records WHERE " + WHERE_SKIPPED + " AND model_number='SCMVAS-M700' LIMIT 2"
|
||||
);
|
||||
const pair2 = await db.query(
|
||||
"SELECT 'RENDERED' AS tag, serial_number, model_number, test_date, test_station, source_file, raw_data " +
|
||||
"FROM test_records WHERE overall_result='PASS' AND log_type='VASLOG' AND forweb_exported_at IS NOT NULL " +
|
||||
"AND model_number='SCMVAS-M700' LIMIT 2"
|
||||
);
|
||||
for (const r of [...pair, ...pair2]) {
|
||||
console.log(' [' + r.tag + '] sn=' + r.serial_number + ' date=' + r.test_date +
|
||||
' station=' + (r.test_station || '-') + ' src=' + r.source_file);
|
||||
console.log(' raw_data: ' + JSON.stringify((r.raw_data||'').replace(/\n/g,'\\n')));
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
console.log('\n--- [7] Accuracy-value magnitude distribution ---');
|
||||
const accMag = await db.query(
|
||||
"SELECT raw_data FROM test_records WHERE " + WHERE_SKIPPED + " LIMIT 50"
|
||||
);
|
||||
const vals = [];
|
||||
for (const r of accMag) {
|
||||
const m = (r.raw_data || '').match(/"(PASS|FAIL)\s*(-?\.?\d+\.?\d*)"/);
|
||||
if (m) vals.push(parseFloat(m[2]));
|
||||
}
|
||||
if (vals.length) {
|
||||
const abs = vals.map(Math.abs).sort((a,b)=>a-b);
|
||||
console.log(' sample count: ' + vals.length);
|
||||
console.log(' min |val|: ' + abs[0]);
|
||||
console.log(' median |val|: ' + abs[Math.floor(abs.length/2)]);
|
||||
console.log(' max |val|: ' + abs[abs.length-1]);
|
||||
}
|
||||
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=180):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_invest.js'
|
||||
with sftp.open(remote,'w') as fh: fh.write(NODE_SCRIPT)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_invest.js')
|
||||
print(out)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('--- STDERR ---')
|
||||
print(err[:2000])
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Actually export one SCMHVAS datasheet and pull it back for visual check."""
|
||||
import base64, subprocess, yaml, paramiko, os
|
||||
|
||||
TEST_SN = '179379-1'
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\live-export'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print(f'=== Live export for {TEST_SN} ===')
|
||||
out, err, rc = ps(c, f'cd C:\\Shares\\testdatadb; & node database/export-datasheets.js --serial {TEST_SN}', to=120)
|
||||
print(f'[rc={rc}]')
|
||||
print('--- STDOUT ---')
|
||||
print(out)
|
||||
if err.strip():
|
||||
print('--- STDERR ---')
|
||||
print(err[:2000])
|
||||
|
||||
print(f'\n=== SFTP pull X:\\For_Web\\{TEST_SN}.TXT ===')
|
||||
sftp = c.open_sftp()
|
||||
try:
|
||||
src = f'X:/For_Web/{TEST_SN}.TXT'
|
||||
dst = os.path.join(LOCAL_OUT, f'{TEST_SN}.TXT')
|
||||
sftp.get(src, dst)
|
||||
print(f'[OK] pulled {src} -> {dst}')
|
||||
print(f'[INFO] size={os.path.getsize(dst)} bytes')
|
||||
finally:
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Parser for multi-line DAT files (DSCLOG, 5BLOG, 8BLOG, PWRLOG, SCTLOG, VASLOG)
|
||||
*
|
||||
* Format:
|
||||
* "MODEL_NUMBER "
|
||||
* measurement1,measurement2,measurement3,measurement4,"PASS/FAIL"
|
||||
* ... (test data lines)
|
||||
* 0
|
||||
* "summary line 1"
|
||||
* ...
|
||||
* "SERIAL-NUM","MM-DD-YYYY"
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* Parse a multi-line DAT file and extract test records
|
||||
* @param {string} filePath - Path to the DAT file
|
||||
* @param {string} logType - Type of log (DSCLOG, 5BLOG, etc.)
|
||||
* @param {string} testStation - Test station identifier (TS-1L, etc.)
|
||||
* @returns {Array} Array of parsed records
|
||||
*/
|
||||
function parseMultilineFile(filePath, logType, testStation = null) {
|
||||
const records = [];
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
const lines = content.split('\n').map(l => l.trim());
|
||||
|
||||
let currentRecord = [];
|
||||
let modelNumber = null;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
|
||||
// Skip empty lines
|
||||
if (!line) continue;
|
||||
|
||||
// Check if it's a serial/date line (format: "SERIAL","DATE")
|
||||
const serialDateMatch = line.match(/^"(\d+-\d+[A-Za-z]?)","(\d{2}-\d{2}-\d{4})"$/);
|
||||
|
||||
if (serialDateMatch) {
|
||||
// This is the end of a record
|
||||
const serialNumber = serialDateMatch[1];
|
||||
const dateStr = serialDateMatch[2];
|
||||
|
||||
if (modelNumber && currentRecord.length > 0) {
|
||||
// Parse date from MM-DD-YYYY to YYYY-MM-DD
|
||||
const [month, day, year] = dateStr.split('-');
|
||||
const testDate = `${year}-${month}-${day}`;
|
||||
|
||||
// Determine overall result from raw data
|
||||
const rawData = currentRecord.join('\n');
|
||||
const overallResult = determineResult(rawData);
|
||||
|
||||
records.push({
|
||||
log_type: logType,
|
||||
model_number: modelNumber.trim(),
|
||||
serial_number: serialNumber,
|
||||
test_date: testDate,
|
||||
test_station: testStation,
|
||||
overall_result: overallResult,
|
||||
raw_data: rawData,
|
||||
source_file: filePath
|
||||
});
|
||||
}
|
||||
|
||||
// Reset for next record
|
||||
currentRecord = [];
|
||||
modelNumber = null;
|
||||
}
|
||||
// Check if this is a model number line
|
||||
// Model numbers: single quoted string with product code (letters+numbers, possibly with dash)
|
||||
// Examples: "DSCA38-1793 ", "SCM5B30-01 ", "8B30-01 "
|
||||
else if (/^"[A-Z0-9]+[A-Z0-9-]*\s*"$/.test(line) && !line.includes(',') && !line.includes('PASS') && !line.includes('FAIL')) {
|
||||
// This is a model number line - start new record
|
||||
if (currentRecord.length > 0 && modelNumber) {
|
||||
// Previous record didn't have serial/date - skip it
|
||||
currentRecord = [];
|
||||
}
|
||||
modelNumber = line.replace(/"/g, '').trim();
|
||||
currentRecord.push(line);
|
||||
} else {
|
||||
// Add line to current record
|
||||
currentRecord.push(line);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error parsing ${filePath}: ${err.message}`);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine overall PASS/FAIL result from raw data
|
||||
*/
|
||||
function determineResult(rawData) {
|
||||
const failCount = (rawData.match(/"FAIL/gi) || []).length;
|
||||
const passCount = (rawData.match(/"PASS/gi) || []).length;
|
||||
|
||||
if (failCount > 0) return 'FAIL';
|
||||
if (passCount > 0) return 'PASS';
|
||||
return 'UNKNOWN';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract test station from file path
|
||||
*/
|
||||
function extractTestStation(filePath) {
|
||||
const match = filePath.match(/TS-\d+[LR]/i);
|
||||
return match ? match[0].toUpperCase() : null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
parseMultilineFile,
|
||||
extractTestStation
|
||||
};
|
||||
@@ -0,0 +1,497 @@
|
||||
/**
|
||||
* Spec Reader - Parses QuickBASIC binary DAT spec files
|
||||
*
|
||||
* Reads model specification data from 4 product family DAT files:
|
||||
* 5BMAIN.DAT (SCM5B family, 160 bytes/record)
|
||||
* 8BMAIN.DAT (8B family, 163 bytes/record)
|
||||
* DSCOUT.DAT (DSCA family, 163 bytes/record)
|
||||
* SCTMAIN.DAT (DSCT family, 121 bytes/record)
|
||||
*
|
||||
* These are QuickBASIC random-access files using TYPE (struct) records.
|
||||
* All values are little-endian: SINGLE = IEEE 754 float (4 bytes),
|
||||
* INTEGER = signed 16-bit (2 bytes), STRING * N = fixed-width ASCII.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Default spec data directory
|
||||
const DEFAULT_SPEC_DIR = path.join(__dirname, '..', 'specdata');
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Binary read helpers
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
function readString(buf, offset, length) {
|
||||
return buf.toString('ascii', offset, offset + length).replace(/\0/g, '').trim();
|
||||
}
|
||||
|
||||
function readSingle(buf, offset) {
|
||||
return buf.readFloatLE(offset);
|
||||
}
|
||||
|
||||
function readInteger(buf, offset) {
|
||||
return buf.readInt16LE(offset);
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// TYPE definitions (field name, type, size)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
const FIELD_TYPES = {
|
||||
STRING17: { size: 17, read: (buf, off) => readString(buf, off, 17) },
|
||||
STRING9: { size: 9, read: (buf, off) => readString(buf, off, 9) },
|
||||
STRING15: { size: 15, read: (buf, off) => readString(buf, off, 15) },
|
||||
STRING14: { size: 14, read: (buf, off) => readString(buf, off, 14) },
|
||||
STRING13: { size: 13, read: (buf, off) => readString(buf, off, 13) },
|
||||
STRING7: { size: 7, read: (buf, off) => readString(buf, off, 7) },
|
||||
SINGLE: { size: 4, read: (buf, off) => readSingle(buf, off) },
|
||||
INTEGER: { size: 2, read: (buf, off) => readInteger(buf, off) },
|
||||
};
|
||||
|
||||
const S15 = 'STRING15';
|
||||
const S14 = 'STRING14';
|
||||
const S13 = 'STRING13';
|
||||
const S7 = 'STRING7';
|
||||
const SNG = 'SINGLE';
|
||||
const INT = 'INTEGER';
|
||||
|
||||
// SCM5B: 160 bytes/record
|
||||
const SCM5B_FIELDS = [
|
||||
['MODNAME', S15], ['SENTYPE', S7],
|
||||
['ISMAXNEXCL', SNG], ['ISMAXFEXCL', SNG], ['OUTRES', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG], ['IEXC', SNG],
|
||||
['RCONV', SNG], ['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG], ['CALTOL', SNG],
|
||||
['VEXC', SNG], ['VEXCACC', SNG], ['EXCLOAD', SNG],
|
||||
['EXCLOADREG', SNG], ['EXCIMAX', SNG], ['LINEAR', SNG],
|
||||
['ACCURACY', SNG], ['TESTFREQ', SNG], ['ATTEN', SNG],
|
||||
['ATTENTOL', SNG], ['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['PSS', SNG], ['OUTNOISE', SNG], ['INPUTRES', SNG],
|
||||
['VOPENINMIN', SNG], ['VOPENINMAX', SNG], ['LEADRERR', SNG],
|
||||
['LINEARIZED', INT], ['OSCALIN', SNG], ['GNCALIN', SNG],
|
||||
['BANDWIDTH', SNG], ['IMATCHTOL', SNG],
|
||||
];
|
||||
|
||||
// 8B: 163 bytes/record (no OUTRES, has OUTSIGTYPE)
|
||||
const B8_FIELDS = [
|
||||
['MODNAME', S15], ['SENTYPE', S7],
|
||||
['ISMAXNEXCL', SNG], ['ISMAXFEXCL', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG], ['IEXC', SNG],
|
||||
['RCONV', SNG], ['OUTSIGTYPE', S7],
|
||||
['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG], ['CALTOL', SNG],
|
||||
['VEXC', SNG], ['VEXCACC', SNG], ['EXCLOAD', SNG],
|
||||
['EXCLOADREG', SNG], ['EXCIMAX', SNG], ['LINEAR', SNG],
|
||||
['ACCURACY', SNG], ['TESTFREQ', SNG], ['ATTEN', SNG],
|
||||
['ATTENTOL', SNG], ['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['PSS', SNG], ['OUTNOISE', SNG], ['INPUTRES', SNG],
|
||||
['VOPENINMIN', SNG], ['VOPENINMAX', SNG], ['LEADRERR', SNG],
|
||||
['LINEARIZED', INT], ['OSCALIN', SNG], ['GNCALIN', SNG],
|
||||
['BANDWIDTH', SNG], ['IMATCHTOL', SNG],
|
||||
];
|
||||
|
||||
// DSCA: 163 bytes/record
|
||||
const DSCA_FIELDS = [
|
||||
['MODNAME', S13], ['SENTYPE', S7],
|
||||
['ISMAXNL', SNG], ['ISMAXFL', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG], ['RCONV', SNG],
|
||||
['MINOUT', SNG], ['MAXOUT', SNG], ['OUTSIGTYPE', S7],
|
||||
['OSCALIN', SNG], ['GNCALIN', SNG], ['CALTOL', SNG],
|
||||
['LOAD1', SNG], ['LINEAR1', SNG], ['ACCURACY1', SNG],
|
||||
['LOAD2', SNG], ['LINEAR2', SNG], ['ACCURACY2', SNG],
|
||||
['LOAD3', SNG], ['LINEAR3', SNG], ['ACCURACY3', SNG],
|
||||
['BANDWIDTH', SNG], ['TESTFREQ', SNG], ['ATTEN', SNG],
|
||||
['ATTENTOL', SNG], ['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['PSS', SNG], ['OUTNOISE', SNG], ['INPUTRES', SNG],
|
||||
['COMPLIANCE', SNG], ['MAXLOAD', SNG], ['ILIMIT', SNG],
|
||||
['PERCOVER', SNG], ['MINVS', SNG], ['MAXVS', SNG],
|
||||
];
|
||||
|
||||
// DSCT: 121 bytes/record (uses INTEGER for some fields)
|
||||
const DSCT_FIELDS = [
|
||||
['MODNAME', S14], ['SENTYPE', S7],
|
||||
['MININ', SNG], ['MAXIN', SNG],
|
||||
['IEXCMFS', SNG], ['IEXCPFS', SNG],
|
||||
['RCONV', SNG], ['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG],
|
||||
['LINEAR', SNG], ['ACCURACY', SNG],
|
||||
['TESTFREQ', INT], ['ATTEN', INT], ['ATTENTOL', INT],
|
||||
['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['OUTNOISE', SNG], ['INPUTRES', SNG],
|
||||
['IOPENTC', SNG], ['LEADRERR', SNG],
|
||||
['LINEARIZED', INT], ['OSCALIN', SNG], ['GNCALIN', SNG],
|
||||
['BANDWIDTH', SNG], ['IMATCHTOL', SNG],
|
||||
['CALTOL', SNG], ['VSEN', SNG],
|
||||
];
|
||||
|
||||
const S9 = 'STRING9';
|
||||
|
||||
// SCM5B45: 119 bytes/record (frequency/counter modules)
|
||||
const SCM5B45_FIELDS = [
|
||||
['MODNAME', S9],
|
||||
['MININ', SNG], ['MAXIN', SNG], ['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['ZHYSAMPL', SNG], ['ZHYSLIM', SNG], ['TTLHYSAMPL', SNG],
|
||||
['TTLLIMHI', SNG], ['TTLLIMLO', SNG], ['MINPW', SNG],
|
||||
['OSCALIN', SNG], ['GNCALIN', SNG], ['CALTOL', SNG],
|
||||
['LINEAR', SNG], ['ACCURACY', SNG],
|
||||
['TESTFREQ', INT], ['ATTEN', INT], ['ATTENTOL', INT],
|
||||
['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['ISMAX', SNG], ['PSS', SNG],
|
||||
['NOISEMFS', SNG], ['NOISETESTPT', SNG], ['NOISEPFS', SNG],
|
||||
['OUTRES', SNG], ['EXCVOLT', SNG],
|
||||
['EXCTOLNL', SNG], ['EXCTOLL', SNG],
|
||||
];
|
||||
|
||||
// SCM5B48: 264 bytes/record (multi-bandwidth modules)
|
||||
const SCM5B48_FIELDS = [
|
||||
['MODNAME', S15], ['SENTYPE', S7],
|
||||
['ISMAXNEXCL', SNG], ['ISMAXFEXCL', SNG], ['OUTRES', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG],
|
||||
['MININ1', SNG], ['MAXIN1', SNG],
|
||||
['MININ2', SNG], ['MAXIN2', SNG],
|
||||
['MININ3', SNG], ['MAXIN3', SNG],
|
||||
['IEXC', SNG], ['IEXC1', SNG], ['IEXC2', SNG],
|
||||
['RCONV', SNG], ['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG], ['CALTOL', SNG],
|
||||
['VEXC', SNG], ['VEXCACC', SNG], ['EXCLOAD', SNG],
|
||||
['EXCLOADREG', SNG], ['EXCIMAX', SNG],
|
||||
['LINEAR', SNG], ['ACCURACY', SNG],
|
||||
['TESTFREQ', SNG], ['TESTFREQ1', SNG], ['TESTFREQ2', SNG], ['TESTFREQ3', SNG], ['TESTFREQ4', SNG],
|
||||
['ATTEN', SNG], ['ATTEN1', SNG], ['ATTEN2', SNG], ['ATTEN3', SNG], ['ATTEN4', SNG],
|
||||
['ATTENTOL', SNG],
|
||||
['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['PSS', SNG], ['PSS1', SNG], ['PSS2', SNG], ['PSS3', SNG],
|
||||
['OUTNOISE', SNG], ['OUTNOISE1', SNG], ['OUTNOISE2', SNG], ['OUTNOISE3', SNG],
|
||||
['INPUTRES', SNG], ['VOPENINMIN', SNG], ['VOPENINMAX', SNG], ['LEADRERR', SNG],
|
||||
['LINEARIZED', INT],
|
||||
['OSCALIN', SNG], ['GNCALIN', SNG],
|
||||
['BANDWIDTH', SNG], ['BANDWIDTH1', SNG], ['BANDWIDTH2', SNG], ['BANDWIDTH3', SNG], ['BANDWIDTH4', SNG],
|
||||
['IMATCHTOL', SNG],
|
||||
];
|
||||
|
||||
// SCM5B49: 93 bytes/record (sample & hold modules)
|
||||
const SCM5B49_FIELDS = [
|
||||
['MODNAME', S9],
|
||||
['MININ', SNG], ['MAXIN', SNG], ['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['MAXSUPPLYNL', SNG], ['MAXSUPPLYFL', SNG], ['LIMITOUT', SNG], ['POWERSEN', SNG],
|
||||
['TESTFREQ', INT], ['ATTEN', INT],
|
||||
['LINEAR0MA', SNG], ['LINEAR50MA', SNG],
|
||||
['ACCURACY0MA', SNG], ['ACCURACY50MA', SNG],
|
||||
['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['NOISEOUT', SNG], ['QINJECT', SNG],
|
||||
['INPUTRES', SNG], ['ACQLIM', SNG],
|
||||
['DROOP', SNG], ['PERCOVER', SNG],
|
||||
];
|
||||
|
||||
// DSCA (TSTDIN1B variant, for DSCMAIN4.DAT): 159 bytes/record
|
||||
const DSCA_DIN_FIELDS = [
|
||||
['MODNAME', S13], ['SENTYPE', S7],
|
||||
['ISMAXNEXCL', SNG], ['ISMAXFEXCL', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG],
|
||||
['IEXCPFS', SNG], ['IEXCMFS', SNG],
|
||||
['RCONV', SNG], ['OUTSIGTYPE', S7],
|
||||
['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG], ['CALTOL', SNG],
|
||||
['VEXC', SNG], ['VEXCACC', SNG], ['EXCLOAD', SNG],
|
||||
['EXCLOADREG', SNG], ['EXCIMAX', SNG],
|
||||
['LINEAR', SNG], ['ACCURACY', SNG],
|
||||
['TESTFREQ', INT], ['ATTEN', INT], ['ATTENTOL', INT],
|
||||
['STEPRMIN', SNG], ['STEPRMAX', SNG],
|
||||
['PSS', SNG], ['OUTNOISE', SNG], ['INPUTRES', SNG],
|
||||
['OPENTC', SNG], ['LEADRERR', SNG],
|
||||
['LINEARIZED', INT], ['OSCALIN', SNG], ['GNCALIN', SNG],
|
||||
['BANDWIDTH', SNG], ['MINVS', SNG], ['MAXVS', SNG],
|
||||
];
|
||||
|
||||
// SCM7B: 170 bytes/record
|
||||
const S17 = 'STRING17';
|
||||
const SCM7B_FIELDS = [
|
||||
['MODNAME', S17], ['SENTYPE', S7],
|
||||
['MINVS', SNG], ['NOMVS', SNG], ['MAXVS', SNG],
|
||||
['VLIM', SNG], ['ILIM', SNG], ['PE', SNG],
|
||||
['ISMAXNEXCL', SNG],
|
||||
['MININ', SNG], ['MAXIN', SNG],
|
||||
['MINOUT', SNG], ['MAXOUT', SNG],
|
||||
['IEXC', SNG], ['EXCIMIN', SNG], ['EXCIMAX', SNG],
|
||||
['LEADRERR', SNG], ['RCONV', SNG],
|
||||
['OSCALPT', SNG], ['GNCALPT', SNG], ['CALTOL', SNG],
|
||||
['ISMAXFEXCL', SNG],
|
||||
['VEXC', SNG], ['VEXCLO', SNG], ['VEXCHI', SNG],
|
||||
['LOOPIMAX', SNG],
|
||||
['LINEAR', SNG], ['ACCURACY', SNG], ['PSS', SNG],
|
||||
['TESTFREQ', INT], ['ATTEN', INT], ['ATTENTOL', INT],
|
||||
['STEPRESP', SNG], ['STEPTOL', SNG],
|
||||
['OUTNOISERMS', SNG], ['OUTNOISEVPK', SNG],
|
||||
['INPUTRES', SNG], ['VOPENTC', SNG],
|
||||
['CJCACC', SNG], ['IBIAS', SNG],
|
||||
];
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Record size calculation
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
function calcRecordSize(fields) {
|
||||
let size = 0;
|
||||
for (const [, type] of fields) {
|
||||
size += FIELD_TYPES[type].size;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Parse a single record from a buffer
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
function parseRecord(buf, offset, fields) {
|
||||
const record = {};
|
||||
let pos = offset;
|
||||
for (const [name, type] of fields) {
|
||||
const ft = FIELD_TYPES[type];
|
||||
record[name] = ft.read(buf, pos);
|
||||
pos += ft.size;
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Parse an entire DAT file into an array of records
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
function parseDatFile(filePath, fields) {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`Spec file not found: ${filePath}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const buf = fs.readFileSync(filePath);
|
||||
const recordSize = calcRecordSize(fields);
|
||||
const numRecords = Math.floor(buf.length / recordSize);
|
||||
const records = [];
|
||||
|
||||
for (let i = 0; i < numRecords; i++) {
|
||||
const offset = i * recordSize;
|
||||
if (offset + recordSize > buf.length) break;
|
||||
|
||||
const record = parseRecord(buf, offset, fields);
|
||||
|
||||
// Skip records with empty, placeholder, or corrupted model names
|
||||
const modname = record.MODNAME;
|
||||
if (!modname || modname.length === 0) continue;
|
||||
// Skip if model name contains non-alphanumeric characters (except dash)
|
||||
if (!/^[A-Za-z0-9-]+$/.test(modname)) continue;
|
||||
// Skip placeholder entries
|
||||
if (/^[XYZ]+$/.test(modname) || /^ZZZZ/.test(modname)) continue;
|
||||
// Skip if MODNAME doesn't start with a known product prefix
|
||||
const upper = modname.toUpperCase();
|
||||
if (!upper.match(/^(SCM5B|5B|SCM7B|7B|8B|DSCA|DSCT|SCT|BOGUS)/)) continue;
|
||||
|
||||
records.push(record);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Family configuration
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
const FAMILIES = {
|
||||
SCM5B: {
|
||||
file: '5BMAIN.DAT',
|
||||
fields: SCM5B_FIELDS,
|
||||
family: 'SCM5B',
|
||||
logType: '5BLOG',
|
||||
},
|
||||
B8: {
|
||||
file: '8BMAIN.DAT',
|
||||
fields: B8_FIELDS,
|
||||
family: '8B',
|
||||
logType: '8BLOG',
|
||||
},
|
||||
DSCA: {
|
||||
file: 'DSCOUT.DAT',
|
||||
fields: DSCA_FIELDS,
|
||||
family: 'DSCA',
|
||||
logType: 'DSCLOG',
|
||||
},
|
||||
DSCT: {
|
||||
file: 'SCTMAIN.DAT',
|
||||
fields: DSCT_FIELDS,
|
||||
family: 'DSCT',
|
||||
logType: 'SCTLOG',
|
||||
},
|
||||
DSCA_DIN: {
|
||||
file: 'DSCMAIN4.DAT',
|
||||
fields: DSCA_DIN_FIELDS,
|
||||
family: 'DSCA',
|
||||
logType: 'DSCLOG',
|
||||
},
|
||||
SCM5B45: {
|
||||
file: '5B45DATA.DAT',
|
||||
fields: SCM5B45_FIELDS,
|
||||
family: 'SCM5B',
|
||||
logType: '5BLOG',
|
||||
},
|
||||
SCM5B48: {
|
||||
file: 'DB5B48.DAT',
|
||||
fields: SCM5B48_FIELDS,
|
||||
family: 'SCM5B',
|
||||
logType: '5BLOG',
|
||||
},
|
||||
SCM5B49: {
|
||||
file: '5B49_2.DAT',
|
||||
fields: SCM5B49_FIELDS,
|
||||
family: 'SCM5B',
|
||||
logType: '5BLOG',
|
||||
},
|
||||
SCM7B: {
|
||||
file: '7BMAIN.DAT',
|
||||
fields: SCM7B_FIELDS,
|
||||
family: 'SCM7B',
|
||||
logType: '7BLOG',
|
||||
},
|
||||
};
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Main API: load all specs into a lookup map
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Load all model specs from binary DAT files.
|
||||
* @param {string} specDir - Directory containing the DAT files
|
||||
* @returns {Map<string, object>} Map of model_number -> spec record (with _family added)
|
||||
*/
|
||||
function loadAllSpecs(specDir) {
|
||||
specDir = specDir || DEFAULT_SPEC_DIR;
|
||||
const specMap = new Map();
|
||||
|
||||
for (const [familyKey, config] of Object.entries(FAMILIES)) {
|
||||
const filePath = path.join(specDir, config.file);
|
||||
const records = parseDatFile(filePath, config.fields);
|
||||
|
||||
for (const record of records) {
|
||||
record._family = config.family;
|
||||
record._logType = config.logType;
|
||||
// Normalize model name for lookup (trim, uppercase)
|
||||
const key = record.MODNAME.toUpperCase().trim();
|
||||
specMap.set(key, record);
|
||||
}
|
||||
|
||||
console.log(`[SPEC] Loaded ${records.length} models from ${config.file} (${config.family})`);
|
||||
}
|
||||
|
||||
console.log(`[SPEC] Total models loaded: ${specMap.size}`);
|
||||
return specMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up specs for a model number.
|
||||
* Tries exact match, then common prefix variations (SCM5B <-> 5B, DSCA <-> DSC).
|
||||
* @param {Map} specMap - Spec map from loadAllSpecs()
|
||||
* @param {string} modelNumber - Model number to look up
|
||||
* @returns {object|null} Spec record or null
|
||||
*/
|
||||
function getSpecs(specMap, modelNumber) {
|
||||
if (!modelNumber) return null;
|
||||
const key = modelNumber.toUpperCase().trim();
|
||||
|
||||
// SCMVAS/SCMHVAS/VAS/HVAS are Accuracy-only; no binary spec file exists for them.
|
||||
// Return a sentinel so export-datasheets.js routes them through the SCMVAS template
|
||||
// instead of skipping on "missing specs".
|
||||
if (/^(SCMVAS|SCMHVAS|VAS|HVAS)-/.test(key)) {
|
||||
return { MODNAME: modelNumber.trim(), _family: 'SCMVAS', _noSpecs: true };
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if (specMap.has(key)) return specMap.get(key);
|
||||
|
||||
// Try adding/removing SCM prefix: "5B41-03" <-> "SCM5B41-03"
|
||||
if (key.startsWith('SCM5B')) {
|
||||
const short = key.replace('SCM5B', '5B');
|
||||
if (specMap.has(short)) return specMap.get(short);
|
||||
} else if (key.startsWith('5B')) {
|
||||
const full = 'SCM' + key;
|
||||
if (specMap.has(full)) return specMap.get(full);
|
||||
}
|
||||
|
||||
// Try adding/removing SCM prefix for 7B
|
||||
if (key.startsWith('SCM7B')) {
|
||||
const short = key.replace('SCM7B', '7B');
|
||||
if (specMap.has(short)) return specMap.get(short);
|
||||
} else if (key.startsWith('7B')) {
|
||||
const full = 'SCM' + key;
|
||||
if (specMap.has(full)) return specMap.get(full);
|
||||
}
|
||||
|
||||
// Try DSCA variations
|
||||
if (key.startsWith('DSCA')) {
|
||||
// Some specs stored without the 'A'
|
||||
const short = key.replace('DSCA', 'DSC');
|
||||
if (specMap.has(short)) return specMap.get(short);
|
||||
}
|
||||
|
||||
// Try partial match on model base (before any suffix like C, D)
|
||||
// e.g., "DSCA30-05C" -> try "DSCA30-05"
|
||||
const baseMatch = key.match(/^(.+?)([A-Z])$/);
|
||||
if (baseMatch) {
|
||||
const base = baseMatch[1];
|
||||
if (specMap.has(base)) return specMap.get(base);
|
||||
// Also try with prefix variations
|
||||
if (base.startsWith('SCM5B')) {
|
||||
const short = base.replace('SCM5B', '5B');
|
||||
if (specMap.has(short)) return specMap.get(short);
|
||||
} else if (base.startsWith('5B')) {
|
||||
if (specMap.has('SCM' + base)) return specMap.get('SCM' + base);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine product family from model number string
|
||||
*/
|
||||
function getFamily(modelNumber) {
|
||||
if (!modelNumber) return null;
|
||||
const m = modelNumber.toUpperCase();
|
||||
// Order matters: SCMHVAS/SCMVAS must match before generic SCM5B-style.
|
||||
if (m.startsWith('SCMHVAS') || m.startsWith('SCMVAS') ||
|
||||
m.startsWith('HVAS') || m.startsWith('VAS-')) return 'SCMVAS';
|
||||
if (m.startsWith('SCM5B') || m.startsWith('5B')) return 'SCM5B';
|
||||
if (m.startsWith('SCM7B') || m.startsWith('7B')) return 'SCM7B';
|
||||
if (m.startsWith('8B')) return '8B';
|
||||
if (m.startsWith('DSCA')) return 'DSCA';
|
||||
if (m.startsWith('DSCT') || m.startsWith('SCT')) return 'DSCT';
|
||||
return null;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// CLI: test the parser
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
if (require.main === module) {
|
||||
const specDir = process.argv[2] || DEFAULT_SPEC_DIR;
|
||||
console.log(`Loading specs from: ${specDir}\n`);
|
||||
|
||||
const specMap = loadAllSpecs(specDir);
|
||||
|
||||
// Print a few examples from each family
|
||||
const examples = {};
|
||||
for (const [key, spec] of specMap) {
|
||||
const fam = spec._family;
|
||||
if (!examples[fam]) examples[fam] = [];
|
||||
if (examples[fam].length < 3) {
|
||||
examples[fam].push(spec);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [fam, specs] of Object.entries(examples)) {
|
||||
console.log(`\n--- ${fam} Examples ---`);
|
||||
for (const s of specs) {
|
||||
console.log(` ${s.MODNAME}: SENTYPE=${s.SENTYPE}, MININ=${s.MININ}, MAXIN=${s.MAXIN}, MINOUT=${s.MINOUT}, MAXOUT=${s.MAXOUT}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { loadAllSpecs, getSpecs, getFamily, FAMILIES };
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Parser for Engineering-Tested SCMHVAS pre-rendered .txt datasheets.
|
||||
*
|
||||
* Source: TS-3R\LOGS\VASLOG\VASLOG - Engineering Tested\*.txt
|
||||
* Each file is a complete, human-readable test datasheet. We extract
|
||||
* metadata for the DB row and keep the full file contents in raw_data
|
||||
* so the export stage can copy it verbatim to X:\For_Web\<SN>.TXT.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Filename examples:
|
||||
// 166590-1.txt -> SN 166590-1
|
||||
// 166590-110042023104524.txt -> SN 166590-1, timestamp 10042023104524
|
||||
// 166594-1010042023090444.txt -> SN 166594-10, timestamp 10042023090444
|
||||
// The trailing MMDDYYYYhhmmss block (14 digits) is optional and must be
|
||||
// stripped. The SN is the remainder; it always has exactly one dash.
|
||||
//
|
||||
// A single greedy regex can't do this reliably because `\d+-\d+` will
|
||||
// swallow part of the 14-digit timestamp. Split into two steps:
|
||||
// (1) detect and peel the trailing 14-digit timestamp, then
|
||||
// (2) validate what remains as a proper SN (`N-N` optionally followed by
|
||||
// one letter). If the remainder doesn't validate, null the SN so the
|
||||
// in-file `SN:` header wins.
|
||||
const SN_RE = /^\d+-\d+[A-Za-z]?$/;
|
||||
|
||||
function parseFilename(fileName) {
|
||||
const base = fileName.replace(/\.txt$/i, '');
|
||||
if (base === fileName) return null; // not a .txt
|
||||
|
||||
const tsMatch = base.match(/^(.+?)(\d{14})$/);
|
||||
let serialCandidate;
|
||||
let timestamp;
|
||||
if (tsMatch) {
|
||||
serialCandidate = tsMatch[1];
|
||||
timestamp = tsMatch[2];
|
||||
} else {
|
||||
serialCandidate = base;
|
||||
timestamp = null;
|
||||
}
|
||||
|
||||
const serialNumber = SN_RE.test(serialCandidate) ? serialCandidate : null;
|
||||
return { serialNumber, timestamp };
|
||||
}
|
||||
|
||||
function extractField(text, label) {
|
||||
const re = new RegExp('^\\s*' + label + ':\\s*(.+?)\\s*$', 'm');
|
||||
const m = text.match(re);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
// MM/DD/YYYY or MM-DD-YYYY -> YYYY-MM-DD (DB canonical)
|
||||
function normalizeDate(dateStr) {
|
||||
if (!dateStr) return null;
|
||||
const m = dateStr.match(/^(\d{1,2})[-/](\d{1,2})[-/](\d{4})$/);
|
||||
if (!m) return null;
|
||||
const mm = m[1].padStart(2, '0');
|
||||
const dd = m[2].padStart(2, '0');
|
||||
return `${m[3]}-${mm}-${dd}`;
|
||||
}
|
||||
|
||||
function extractAccuracyStatus(text) {
|
||||
// Line format: " Accuracy 0.007% +/- 0.03% PASS"
|
||||
const m = text.match(/^\s*Accuracy\s+\S+\s+\S+(?:\s+\S+)?\s+(PASS|FAIL)\s*$/mi);
|
||||
return m ? m[1].toUpperCase() : null;
|
||||
}
|
||||
|
||||
function parseVaslogEngTxt(filePath, testStation = null) {
|
||||
const records = [];
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(filePath)) return records;
|
||||
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
const baseName = path.basename(filePath);
|
||||
|
||||
const parsedName = parseFilename(baseName);
|
||||
if (!parsedName) return records;
|
||||
|
||||
const modelNumber = extractField(content, 'Model');
|
||||
const dateRaw = extractField(content, 'Date');
|
||||
const snFromFile = extractField(content, 'SN');
|
||||
const testDate = normalizeDate(dateRaw);
|
||||
const result = extractAccuracyStatus(content) || 'PASS';
|
||||
|
||||
if (!modelNumber || !testDate) return records;
|
||||
|
||||
// Prefer the in-file SN: header. Fall back to filename-derived SN
|
||||
// only if it validated against SN_RE (parsedName.serialNumber is
|
||||
// null on pathological names, which forces the header to win).
|
||||
const serialNumber = snFromFile || parsedName.serialNumber;
|
||||
if (!serialNumber) return records;
|
||||
|
||||
records.push({
|
||||
log_type: 'VASLOG_ENG',
|
||||
model_number: modelNumber.trim(),
|
||||
serial_number: serialNumber.trim(),
|
||||
test_date: testDate,
|
||||
test_station: testStation,
|
||||
overall_result: result,
|
||||
raw_data: content,
|
||||
source_file: filePath,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error parsing ${filePath}: ${err.message}`);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
module.exports = { parseVaslogEngTxt, parseFilename };
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Redeploy the patched templates/datasheet-exact.js only.
|
||||
|
||||
Backs up the current AD2 copy as .bak-20260412b (different suffix from the
|
||||
main deploy earlier today) then overwrites.
|
||||
"""
|
||||
import base64, os, subprocess, yaml, paramiko
|
||||
|
||||
HOST='192.168.0.6'; USER='sysadmin'
|
||||
LOCAL = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\implementation\templates\datasheet-exact.js'
|
||||
REMOTE = 'C:/Shares/testdatadb/templates/datasheet-exact.js'
|
||||
BACKUP_SUFFIX = '.bak-20260412b'
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
# Verify remote exists
|
||||
try:
|
||||
sz = sftp.stat(REMOTE).st_size
|
||||
print(f'[OK] remote exists: {REMOTE} ({sz} bytes)')
|
||||
except IOError:
|
||||
raise SystemExit(f'[FAIL] remote missing: {REMOTE}')
|
||||
|
||||
# Backup
|
||||
backup_path = REMOTE + BACKUP_SUFFIX
|
||||
with sftp.open(REMOTE, 'rb') as src:
|
||||
data = src.read()
|
||||
with sftp.open(backup_path, 'wb') as dst:
|
||||
dst.write(data)
|
||||
print(f'[OK] backup: {backup_path} ({len(data)} bytes)')
|
||||
|
||||
# Upload new
|
||||
sftp.put(LOCAL, REMOTE)
|
||||
new_sz = os.path.getsize(LOCAL)
|
||||
print(f'[OK] uploaded: {LOCAL} -> {REMOTE} ({new_sz} bytes)')
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Restart testdatadb service, rerun backfill on remaining ~438 records, verify."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
HOST='192.168.0.6'; USER='sysadmin'
|
||||
|
||||
NODE_BACKFILL = r'''
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database/db');
|
||||
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('./templates/datasheet-exact');
|
||||
|
||||
const OUTPUT_DIR = '\\\\ad2\\webshare\\For_Web';
|
||||
|
||||
(async () => {
|
||||
if (!fs.existsSync(OUTPUT_DIR)) { console.error('[FAIL] output dir not reachable'); process.exit(1); }
|
||||
const specMap = loadAllSpecs();
|
||||
const where = "overall_result='PASS' AND forweb_exported_at IS NULL " +
|
||||
"AND ((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%') OR log_type='VASLOG_ENG')";
|
||||
const rows = await db.query('SELECT * FROM test_records WHERE ' + where + ' ORDER BY test_date DESC');
|
||||
console.log('[INFO] ' + rows.length + ' records to process');
|
||||
|
||||
let rendered = 0, passthrough = 0, skipped = 0, errors = 0;
|
||||
const batchIds = [];
|
||||
async function flush() {
|
||||
if (!batchIds.length) return;
|
||||
const now = new Date().toISOString();
|
||||
await db.transaction(async tx => {
|
||||
for (const id of batchIds) await tx.execute('UPDATE test_records SET forweb_exported_at=$1 WHERE id=$2',[now,id]);
|
||||
});
|
||||
batchIds.length = 0;
|
||||
}
|
||||
for (const r of rows) {
|
||||
try {
|
||||
const outPath = path.join(OUTPUT_DIR, r.serial_number + '.TXT');
|
||||
if (r.log_type === 'VASLOG_ENG') {
|
||||
if (r.source_file && fs.existsSync(r.source_file)) fs.copyFileSync(r.source_file, outPath);
|
||||
else fs.writeFileSync(outPath, r.raw_data || '', 'utf8');
|
||||
passthrough++;
|
||||
} else {
|
||||
const specs = getSpecs(specMap, r.model_number);
|
||||
if (!specs) { skipped++; continue; }
|
||||
const txt = generateExactDatasheet(r, specs);
|
||||
if (!txt) { skipped++; continue; }
|
||||
fs.writeFileSync(outPath, txt, 'utf8');
|
||||
rendered++;
|
||||
}
|
||||
batchIds.push(r.id);
|
||||
if (batchIds.length >= 100) { await flush(); process.stdout.write('[PROGRESS] ' + (rendered+passthrough) + '/' + rows.length + '\n'); }
|
||||
} catch (e) { errors++; console.error('[ERR] ' + r.serial_number + ': ' + e.message); }
|
||||
}
|
||||
await flush();
|
||||
console.log('\n========================================');
|
||||
console.log('Straggler Backfill Complete');
|
||||
console.log('========================================');
|
||||
console.log('Rendered: ' + rendered);
|
||||
console.log('Passthrough: ' + passthrough);
|
||||
console.log('Skipped: ' + skipped);
|
||||
console.log('Errors: ' + errors);
|
||||
|
||||
// Post-run count
|
||||
const remaining = await db.queryOne("SELECT COUNT(*) c FROM test_records WHERE " + where);
|
||||
console.log('Remaining backlog: ' + remaining.c);
|
||||
|
||||
// Sample a plain-decimal-derived datasheet to verify render
|
||||
const sample = await db.queryOne(
|
||||
"SELECT serial_number, model_number FROM test_records WHERE forweb_exported_at IS NOT NULL " +
|
||||
"AND raw_data LIKE '%PASS .%' AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"ORDER BY forweb_exported_at DESC LIMIT 1"
|
||||
);
|
||||
if (sample) console.log('Plain-decimal sample just rendered: SN=' + sample.serial_number + ' model=' + sample.model_number);
|
||||
|
||||
await db.close();
|
||||
})().catch(e => { console.error('[FATAL] ' + e.message); process.exit(1); });
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=1800):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print('=== STEP 1: restart testdatadb ===', flush=True)
|
||||
out, err, rc = ps(c, r'Restart-Service testdatadb -Force; Start-Sleep -Seconds 3; Get-Service testdatadb | Select Name,Status | Format-Table -AutoSize | Out-String', to=60)
|
||||
print(out, flush=True)
|
||||
|
||||
print('=== STEP 2: deploy and run backfill node script ===', flush=True)
|
||||
sftp = c.open_sftp()
|
||||
remote_js = 'C:/Shares/testdatadb/_backfill_stragglers.js'
|
||||
with sftp.open(remote_js, 'w') as fh: fh.write(NODE_BACKFILL)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_backfill_stragglers.js')
|
||||
print(f'[rc={rc}]', flush=True)
|
||||
print(out, flush=True)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('--- STDERR ---', flush=True)
|
||||
print(err[:2000], flush=True)
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote_js)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Restart testdatadb service on AD2 and verify it comes back up healthy."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print('=== Restart testdatadb ===')
|
||||
out, err, rc = ps(c, r'Restart-Service testdatadb -Force; Start-Sleep -Seconds 3; Get-Service testdatadb | Select Name,Status | Format-Table -AutoSize | Out-String', to=60)
|
||||
print(out)
|
||||
|
||||
print('=== Service port probe (common node app ports) ===')
|
||||
out, err, rc = ps(c, r'foreach ($p in @(3000,3001,3002,8000,8001,8002,8080,5000)) { $r = Test-NetConnection -ComputerName localhost -Port $p -InformationLevel Quiet -WarningAction SilentlyContinue; if ($r) { Write-Host "[OPEN] $p" } }')
|
||||
print(out)
|
||||
|
||||
print('=== Listening ports on AD2 matching node ===')
|
||||
out, err, rc = ps(c, r'Get-NetTCPConnection -State Listen | ForEach-Object { $proc = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue; if ($proc -and $proc.Name -match "node") { "{0,-8} {1}" -f $_.LocalPort, $proc.Path } } | Sort-Object -Unique')
|
||||
print(out)
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Local wrapper around deploy-to-ad2.py.
|
||||
|
||||
Reason: the approved deploy script fetches the AD2 password via
|
||||
`bash D:/vault/scripts/vault.sh get-field ...`, which internally pipes
|
||||
through `yq`. In Claude Code's sandboxed bash env, `yq` raises Permission
|
||||
denied. This wrapper monkey-patches `get_ad2_password` to call `sops`
|
||||
directly and parse the YAML with PyYAML -- the underlying file (and
|
||||
secret) is unchanged.
|
||||
|
||||
Also strips a stale shell-escape backslash before the `!` in the vault
|
||||
entry's password field. That vault entry needs cleanup separately; until
|
||||
then this is the workaround.
|
||||
|
||||
Usage: python run-deploy-local.py [--dry-run]
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DEPLOY_PATH = os.path.join(HERE, 'deploy-to-ad2.py')
|
||||
|
||||
|
||||
def _get_pwd_via_sops() -> str:
|
||||
r = subprocess.run(
|
||||
['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True,
|
||||
)
|
||||
data = yaml.safe_load(r.stdout)
|
||||
return data['credentials']['password'].replace('\\', '')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
spec = importlib.util.spec_from_file_location('deploy_to_ad2', DEPLOY_PATH)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
mod.get_ad2_password = _get_pwd_via_sops
|
||||
return mod.main()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,910 @@
|
||||
/**
|
||||
* Exact-Match Datasheet Formatter
|
||||
*
|
||||
* Generates TXT datasheets matching the original QuickBASIC DATASHEETWRITE output.
|
||||
* Requires a DB record (with raw_data) and model specs from spec-reader.
|
||||
*/
|
||||
|
||||
const { getFamily } = require('../parsers/spec-reader');
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// DATA LINES: parameter names and units per family
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
const DATA_LINES = {
|
||||
SCM5B: [
|
||||
['Supply Current, Nom', 'mA'], // 1
|
||||
['Supply Current, Max', 'mA'], // 2
|
||||
['Exc. Current #1', 'uA'], // 3
|
||||
['Exc. Current #2', 'uA'], // 4
|
||||
['Exc. Current Match', 'uA'], // 5
|
||||
['Output Resistance', 'ohms'], // 6
|
||||
['CJC Gain', 'uV/C'], // 7
|
||||
['Exc. Voltage', 'V'], // 8
|
||||
['Exc. Load Reg.', 'ppm/mA'], // 9
|
||||
['Vout Reg. w/ Load', '%'], // 10
|
||||
['Exc. Current Limit', 'mA'], // 11
|
||||
['Linearity', '%'], // 12
|
||||
['Accuracy', '%'], // 13
|
||||
['Lead R Effect', 'C/ohm'], // 14
|
||||
['Supply Sensitivity', 'uV/%'], // 15
|
||||
['Input Resistance', 'Mohms'], // 16
|
||||
['Open Input Response', 'V'], // 17
|
||||
['Frequency Response', 'dB'], // 18
|
||||
['Step Response', '%'], // 19
|
||||
['Output Noise', 'uVrms'], // 20
|
||||
['Over-range Response', 'V'], // 21
|
||||
],
|
||||
'8B': [
|
||||
['Supply Current, Nom', 'mA'],
|
||||
['Supply Current, Max', 'mA'],
|
||||
['Exc. Current #1', 'uA'],
|
||||
['Exc. Current #2', 'uA'],
|
||||
['Exc. Current Match', 'uA'],
|
||||
['Output Resistance', 'ohms'],
|
||||
['CJC Gain', 'uV/C'],
|
||||
['Exc. Voltage', 'V'],
|
||||
['Exc. Load Reg.', 'ppm/mA'],
|
||||
['Vout Reg. w/ Load', '%'],
|
||||
['Exc. Current Limit', 'mA'],
|
||||
['Linearity', '%'],
|
||||
['Accuracy', '%'],
|
||||
['Lead R Effect', 'C/ohm'],
|
||||
['Supply Sensitivity', 'ppm/%'],
|
||||
['Input Resistance', 'Mohms'],
|
||||
['Open Input Response', 'V'],
|
||||
['Frequency Response', 'dB'],
|
||||
['Step Response', '%'],
|
||||
['Output Noise', 'uVrms'],
|
||||
['Over-range Response', 'V'],
|
||||
],
|
||||
DSCA: [
|
||||
['Supply Current, Nom', 'mA'],
|
||||
['Supply Current @ Max Load', 'mA'],
|
||||
['Linearity, 0mA Load', '%'],
|
||||
['Accuracy, 0mA Load', '%'],
|
||||
['Linearity, 5mA Load', '%'],
|
||||
['Accuracy, 5mA Load', '%'],
|
||||
['Linearity, 50mA Load', '%'],
|
||||
['Accuracy, 50mA Load', '%'],
|
||||
['Positive Current Limit', 'mA'],
|
||||
['Negative Current Limit', 'mA'],
|
||||
['Overrange', '%'],
|
||||
['Power Supply Sensitivity', '%/%'],
|
||||
['Input Resistance', 'Mohms'],
|
||||
['Frequency Response', 'dB'],
|
||||
['Step Response', '%'],
|
||||
['Output Noise', ''],
|
||||
['Compliance', '%'],
|
||||
['Accuracy @ 5 ohm load', '%'],
|
||||
],
|
||||
SCM7B: [
|
||||
['Supply Current', 'mA'], // 1
|
||||
['Supply Current w/ Load', 'mA'], // 2
|
||||
['Bias Current', 'nA'], // 3
|
||||
['Input Resistance', 'kohms'], // 4
|
||||
['Offset Calibration', 'mV'], // 5
|
||||
['Gain Calibration', 'mV'], // 6
|
||||
['Linearity/Conformity', '%'], // 7
|
||||
['Accuracy', '%'], // 8
|
||||
['VLoop @ 0 mA (Vs = 18V)', 'V'], // 9
|
||||
[' (Vs = 35V)', 'V'], // 10
|
||||
['VLoop @ 4 mA (Vs = 18V)', 'V'], // 11
|
||||
[' (Vs = 35V)', 'V'], // 12
|
||||
['VLoop @ 20mA (Vs = 18V)', 'V'], // 13
|
||||
[' (Vs = 35V)', 'V'], // 14
|
||||
['VLoop Peak Ripple', 'mV'], // 15
|
||||
['High Excitation Current', 'uA'], // 16
|
||||
['Low Excitation Current', 'uA'], // 17
|
||||
['Output Effective Power', 'mW'], // 18
|
||||
['Supply Sensitivity', '%/%Vs'], // 19
|
||||
['Open Sensor Response', 'V'], // 20
|
||||
['Lead Resistance Effect', 'C/ohm'], // 21
|
||||
['CJC Gain', 'uV/C'], // 22
|
||||
['100kHz Output Noise', 'uVrms'], // 23
|
||||
['Attenuation', 'dB'], // 24
|
||||
['150ms Step Response', 'V'], // 25
|
||||
['Output Noise', 'mVpk'], // 26
|
||||
['Over-Range', 'V'], // 27
|
||||
['Under-Range', 'V'], // 28
|
||||
['Open Loop Detect', 'mA'], // 29
|
||||
['Error @ Max Rload', '%'], // 30
|
||||
['Pass-Through Error', '%'], // 31
|
||||
],
|
||||
SCMVAS: [
|
||||
['Accuracy', '%'],
|
||||
],
|
||||
DSCT: [
|
||||
['Under-range Limit', 'mA'],
|
||||
['Over-range Limit', 'mA'],
|
||||
['Error @ Vloop = 10.8V', '%'],
|
||||
['Error @ Vloop = 60V', '%'],
|
||||
['Minus f.s. Exc. Current', 'uA'],
|
||||
['Plus f.s. Exc. Current', 'uA'],
|
||||
['Current Source Matching', '%'],
|
||||
['Linearity / Conformity', '%'],
|
||||
['Accuracy', '%'],
|
||||
['Lead Resistance Effects', 'C/ohm'],
|
||||
['Loop Voltage Sensitivity', '%/V'],
|
||||
['Input Resistance', 'Mohm'],
|
||||
['Open Thermocouple Response', 'mA'],
|
||||
['Frequency Response', 'dB'],
|
||||
['Step Response', '%'],
|
||||
['Output Noise', 'uArms'],
|
||||
],
|
||||
};
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Sensor type number mapping (for input column headers)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function getSensorNum(sentype) {
|
||||
if (!sentype) return 1;
|
||||
const s = sentype.toUpperCase().trim();
|
||||
if (s === 'V' || s === 'MV') return 1;
|
||||
if (s === 'MA') return 2;
|
||||
if (s.includes('JTC') || s === 'J') return 3;
|
||||
if (s.includes('KTC') || s === 'K') return 4;
|
||||
if (s.includes('TTC') || s === 'T') return 5;
|
||||
if (s.includes('ETC') || s === 'E' || s.includes('RTC') || s.includes('STC') || s.includes('NTC') || s.includes('BTC')) return 6;
|
||||
if (s.includes('RTD')) return 7;
|
||||
if (s === 'FBRIDGE' || s === 'HBRIDGE') return 8;
|
||||
if (s === '2WTX') return 9;
|
||||
return 1; // default voltage
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Parse raw_data from DB record
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function parseRawData(rawData, family) {
|
||||
if (!rawData) return null;
|
||||
|
||||
const lines = rawData.split('\n').map(l => l.trim()).filter(l => l.length > 0);
|
||||
if (lines.length < 8) return null;
|
||||
|
||||
const result = {
|
||||
modelLine: '',
|
||||
accuracy: [], // 5 points: { stim, calc, meas, error, status }
|
||||
stepResponse: 0,
|
||||
statusEntries: [],
|
||||
};
|
||||
|
||||
let lineIdx = 0;
|
||||
|
||||
// Line 0: model name (quoted)
|
||||
result.modelLine = lines[lineIdx++].replace(/"/g, '').trim();
|
||||
|
||||
// Lines 1-5: accuracy points
|
||||
for (let i = 0; i < 5 && lineIdx < lines.length; i++) {
|
||||
const parts = parseCSVLine(lines[lineIdx++]);
|
||||
if (parts.length >= 5) {
|
||||
result.accuracy.push({
|
||||
stim: parseFloat(parts[0]),
|
||||
calc: parseFloat(parts[1]),
|
||||
meas: parseFloat(parts[2]),
|
||||
error: parseFloat(parts[3]),
|
||||
status: parts[4].replace(/"/g, '').trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Next line: step response / placeholders
|
||||
if (lineIdx < lines.length) {
|
||||
const parts = parseCSVLine(lines[lineIdx++]);
|
||||
// SCM5B/8B: "0","0",value DSCT: just value
|
||||
const lastVal = parts[parts.length - 1];
|
||||
result.stepResponse = parseFloat(lastVal) || 0;
|
||||
}
|
||||
|
||||
// Remaining lines: STATUS groups
|
||||
// SCM5B/8B: groups of 5, DSCT: groups of 4
|
||||
const groupSize = (family === 'DSCT') ? 4 : 5;
|
||||
while (lineIdx < lines.length) {
|
||||
const line = lines[lineIdx];
|
||||
// Stop if we hit the serial/date line
|
||||
if (line.match(/^"\d+-\d+[A-Za-z]?","/)) break;
|
||||
const parts = parseCSVLine(line);
|
||||
for (const p of parts) {
|
||||
result.statusEntries.push(p.replace(/"/g, ''));
|
||||
}
|
||||
lineIdx++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Simple CSV parser that handles quoted strings
|
||||
function parseCSVLine(line) {
|
||||
const parts = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (ch === '"') {
|
||||
inQuotes = !inQuotes;
|
||||
} else if (ch === ',' && !inQuotes) {
|
||||
parts.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
}
|
||||
parts.push(current.trim());
|
||||
return parts;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Format measured value from STATUS entry
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Format a number matching QuickBASIC STR$() behavior:
|
||||
* - Positive numbers get a leading space
|
||||
* - Leading zeros before decimal are dropped (0.03 -> .03)
|
||||
* - Rounds to 6 significant digits to clean IEEE 754 artifacts
|
||||
*/
|
||||
function r(val, fixedDecimals) {
|
||||
if (val == null || isNaN(val)) return '0';
|
||||
const rounded = parseFloat(val.toPrecision(6));
|
||||
let str;
|
||||
if (fixedDecimals != null) {
|
||||
str = rounded.toFixed(fixedDecimals);
|
||||
} else {
|
||||
str = String(rounded);
|
||||
}
|
||||
// QB STR$() drops leading zero: "0.03" -> ".03"
|
||||
str = str.replace(/^0\./, '.').replace(/^-0\./, '-.');
|
||||
// QB STR$() prepends space for positive numbers
|
||||
if (rounded >= 0 && !str.startsWith(' ')) {
|
||||
str = ' ' + str;
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse STATUS$ entry and format measured value matching QB PRINT USING.
|
||||
* QB format strings all produce exactly 6 characters for the number:
|
||||
* "0" -> "###### &" (integer, 6 digits)
|
||||
* "1" -> "####.# &" (1 decimal, 6 chars)
|
||||
* "2" -> "####.# &" (same as 1)
|
||||
* "3" -> "##.### &" (3 decimals, 6 chars)
|
||||
* "4" -> "#.#### &" (4 decimals, 6 chars)
|
||||
*/
|
||||
function formatMeasured(statusStr) {
|
||||
if (!statusStr || statusStr.length <= 4) return null;
|
||||
|
||||
const passFail = statusStr.substring(0, 4); // "PASS" or "FAIL"
|
||||
const decimalDigit = statusStr[statusStr.length - 1];
|
||||
const valueStr = statusStr.substring(5, statusStr.length - 1).trim();
|
||||
const value = parseFloat(valueStr);
|
||||
|
||||
if (isNaN(value)) return { passFail, formatted: valueStr, width: 6 };
|
||||
|
||||
// QB PRINT USING: right-justified in 6 character positions
|
||||
// Negative sign takes one digit position
|
||||
let formatted;
|
||||
switch (decimalDigit) {
|
||||
case '0': formatted = Math.round(value).toString().padStart(6); break;
|
||||
case '1': formatted = value.toFixed(1).padStart(6); break;
|
||||
case '2': formatted = value.toFixed(1).padStart(6); break;
|
||||
case '3': formatted = value.toFixed(3).padStart(6); break;
|
||||
case '4': formatted = value.toFixed(4).padStart(6); break;
|
||||
default: formatted = value.toFixed(1).padStart(6); break;
|
||||
}
|
||||
|
||||
return { passFail, formatted, value };
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Format TSPEC display string from spec values
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function buildTSpecs(specs, family, stepResponse) {
|
||||
if (!specs) return [];
|
||||
const tspecs = [];
|
||||
|
||||
if (family === 'SCM5B' || family === '8B') {
|
||||
tspecs[1] = ' < ' + r(specs.ISMAXNEXCL);
|
||||
tspecs[2] = ' < ' + r(specs.ISMAXFEXCL);
|
||||
tspecs[3] = ' ' + r(specs.IEXC);
|
||||
tspecs[4] = ' ' + r(specs.IEXC);
|
||||
const imatchtol = (specs.IMATCHTOL || 0) / 100;
|
||||
tspecs[5] = '+/-' + r(specs.IEXC * imatchtol, 0);
|
||||
tspecs[6] = family === '8B' ? ' < 50' : ' < ' + r(specs.OUTRES || 55);
|
||||
tspecs[7] = ''; // CJC gain - computed from polynomial, skip for now
|
||||
if (specs.VEXC) {
|
||||
const vexcAcc = Math.round(specs.VEXCACC / 100 * specs.VEXC * 1000) / 1000;
|
||||
tspecs[8] = r(specs.VEXC, 1) + '+/-' + r(vexcAcc, 3);
|
||||
} else {
|
||||
tspecs[8] = '';
|
||||
}
|
||||
tspecs[9] = '+/-' + r(specs.EXCLOADREG);
|
||||
const acc125 = Math.round((specs.ACCURACY * 1.25) * 100) / 100;
|
||||
tspecs[10] = '+/-' + r(acc125);
|
||||
tspecs[11] = ' < ' + r(specs.EXCIMAX);
|
||||
tspecs[12] = '+/-' + r(specs.LINEAR);
|
||||
tspecs[13] = '+/-' + r(specs.ACCURACY);
|
||||
tspecs[14] = '+/-' + r(stepResponse || 0, 1);
|
||||
tspecs[15] = '+/-' + r(specs.PSS || 0);
|
||||
tspecs[16] = ' >=' + r(specs.INPUTRES);
|
||||
if (specs.VOPENINMIN != null && specs.VOPENINMAX != null) {
|
||||
tspecs[17] = r(specs.VOPENINMIN, 2) + ' to ' + r(specs.VOPENINMAX, 2);
|
||||
} else {
|
||||
tspecs[17] = '';
|
||||
}
|
||||
tspecs[18] = r(specs.ATTEN) + '+/-' + r(specs.ATTENTOL);
|
||||
tspecs[19] = r(specs.STEPRMIN || 0) + ' to ' + r(specs.STEPRMAX || 0);
|
||||
tspecs[20] = ' < ' + r(specs.OUTNOISE);
|
||||
tspecs[21] = tspecs[17]; // duplicate
|
||||
} else if (family === 'DSCA') {
|
||||
tspecs[1] = ' < ' + r(specs.ISMAXNL || 0);
|
||||
tspecs[2] = ' < ' + r(specs.ISMAXFL || 0);
|
||||
tspecs[3] = '+/-' + r(specs.LINEAR1 || 0);
|
||||
tspecs[4] = '+/-' + r(specs.ACCURACY1 || 0);
|
||||
tspecs[5] = '+/-' + r(specs.LINEAR2 || 0);
|
||||
tspecs[6] = '+/-' + r(specs.ACCURACY2 || 0);
|
||||
tspecs[7] = '+/-' + r(specs.LINEAR3 || 0);
|
||||
tspecs[8] = '+/-' + r(specs.ACCURACY3 || 0);
|
||||
tspecs[9] = ' < ' + r(specs.ILIMIT || 0);
|
||||
tspecs[10] = ' > ' + r(-(specs.ILIMIT || 0));
|
||||
tspecs[11] = ' > ' + r(specs.PERCOVER || 0);
|
||||
tspecs[12] = '+/-' + r(specs.PSS || 0);
|
||||
tspecs[13] = ' >=' + r(specs.INPUTRES || 0);
|
||||
tspecs[14] = r(specs.ATTEN || 0) + '+/-' + r(specs.ATTENTOL || 0);
|
||||
tspecs[15] = r(specs.STEPRMIN || 0) + ' to ' + r(specs.STEPRMAX || 0);
|
||||
tspecs[16] = ' <=' + r(specs.OUTNOISE || 0);
|
||||
tspecs[17] = '+/-' + r(specs.COMPLIANCE || 0);
|
||||
tspecs[18] = '+/-' + r((specs.ACCURACY1 || 0) * 2);
|
||||
} else if (family === 'DSCT') {
|
||||
tspecs[1] = ''; // computed at runtime
|
||||
tspecs[2] = ''; // computed at runtime
|
||||
tspecs[3] = ' < 1';
|
||||
tspecs[4] = ' < 1';
|
||||
const iexcmTol = specs.MODNAME && specs.MODNAME.startsWith('DSCT') ? 0.05 : 0.02;
|
||||
tspecs[5] = Math.round(specs.IEXCMFS || 0) + '+/-' + Math.round((specs.IEXCMFS || 0) * iexcmTol);
|
||||
tspecs[6] = Math.round(specs.IEXCPFS || 0) + '+/-' + Math.round((specs.IEXCPFS || 0) * iexcmTol);
|
||||
tspecs[7] = '+/-' + r(specs.IMATCHTOL || 0);
|
||||
tspecs[8] = '+/- ' + r(specs.LINEAR || 0);
|
||||
tspecs[9] = '+/- ' + r(specs.ACCURACY || 0);
|
||||
tspecs[10] = '+/-' + r(stepResponse || 0, 1);
|
||||
tspecs[11] = '+/-' + r(specs.VSEN || 0);
|
||||
tspecs[12] = ' >=' + r(specs.INPUTRES || 0);
|
||||
const iopentc = specs.IOPENTC || 0;
|
||||
const maxout = specs.MAXOUT || 20;
|
||||
tspecs[13] = (iopentc > maxout ? ' > ' : ' < ') + r(iopentc);
|
||||
tspecs[14] = r(specs.ATTEN || 0) + '+/-' + r(specs.ATTENTOL || 0);
|
||||
tspecs[15] = r(specs.STEPRMIN || 0) + ' to ' + r(specs.STEPRMAX || 0);
|
||||
tspecs[16] = ' < ' + r(specs.OUTNOISE || 0);
|
||||
} else if (family === 'SCM7B') {
|
||||
const orange = (specs.MAXOUT || 5) - (specs.MINOUT || 0);
|
||||
tspecs[1] = '< ' + r(specs.ISMAXNEXCL + 6);
|
||||
tspecs[2] = '< ' + r(specs.ISMAXFEXCL + 6);
|
||||
tspecs[3] = '+/-' + r(specs.IBIAS || 0);
|
||||
tspecs[4] = ' > ' + r(specs.INPUTRES || 0);
|
||||
const calTol = 20 * orange * (specs.CALTOL || 0);
|
||||
tspecs[5] = '+/-' + r(calTol);
|
||||
tspecs[6] = '+/-' + r(calTol);
|
||||
tspecs[7] = '+/-' + r(specs.LINEAR || 0);
|
||||
tspecs[8] = '+/-' + r(specs.ACCURACY || 0);
|
||||
if (specs.VEXC) {
|
||||
const vexc5 = specs.VEXC * 0.05;
|
||||
tspecs[9] = r(specs.VEXC) + ' +/-' + r(vexc5);
|
||||
tspecs[10] = tspecs[9];
|
||||
}
|
||||
if (specs.VEXCLO) {
|
||||
const vlo5 = specs.VEXCLO * 0.05;
|
||||
tspecs[11] = r(specs.VEXCLO) + ' +/-' + r(vlo5);
|
||||
tspecs[12] = tspecs[11];
|
||||
}
|
||||
if (specs.VEXCHI) {
|
||||
const vhi5 = specs.VEXCHI * 0.05;
|
||||
tspecs[13] = r(specs.VEXCHI) + ' +/-' + r(vhi5);
|
||||
tspecs[14] = tspecs[13];
|
||||
}
|
||||
tspecs[15] = ' < 50';
|
||||
tspecs[16] = ' < ' + r(specs.EXCIMAX || 0);
|
||||
tspecs[17] = ' > ' + r(specs.EXCIMIN || 0);
|
||||
tspecs[18] = ' > ' + r(specs.PE || 0);
|
||||
tspecs[19] = '+/-' + r(specs.PSS || 0);
|
||||
tspecs[20] = ''; // Open TC - needs runtime calc
|
||||
tspecs[21] = '+/-' + r(specs.LEADRERR || 0);
|
||||
tspecs[22] = ''; // CJC - needs seebeck polynomial
|
||||
tspecs[23] = ' < ' + r(specs.OUTNOISERMS || 0);
|
||||
tspecs[24] = r(specs.ATTEN || 0) + '+/-' + r(specs.ATTENTOL || 0);
|
||||
// Step response
|
||||
if (specs.STEPRESP && specs.STEPTOL) {
|
||||
const lowV = specs.STEPRESP - specs.STEPTOL;
|
||||
const highV = specs.STEPRESP + specs.STEPTOL;
|
||||
tspecs[25] = r(lowV) + ' to ' + r(highV);
|
||||
} else {
|
||||
tspecs[25] = '';
|
||||
}
|
||||
tspecs[26] = ' < ' + r(specs.OUTNOISEVPK || 0);
|
||||
tspecs[27] = '+5 to +5.8';
|
||||
tspecs[28] = '-.9 to +1';
|
||||
tspecs[29] = '0';
|
||||
tspecs[30] = ''; // Compliance - needs runtime calc
|
||||
tspecs[31] = '+/-' + r(specs.ACCURACY || 0);
|
||||
}
|
||||
|
||||
return tspecs;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Format accuracy value based on sensor type
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function formatAccuracyLine(point, sensorNum, maxIn) {
|
||||
let stimStr;
|
||||
if (sensorNum >= 3 && sensorNum <= 6) {
|
||||
// Temperature: +####.##
|
||||
stimStr = formatSigned(point.stim, 2, 8);
|
||||
} else if (sensorNum === 7) {
|
||||
// Resistance: #####.##
|
||||
stimStr = point.stim.toFixed(2).padStart(8);
|
||||
} else {
|
||||
// Voltage/Current: +###.###
|
||||
const scale = (maxIn != null && maxIn < 1) ? 1000 : 1;
|
||||
stimStr = formatSigned(point.stim * scale, 3, 8);
|
||||
}
|
||||
|
||||
const calcStr = formatSigned(point.calc, 3, 7);
|
||||
const measStr = formatSigned(point.meas, 3, 7);
|
||||
const errorStr = formatSigned(point.error, 3, 8);
|
||||
|
||||
return ' ' + stimStr + ' ' + calcStr + ' ' + measStr + ' ' + errorStr + ' ' + point.status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set text at a specific column position (0-indexed) in a string.
|
||||
* Pads with spaces if the string is shorter than the target column.
|
||||
*/
|
||||
function setCol(str, col, text) {
|
||||
while (str.length < col) str += ' ';
|
||||
return str + text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad string to reach a column position (for inline TAB simulation).
|
||||
* Returns spaces needed to reach the column from current position.
|
||||
*/
|
||||
function padToCol(str, col) {
|
||||
const needed = col - str.length;
|
||||
return needed > 0 ? ' '.repeat(needed) : ' ';
|
||||
}
|
||||
|
||||
function formatSigned(val, decimals, width) {
|
||||
const sign = val >= 0 ? '+' : '';
|
||||
const str = sign + val.toFixed(decimals);
|
||||
return str.padStart(width);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Main: generate exact-match TXT datasheet
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Generate an exact-match TXT datasheet from a DB record and model specs.
|
||||
* @param {object} record - DB record with raw_data, model_number, serial_number, test_date
|
||||
* @param {object} specs - Model spec record from spec-reader
|
||||
* @returns {string|null} Formatted TXT datasheet, or null if data is insufficient
|
||||
*/
|
||||
function generateExactDatasheet(record, specs) {
|
||||
const family = getFamily(record.model_number);
|
||||
if (!family) return null;
|
||||
|
||||
if (family === 'SCMVAS') {
|
||||
return generateSCMVASDatasheet(record);
|
||||
}
|
||||
|
||||
const parsed = (family === 'SCM7B')
|
||||
? parse7BRawData(record.raw_data)
|
||||
: parseRawData(record.raw_data, family);
|
||||
if (!parsed) return null;
|
||||
if (family !== 'SCM7B' && parsed.accuracy.length < 5) return null;
|
||||
|
||||
const dataLines = DATA_LINES[family];
|
||||
if (!dataLines) return null;
|
||||
|
||||
const sentype = specs ? specs.SENTYPE : '';
|
||||
const sensorNum = getSensorNum(sentype);
|
||||
const maxIn = specs ? specs.MAXIN : 10;
|
||||
const tspecs = specs ? buildTSpecs(specs, family, parsed.stepResponse) : [];
|
||||
|
||||
// Format test date from YYYY-MM-DD to MM-DD-YYYY
|
||||
const dateParts = (record.test_date || '').split('-');
|
||||
const dateStr = dateParts.length === 3
|
||||
? `${dateParts[1]}-${dateParts[2]}-${dateParts[0]}`
|
||||
: record.test_date || '';
|
||||
|
||||
let modelName = specs ? specs.MODNAME : record.model_number;
|
||||
// 7B header prepends "SCM" to the model name
|
||||
if (family === 'SCM7B' && !modelName.toUpperCase().startsWith('SCM')) {
|
||||
modelName = 'SCM' + modelName;
|
||||
}
|
||||
|
||||
const lines = [];
|
||||
const TAB5 = ' '; // 4 spaces = TAB(5) in QB (0-indexed)
|
||||
|
||||
// ---- Header ----
|
||||
lines.push(TAB5 + 'DATAFORTH CORPORATION Phone: (520) 741-1404');
|
||||
lines.push(TAB5 + '3331 E. Hemisphere Loop Fax: (520) 741-0762');
|
||||
lines.push(TAB5 + 'Tucson, AZ 85706 USA email: info@dataforth.com');
|
||||
lines.push('');
|
||||
lines.push(' TEST DATA SHEET');
|
||||
lines.push(TAB5 + '~'.repeat(71));
|
||||
// QB: PRINT #9, TAB(5); "Date: "; DATE$
|
||||
// PRINT #9, TAB(5); "Model: "; SPECS.MODNAME
|
||||
// PRINT #9, TAB(5); "SN: "; TAB(12); SN$
|
||||
lines.push(TAB5 + 'Date: ' + dateStr);
|
||||
lines.push(TAB5 + 'Model: ' + modelName);
|
||||
let snLine = TAB5 + 'SN: ';
|
||||
snLine = setCol(snLine, 11, record.serial_number); // TAB(12) = index 11
|
||||
lines.push(snLine);
|
||||
lines.push('');
|
||||
|
||||
// ---- Accuracy Test ----
|
||||
// 7B CSV format doesn't include individual accuracy test points (only error pcts in LOGIT)
|
||||
// The accuracy data is only in the SHT files, not the DAT files
|
||||
if (family === 'SCM7B') {
|
||||
// Skip accuracy section entirely for 7B — data not available from DAT format
|
||||
} else {
|
||||
lines.push(' ACCURACY TEST');
|
||||
lines.push('');
|
||||
lines.push(' Calculated Measured');
|
||||
|
||||
// Input column header based on sensor type
|
||||
let inputHeader;
|
||||
if (sensorNum >= 3 && sensorNum <= 6) {
|
||||
inputHeader = ' Temp. (C)';
|
||||
} else if (sensorNum === 2 || sensorNum === 9) {
|
||||
inputHeader = ' Iin (mA)';
|
||||
} else if (sensorNum === 7) {
|
||||
inputHeader = ' Rin (ohms)';
|
||||
} else {
|
||||
inputHeader = (maxIn != null && maxIn < 1) ? ' Vin (mV)' : ' Vin (V)';
|
||||
}
|
||||
lines.push(' ' + inputHeader + ' Vout (V) Vout (V)* Error (%) Status');
|
||||
lines.push(TAB5 + '========== ========== ========== ========= ========');
|
||||
|
||||
for (const point of parsed.accuracy) {
|
||||
lines.push(formatAccuracyLine(point, sensorNum, maxIn));
|
||||
}
|
||||
lines.push('');
|
||||
} // end accuracy section conditional
|
||||
|
||||
// ---- Final Test Results ----
|
||||
// QB column positions (1-indexed): TAB(31), TAB(47), TAB(60-speclen), TAB(61), TAB(71)
|
||||
lines.push(' FINAL TEST RESULTS');
|
||||
lines.push('');
|
||||
// QB: TAB(12); "Parameter"; TAB(30); "Measured Value"; TAB(51); "Specification "; TAB(70); "Status"
|
||||
let hdr1 = setCol('', 11, 'Parameter');
|
||||
hdr1 = setCol(hdr1, 29, 'Measured Value');
|
||||
hdr1 = setCol(hdr1, 50, 'Specification ');
|
||||
hdr1 = setCol(hdr1, 69, 'Status');
|
||||
lines.push(hdr1);
|
||||
// QB: TAB(5); "======================="; TAB(30); "==============="; TAB(47); "====================="; TAB(70); "======"
|
||||
let hdr2 = setCol('', 4, '=======================');
|
||||
hdr2 = setCol(hdr2, 29, '===============');
|
||||
hdr2 = setCol(hdr2, 46, '=====================');
|
||||
hdr2 = setCol(hdr2, 69, '======');
|
||||
lines.push(hdr2);
|
||||
|
||||
for (let i = 0; i < dataLines.length && i < parsed.statusEntries.length; i++) {
|
||||
const status = parsed.statusEntries[i];
|
||||
if (!status || status.length <= 4) continue; // Skip if no measured data
|
||||
|
||||
const [paramName, paramUnit] = dataLines[i];
|
||||
let unit = paramUnit;
|
||||
|
||||
// Unit overrides per QB logic
|
||||
if (family === 'SCM5B' || family === '8B') {
|
||||
if (i === 13 && sensorNum === 7) unit = 'ohm/ohm';
|
||||
if (i === 14 && (sensorNum === 5 || sensorNum === 6)) unit = 'C/V';
|
||||
}
|
||||
|
||||
const measured = formatMeasured(status);
|
||||
if (!measured) continue;
|
||||
|
||||
// Build line matching QB TAB positions (converting to 0-indexed for string ops)
|
||||
// TAB(5): parameter name
|
||||
// TAB(31): measured value (6 chars right-justified) + space + unit
|
||||
// TAB(60-speclen): spec string right-aligned to end at col 60
|
||||
// TAB(61): unit
|
||||
// TAB(71): PASS/FAIL
|
||||
let line = '';
|
||||
line = setCol(line, 4, paramName); // TAB(5) = index 4
|
||||
line = setCol(line, 30, measured.formatted + ' ' + unit); // TAB(31) = index 30
|
||||
|
||||
const tspec = tspecs[i + 1]; // 1-indexed in TSPECS
|
||||
if (tspec) {
|
||||
const specLen = tspec.length;
|
||||
line = setCol(line, 59 - specLen, tspec); // TAB(60-speclen)
|
||||
line = setCol(line, 60, unit); // TAB(61) = index 60
|
||||
}
|
||||
line = setCol(line, 70, measured.passFail); // TAB(71) = index 70
|
||||
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
// ---- Footer ----
|
||||
// 240 VAC / Hi-Pot (conditional by family/model)
|
||||
if (family === 'SCM5B') {
|
||||
const mn = (modelName || '').trim();
|
||||
if (!mn.startsWith('SCM5BPT') && !mn.startsWith('SCM5B-1369')) {
|
||||
lines.push(TAB5 + '240 VAC Withstand' + ''.padEnd(49) + 'PASS');
|
||||
lines.push(TAB5 + 'Hi-Pot' + ''.padEnd(60) + 'PASS');
|
||||
}
|
||||
} else if (family === '8B') {
|
||||
const mn = (modelName || '').trim();
|
||||
if (!mn.startsWith('8BPT')) {
|
||||
lines.push(TAB5 + 'VAC Withstand' + ''.padEnd(53) + 'PASS');
|
||||
lines.push(TAB5 + 'Hi-Pot' + ''.padEnd(60) + 'PASS');
|
||||
}
|
||||
} else if (family === 'SCM7B') {
|
||||
const mn = (modelName || '').toUpperCase();
|
||||
if (!mn.includes('7BPT')) {
|
||||
let vac = setCol(TAB5 + '120VAC Withstand', 70, 'PASS');
|
||||
lines.push(vac);
|
||||
let hp = setCol(TAB5 + 'Hi-Pot', 70, 'PASS');
|
||||
lines.push(hp);
|
||||
}
|
||||
} else if (family === 'DSCA') {
|
||||
lines.push(TAB5 + '240VAC Withstand' + ''.padEnd(50) + 'PASS');
|
||||
lines.push(TAB5 + 'Hi-Pot' + ''.padEnd(60) + 'PASS');
|
||||
} else if (family === 'DSCT') {
|
||||
lines.push(TAB5 + '240 VAC Withstand' + ''.padEnd(49) + 'PASS');
|
||||
lines.push(TAB5 + 'Hi-Pot' + ''.padEnd(60) + 'PASS');
|
||||
}
|
||||
|
||||
// Underline + Check List
|
||||
lines.push(TAB5 + '_'.repeat(71));
|
||||
if (family === 'SCM7B') {
|
||||
lines.push(' Packing Check List');
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB5 + 'Module Appearance: _____', 44, 'Mounting Screw: _____'));
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB5 + 'Pins Straight: _____', 44, 'Module Header: _____'));
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB5 + 'Tested by: _____________', 44, 'QC: _______________'));
|
||||
} else if (family !== 'DSCA') {
|
||||
lines.push(' Check List');
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB5 + 'Module Appearance: __X__', 44, 'Mounting Screw: __X__'));
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB5 + 'Pins Straight: __X__', 44, 'Module Header: __X__'));
|
||||
}
|
||||
|
||||
// DSCA current output load note
|
||||
if (family === 'DSCA' && specs && specs.OUTSIGTYPE && specs.OUTSIGTYPE.trim().toUpperCase() === 'CURRENT') {
|
||||
lines.push(TAB5 + 'Standard output load for test is 250 ohms.');
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push(TAB5 + 'It is hereby certified that the above product is in conformance with');
|
||||
lines.push(TAB5 + 'all requirements to the extent specified. This product is not');
|
||||
lines.push(TAB5 + 'authorized or warranted for use in life support devices and/or systems.');
|
||||
lines.push('');
|
||||
lines.push(TAB5 + '* NIST traceable calibration certificates support Measured Value data.');
|
||||
lines.push(TAB5 + ' Calibration services are available through ANSI/NCSL Z540-1 and');
|
||||
lines.push(TAB5 + ' ISO Guide 25 Certified Metrology Labs.');
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\r\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse 7B raw_data (single CSV line format)
|
||||
* Format: STAGE: MODEL,SN,DATE,VERSION,DMMSERIAL,val1,...val31,err1,...errN
|
||||
* val=9999 means not tested, [val] means FAIL
|
||||
*/
|
||||
function parse7BRawData(rawData) {
|
||||
if (!rawData) return null;
|
||||
|
||||
const match = rawData.match(/^([A-Z-]+):\s*(.*)$/);
|
||||
if (!match) return null;
|
||||
|
||||
const parts = match[2].split(',');
|
||||
if (parts.length < 36) return null; // model + sn + date + version + dmmserial + 31 values minimum
|
||||
|
||||
const result = {
|
||||
modelLine: parts[0].trim(),
|
||||
accuracy: [],
|
||||
stepResponse: 0,
|
||||
statusEntries: [],
|
||||
};
|
||||
|
||||
// Values start at index 5 (after model, sn, date, version, dmmserial)
|
||||
for (let i = 0; i < 31; i++) {
|
||||
const rawVal = (parts[5 + i] || '').trim();
|
||||
|
||||
if (rawVal === '9999' || rawVal === '') {
|
||||
// Not tested - push short "PASS" (will be skipped by formatter)
|
||||
result.statusEntries.push('PASS');
|
||||
} else if (rawVal.startsWith('[')) {
|
||||
// FAIL - bracketed value
|
||||
const val = rawVal.replace(/[\[\]]/g, '').trim();
|
||||
const numVal = parseFloat(val);
|
||||
if (isNaN(numVal) || numVal === 0) {
|
||||
result.statusEntries.push('FAIL');
|
||||
} else {
|
||||
const decimals = guessDecimals(numVal);
|
||||
result.statusEntries.push('FAIL ' + val + decimals);
|
||||
}
|
||||
} else {
|
||||
// PASS with value
|
||||
const numVal = parseFloat(rawVal);
|
||||
if (isNaN(numVal)) {
|
||||
result.statusEntries.push('PASS');
|
||||
} else {
|
||||
const decimals = guessDecimals(numVal);
|
||||
result.statusEntries.push('PASS ' + rawVal.trim() + decimals);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Error percentages follow the 31 values - these are the accuracy test point errors
|
||||
const errorStart = 5 + 31;
|
||||
for (let i = errorStart; i < parts.length; i++) {
|
||||
const val = parseFloat((parts[i] || '').trim());
|
||||
if (!isNaN(val)) {
|
||||
result.accuracy.push({
|
||||
stim: 0, // Stimulus not stored in 7B CSV format
|
||||
calc: 0,
|
||||
meas: 0,
|
||||
error: val * 100, // Convert fraction to percentage
|
||||
status: 'PASS',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Guess the decimal format digit based on value magnitude
|
||||
*/
|
||||
function guessDecimals(val) {
|
||||
const abs = Math.abs(val);
|
||||
if (abs === 0) return '0';
|
||||
if (abs >= 100) return '0';
|
||||
if (abs >= 10) return '1';
|
||||
if (abs >= 1) return '1';
|
||||
if (abs >= 0.1) return '3';
|
||||
return '4';
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SCMVAS / SCMHVAS: Accuracy-only datasheet (no spec lookup)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// QB's STR$() emits SINGLE values in two formats depending on magnitude:
|
||||
// (1) scientific with a trailing test-status digit: "PASS-7.005501E-033"
|
||||
// (the trailing single digit is a status code, dropped)
|
||||
// (2) plain decimal without status digit: "PASS .01599373" or "PASS-.00499773"
|
||||
// Both are already in percent units (not fractions). Try scientific first,
|
||||
// then plain-decimal as fallback.
|
||||
const SCMVAS_ACCURACY_RE_SCI = /^(PASS|FAIL)\s*(-?\d+\.?\d*E[+-]?\d{2})\d?$/i;
|
||||
const SCMVAS_ACCURACY_RE_PLAIN = /^(PASS|FAIL)\s*(-?\.?\d+\.?\d*)$/i;
|
||||
|
||||
function extractSCMVASAccuracy(rawData) {
|
||||
if (!rawData) return null;
|
||||
// Scan every quoted string in raw_data for a PASS/FAIL + float value.
|
||||
// raw_data lines look like: "PASS-7.005501E-033","","","" — so we extract
|
||||
// each quoted token and test it against the regex.
|
||||
const tokens = rawData.match(/"[^"]*"/g) || [];
|
||||
for (const tok of tokens) {
|
||||
const inner = tok.slice(1, -1).trim();
|
||||
if (!inner) continue;
|
||||
const m = inner.match(SCMVAS_ACCURACY_RE_SCI) || inner.match(SCMVAS_ACCURACY_RE_PLAIN);
|
||||
if (m) {
|
||||
const passFail = m[1].toUpperCase();
|
||||
const value = parseFloat(m[2]);
|
||||
if (isNaN(value)) return null;
|
||||
return { passFail, value };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatSCMVASAccuracyDisplay(value) {
|
||||
const abs = Math.abs(value);
|
||||
let str = abs.toFixed(3);
|
||||
// Trim trailing zeros after decimal, but preserve at least one digit.
|
||||
if (str.indexOf('.') >= 0) {
|
||||
str = str.replace(/0+$/, '').replace(/\.$/, '');
|
||||
}
|
||||
return str + '%';
|
||||
}
|
||||
|
||||
function formatSCMVASDate(testDate) {
|
||||
if (!testDate) return '';
|
||||
// Accept YYYY-MM-DD (DB), MM-DD-YYYY or MM/DD/YYYY (raw). Normalize to MM/DD/YYYY.
|
||||
const s = String(testDate).trim();
|
||||
let m = s.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (m) return `${m[2]}/${m[3]}/${m[1]}`;
|
||||
m = s.match(/^(\d{2})[-/](\d{2})[-/](\d{4})$/);
|
||||
if (m) return `${m[1]}/${m[2]}/${m[3]}`;
|
||||
return s;
|
||||
}
|
||||
|
||||
function generateSCMVASDatasheet(record) {
|
||||
const acc = extractSCMVASAccuracy(record.raw_data);
|
||||
if (!acc) return null;
|
||||
|
||||
const TAB8 = ' ';
|
||||
const modelName = (record.model_number || '').trim();
|
||||
const sn = (record.serial_number || '').trim();
|
||||
const dateStr = formatSCMVASDate(record.test_date);
|
||||
const measured = formatSCMVASAccuracyDisplay(acc.value);
|
||||
const status = acc.passFail;
|
||||
|
||||
const lines = [];
|
||||
|
||||
// Header
|
||||
lines.push(TAB8 + 'Dataforth Corporation Phone number: (520) 741-1404');
|
||||
lines.push(TAB8 + '3331 E. Hemisphere Loop Fax: (520) 741-0762');
|
||||
lines.push(TAB8 + 'Tucson, AZ 85706 USA Email: info@dataforth.com');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
lines.push('');
|
||||
lines.push(' TEST DATA SHEET');
|
||||
lines.push(TAB8 + '~'.repeat(71));
|
||||
lines.push(TAB8 + 'Date: ' + dateStr);
|
||||
lines.push(TAB8 + 'Model: ' + modelName);
|
||||
lines.push(TAB8 + 'SN: ' + sn);
|
||||
// Section header: centered "FINAL TEST RESULTS" padded to column 77 to match golden samples.
|
||||
lines.push(' FINAL TEST RESULTS ');
|
||||
lines.push(TAB8 + '~'.repeat(71));
|
||||
|
||||
// Results table: columns at 8, 28, 48, 68
|
||||
let hdr = TAB8 + 'Parameter';
|
||||
hdr = setCol(hdr, 28, 'Measured Value');
|
||||
hdr = setCol(hdr, 48, 'Specification');
|
||||
hdr = setCol(hdr, 68, 'Status');
|
||||
lines.push(hdr);
|
||||
|
||||
let sep = TAB8 + '================';
|
||||
sep = setCol(sep, 28, '==============');
|
||||
sep = setCol(sep, 48, '=============');
|
||||
sep = setCol(sep, 68, '======');
|
||||
lines.push(sep);
|
||||
|
||||
let row = TAB8 + 'Accuracy';
|
||||
row = setCol(row, 28, measured);
|
||||
row = setCol(row, 48, '+/- 0.03%');
|
||||
row = setCol(row, 68, status);
|
||||
lines.push(row);
|
||||
|
||||
lines.push(TAB8);
|
||||
lines.push(TAB8 + '_'.repeat(71));
|
||||
lines.push(' Check List');
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB8 + 'Module Appearance: __X__', 48, 'Mounting Screw: __X__'));
|
||||
lines.push('');
|
||||
lines.push(setCol(TAB8 + 'Pins Straight: __X__', 48, 'Module Header: __X__'));
|
||||
lines.push('');
|
||||
lines.push(TAB8 + 'It is hereby certified that the above product is in conformance with');
|
||||
lines.push(TAB8 + 'all requirements to the extent specified. This product is not');
|
||||
lines.push(TAB8 + 'authorized or warranted for use in life support devices and/or systems.');
|
||||
lines.push('');
|
||||
lines.push(TAB8 + '* NIST traceable calibration certificates support Measured Value data.');
|
||||
lines.push(TAB8 + 'Calibration services are available through ANSI/NCSL Z540-1 and');
|
||||
lines.push(TAB8 + 'ISO Guide 25 Certified Metrology Labs.');
|
||||
lines.push(TAB8);
|
||||
lines.push(TAB8);
|
||||
|
||||
return lines.join('\r\n');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateExactDatasheet,
|
||||
generateSCMVASDatasheet,
|
||||
extractSCMVASAccuracy,
|
||||
parseRawData,
|
||||
parse7BRawData,
|
||||
DATA_LINES,
|
||||
};
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Local test harness for the SCMVAS/SCMHVAS datasheet pipeline extension.
|
||||
*
|
||||
* Loads samples/vaslog-dat/HVAS-M04.DAT, parses it through the updated
|
||||
* multiline parser (no DB), feeds each parsed record through
|
||||
* generateSCMVASDatasheet(), and prints the output for visual comparison
|
||||
* against samples/corrected-hvas and samples/vaslog-engtxt.
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const { parseMultilineFile } = require('./parsers/multiline');
|
||||
const { generateSCMVASDatasheet, extractSCMVASAccuracy } = require('./templates/datasheet-exact');
|
||||
const { parseVaslogEngTxt } = require('./parsers/vaslog-engtxt');
|
||||
|
||||
const RESEARCH_DIR = path.join(__dirname, '..', 'scmvas-hvas-research');
|
||||
const DAT_SAMPLE = path.join(RESEARCH_DIR, 'samples', 'vaslog-dat', 'HVAS-M04.DAT');
|
||||
const ENG_SAMPLE_DIR = path.join(RESEARCH_DIR, 'samples', 'vaslog-engtxt');
|
||||
const GOLDEN_SAMPLE = path.join(RESEARCH_DIR, 'samples', 'vaslog-engtxt', '166590-110042023104524.txt');
|
||||
|
||||
function hr(title) {
|
||||
console.log('');
|
||||
console.log('='.repeat(78));
|
||||
console.log(title);
|
||||
console.log('='.repeat(78));
|
||||
}
|
||||
|
||||
function testAccuracyExtraction() {
|
||||
hr('[TEST] Accuracy extraction regex');
|
||||
const cases = [
|
||||
{ raw: '"PASS-7.005501E-033"', expect: { passFail: 'PASS', approx: 0.007 } },
|
||||
{ raw: '"PASS 4.988443E-033"', expect: { passFail: 'PASS', approx: 0.005 } },
|
||||
{ raw: '"PASS 1.524978E-023"', expect: { passFail: 'PASS', approx: 0.015 } },
|
||||
{ raw: '"FAIL 2.500000E-013"', expect: { passFail: 'FAIL', approx: 0.25 } },
|
||||
{ raw: '"PASS-1.254585E-033"', expect: { passFail: 'PASS', approx: 0.001 } },
|
||||
// Plain-decimal variants (QB STR$ emits these for values above its
|
||||
// scientific-notation threshold). Observed in ~1.6% of historical records.
|
||||
{ raw: '"PASS .01599373"', expect: { passFail: 'PASS', approx: 0.016 } },
|
||||
{ raw: '"PASS .02399053"', expect: { passFail: 'PASS', approx: 0.024 } },
|
||||
{ raw: '"PASS-.00499773"', expect: { passFail: 'PASS', approx: 0.005 } },
|
||||
{ raw: '"FAIL .05000000"', expect: { passFail: 'FAIL', approx: 0.050 } },
|
||||
];
|
||||
for (const c of cases) {
|
||||
const got = extractSCMVASAccuracy(c.raw);
|
||||
const ok = got && got.passFail === c.expect.passFail && Math.abs(Math.abs(got.value) - c.expect.approx) < 0.001;
|
||||
console.log(` ${ok ? '[OK] ' : '[FAIL]'} ${c.raw.padEnd(28)} -> ${JSON.stringify(got)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function testDatParsingAndGeneration() {
|
||||
hr(`[TEST] Parse ${path.basename(DAT_SAMPLE)} + generate datasheets`);
|
||||
|
||||
if (!fs.existsSync(DAT_SAMPLE)) {
|
||||
console.log(`[FAIL] sample not found: ${DAT_SAMPLE}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const records = parseMultilineFile(DAT_SAMPLE, 'VASLOG', 'TS-3R');
|
||||
console.log(`[INFO] parsed ${records.length} records`);
|
||||
|
||||
records.forEach((r, idx) => {
|
||||
console.log('');
|
||||
console.log('-'.repeat(78));
|
||||
console.log(`[REC ${idx + 1}] model=${r.model_number} sn=${r.serial_number} date=${r.test_date} result=${r.overall_result}`);
|
||||
console.log('-'.repeat(78));
|
||||
const txt = generateSCMVASDatasheet(r);
|
||||
if (!txt) {
|
||||
console.log('[WARN] datasheet generation returned null');
|
||||
return;
|
||||
}
|
||||
console.log(txt);
|
||||
});
|
||||
}
|
||||
|
||||
function testEngTxtPassthrough() {
|
||||
hr('[TEST] Engineering-Tested .txt parser');
|
||||
|
||||
if (!fs.existsSync(ENG_SAMPLE_DIR)) {
|
||||
console.log(`[FAIL] sample dir not found: ${ENG_SAMPLE_DIR}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(ENG_SAMPLE_DIR)
|
||||
.filter(n => n.toLowerCase().endsWith('.txt'))
|
||||
.slice(0, 3)
|
||||
.map(n => path.join(ENG_SAMPLE_DIR, n));
|
||||
|
||||
for (const f of files) {
|
||||
const recs = parseVaslogEngTxt(f, 'TS-3R');
|
||||
console.log('');
|
||||
console.log(`[INFO] ${path.basename(f)} -> ${recs.length} record(s)`);
|
||||
for (const r of recs) {
|
||||
console.log(` log_type=${r.log_type} model=${r.model_number} sn=${r.serial_number} date=${r.test_date} result=${r.overall_result}`);
|
||||
console.log(` raw_data bytes=${r.raw_data.length}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function testGoldenComparison() {
|
||||
hr('[TEST] Golden comparison (mock a record that matches 166590-1)');
|
||||
|
||||
if (!fs.existsSync(GOLDEN_SAMPLE)) {
|
||||
console.log(`[FAIL] golden not found: ${GOLDEN_SAMPLE}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build a synthetic record with the same fields the VASLOG import would
|
||||
// produce if 166590-1 had been logged through the production pipeline.
|
||||
const mock = {
|
||||
log_type: 'VASLOG',
|
||||
model_number: 'SCMHVAS-M0200',
|
||||
serial_number: '166590-1',
|
||||
test_date: '2023-10-04',
|
||||
overall_result: 'PASS',
|
||||
raw_data: [
|
||||
'"SCMHVAS-M0200 "',
|
||||
'0,0,0,0,""',
|
||||
'0,0,0,0,""',
|
||||
'0,0,0,0,""',
|
||||
'0,0,0,0,""',
|
||||
'0,0,0,0,""',
|
||||
'0',
|
||||
'"","","",""',
|
||||
'"","","",""',
|
||||
'"PASS-7.005501E-033","","",""',
|
||||
'"","","",""',
|
||||
'"166590-1","10-04-2023"',
|
||||
].join('\n'),
|
||||
};
|
||||
|
||||
const generated = generateSCMVASDatasheet(mock);
|
||||
const golden = fs.readFileSync(GOLDEN_SAMPLE, 'utf8');
|
||||
|
||||
console.log('');
|
||||
console.log('--- GENERATED ---');
|
||||
console.log(generated);
|
||||
console.log('');
|
||||
console.log('--- GOLDEN ---');
|
||||
console.log(golden);
|
||||
|
||||
const genLines = generated.split(/\r?\n/);
|
||||
const goldLines = golden.split(/\r?\n/);
|
||||
console.log('');
|
||||
console.log(`[INFO] generated lines=${genLines.length} golden lines=${goldLines.length}`);
|
||||
const max = Math.max(genLines.length, goldLines.length);
|
||||
let diffs = 0;
|
||||
for (let i = 0; i < max; i++) {
|
||||
const g = genLines[i] || '';
|
||||
const d = goldLines[i] || '';
|
||||
if (g !== d) {
|
||||
diffs++;
|
||||
if (diffs <= 8) {
|
||||
console.log(`[DIFF] line ${i + 1}:`);
|
||||
console.log(` gen: [${g}] (len ${g.length})`);
|
||||
console.log(` gld: [${d}] (len ${d.length})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`[INFO] total differing lines: ${diffs}`);
|
||||
}
|
||||
|
||||
testAccuracyExtraction();
|
||||
testDatParsingAndGeneration();
|
||||
testEngTxtPassthrough();
|
||||
testGoldenComparison();
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Run export-datasheets.js --dry-run --serial for a known SCMHVAS record.
|
||||
|
||||
Pick a serial that's guaranteed in the DB (from HVAS-M01.DAT samples we
|
||||
pulled earlier: 179379-1 SCMHVAS-M0100).
|
||||
"""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
TEST_SERIALS = ['179379-1', '179379-2', '168630-9']
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
# Confirm serials are in the DB
|
||||
print('=== DB presence check ===')
|
||||
serials_list = "','".join(TEST_SERIALS)
|
||||
sql = f"SELECT serial_number, model_number, log_type, test_date, overall_result, forweb_exported_at FROM test_records WHERE serial_number IN ('{serials_list}') ORDER BY serial_number;"
|
||||
out, err, rc = ps(c, f'cd C:\\Shares\\testdatadb; & node -e "const db=require(\'./database/db\');(async()=>{{const r=await db.query(`{sql}`);console.log(JSON.stringify(r,null,2));await db.close();}})();"')
|
||||
print(out[:3000])
|
||||
if err: print('STDERR:', err[:500])
|
||||
|
||||
# Dry-run export for first serial
|
||||
sn = TEST_SERIALS[0]
|
||||
print(f'\n=== Dry-run export for {sn} ===')
|
||||
out, err, rc = ps(c, f'cd C:\\Shares\\testdatadb; & node database/export-datasheets.js --dry-run --serial {sn}', to=120)
|
||||
print(out[:3000])
|
||||
if err: print('STDERR:', err[:500])
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Verify \\ad2\webshare\For_Web is writable from SSH session (task #12 approach)."""
|
||||
import base64, subprocess, yaml, paramiko
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
print('=== UNC access probe ===')
|
||||
out, err, rc = ps(c, r'Test-Path "\\ad2\webshare\For_Web"; Test-Path "\\localhost\webshare\For_Web"')
|
||||
print(out)
|
||||
|
||||
print('=== Count existing For_Web files ===')
|
||||
out, err, rc = ps(c, r'Get-ChildItem "\\ad2\webshare\For_Web" -File -Filter *.TXT -ErrorAction SilentlyContinue | Measure-Object | Select-Object Count | Format-Table -AutoSize')
|
||||
print(out)
|
||||
|
||||
print('=== Write test ===')
|
||||
out, err, rc = ps(c, r'$f = "\\ad2\webshare\For_Web\_sshwrite_test.txt"; Set-Content -Path $f -Value "ssh session write test 2026-04-12"; if (Test-Path $f) { Write-Host "[OK] write succeeded"; Remove-Item $f; Write-Host "[OK] cleanup" } else { Write-Host "[FAIL]" }')
|
||||
print(out)
|
||||
if err.strip() and 'CLIXML' not in err:
|
||||
print('STDERR:', err[:400])
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Pull a few just-backfilled files for byte-level verification."""
|
||||
import base64, os, subprocess, yaml, paramiko
|
||||
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\backfill-verify'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
NODE_QUERY = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const rows = await db.query(
|
||||
"SELECT serial_number, model_number, log_type, source_file FROM test_records " +
|
||||
"WHERE forweb_exported_at IS NOT NULL " +
|
||||
"AND ((model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') OR log_type='VASLOG_ENG') " +
|
||||
"ORDER BY forweb_exported_at DESC LIMIT 5"
|
||||
);
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=120):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote = 'C:/Shares/testdatadb/_q.js'
|
||||
with sftp.open(remote,'w') as fh:
|
||||
fh.write(NODE_QUERY)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_q.js')
|
||||
import json
|
||||
# Extract JSON from output
|
||||
start = out.find('[')
|
||||
rows = json.loads(out[start:out.rfind(']')+1])
|
||||
print(f'[INFO] {len(rows)} recently-exported records')
|
||||
|
||||
sftp = c.open_sftp()
|
||||
for r in rows:
|
||||
sn = r['serial_number']
|
||||
model = r['model_number']
|
||||
ltype = r['log_type']
|
||||
src_file = r.get('source_file', '')
|
||||
# Pull the exported file from For_Web
|
||||
export_remote = f'//ad2/webshare/For_Web/{sn}.TXT'
|
||||
# Can't SFTP via UNC directly; PowerShell read back
|
||||
# Use a fresh exec_command to get the content
|
||||
out2, err2, rc2 = ps(c, fr'Get-Content -Raw -LiteralPath "\\ad2\webshare\For_Web\{sn}.TXT" -ErrorAction SilentlyContinue')
|
||||
local_exp = os.path.join(LOCAL_OUT, f'{sn}-exported.TXT')
|
||||
with open(local_exp, 'w', encoding='utf-8', newline='') as fh:
|
||||
fh.write(out2)
|
||||
print(f'[INFO] {sn} ({model} / {ltype}) exported size={len(out2)} bytes')
|
||||
|
||||
# If it's a passthrough, also pull the source file for diff
|
||||
if ltype == 'VASLOG_ENG' and src_file:
|
||||
src_posix = src_file.replace('\\','/')
|
||||
try:
|
||||
local_src = os.path.join(LOCAL_OUT, f'{sn}-source.txt')
|
||||
sftp.get(src_posix, local_src)
|
||||
# Compare byte-for-byte
|
||||
with open(local_src, 'rb') as f1, open(local_exp, 'rb') as f2:
|
||||
# The exported came through PowerShell Get-Content which may have
|
||||
# mangled line endings; load source byte-for-byte for reference
|
||||
pass
|
||||
print(f' [INFO] source pulled: {local_src}')
|
||||
except Exception as e:
|
||||
print(f' [WARN] source pull fail: {e}')
|
||||
sftp.close()
|
||||
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
# Byte-level compare for the first VASLOG_ENG
|
||||
print('\n=== Byte-level compare ===')
|
||||
for fn in os.listdir(LOCAL_OUT):
|
||||
if fn.endswith('-source.txt'):
|
||||
sn = fn.replace('-source.txt','')
|
||||
src = os.path.join(LOCAL_OUT, fn)
|
||||
exp = os.path.join(LOCAL_OUT, f'{sn}-exported.TXT')
|
||||
if os.path.exists(exp):
|
||||
with open(src, 'rb') as f1, open(exp, 'rb') as f2:
|
||||
s = f1.read(); e = f2.read()
|
||||
print(f'{sn}: src={len(s)}B exp={len(e)}B identical={s == e}')
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Byte-exact verification of backfilled files via a temp copy on AD2."""
|
||||
import base64, os, subprocess, yaml, paramiko
|
||||
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\backfill-verify'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
NODE_QUERY = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const rows = await db.query(
|
||||
"SELECT serial_number, model_number, log_type, source_file FROM test_records " +
|
||||
"WHERE forweb_exported_at IS NOT NULL " +
|
||||
"AND log_type='VASLOG_ENG' ORDER BY forweb_exported_at DESC LIMIT 3"
|
||||
);
|
||||
console.log(JSON.stringify(rows));
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote_q = 'C:/Shares/testdatadb/_q.js'
|
||||
with sftp.open(remote_q,'w') as fh: fh.write(NODE_QUERY)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_q.js')
|
||||
import json
|
||||
rows = json.loads(out[out.find('['):out.rfind(']')+1])
|
||||
print(f'[INFO] verifying {len(rows)} VASLOG_ENG records')
|
||||
|
||||
# Copy exported files to C:\Users\sysadmin\Documents for SFTP
|
||||
tmp_dir = 'C:/Users/sysadmin/Documents/verify'
|
||||
ps(c, f'New-Item -ItemType Directory -Force -Path "{tmp_dir}" | Out-Null')
|
||||
|
||||
sftp = c.open_sftp()
|
||||
for r in rows:
|
||||
sn = r['serial_number']
|
||||
src_file = r['source_file']
|
||||
# Copy exported file to tmp
|
||||
ps(c, fr'Copy-Item -LiteralPath "\\ad2\webshare\For_Web\{sn}.TXT" -Destination "{tmp_dir}\{sn}-exp.TXT" -Force')
|
||||
# Also copy source file to tmp (for byte-exact SFTP)
|
||||
ps(c, fr'Copy-Item -LiteralPath "{src_file}" -Destination "{tmp_dir}\{sn}-src.txt" -Force')
|
||||
|
||||
local_exp = os.path.join(LOCAL_OUT, f'{sn}-exp.TXT')
|
||||
local_src = os.path.join(LOCAL_OUT, f'{sn}-src.txt')
|
||||
sftp.get(f'{tmp_dir}/{sn}-exp.TXT', local_exp)
|
||||
sftp.get(f'{tmp_dir}/{sn}-src.txt', local_src)
|
||||
|
||||
with open(local_exp, 'rb') as f: exp = f.read()
|
||||
with open(local_src, 'rb') as f: src = f.read()
|
||||
same = exp == src
|
||||
print(f' {sn} ({r["model_number"]}): src={len(src)}B exp={len(exp)}B identical={same}')
|
||||
if not same:
|
||||
print(f' first diff byte: {next((i for i,(a,b) in enumerate(zip(src,exp)) if a != b), min(len(src),len(exp)))}')
|
||||
sftp.close()
|
||||
|
||||
# Cleanup
|
||||
ps(c, fr'Remove-Item -LiteralPath "{tmp_dir}" -Recurse -Force')
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote_q)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Pull the plain-decimal-derived datasheet (SN 66260-12) for visual check."""
|
||||
import base64, os, subprocess, yaml, paramiko
|
||||
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\backfill-verify'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
tmp_dir = 'C:/Users/sysadmin/Documents/verify'
|
||||
ps(c, f'New-Item -ItemType Directory -Force -Path "{tmp_dir}" | Out-Null')
|
||||
sn = '66260-12'
|
||||
ps(c, fr'Copy-Item -LiteralPath "\\ad2\webshare\For_Web\{sn}.TXT" -Destination "{tmp_dir}\{sn}.TXT" -Force')
|
||||
sftp = c.open_sftp()
|
||||
local = os.path.join(LOCAL_OUT, f'{sn}-plain.TXT')
|
||||
sftp.get(f'{tmp_dir}/{sn}.TXT', local)
|
||||
sftp.close()
|
||||
with open(local, 'rb') as f: data = f.read()
|
||||
print(f'size={len(data)} bytes')
|
||||
print(data.decode('utf-8','replace'))
|
||||
ps(c, fr'Remove-Item -LiteralPath "{tmp_dir}" -Recurse -Force')
|
||||
finally:
|
||||
c.close()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Pull one rendered SCMVAS datasheet for visual check."""
|
||||
import base64, os, subprocess, yaml, paramiko
|
||||
|
||||
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\backfill-verify'
|
||||
os.makedirs(LOCAL_OUT, exist_ok=True)
|
||||
|
||||
NODE_QUERY = r'''
|
||||
const db = require('./database/db');
|
||||
(async () => {
|
||||
const rows = await db.query(
|
||||
"SELECT serial_number, model_number FROM test_records " +
|
||||
"WHERE forweb_exported_at IS NOT NULL AND log_type='VASLOG' " +
|
||||
"AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
|
||||
"ORDER BY forweb_exported_at DESC LIMIT 3"
|
||||
);
|
||||
console.log(JSON.stringify(rows));
|
||||
await db.close();
|
||||
})();
|
||||
'''
|
||||
|
||||
def pwd():
|
||||
r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
|
||||
capture_output=True, text=True, timeout=30, check=True)
|
||||
return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')
|
||||
|
||||
def ps(c, cmd, to=60):
|
||||
enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
|
||||
stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
|
||||
return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()
|
||||
|
||||
c = paramiko.SSHClient()
|
||||
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
|
||||
try:
|
||||
sftp = c.open_sftp()
|
||||
remote_q = 'C:/Shares/testdatadb/_q.js'
|
||||
with sftp.open(remote_q,'w') as fh: fh.write(NODE_QUERY)
|
||||
sftp.close()
|
||||
|
||||
out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_q.js')
|
||||
import json
|
||||
rows = json.loads(out[out.find('['):out.rfind(']')+1])
|
||||
|
||||
tmp_dir = 'C:/Users/sysadmin/Documents/verify'
|
||||
ps(c, f'New-Item -ItemType Directory -Force -Path "{tmp_dir}" | Out-Null')
|
||||
|
||||
sftp = c.open_sftp()
|
||||
for r in rows[:1]:
|
||||
sn = r['serial_number']; model = r['model_number']
|
||||
ps(c, fr'Copy-Item -LiteralPath "\\ad2\webshare\For_Web\{sn}.TXT" -Destination "{tmp_dir}\{sn}.TXT" -Force')
|
||||
local = os.path.join(LOCAL_OUT, f'{sn}-rendered.TXT')
|
||||
sftp.get(f'{tmp_dir}/{sn}.TXT', local)
|
||||
print(f'=== {sn} ({model}) ===')
|
||||
with open(local, 'rb') as f:
|
||||
data = f.read()
|
||||
print(f'size={len(data)} bytes')
|
||||
print(data.decode('utf-8','replace'))
|
||||
sftp.close()
|
||||
|
||||
ps(c, fr'Remove-Item -LiteralPath "{tmp_dir}" -Recurse -Force')
|
||||
sftp = c.open_sftp()
|
||||
try: sftp.remove(remote_q)
|
||||
except Exception: pass
|
||||
sftp.close()
|
||||
finally:
|
||||
c.close()
|
||||
Reference in New Issue
Block a user