Add SCMVAS/SCMHVAS datasheet pipeline extension (Dataforth)
Extends the Test Datasheet Pipeline on AD2:C:\Shares\testdatadb to
generate web-published datasheets for the SCMVAS-Mxxx (obsolete) and
SCMHVAS-Mxxxx (replacement) High Voltage Input Module product lines.
Both are tested either with the existing TESTHV3 software (production
VASLOG .DAT logs) or in Engineering with plain .txt output.
Key changes on AD2 (all deployed 2026-04-12 with dated backups):
- parsers/spec-reader.js: getSpecs() returns a `{_family:'SCMVAS',
_noSpecs:true}` sentinel for SCMVAS/SCMHVAS/VAS-M/HVAS-M model prefixes
so the export pipeline does not silently skip them for missing specs.
- templates/datasheet-exact.js: new Accuracy-only template branch
(generateSCMVASDatasheet + helpers) that mirrors the existing shipped
format byte-for-byte. Extraction regex covers both QuickBASIC STR$()
output formats: scientific-with-trailing-status-digit (98.4% of
records) and plain-decimal (1.6% of records above QB's threshold).
- parsers/vaslog-engtxt.js (new): parses the Engineering-Tested .txt
files in TS-3R\LOGS\VASLOG\VASLOG - Engineering Tested\. Filename SN
regex strips optional trailing 14-digit timestamp; in-file "SN:"
header is the authoritative source when the filename is malformed.
- database/import.js: LOG_TYPES grows a VASLOG_ENG entry with
subfolder + recursive flags. Pre-existing 7 log types keep their
implicit recursive=true behaviour (config.recursive !== false).
importFiles() routes VASLOG_ENG paths before the generic loop so a
VASLOG - Engineering Tested/*.txt path does not mis-dispatch to the
multiline parser.
- database/export-datasheets.js: VASLOG_ENG records are written
verbatim via fs.copyFileSync(source_file, For_Web/<SN>.TXT) for true
byte-level pass-through, with a graceful raw_data fallback when the
source file is no longer on disk.
Deploy outcome:
- 27,503 SCMVAS/SCMHVAS datasheets rendered (27,065 from scientific +
438 from plain-decimal PASS lines, post-patch rerun)
- 434 Engineering-Tested .txt files pass-through-copied to For_Web
- 0 errors across both batches
Repo layout added here:
- scmvas-hvas-research/: discovery artifacts (source .BAS, hvin.dat,
sample .DAT + .txt, binary-format notes, IMPLEMENTATION_PLAN.md)
- implementation/: staged final code + deploy helpers + local test
harness + per-step verification scripts
- backups/pre-deploy-20260412/: independent local snapshot of the 4
AD2 files replaced, pulled byte-for-byte before deploy
All helper scripts fetch the AD2 password at runtime from the SOPS
vault (clients/dataforth/ad2.sops.yaml). None of the committed files
contain the plaintext credential. Known vault-entry hygiene issue
(stale shell-escape backslash before the `!`) is documented in the
fetcher comments and stripped at read-time; flagged separately for
cleanup.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Archive For_Web Files
|
||||
*
|
||||
* Moves files older than the current year into year-based subfolders.
|
||||
* e.g., X:\For_Web\2024\12345-1.TXT
|
||||
*
|
||||
* The TestDataSheetUploader only uploads files modified in the current year,
|
||||
* so archived files won't be re-uploaded. Keeps the active folder small and fast.
|
||||
*
|
||||
* Usage:
|
||||
* node archive-for-web.js Archive all pre-current-year files
|
||||
* node archive-for-web.js --dry-run Show what would be moved
|
||||
* node archive-for-web.js --year 2024 Only archive files from 2024
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const FOR_WEB = 'X:\\For_Web';
|
||||
|
||||
function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const yearIdx = args.indexOf('--year');
|
||||
const targetYear = yearIdx >= 0 ? parseInt(args[yearIdx + 1]) : null;
|
||||
const currentYear = new Date().getFullYear();
|
||||
|
||||
console.log('========================================');
|
||||
console.log('Archive For_Web Files');
|
||||
console.log('========================================');
|
||||
console.log(`Source: ${FOR_WEB}`);
|
||||
console.log(`Current year: ${currentYear}`);
|
||||
console.log(`Dry run: ${dryRun}`);
|
||||
if (targetYear) console.log(`Target year: ${targetYear}`);
|
||||
console.log(`Start: ${new Date().toISOString()}`);
|
||||
console.log('');
|
||||
|
||||
if (!fs.existsSync(FOR_WEB)) {
|
||||
console.error('ERROR: For_Web directory not found');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Scan files
|
||||
console.log('Scanning files...');
|
||||
const entries = fs.readdirSync(FOR_WEB, { withFileTypes: true });
|
||||
|
||||
const yearCounts = {};
|
||||
let scanned = 0;
|
||||
let toMove = 0;
|
||||
let moved = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
scanned++;
|
||||
|
||||
if (scanned % 50000 === 0) {
|
||||
process.stdout.write(`\rScanned: ${scanned}`);
|
||||
}
|
||||
|
||||
const filePath = path.join(FOR_WEB, entry.name);
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(filePath);
|
||||
} catch (err) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileYear = stat.mtime.getFullYear();
|
||||
|
||||
// Skip current year files
|
||||
if (fileYear >= currentYear) continue;
|
||||
|
||||
// If targeting a specific year, skip others
|
||||
if (targetYear && fileYear !== targetYear) continue;
|
||||
|
||||
yearCounts[fileYear] = (yearCounts[fileYear] || 0) + 1;
|
||||
toMove++;
|
||||
|
||||
if (!dryRun) {
|
||||
// Create year subdirectory if needed
|
||||
const yearDir = path.join(FOR_WEB, String(fileYear));
|
||||
if (!fs.existsSync(yearDir)) {
|
||||
fs.mkdirSync(yearDir);
|
||||
console.log(`\nCreated directory: ${yearDir}`);
|
||||
}
|
||||
|
||||
const destPath = path.join(yearDir, entry.name);
|
||||
try {
|
||||
fs.renameSync(filePath, destPath);
|
||||
moved++;
|
||||
} catch (err) {
|
||||
// If rename fails (cross-device), try copy+delete
|
||||
try {
|
||||
fs.copyFileSync(filePath, destPath);
|
||||
fs.unlinkSync(filePath);
|
||||
moved++;
|
||||
} catch (err2) {
|
||||
console.error(`\nERROR moving ${entry.name}: ${err2.message}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
if (moved % 10000 === 0) {
|
||||
process.stdout.write(`\rMoved: ${moved}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
console.log('========================================');
|
||||
console.log('Archive Summary');
|
||||
console.log('========================================');
|
||||
console.log(`Files scanned: ${scanned}`);
|
||||
console.log(`Files to archive: ${toMove}`);
|
||||
|
||||
if (Object.keys(yearCounts).length > 0) {
|
||||
console.log('\nBy year:');
|
||||
for (const [year, count] of Object.entries(yearCounts).sort()) {
|
||||
console.log(` ${year}: ${count.toLocaleString()} files`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
console.log(`\nFiles moved: ${moved}`);
|
||||
console.log(`Errors: ${errors}`);
|
||||
}
|
||||
|
||||
console.log(`\nEnd: ${new Date().toISOString()}`);
|
||||
}
|
||||
|
||||
run();
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* PostgreSQL Database Abstraction Layer
|
||||
*
|
||||
* Provides a connection pool and helper methods for the TestDataDB app.
|
||||
* Replaces better-sqlite3 singleton with pg.Pool.
|
||||
*
|
||||
* Environment variables (all optional, defaults connect to local PG):
|
||||
* PGHOST (default: localhost)
|
||||
* PGPORT (default: 5432)
|
||||
* PGUSER (default: testdatadb_app)
|
||||
* PGPASSWORD (default: DfTestDB2026!)
|
||||
* PGDATABASE (default: testdatadb)
|
||||
*/
|
||||
|
||||
const { Pool } = require('pg');
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.PGHOST || 'localhost',
|
||||
port: parseInt(process.env.PGPORT || '5432', 10),
|
||||
user: process.env.PGUSER || 'testdatadb_app',
|
||||
password: process.env.PGPASSWORD || 'DfTestDB2026!',
|
||||
database: process.env.PGDATABASE || 'testdatadb',
|
||||
max: 20,
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: 5000,
|
||||
});
|
||||
|
||||
pool.on('error', (err) => {
|
||||
console.error(`[${new Date().toISOString()}] [PG POOL ERROR] ${err.message}`);
|
||||
});
|
||||
|
||||
/**
|
||||
* Convert SQLite-style ? placeholders to PostgreSQL $1, $2, ... placeholders.
|
||||
* Skips ? inside single-quoted strings.
|
||||
*/
|
||||
function convertPlaceholders(sql) {
|
||||
let idx = 0;
|
||||
let inString = false;
|
||||
let result = '';
|
||||
for (let i = 0; i < sql.length; i++) {
|
||||
const ch = sql[i];
|
||||
if (ch === "'" && (i === 0 || sql[i - 1] !== '\\')) {
|
||||
inString = !inString;
|
||||
result += ch;
|
||||
} else if (ch === '?' && !inString) {
|
||||
idx++;
|
||||
result += '$' + idx;
|
||||
} else {
|
||||
result += ch;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query, return all rows.
|
||||
* @param {string} sql - SQL with ? or $N placeholders
|
||||
* @param {Array} params - Parameter values
|
||||
* @returns {Promise<Array>} rows
|
||||
*/
|
||||
async function query(sql, params = []) {
|
||||
const pgSql = sql.includes('?') ? convertPlaceholders(sql) : sql;
|
||||
const result = await pool.query(pgSql, params);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query, return the first row or null.
|
||||
*/
|
||||
async function queryOne(sql, params = []) {
|
||||
const rows = await query(sql, params);
|
||||
return rows[0] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a statement (INSERT/UPDATE/DELETE), return { rowCount }.
|
||||
*/
|
||||
async function execute(sql, params = []) {
|
||||
const pgSql = sql.includes('?') ? convertPlaceholders(sql) : sql;
|
||||
const result = await pool.query(pgSql, params);
|
||||
return { rowCount: result.rowCount, rows: result.rows };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a function inside a transaction.
|
||||
* The callback receives a client with query/execute helpers.
|
||||
* @param {Function} fn - async (client) => result
|
||||
* @returns {Promise<*>} result of fn
|
||||
*/
|
||||
async function transaction(fn) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
const txClient = {
|
||||
async query(sql, params = []) {
|
||||
const pgSql = sql.includes('?') ? convertPlaceholders(sql) : sql;
|
||||
const result = await client.query(pgSql, params);
|
||||
return result.rows;
|
||||
},
|
||||
async queryOne(sql, params = []) {
|
||||
const rows = await txClient.query(sql, params);
|
||||
return rows[0] || null;
|
||||
},
|
||||
async execute(sql, params = []) {
|
||||
const pgSql = sql.includes('?') ? convertPlaceholders(sql) : sql;
|
||||
const result = await client.query(pgSql, params);
|
||||
return { rowCount: result.rowCount, rows: result.rows };
|
||||
},
|
||||
// Direct pg client access for COPY or other advanced operations
|
||||
raw: client,
|
||||
};
|
||||
|
||||
const result = await fn(txClient);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the pool (for graceful shutdown).
|
||||
*/
|
||||
async function close() {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the raw pool (for advanced use like COPY).
|
||||
*/
|
||||
function getPool() {
|
||||
return pool;
|
||||
}
|
||||
|
||||
module.exports = { query, queryOne, execute, transaction, close, getPool, convertPlaceholders };
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Export Datasheets
|
||||
*
|
||||
* Generates TXT datasheets for unexported PASS records and writes them to X:\For_Web\.
|
||||
* Updates forweb_exported_at after successful export.
|
||||
*
|
||||
* Usage:
|
||||
* node export-datasheets.js Export all pending (batch mode)
|
||||
* node export-datasheets.js --limit 100 Export up to 100 records
|
||||
* node export-datasheets.js --file <paths> Export records matching specific source files
|
||||
* node export-datasheets.js --serial 178439-1 Export a specific serial number
|
||||
* node export-datasheets.js --dry-run Show what would be exported without writing
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./db');
|
||||
|
||||
const { loadAllSpecs, getSpecs } = require('../parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('../templates/datasheet-exact');
|
||||
|
||||
// Configuration
|
||||
const OUTPUT_DIR = 'X:\\For_Web';
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
async function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const limitIdx = args.indexOf('--limit');
|
||||
const limit = limitIdx >= 0 ? parseInt(args[limitIdx + 1]) : 0;
|
||||
const serialIdx = args.indexOf('--serial');
|
||||
const serial = serialIdx >= 0 ? args[serialIdx + 1] : null;
|
||||
const fileIdx = args.indexOf('--file');
|
||||
const files = fileIdx >= 0 ? args.slice(fileIdx + 1).filter(f => !f.startsWith('--')) : null;
|
||||
|
||||
console.log('========================================');
|
||||
console.log('Datasheet Export');
|
||||
console.log('========================================');
|
||||
console.log(`Output: ${OUTPUT_DIR}`);
|
||||
console.log(`Dry run: ${dryRun}`);
|
||||
if (limit) console.log(`Limit: ${limit}`);
|
||||
if (serial) console.log(`Serial: ${serial}`);
|
||||
console.log(`Start: ${new Date().toISOString()}`);
|
||||
|
||||
if (!dryRun && !fs.existsSync(OUTPUT_DIR)) {
|
||||
console.error(`ERROR: Output directory does not exist: ${OUTPUT_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('\nLoading model specs...');
|
||||
const specMap = loadAllSpecs();
|
||||
|
||||
// Build query
|
||||
const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
if (serial) {
|
||||
paramIdx++;
|
||||
conditions.push(`serial_number = $${paramIdx}`);
|
||||
params.push(serial);
|
||||
}
|
||||
|
||||
if (files && files.length > 0) {
|
||||
const placeholders = files.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
|
||||
conditions.push(`source_file IN (${placeholders})`);
|
||||
params.push(...files);
|
||||
}
|
||||
|
||||
let sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')} ORDER BY test_date DESC`;
|
||||
|
||||
if (limit) {
|
||||
paramIdx++;
|
||||
sql += ` LIMIT $${paramIdx}`;
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
const records = await db.query(sql, params);
|
||||
console.log(`\nFound ${records.length} records to export`);
|
||||
|
||||
if (records.length === 0) {
|
||||
console.log('Nothing to export.');
|
||||
await db.close();
|
||||
return { exported: 0, skipped: 0, errors: 0 };
|
||||
}
|
||||
|
||||
let exported = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let noSpecs = 0;
|
||||
let pendingUpdates = [];
|
||||
|
||||
for (const record of records) {
|
||||
try {
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) {
|
||||
noSpecs++;
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const filename = record.serial_number + '.TXT';
|
||||
const outputPath = path.join(OUTPUT_DIR, filename);
|
||||
|
||||
if (dryRun) {
|
||||
console.log(` [DRY RUN] Would write: ${filename}`);
|
||||
exported++;
|
||||
} else {
|
||||
fs.writeFileSync(outputPath, txt, 'utf8');
|
||||
pendingUpdates.push(record.id);
|
||||
exported++;
|
||||
|
||||
// Batch commit
|
||||
if (pendingUpdates.length >= BATCH_SIZE) {
|
||||
await flushUpdates(pendingUpdates);
|
||||
pendingUpdates = [];
|
||||
process.stdout.write(`\r Exported: ${exported} / ${records.length}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`\n ERROR exporting ${record.serial_number}: ${err.message}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining updates
|
||||
if (pendingUpdates.length > 0) {
|
||||
await flushUpdates(pendingUpdates);
|
||||
}
|
||||
|
||||
console.log(`\n\n========================================`);
|
||||
console.log(`Export Complete`);
|
||||
console.log(`========================================`);
|
||||
console.log(`Exported: ${exported}`);
|
||||
console.log(`Skipped: ${skipped} (${noSpecs} missing specs)`);
|
||||
console.log(`Errors: ${errors}`);
|
||||
console.log(`End: ${new Date().toISOString()}`);
|
||||
|
||||
await db.close();
|
||||
return { exported, skipped, errors };
|
||||
}
|
||||
|
||||
async function flushUpdates(ids) {
|
||||
const now = new Date().toISOString();
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const id of ids) {
|
||||
await txClient.execute(
|
||||
'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
|
||||
[now, id]
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Export function for use by import.js (no db argument -- uses shared pool)
|
||||
async function exportNewRecords(specMap, filePaths) {
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
console.log(`[EXPORT] Output directory not found: ${OUTPUT_DIR}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
if (filePaths && filePaths.length > 0) {
|
||||
const placeholders = filePaths.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
|
||||
conditions.push(`source_file IN (${placeholders})`);
|
||||
params.push(...filePaths);
|
||||
}
|
||||
|
||||
const sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')}`;
|
||||
const records = await db.query(sql, params);
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
let exported = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const record of records) {
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) continue;
|
||||
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) continue;
|
||||
|
||||
const filename = record.serial_number + '.TXT';
|
||||
const outputPath = path.join(OUTPUT_DIR, filename);
|
||||
|
||||
try {
|
||||
fs.writeFileSync(outputPath, txt, 'utf8');
|
||||
await txClient.execute(
|
||||
'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
|
||||
[new Date().toISOString(), record.id]
|
||||
);
|
||||
exported++;
|
||||
} catch (err) {
|
||||
console.error(`[EXPORT] Error writing ${filename}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[EXPORT] Generated ${exported} datasheet(s)`);
|
||||
return exported;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = { exportNewRecords };
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Generate PDF datasheets for specific serial numbers
|
||||
* For Quatronix customer request - 70 datasheets needed urgently
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
const PDFDocument = require('pdfkit');
|
||||
|
||||
const { loadAllSpecs, getSpecs } = require('../parsers/spec-reader');
|
||||
const { generateExactDatasheet } = require('../templates/datasheet-exact');
|
||||
|
||||
const DB_PATH = path.join(__dirname, 'testdata.db');
|
||||
const OUTPUT_DIR = process.argv[2] || path.join(process.env.USERPROFILE, 'Desktop', 'Quatronix-Datasheets');
|
||||
|
||||
// Build the list of needed serial numbers
|
||||
const needed = [
|
||||
// SCM5B34-03: 177368-6~15
|
||||
...Array.from({length:10}, (_,i) => '177368-' + (i+6)),
|
||||
// SCM5B35-02: 177625-6~10
|
||||
...Array.from({length:5}, (_,i) => '177625-' + (i+6)),
|
||||
// SCM5B38-05: 177963-6
|
||||
'177963-6',
|
||||
// SCM5B392-11: 177199-13
|
||||
'177199-13',
|
||||
// SCM5B40-03: 178444-1
|
||||
'178444-1',
|
||||
// SCM5B41-02: 178362-1
|
||||
'178362-1',
|
||||
// SCM5B42-02: 177299-4, 177299-5
|
||||
'177299-4', '177299-5',
|
||||
// SCM5B45-02D: 178607-1
|
||||
'178607-1',
|
||||
// SCM5B45-04: 178385-4~8
|
||||
...Array.from({length:5}, (_,i) => '178385-' + (i+4)),
|
||||
// SCM5B48-01: 177593-1
|
||||
'177593-1',
|
||||
// SCM5B49-05: 177000-15
|
||||
'177000-15',
|
||||
// DSCA30-05C: 176566-2
|
||||
'176566-2',
|
||||
// DSCA38-19C: 178001-22, 178001-23
|
||||
'178001-22', '178001-23',
|
||||
// DSCA41-02: 178135-2
|
||||
'178135-2',
|
||||
// DSCA38-1468: 178595-1
|
||||
'178595-1',
|
||||
// SCM5B41-02: 177012-1~30
|
||||
...Array.from({length:30}, (_,i) => '177012-' + (i+1)),
|
||||
// SCM5B47S-10: 178768-8
|
||||
'178768-8',
|
||||
// SCM5B45-04D: 177207-4~7
|
||||
...Array.from({length:4}, (_,i) => '177207-' + (i+4)),
|
||||
// 8B51-12: 178601-6~9
|
||||
...Array.from({length:4}, (_,i) => '178601-' + (i+6)),
|
||||
];
|
||||
|
||||
async function generatePdf(txt, outputPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const doc = new PDFDocument({
|
||||
size: 'LETTER',
|
||||
margins: { top: 36, bottom: 36, left: 36, right: 36 }
|
||||
});
|
||||
const stream = fs.createWriteStream(outputPath);
|
||||
stream.on('finish', resolve);
|
||||
stream.on('error', reject);
|
||||
doc.pipe(stream);
|
||||
doc.font('Courier').fontSize(9.5);
|
||||
const lines = txt.split(/\r?\n/);
|
||||
for (const line of lines) {
|
||||
doc.text(line, { lineGap: 1 });
|
||||
}
|
||||
doc.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function run() {
|
||||
console.log('========================================');
|
||||
console.log('Generate Customer PDFs');
|
||||
console.log('========================================');
|
||||
console.log(`Output: ${OUTPUT_DIR}`);
|
||||
console.log(`Serial numbers: ${needed.length}`);
|
||||
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
const specMap = loadAllSpecs();
|
||||
const db = new Database(DB_PATH, { readonly: true });
|
||||
|
||||
let generated = 0;
|
||||
let notFound = [];
|
||||
let noSpecs = [];
|
||||
let errors = [];
|
||||
|
||||
for (const sn of needed) {
|
||||
const record = db.prepare(
|
||||
"SELECT * FROM test_records WHERE serial_number = ? AND overall_result = 'PASS' LIMIT 1"
|
||||
).get(sn);
|
||||
|
||||
if (!record) {
|
||||
notFound.push(sn);
|
||||
continue;
|
||||
}
|
||||
|
||||
const specs = getSpecs(specMap, record.model_number);
|
||||
if (!specs) {
|
||||
noSpecs.push(sn + ' (' + record.model_number + ')');
|
||||
continue;
|
||||
}
|
||||
|
||||
const txt = generateExactDatasheet(record, specs);
|
||||
if (!txt) {
|
||||
errors.push(sn + ' (format failed)');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Write TXT
|
||||
fs.writeFileSync(path.join(OUTPUT_DIR, sn + '.TXT'), txt, 'utf8');
|
||||
|
||||
// Write PDF
|
||||
try {
|
||||
await generatePdf(txt, path.join(OUTPUT_DIR, sn + '.pdf'));
|
||||
generated++;
|
||||
process.stdout.write(`\rGenerated: ${generated}`);
|
||||
} catch (err) {
|
||||
errors.push(sn + ' (PDF: ' + err.message + ')');
|
||||
}
|
||||
}
|
||||
|
||||
db.close();
|
||||
|
||||
console.log('\n\n========================================');
|
||||
console.log('Results');
|
||||
console.log('========================================');
|
||||
console.log(`Generated: ${generated} (TXT + PDF)`);
|
||||
if (notFound.length > 0) {
|
||||
console.log(`\nNot in database (${notFound.length}):`);
|
||||
notFound.forEach(s => console.log(' ' + s));
|
||||
}
|
||||
if (noSpecs.length > 0) {
|
||||
console.log(`\nNo spec data (${noSpecs.length}):`);
|
||||
noSpecs.forEach(s => console.log(' ' + s));
|
||||
}
|
||||
if (errors.length > 0) {
|
||||
console.log(`\nErrors (${errors.length}):`);
|
||||
errors.forEach(s => console.log(' ' + s));
|
||||
}
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* Work Order Report Importer
|
||||
*
|
||||
* Imports work order status reports from TS-XX/Reports/ into PostgreSQL.
|
||||
* Links work order numbers to existing test records.
|
||||
*
|
||||
* Usage:
|
||||
* node import-work-orders.js Full import from all stations
|
||||
* node import-work-orders.js --file <paths> Import specific report files
|
||||
* node import-work-orders.js --station TS-4L Import from one station
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./db');
|
||||
const { parseWoReport } = require('../parsers/wo-report');
|
||||
|
||||
const TEST_PATH = 'C:\\Shares\\test';
|
||||
|
||||
async function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const stationIdx = args.indexOf('--station');
|
||||
const targetStation = stationIdx >= 0 ? args[stationIdx + 1] : null;
|
||||
const fileIdx = args.indexOf('--file');
|
||||
const specificFiles = fileIdx >= 0 ? args.slice(fileIdx + 1).filter(f => !f.startsWith('--')) : null;
|
||||
|
||||
console.log('========================================');
|
||||
console.log('Work Order Report Import');
|
||||
console.log('========================================');
|
||||
console.log(`Start: ${new Date().toISOString()}`);
|
||||
|
||||
let files = [];
|
||||
|
||||
if (specificFiles && specificFiles.length > 0) {
|
||||
files = specificFiles;
|
||||
} else {
|
||||
try {
|
||||
const stationDirs = fs.readdirSync(TEST_PATH, { withFileTypes: true })
|
||||
.filter(d => d.isDirectory() && d.name.match(/^TS-/i))
|
||||
.filter(d => !targetStation || d.name.toUpperCase() === targetStation.toUpperCase())
|
||||
.map(d => d.name);
|
||||
|
||||
for (const station of stationDirs) {
|
||||
const reportsDir = path.join(TEST_PATH, station, 'Reports');
|
||||
if (!fs.existsSync(reportsDir)) continue;
|
||||
|
||||
const reportFiles = fs.readdirSync(reportsDir)
|
||||
.filter(f => f.toUpperCase().endsWith('.TXT'))
|
||||
.map(f => path.join(reportsDir, f));
|
||||
|
||||
files.push(...reportFiles);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error scanning stations:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Found ${files.length} report files to import`);
|
||||
|
||||
let woCount = 0;
|
||||
let lineCount = 0;
|
||||
let linkedCount = 0;
|
||||
let errors = 0;
|
||||
|
||||
const BATCH_SIZE = 500;
|
||||
let batch = [];
|
||||
|
||||
for (const filePath of files) {
|
||||
try {
|
||||
const wo = parseWoReport(filePath);
|
||||
if (!wo.wo_number) continue;
|
||||
batch.push({ wo, woLines: wo.lines });
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
const result = await processBatch(batch);
|
||||
woCount += result.woCount;
|
||||
lineCount += result.lineCount;
|
||||
linkedCount += result.linkedCount;
|
||||
batch = [];
|
||||
process.stdout.write(`\rProcessed: ${woCount} WOs, ${lineCount} lines`);
|
||||
}
|
||||
} catch (err) {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if (batch.length > 0) {
|
||||
const result = await processBatch(batch);
|
||||
woCount += result.woCount;
|
||||
lineCount += result.lineCount;
|
||||
linkedCount += result.linkedCount;
|
||||
}
|
||||
|
||||
// Bulk update work_order on test_records from serial number pattern
|
||||
console.log('\n\nBulk-linking test records by serial number pattern...');
|
||||
const bulkResult = await db.execute(`
|
||||
UPDATE test_records
|
||||
SET work_order = CASE
|
||||
WHEN serial_number LIKE '%-%'
|
||||
THEN SPLIT_PART(serial_number, '-', 1)
|
||||
ELSE serial_number
|
||||
END
|
||||
WHERE work_order IS NULL
|
||||
`);
|
||||
console.log(`Bulk-linked ${bulkResult.rowCount} test records`);
|
||||
|
||||
console.log('\n========================================');
|
||||
console.log('Import Complete');
|
||||
console.log('========================================');
|
||||
console.log(`Work orders imported: ${woCount}`);
|
||||
console.log(`Test lines imported: ${lineCount}`);
|
||||
console.log(`Test records linked: ${linkedCount}`);
|
||||
console.log(`Errors: ${errors}`);
|
||||
console.log(`End: ${new Date().toISOString()}`);
|
||||
|
||||
await db.close();
|
||||
}
|
||||
|
||||
async function processBatch(items) {
|
||||
let woCount = 0;
|
||||
let lineCount = 0;
|
||||
let linkedCount = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const { wo, woLines } of items) {
|
||||
await txClient.execute(
|
||||
`INSERT INTO work_orders
|
||||
(wo_number, wo_date, program, version, lib_version, test_station, source_file)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (wo_number, test_station)
|
||||
DO UPDATE SET wo_date = EXCLUDED.wo_date, program = EXCLUDED.program,
|
||||
version = EXCLUDED.version, lib_version = EXCLUDED.lib_version,
|
||||
source_file = EXCLUDED.source_file`,
|
||||
[wo.wo_number, wo.wo_date, wo.program, wo.version, wo.lib_version, wo.station, wo.source_file]
|
||||
);
|
||||
woCount++;
|
||||
|
||||
for (const line of woLines) {
|
||||
const result = await txClient.execute(
|
||||
`INSERT INTO work_order_lines
|
||||
(wo_number, serial_number, status, model_number, ds_filename, test_date, test_time, test_station)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (wo_number, serial_number, test_date, test_time) DO NOTHING`,
|
||||
[wo.wo_number, line.serial_number, line.status, line.model_number,
|
||||
line.ds_filename, line.test_date, line.test_time, wo.station]
|
||||
);
|
||||
if (result.rowCount > 0) lineCount++;
|
||||
|
||||
// Link to test_records
|
||||
const linked = await txClient.execute(
|
||||
'UPDATE test_records SET work_order = $1 WHERE serial_number = $2 AND work_order IS NULL',
|
||||
[wo.wo_number, line.serial_number]
|
||||
);
|
||||
if (linked.rowCount > 0) linkedCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { woCount, lineCount, linkedCount };
|
||||
}
|
||||
|
||||
// Export for use by sync script
|
||||
async function importReportFiles(filePaths) {
|
||||
if (!filePaths || filePaths.length === 0) return 0;
|
||||
|
||||
let imported = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
const wo = parseWoReport(filePath);
|
||||
if (!wo.wo_number) continue;
|
||||
|
||||
await txClient.execute(
|
||||
`INSERT INTO work_orders
|
||||
(wo_number, wo_date, program, version, lib_version, test_station, source_file)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (wo_number, test_station)
|
||||
DO UPDATE SET wo_date = EXCLUDED.wo_date, program = EXCLUDED.program,
|
||||
version = EXCLUDED.version, lib_version = EXCLUDED.lib_version,
|
||||
source_file = EXCLUDED.source_file`,
|
||||
[wo.wo_number, wo.wo_date, wo.program, wo.version, wo.lib_version, wo.station, wo.source_file]
|
||||
);
|
||||
|
||||
for (const line of wo.lines) {
|
||||
await txClient.execute(
|
||||
`INSERT INTO work_order_lines
|
||||
(wo_number, serial_number, status, model_number, ds_filename, test_date, test_time, test_station)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (wo_number, serial_number, test_date, test_time) DO NOTHING`,
|
||||
[wo.wo_number, line.serial_number, line.status, line.model_number,
|
||||
line.ds_filename, line.test_date, line.test_time, wo.station]
|
||||
);
|
||||
await txClient.execute(
|
||||
'UPDATE test_records SET work_order = $1 WHERE serial_number = $2 AND work_order IS NULL',
|
||||
[wo.wo_number, line.serial_number]
|
||||
);
|
||||
}
|
||||
imported++;
|
||||
} catch (err) {
|
||||
// skip bad files
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[WO] Imported ${imported} work order report(s)`);
|
||||
return imported;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
run().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = { importReportFiles };
|
||||
@@ -0,0 +1,367 @@
|
||||
/**
|
||||
* Data Import Script
|
||||
* Imports test data from DAT and SHT files into PostgreSQL database
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./db');
|
||||
|
||||
const { parseMultilineFile, extractTestStation } = require('../parsers/multiline');
|
||||
const { parseCsvFile } = require('../parsers/csvline');
|
||||
const { parseShtFile } = require('../parsers/shtfile');
|
||||
|
||||
// Data source paths
|
||||
const TEST_PATH = 'C:/Shares/test';
|
||||
const RECOVERY_PATH = 'C:/Shares/Recovery-TEST';
|
||||
const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS');
|
||||
|
||||
// Log types and their parsers
|
||||
const LOG_TYPES = {
|
||||
'DSCLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'5BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'8BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'PWRLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'SCTLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'VASLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'7BLOG': { parser: 'csvline', ext: '.DAT' }
|
||||
};
|
||||
|
||||
// Find all files of a specific type in a directory
|
||||
function findFiles(dir, pattern, recursive = true) {
|
||||
const results = [];
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const items = fs.readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item.name);
|
||||
|
||||
if (item.isDirectory() && recursive) {
|
||||
results.push(...findFiles(fullPath, pattern, recursive));
|
||||
} else if (item.isFile()) {
|
||||
if (pattern.test(item.name)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Ignore permission errors
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Parse records from a file (sync -- file I/O only)
|
||||
function parseFile(filePath, logType, parser) {
|
||||
const testStation = extractTestStation(filePath);
|
||||
|
||||
switch (parser) {
|
||||
case 'multiline':
|
||||
return parseMultilineFile(filePath, logType, testStation);
|
||||
case 'csvline':
|
||||
return parseCsvFile(filePath, testStation);
|
||||
case 'shtfile':
|
||||
return parseShtFile(filePath, testStation);
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Batch insert records into PostgreSQL
|
||||
async function insertBatch(txClient, records) {
|
||||
let imported = 0;
|
||||
for (const record of records) {
|
||||
try {
|
||||
const result = await txClient.execute(
|
||||
`INSERT INTO test_records
|
||||
(log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (log_type, model_number, serial_number, test_date, test_station)
|
||||
DO UPDATE SET raw_data = EXCLUDED.raw_data, overall_result = EXCLUDED.overall_result`,
|
||||
[
|
||||
record.log_type,
|
||||
record.model_number,
|
||||
record.serial_number,
|
||||
record.test_date,
|
||||
record.test_station,
|
||||
record.overall_result,
|
||||
record.raw_data,
|
||||
record.source_file
|
||||
]
|
||||
);
|
||||
if (result.rowCount > 0) imported++;
|
||||
} catch (err) {
|
||||
// Constraint error - skip
|
||||
}
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
|
||||
// Import records from a file
|
||||
async function importFile(txClient, filePath, logType, parser) {
|
||||
let records = [];
|
||||
|
||||
try {
|
||||
records = parseFile(filePath, logType, parser);
|
||||
const imported = await insertBatch(txClient, records);
|
||||
return { total: records.length, imported };
|
||||
} catch (err) {
|
||||
console.error(`Error importing ${filePath}: ${err.message}`);
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// Import from HISTLOGS (master consolidated logs)
|
||||
async function importHistlogs(txClient) {
|
||||
console.log('\n=== Importing from HISTLOGS ===');
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const logDir = path.join(HISTLOGS_PATH, logType);
|
||||
|
||||
if (!fs.existsSync(logDir)) {
|
||||
console.log(` ${logType}: directory not found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
|
||||
console.log(` ${logType}: found ${files.length} files`);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from test station logs
|
||||
async function importStationLogs(txClient, basePath, label) {
|
||||
console.log(`\n=== Importing from ${label} ===`);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
const stationPattern = /^TS-\d+[LR]?$/i;
|
||||
let stations = [];
|
||||
|
||||
try {
|
||||
const items = fs.readdirSync(basePath, { withFileTypes: true });
|
||||
stations = items
|
||||
.filter(i => i.isDirectory() && stationPattern.test(i.name))
|
||||
.map(i => i.name);
|
||||
} catch (err) {
|
||||
console.log(` Error reading ${basePath}: ${err.message}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(` Found stations: ${stations.join(', ')}`);
|
||||
|
||||
for (const station of stations) {
|
||||
const logsDir = path.join(basePath, station, 'LOGS');
|
||||
|
||||
if (!fs.existsSync(logsDir)) continue;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const logDir = path.join(logsDir, logType);
|
||||
|
||||
if (!fs.existsSync(logDir)) continue;
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also import SHT files
|
||||
const shtFiles = findFiles(basePath, /\.SHT$/i, true);
|
||||
console.log(` Found ${shtFiles.length} SHT files`);
|
||||
|
||||
for (const file of shtFiles) {
|
||||
const { total, imported } = await importFile(txClient, file, 'SHT', 'shtfile');
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from Recovery-TEST backups (newest first)
|
||||
async function importRecoveryBackups(txClient) {
|
||||
console.log('\n=== Importing from Recovery-TEST backups ===');
|
||||
|
||||
if (!fs.existsSync(RECOVERY_PATH)) {
|
||||
console.log(' Recovery-TEST directory not found');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true })
|
||||
.filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name))
|
||||
.map(i => i.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
|
||||
console.log(` Found backup dates: ${backups.join(', ')}`);
|
||||
|
||||
let totalImported = 0;
|
||||
|
||||
for (const backup of backups) {
|
||||
const backupPath = path.join(RECOVERY_PATH, backup);
|
||||
const imported = await importStationLogs(txClient, backupPath, `Recovery-TEST/${backup}`);
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Main import function
|
||||
async function runImport() {
|
||||
console.log('========================================');
|
||||
console.log('Test Data Import');
|
||||
console.log('========================================');
|
||||
console.log(`Start time: ${new Date().toISOString()}`);
|
||||
|
||||
let grandTotal = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
grandTotal += await importHistlogs(txClient);
|
||||
grandTotal += await importRecoveryBackups(txClient);
|
||||
grandTotal += await importStationLogs(txClient, TEST_PATH, 'test');
|
||||
});
|
||||
|
||||
const stats = await db.queryOne('SELECT COUNT(*) as count FROM test_records');
|
||||
|
||||
console.log('\n========================================');
|
||||
console.log('Import Complete');
|
||||
console.log('========================================');
|
||||
console.log(`Total records in database: ${stats.count}`);
|
||||
console.log(`End time: ${new Date().toISOString()}`);
|
||||
|
||||
await db.close();
|
||||
}
|
||||
|
||||
// Import a single file (for incremental imports from sync)
|
||||
async function importSingleFile(filePath) {
|
||||
console.log(`Importing: ${filePath}`);
|
||||
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Unknown log type for: ${filePath}`);
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
let result;
|
||||
await db.transaction(async (txClient) => {
|
||||
result = await importFile(txClient, filePath, logType, parser);
|
||||
});
|
||||
|
||||
console.log(` Imported ${result.imported} of ${result.total} records`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Import multiple files (for batch incremental imports)
|
||||
async function importFiles(filePaths) {
|
||||
console.log(`\n========================================`);
|
||||
console.log(`Incremental Import: ${filePaths.length} files`);
|
||||
console.log(`========================================`);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
await db.transaction(async (txClient) => {
|
||||
for (const filePath of filePaths) {
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Skipping unknown type: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const { total, imported } = await importFile(txClient, filePath, logType, parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
console.log(` ${path.basename(filePath)}: ${imported}/${total} records`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
|
||||
// Export datasheets for newly imported records
|
||||
if (totalImported > 0) {
|
||||
try {
|
||||
const { loadAllSpecs } = require('../parsers/spec-reader');
|
||||
const { exportNewRecords } = require('./export-datasheets');
|
||||
const specMap = loadAllSpecs();
|
||||
await exportNewRecords(specMap, filePaths);
|
||||
} catch (err) {
|
||||
console.error(`[EXPORT] Datasheet export failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { total: totalRecords, imported: totalImported };
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length > 0 && args[0] === '--file') {
|
||||
const files = args.slice(1);
|
||||
if (files.length === 0) {
|
||||
console.log('Usage: node import.js --file <file1> [file2] ...');
|
||||
process.exit(1);
|
||||
}
|
||||
importFiles(files).then(() => db.close()).catch(console.error);
|
||||
} else if (args.length > 0 && args[0] === '--help') {
|
||||
console.log('Usage:');
|
||||
console.log(' node import.js Full import from all sources');
|
||||
console.log(' node import.js --file <f> Import specific file(s)');
|
||||
process.exit(0);
|
||||
} else {
|
||||
runImport().catch(console.error);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { runImport, importSingleFile, importFiles };
|
||||
@@ -0,0 +1,397 @@
|
||||
/**
|
||||
* Data Import Script
|
||||
* Imports test data from DAT and SHT files into SQLite database
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const { parseMultilineFile, extractTestStation } = require('../parsers/multiline');
|
||||
const { parseCsvFile } = require('../parsers/csvline');
|
||||
const { parseShtFile } = require('../parsers/shtfile');
|
||||
|
||||
// Configuration
|
||||
const DB_PATH = path.join(__dirname, 'testdata.db');
|
||||
const SCHEMA_PATH = path.join(__dirname, 'schema.sql');
|
||||
|
||||
// Data source paths
|
||||
const TEST_PATH = 'C:/Shares/test';
|
||||
const RECOVERY_PATH = 'C:/Shares/Recovery-TEST';
|
||||
const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS');
|
||||
|
||||
// Log types and their parsers
|
||||
const LOG_TYPES = {
|
||||
'DSCLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'5BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'8BLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'PWRLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'SCTLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'VASLOG': { parser: 'multiline', ext: '.DAT' },
|
||||
'7BLOG': { parser: 'csvline', ext: '.DAT' }
|
||||
};
|
||||
|
||||
// Initialize database
|
||||
function initDatabase() {
|
||||
console.log('Initializing database...');
|
||||
const db = new Database(DB_PATH);
|
||||
|
||||
// Read and execute schema
|
||||
const schema = fs.readFileSync(SCHEMA_PATH, 'utf8');
|
||||
db.exec(schema);
|
||||
|
||||
console.log('Database initialized.');
|
||||
return db;
|
||||
}
|
||||
|
||||
// Prepare insert statement
|
||||
// Uses INSERT OR REPLACE so re-tested devices keep the latest result
|
||||
// UNIQUE constraint: (log_type, model_number, serial_number, test_date, test_station)
|
||||
function prepareInsert(db) {
|
||||
return db.prepare(`
|
||||
INSERT OR REPLACE INTO test_records
|
||||
(log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
}
|
||||
|
||||
// Find all files of a specific type in a directory
|
||||
function findFiles(dir, pattern, recursive = true) {
|
||||
const results = [];
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const items = fs.readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item.name);
|
||||
|
||||
if (item.isDirectory() && recursive) {
|
||||
results.push(...findFiles(fullPath, pattern, recursive));
|
||||
} else if (item.isFile()) {
|
||||
if (pattern.test(item.name)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Ignore permission errors
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Import records from a file
|
||||
function importFile(db, insertStmt, filePath, logType, parser) {
|
||||
let records = [];
|
||||
const testStation = extractTestStation(filePath);
|
||||
|
||||
try {
|
||||
switch (parser) {
|
||||
case 'multiline':
|
||||
records = parseMultilineFile(filePath, logType, testStation);
|
||||
break;
|
||||
case 'csvline':
|
||||
records = parseCsvFile(filePath, testStation);
|
||||
break;
|
||||
case 'shtfile':
|
||||
records = parseShtFile(filePath, testStation);
|
||||
break;
|
||||
}
|
||||
|
||||
let imported = 0;
|
||||
for (const record of records) {
|
||||
try {
|
||||
const result = insertStmt.run(
|
||||
record.log_type,
|
||||
record.model_number,
|
||||
record.serial_number,
|
||||
record.test_date,
|
||||
record.test_station,
|
||||
record.overall_result,
|
||||
record.raw_data,
|
||||
record.source_file
|
||||
);
|
||||
if (result.changes > 0) imported++;
|
||||
} catch (err) {
|
||||
// Duplicate or constraint error - skip
|
||||
}
|
||||
}
|
||||
|
||||
return { total: records.length, imported };
|
||||
} catch (err) {
|
||||
console.error(`Error importing ${filePath}: ${err.message}`);
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// Import from HISTLOGS (master consolidated logs)
|
||||
function importHistlogs(db, insertStmt) {
|
||||
console.log('\n=== Importing from HISTLOGS ===');
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const logDir = path.join(HISTLOGS_PATH, logType);
|
||||
|
||||
if (!fs.existsSync(logDir)) {
|
||||
console.log(` ${logType}: directory not found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
|
||||
console.log(` ${logType}: found ${files.length} files`);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = importFile(db, insertStmt, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from test station logs
|
||||
function importStationLogs(db, insertStmt, basePath, label) {
|
||||
console.log(`\n=== Importing from ${label} ===`);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
// Find all test station directories (TS-1, TS-27, TS-8L, TS-10R, etc.)
|
||||
const stationPattern = /^TS-\d+[LR]?$/i;
|
||||
let stations = [];
|
||||
|
||||
try {
|
||||
const items = fs.readdirSync(basePath, { withFileTypes: true });
|
||||
stations = items
|
||||
.filter(i => i.isDirectory() && stationPattern.test(i.name))
|
||||
.map(i => i.name);
|
||||
} catch (err) {
|
||||
console.log(` Error reading ${basePath}: ${err.message}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(` Found stations: ${stations.join(', ')}`);
|
||||
|
||||
for (const station of stations) {
|
||||
const logsDir = path.join(basePath, station, 'LOGS');
|
||||
|
||||
if (!fs.existsSync(logsDir)) continue;
|
||||
|
||||
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
||||
const logDir = path.join(logsDir, logType);
|
||||
|
||||
if (!fs.existsSync(logDir)) continue;
|
||||
|
||||
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), false);
|
||||
|
||||
for (const file of files) {
|
||||
const { total, imported } = importFile(db, insertStmt, file, logType, config.parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also import SHT files
|
||||
const shtFiles = findFiles(basePath, /\.SHT$/i, true);
|
||||
console.log(` Found ${shtFiles.length} SHT files`);
|
||||
|
||||
for (const file of shtFiles) {
|
||||
const { total, imported } = importFile(db, insertStmt, file, 'SHT', 'shtfile');
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Import from Recovery-TEST backups (newest first)
|
||||
function importRecoveryBackups(db, insertStmt) {
|
||||
console.log('\n=== Importing from Recovery-TEST backups ===');
|
||||
|
||||
if (!fs.existsSync(RECOVERY_PATH)) {
|
||||
console.log(' Recovery-TEST directory not found');
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get backup dates, sort newest first
|
||||
const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true })
|
||||
.filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name))
|
||||
.map(i => i.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
|
||||
console.log(` Found backup dates: ${backups.join(', ')}`);
|
||||
|
||||
let totalImported = 0;
|
||||
|
||||
for (const backup of backups) {
|
||||
const backupPath = path.join(RECOVERY_PATH, backup);
|
||||
const imported = importStationLogs(db, insertStmt, backupPath, `Recovery-TEST/${backup}`);
|
||||
totalImported += imported;
|
||||
}
|
||||
|
||||
return totalImported;
|
||||
}
|
||||
|
||||
// Main import function
|
||||
async function runImport() {
|
||||
console.log('========================================');
|
||||
console.log('Test Data Import');
|
||||
console.log('========================================');
|
||||
console.log(`Database: ${DB_PATH}`);
|
||||
console.log(`Start time: ${new Date().toISOString()}`);
|
||||
|
||||
const db = initDatabase();
|
||||
const insertStmt = prepareInsert(db);
|
||||
|
||||
let grandTotal = 0;
|
||||
|
||||
// Use transaction for performance
|
||||
const importAll = db.transaction(() => {
|
||||
// 1. Import HISTLOGS first (authoritative)
|
||||
grandTotal += importHistlogs(db, insertStmt);
|
||||
|
||||
// 2. Import Recovery backups (newest first)
|
||||
grandTotal += importRecoveryBackups(db, insertStmt);
|
||||
|
||||
// 3. Import current test folder
|
||||
grandTotal += importStationLogs(db, insertStmt, TEST_PATH, 'test');
|
||||
});
|
||||
|
||||
importAll();
|
||||
|
||||
// Get final stats
|
||||
const stats = db.prepare('SELECT COUNT(*) as count FROM test_records').get();
|
||||
|
||||
console.log('\n========================================');
|
||||
console.log('Import Complete');
|
||||
console.log('========================================');
|
||||
console.log(`Total records in database: ${stats.count}`);
|
||||
console.log(`End time: ${new Date().toISOString()}`);
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
// Import a single file (for incremental imports from sync)
|
||||
function importSingleFile(filePath) {
|
||||
console.log(`Importing: ${filePath}`);
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
const insertStmt = prepareInsert(db);
|
||||
|
||||
// Determine log type from path
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
// Check for SHT files
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Unknown log type for: ${filePath}`);
|
||||
db.close();
|
||||
return { total: 0, imported: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
const result = importFile(db, insertStmt, filePath, logType, parser);
|
||||
|
||||
console.log(` Imported ${result.imported} of ${result.total} records`);
|
||||
db.close();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Import multiple files (for batch incremental imports)
|
||||
function importFiles(filePaths) {
|
||||
console.log(`\n========================================`);
|
||||
console.log(`Incremental Import: ${filePaths.length} files`);
|
||||
console.log(`========================================`);
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
const insertStmt = prepareInsert(db);
|
||||
|
||||
let totalImported = 0;
|
||||
let totalRecords = 0;
|
||||
|
||||
const importBatch = db.transaction(() => {
|
||||
for (const filePath of filePaths) {
|
||||
// Determine log type from path
|
||||
let logType = null;
|
||||
let parser = null;
|
||||
|
||||
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
||||
if (filePath.includes(type)) {
|
||||
logType = type;
|
||||
parser = config.parser;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!logType) {
|
||||
if (/\.SHT$/i.test(filePath)) {
|
||||
logType = 'SHT';
|
||||
parser = 'shtfile';
|
||||
} else {
|
||||
console.log(` Skipping unknown type: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const { total, imported } = importFile(db, insertStmt, filePath, logType, parser);
|
||||
totalRecords += total;
|
||||
totalImported += imported;
|
||||
console.log(` ${path.basename(filePath)}: ${imported}/${total} records`);
|
||||
}
|
||||
});
|
||||
|
||||
importBatch();
|
||||
|
||||
console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`);
|
||||
db.close();
|
||||
|
||||
return { total: totalRecords, imported: totalImported };
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
// Check for command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length > 0 && args[0] === '--file') {
|
||||
// Import specific file(s)
|
||||
const files = args.slice(1);
|
||||
if (files.length === 0) {
|
||||
console.log('Usage: node import.js --file <file1> [file2] ...');
|
||||
process.exit(1);
|
||||
}
|
||||
importFiles(files);
|
||||
} else if (args.length > 0 && args[0] === '--help') {
|
||||
console.log('Usage:');
|
||||
console.log(' node import.js Full import from all sources');
|
||||
console.log(' node import.js --file <f> Import specific file(s)');
|
||||
process.exit(0);
|
||||
} else {
|
||||
// Full import
|
||||
runImport().catch(console.error);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { runImport, importSingleFile, importFiles };
|
||||
@@ -0,0 +1,260 @@
|
||||
/**
|
||||
* SQLite to PostgreSQL Data Migration
|
||||
*
|
||||
* Streams all data from the SQLite testdata.db into PostgreSQL.
|
||||
* Uses batch INSERTs for performance.
|
||||
*
|
||||
* Usage:
|
||||
* node migrate-data.js Migrate all tables
|
||||
* node migrate-data.js --skip-tsvector Skip tsvector rebuild (faster, trigger handles it)
|
||||
* node migrate-data.js --table test_records Migrate only one table
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
const db = require('./db');
|
||||
|
||||
const SQLITE_PATH = path.join(__dirname, 'testdata.db');
|
||||
const BATCH_SIZE = 5000;
|
||||
|
||||
async function migrateTestRecords(sqlite) {
|
||||
console.log('\n--- Migrating test_records ---');
|
||||
|
||||
const total = sqlite.prepare('SELECT COUNT(*) as cnt FROM test_records').get().cnt;
|
||||
console.log(` Source records: ${total.toLocaleString()}`);
|
||||
|
||||
// Disable triggers during bulk load for performance
|
||||
await db.execute('ALTER TABLE test_records DISABLE TRIGGER trg_search_vector');
|
||||
|
||||
const stmt = sqlite.prepare('SELECT * FROM test_records ORDER BY id');
|
||||
let migrated = 0;
|
||||
let batch = [];
|
||||
|
||||
for (const row of stmt.iterate()) {
|
||||
batch.push(row);
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
await insertTestRecordsBatch(batch);
|
||||
migrated += batch.length;
|
||||
batch = [];
|
||||
process.stdout.write(`\r Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if (batch.length > 0) {
|
||||
await insertTestRecordsBatch(batch);
|
||||
migrated += batch.length;
|
||||
}
|
||||
|
||||
console.log(`\r Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
|
||||
|
||||
// Rebuild search_vector for all rows
|
||||
console.log(' Rebuilding search_vector (this may take a few minutes)...');
|
||||
await db.execute(`
|
||||
UPDATE test_records SET search_vector = to_tsvector('english',
|
||||
COALESCE(serial_number, '') || ' ' ||
|
||||
COALESCE(model_number, '') || ' ' ||
|
||||
COALESCE(raw_data, '')
|
||||
)
|
||||
`);
|
||||
console.log(' search_vector rebuilt.');
|
||||
|
||||
// Re-enable trigger
|
||||
await db.execute('ALTER TABLE test_records ENABLE TRIGGER trg_search_vector');
|
||||
|
||||
// Reset sequence to max id
|
||||
await db.execute(`SELECT setval('test_records_id_seq', (SELECT COALESCE(MAX(id), 1) FROM test_records))`);
|
||||
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function insertTestRecordsBatch(batch) {
|
||||
// Build a multi-row INSERT
|
||||
const cols = ['id', 'log_type', 'model_number', 'serial_number', 'test_date',
|
||||
'test_station', 'overall_result', 'raw_data', 'source_file',
|
||||
'import_date', 'datasheet_exported_at', 'forweb_exported_at', 'work_order'];
|
||||
|
||||
const values = [];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
for (const row of batch) {
|
||||
const placeholders = cols.map(() => {
|
||||
paramIdx++;
|
||||
return `$${paramIdx}`;
|
||||
});
|
||||
values.push(`(${placeholders.join(',')})`);
|
||||
|
||||
params.push(
|
||||
row.id,
|
||||
row.log_type,
|
||||
row.model_number,
|
||||
row.serial_number,
|
||||
row.test_date,
|
||||
row.test_station,
|
||||
row.overall_result,
|
||||
row.raw_data,
|
||||
row.source_file,
|
||||
row.import_date,
|
||||
row.datasheet_exported_at,
|
||||
row.forweb_exported_at,
|
||||
row.work_order
|
||||
);
|
||||
}
|
||||
|
||||
const sql = `INSERT INTO test_records (${cols.join(',')})
|
||||
VALUES ${values.join(',')}
|
||||
ON CONFLICT (log_type, model_number, serial_number, test_date, test_station)
|
||||
DO NOTHING`;
|
||||
|
||||
await db.execute(sql, params);
|
||||
}
|
||||
|
||||
async function migrateWorkOrders(sqlite) {
|
||||
console.log('\n--- Migrating work_orders ---');
|
||||
|
||||
const rows = sqlite.prepare('SELECT * FROM work_orders ORDER BY id').all();
|
||||
console.log(` Source records: ${rows.length.toLocaleString()}`);
|
||||
|
||||
let migrated = 0;
|
||||
|
||||
const cols = ['wo_number', 'wo_date', 'program', 'version',
|
||||
'lib_version', 'test_station', 'source_file', 'import_date'];
|
||||
|
||||
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
||||
const batch = rows.slice(i, i + BATCH_SIZE);
|
||||
const values = [];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
for (const row of batch) {
|
||||
const placeholders = cols.map(() => { paramIdx++; return `$${paramIdx}`; });
|
||||
values.push(`(${placeholders.join(',')})`);
|
||||
params.push(row.wo_number, row.wo_date, row.program, row.version,
|
||||
row.lib_version, row.test_station, row.source_file, row.import_date);
|
||||
}
|
||||
|
||||
await db.execute(
|
||||
`INSERT INTO work_orders (${cols.join(',')}) VALUES ${values.join(',')}
|
||||
ON CONFLICT (wo_number, test_station) DO NOTHING`,
|
||||
params
|
||||
);
|
||||
migrated += batch.length;
|
||||
}
|
||||
|
||||
console.log(` Migrated: ${migrated.toLocaleString()}`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function migrateWorkOrderLines(sqlite) {
|
||||
console.log('\n--- Migrating work_order_lines ---');
|
||||
|
||||
const total = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_order_lines').get().cnt;
|
||||
console.log(` Source records: ${total.toLocaleString()}`);
|
||||
|
||||
const stmt = sqlite.prepare('SELECT * FROM work_order_lines ORDER BY id');
|
||||
let migrated = 0;
|
||||
let batch = [];
|
||||
|
||||
for (const row of stmt.iterate()) {
|
||||
batch.push(row);
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
await insertWoLinesBatch(batch);
|
||||
migrated += batch.length;
|
||||
batch = [];
|
||||
process.stdout.write(`\r Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.length > 0) {
|
||||
await insertWoLinesBatch(batch);
|
||||
migrated += batch.length;
|
||||
}
|
||||
|
||||
console.log(`\r Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function insertWoLinesBatch(batch) {
|
||||
const cols = ['wo_number', 'serial_number', 'status', 'model_number',
|
||||
'ds_filename', 'test_date', 'test_time', 'test_station'];
|
||||
const values = [];
|
||||
const params = [];
|
||||
let paramIdx = 0;
|
||||
|
||||
for (const row of batch) {
|
||||
const placeholders = cols.map(() => { paramIdx++; return `$${paramIdx}`; });
|
||||
values.push(`(${placeholders.join(',')})`);
|
||||
params.push(row.wo_number, row.serial_number, row.status,
|
||||
row.model_number, row.ds_filename, row.test_date, row.test_time, row.test_station);
|
||||
}
|
||||
|
||||
await db.execute(
|
||||
`INSERT INTO work_order_lines (${cols.join(',')}) VALUES ${values.join(',')}
|
||||
ON CONFLICT (wo_number, serial_number, test_date, test_time) DO NOTHING`,
|
||||
params
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const tableArg = args.indexOf('--table');
|
||||
const targetTable = tableArg >= 0 ? args[tableArg + 1] : null;
|
||||
|
||||
console.log('========================================');
|
||||
console.log('SQLite -> PostgreSQL Data Migration');
|
||||
console.log('========================================');
|
||||
console.log(`SQLite: ${SQLITE_PATH}`);
|
||||
console.log(`Start: ${new Date().toISOString()}`);
|
||||
|
||||
const sqlite = new Database(SQLITE_PATH, { readonly: true });
|
||||
|
||||
try {
|
||||
if (!targetTable || targetTable === 'test_records') {
|
||||
await migrateTestRecords(sqlite);
|
||||
}
|
||||
if (!targetTable || targetTable === 'work_orders') {
|
||||
await migrateWorkOrders(sqlite);
|
||||
}
|
||||
if (!targetTable || targetTable === 'work_order_lines') {
|
||||
await migrateWorkOrderLines(sqlite);
|
||||
}
|
||||
|
||||
// VACUUM ANALYZE
|
||||
console.log('\n--- Running VACUUM ANALYZE ---');
|
||||
await db.execute('VACUUM ANALYZE test_records');
|
||||
await db.execute('VACUUM ANALYZE work_orders');
|
||||
await db.execute('VACUUM ANALYZE work_order_lines');
|
||||
console.log(' Done.');
|
||||
|
||||
// Verify counts
|
||||
console.log('\n--- Verification ---');
|
||||
const pgTestCount = await db.queryOne('SELECT COUNT(*) as cnt FROM test_records');
|
||||
const pgWoCount = await db.queryOne('SELECT COUNT(*) as cnt FROM work_orders');
|
||||
const pgWolCount = await db.queryOne('SELECT COUNT(*) as cnt FROM work_order_lines');
|
||||
|
||||
const sqliteTestCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM test_records').get().cnt;
|
||||
const sqliteWoCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_orders').get().cnt;
|
||||
const sqliteWolCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_order_lines').get().cnt;
|
||||
|
||||
console.log(` test_records: SQLite=${sqliteTestCount.toLocaleString()} PG=${parseInt(pgTestCount.cnt).toLocaleString()} ${parseInt(pgTestCount.cnt) === sqliteTestCount ? '[OK]' : '[MISMATCH]'}`);
|
||||
console.log(` work_orders: SQLite=${sqliteWoCount.toLocaleString()} PG=${parseInt(pgWoCount.cnt).toLocaleString()} ${parseInt(pgWoCount.cnt) === sqliteWoCount ? '[OK]' : '[MISMATCH]'}`);
|
||||
console.log(` work_order_lines: SQLite=${sqliteWolCount.toLocaleString()} PG=${parseInt(pgWolCount.cnt).toLocaleString()} ${parseInt(pgWolCount.cnt) === sqliteWolCount ? '[OK]' : '[MISMATCH]'}`);
|
||||
|
||||
} finally {
|
||||
sqlite.close();
|
||||
await db.close();
|
||||
}
|
||||
|
||||
console.log(`\n========================================`);
|
||||
console.log(`Migration Complete`);
|
||||
console.log(`========================================`);
|
||||
console.log(`End: ${new Date().toISOString()}`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Migration failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
-- TestDataDB PostgreSQL Schema
|
||||
-- Migrated from SQLite schema.sql
|
||||
-- PostgreSQL 18 on AD2 (192.168.0.6)
|
||||
|
||||
-- Main test records table
|
||||
CREATE TABLE IF NOT EXISTS test_records (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
log_type TEXT NOT NULL,
|
||||
model_number TEXT NOT NULL,
|
||||
serial_number TEXT NOT NULL,
|
||||
test_date TEXT NOT NULL,
|
||||
test_station TEXT,
|
||||
overall_result TEXT,
|
||||
raw_data TEXT,
|
||||
source_file TEXT,
|
||||
import_date TIMESTAMPTZ DEFAULT NOW(),
|
||||
datasheet_exported_at TIMESTAMPTZ DEFAULT NULL,
|
||||
forweb_exported_at TIMESTAMPTZ DEFAULT NULL,
|
||||
work_order TEXT DEFAULT NULL,
|
||||
search_vector tsvector,
|
||||
UNIQUE(log_type, model_number, serial_number, test_date, test_station)
|
||||
);
|
||||
|
||||
-- Indexes for fast searching
|
||||
CREATE INDEX IF NOT EXISTS idx_serial ON test_records(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_model ON test_records(model_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_date ON test_records(test_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_serial ON test_records(model_number, serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_result ON test_records(overall_result);
|
||||
CREATE INDEX IF NOT EXISTS idx_log_type ON test_records(log_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_test_wo ON test_records(work_order);
|
||||
|
||||
-- Partial index for unexported PASS records (speeds up export queries)
|
||||
CREATE INDEX IF NOT EXISTS idx_unexported_pass ON test_records(overall_result, forweb_exported_at)
|
||||
WHERE overall_result = 'PASS' AND forweb_exported_at IS NULL;
|
||||
|
||||
-- GIN index for full-text search (replaces SQLite FTS5 virtual table)
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector ON test_records USING GIN(search_vector);
|
||||
|
||||
-- Trigger function to maintain search_vector on INSERT/UPDATE
|
||||
CREATE OR REPLACE FUNCTION update_search_vector() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector := to_tsvector('english',
|
||||
COALESCE(NEW.serial_number, '') || ' ' ||
|
||||
COALESCE(NEW.model_number, '') || ' ' ||
|
||||
COALESCE(NEW.raw_data, '')
|
||||
);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Drop trigger if exists, then create
|
||||
DROP TRIGGER IF EXISTS trg_search_vector ON test_records;
|
||||
CREATE TRIGGER trg_search_vector
|
||||
BEFORE INSERT OR UPDATE ON test_records
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_search_vector();
|
||||
|
||||
-- Work orders table
|
||||
CREATE TABLE IF NOT EXISTS work_orders (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
wo_number TEXT NOT NULL,
|
||||
wo_date TEXT,
|
||||
program TEXT,
|
||||
version TEXT,
|
||||
lib_version TEXT,
|
||||
test_station TEXT,
|
||||
source_file TEXT,
|
||||
import_date TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(wo_number, test_station)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wo_number ON work_orders(wo_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_wo_station ON work_orders(test_station);
|
||||
|
||||
-- Work order lines table
|
||||
CREATE TABLE IF NOT EXISTS work_order_lines (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
wo_number TEXT NOT NULL,
|
||||
serial_number TEXT NOT NULL,
|
||||
status TEXT,
|
||||
model_number TEXT,
|
||||
ds_filename TEXT,
|
||||
test_date TEXT,
|
||||
test_time TEXT,
|
||||
test_station TEXT,
|
||||
UNIQUE(wo_number, serial_number, test_date, test_time)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wol_wo ON work_order_lines(wo_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_wol_serial ON work_order_lines(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_wol_model ON work_order_lines(model_number);
|
||||
|
||||
-- Grant permissions to app role
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO testdatadb_app;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO testdatadb_app;
|
||||
@@ -0,0 +1,54 @@
|
||||
-- Test Data Database Schema
|
||||
-- SQLite database for storing and searching test records
|
||||
|
||||
-- Main test records table
|
||||
CREATE TABLE IF NOT EXISTS test_records (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
log_type TEXT NOT NULL, -- DSCLOG, 5BLOG, 7BLOG, 8BLOG, PWRLOG, SCTLOG, VASLOG, SHT
|
||||
model_number TEXT NOT NULL, -- DSCA38-1793, SCM5B30-01, etc.
|
||||
serial_number TEXT NOT NULL, -- 176923-1, 105840-2, etc.
|
||||
test_date TEXT NOT NULL, -- Test date (YYYY-MM-DD format)
|
||||
test_station TEXT, -- TS-1L, TS-3R, etc.
|
||||
overall_result TEXT, -- PASS/FAIL
|
||||
raw_data TEXT, -- Full original record
|
||||
source_file TEXT, -- Original file path
|
||||
import_date TEXT DEFAULT (datetime('now')),
|
||||
datasheet_exported_at TEXT DEFAULT NULL,
|
||||
forweb_exported_at TEXT DEFAULT NULL,
|
||||
UNIQUE(log_type, model_number, serial_number, test_date, test_station)
|
||||
);
|
||||
|
||||
-- Indexes for fast searching
|
||||
CREATE INDEX IF NOT EXISTS idx_serial ON test_records(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_model ON test_records(model_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_date ON test_records(test_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_model_serial ON test_records(model_number, serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_result ON test_records(overall_result);
|
||||
CREATE INDEX IF NOT EXISTS idx_log_type ON test_records(log_type);
|
||||
|
||||
-- Full-text search virtual table
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS test_records_fts USING fts5(
|
||||
serial_number,
|
||||
model_number,
|
||||
raw_data,
|
||||
content='test_records',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
-- Triggers to keep FTS index in sync
|
||||
CREATE TRIGGER IF NOT EXISTS test_records_ai AFTER INSERT ON test_records BEGIN
|
||||
INSERT INTO test_records_fts(rowid, serial_number, model_number, raw_data)
|
||||
VALUES (new.id, new.serial_number, new.model_number, new.raw_data);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS test_records_ad AFTER DELETE ON test_records BEGIN
|
||||
INSERT INTO test_records_fts(test_records_fts, rowid, serial_number, model_number, raw_data)
|
||||
VALUES ('delete', old.id, old.serial_number, old.model_number, old.raw_data);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS test_records_au AFTER UPDATE ON test_records BEGIN
|
||||
INSERT INTO test_records_fts(test_records_fts, rowid, serial_number, model_number, raw_data)
|
||||
VALUES ('delete', old.id, old.serial_number, old.model_number, old.raw_data);
|
||||
INSERT INTO test_records_fts(rowid, serial_number, model_number, raw_data)
|
||||
VALUES (new.id, new.serial_number, new.model_number, new.raw_data);
|
||||
END;
|
||||
Reference in New Issue
Block a user